Skip to content

Commit ae1f91a

Browse files
CohenArthurCohenArthur
committed
transcribe: Move substitute_metavar in its own function
1 parent 265c223 commit ae1f91a

File tree

3 files changed

+166
-47
lines changed

3 files changed

+166
-47
lines changed

‎gcc/rust/ast/rust-ast.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,7 @@ class Token : public TokenTree, public MacroMatch
204204
std::vector<std::unique_ptr<Token> > to_token_stream () const override;
205205

206206
TokenId get_id () const { return tok_ref->get_id (); }
207+
const std::string &get_str () const { return tok_ref->get_str (); }
207208

208209
Location get_locus () const { return tok_ref->get_locus (); }
209210

‎gcc/rust/expand/rust-macro-expand.cc

Lines changed: 131 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -3754,10 +3754,10 @@ MacroExpander::transcribe_rule (
37543754
= substitute_tokens (invoc_stream, macro_rule_tokens, matched_fragments);
37553755

37563756
// // handy for debugging
3757-
// for (auto &tok : substituted_tokens)
3758-
// {
3759-
// rust_debug ("tok: [%s]", tok->as_string ().c_str ());
3760-
// }
3757+
for (auto &tok : substituted_tokens)
3758+
{
3759+
rust_debug ("tok: [%s]", tok->as_string ().c_str ());
3760+
}
37613761

37623762
// parse it to an ASTFragment
37633763
MacroInvocLexer lex (std::move (substituted_tokens));
@@ -3874,6 +3874,67 @@ MacroExpander::transcribe_rule (
38743874
return AST::ASTFragment (std::move (nodes));
38753875
}
38763876

3877+
std::vector<std::unique_ptr<AST::Token>>
3878+
MacroExpander::substitute_metavar (
3879+
std::vector<std::unique_ptr<AST::Token>> &input,
3880+
std::map<std::string, MatchedFragment> &fragments,
3881+
std::unique_ptr<AST::Token> &metavar)
3882+
{
3883+
auto metavar_name = metavar->get_str ();
3884+
3885+
rust_debug ("expanding metavar: %s", metavar_name.c_str ());
3886+
std::vector<std::unique_ptr<AST::Token>> expanded;
3887+
auto it = fragments.find (metavar_name);
3888+
if (it == fragments.end ())
3889+
{
3890+
// Return a copy of the original token
3891+
expanded.push_back (metavar->clone_token ());
3892+
}
3893+
else
3894+
{
3895+
// Replace
3896+
MatchedFragment &frag = it->second;
3897+
for (size_t offs = frag.token_offset_begin; offs < frag.token_offset_end;
3898+
offs++)
3899+
{
3900+
auto &tok = input.at (offs);
3901+
expanded.push_back (tok->clone_token ());
3902+
}
3903+
}
3904+
3905+
return expanded;
3906+
}
3907+
3908+
std::pair<std::vector<std::unique_ptr<AST::Token>>, size_t>
3909+
MacroExpander::substitute_token (
3910+
std::vector<std::unique_ptr<AST::Token>> &input,
3911+
std::map<std::string, MatchedFragment> &fragments,
3912+
std::unique_ptr<AST::Token> &token)
3913+
{
3914+
switch (token->get_id ())
3915+
{
3916+
case IDENTIFIER:
3917+
rust_debug ("expanding metavar");
3918+
return {substitute_metavar (input, fragments, token), 1};
3919+
case LEFT_PAREN:
3920+
rust_debug ("expanding repetition");
3921+
break;
3922+
// TODO: We need to check if the $ was alone. In that case, do
3923+
// not error out: Simply act as if there was an empty identifier
3924+
// with no associated fragment and paste the dollar sign in the
3925+
// transcription. Unsure how to do that since we always have at
3926+
// least the closing curly brace after an empty $...
3927+
default:
3928+
rust_error_at (token->get_locus (),
3929+
"unexpected token in macro transcribe: expected "
3930+
"%<(%> or identifier after %<$%>, got %<%s%>",
3931+
get_token_description (token->get_id ()));
3932+
}
3933+
3934+
// FIXME: gcc_unreachable() error case?
3935+
return {std::vector<std::unique_ptr<AST::Token>> (), 0};
3936+
}
3937+
38773938
std::vector<std::unique_ptr<AST::Token>>
38783939
MacroExpander::substitute_tokens (
38793940
std::vector<std::unique_ptr<AST::Token>> &input,
@@ -3882,60 +3943,83 @@ MacroExpander::substitute_tokens (
38823943
{
38833944
std::vector<std::unique_ptr<AST::Token>> replaced_tokens;
38843945

3946+
// for token in macro
3947+
// if token == ?:
3948+
// // That's not always true: If it's a left paren, it's repetition
3949+
// // We probably want to store the matched amount in the fragment so
3950+
// // we can expand it here
3951+
// id = next_token();
3952+
// frag = fragment.find(id);
3953+
38853954
for (size_t i = 0; i < macro.size (); i++)
38863955
{
38873956
auto &tok = macro.at (i);
38883957
if (tok->get_id () == DOLLAR_SIGN)
38893958
{
3890-
std::vector<std::unique_ptr<AST::Token>> parsed_toks;
3959+
auto &next_tok = macro.at (i + 1);
3960+
// Aaaaah, if only we had C++17 :)
3961+
// auto [expanded, tok_to_skip] = ...
3962+
auto p = substitute_token (input, fragments, next_tok);
3963+
auto expanded = std::move (p.first);
3964+
auto tok_to_skip = p.second;
38913965

3892-
std::string ident;
3893-
for (size_t offs = i; i < macro.size (); offs++)
3894-
{
3895-
auto &tok = macro.at (offs);
3896-
if (tok->get_id () == DOLLAR_SIGN && offs == i)
3897-
{
3898-
parsed_toks.push_back (tok->clone_token ());
3899-
}
3900-
else if (tok->get_id () == IDENTIFIER)
3901-
{
3902-
rust_assert (tok->as_string ().size () == 1);
3903-
ident.push_back (tok->as_string ().at (0));
3904-
parsed_toks.push_back (tok->clone_token ());
3905-
}
3906-
else
3907-
{
3908-
break;
3909-
}
3910-
}
3966+
i += tok_to_skip;
39113967

3912-
// lookup the ident
3913-
auto it = fragments.find (ident);
3914-
if (it == fragments.end ())
3915-
{
3916-
// just leave the tokens in
3917-
for (auto &tok : parsed_toks)
3918-
{
3919-
replaced_tokens.push_back (tok->clone_token ());
3920-
}
3921-
}
3922-
else
3923-
{
3924-
// replace
3925-
MatchedFragment &frag = it->second;
3926-
for (size_t offs = frag.token_offset_begin;
3927-
offs < frag.token_offset_end; offs++)
3928-
{
3929-
auto &tok = input.at (offs);
3930-
replaced_tokens.push_back (tok->clone_token ());
3931-
}
3932-
}
3933-
i += parsed_toks.size () - 1;
3968+
for (auto &token : expanded)
3969+
replaced_tokens.emplace_back (token->clone_token ());
39343970
}
39353971
else
39363972
{
3937-
replaced_tokens.push_back (tok->clone_token ());
3973+
replaced_tokens.emplace_back (tok->clone_token ());
39383974
}
3975+
3976+
// std::vector<std::unique_ptr<AST::Token>> parsed_toks;
3977+
3978+
// std::string ident;
3979+
// for (size_t offs = i; i < macro.size (); offs++)
3980+
// {
3981+
// auto &tok = macro.at (offs);
3982+
// if (tok->get_id () == DOLLAR_SIGN && offs == i)
3983+
// {
3984+
// parsed_toks.push_back (tok->clone_token ());
3985+
// }
3986+
// else if (tok->get_id () == IDENTIFIER)
3987+
// {
3988+
// rust_assert (tok->as_string ().size () == 1);
3989+
// ident.push_back (tok->as_string ().at (0));
3990+
// parsed_toks.push_back (tok->clone_token ());
3991+
// }
3992+
// else
3993+
// {
3994+
// break;
3995+
// }
3996+
// }
3997+
3998+
// // lookup the ident
3999+
// auto it = fragments.find (ident);
4000+
// if (it == fragments.end ())
4001+
// {
4002+
// // just leave the tokens in
4003+
// for (auto &tok : parsed_toks)
4004+
// {
4005+
// replaced_tokens.push_back (tok->clone_token ());
4006+
// }
4007+
// }
4008+
// else
4009+
// {
4010+
// // replace
4011+
// MatchedFragment &frag = it->second;
4012+
// for (size_t offs = frag.token_offset_begin;
4013+
// offs < frag.token_offset_end; offs++)
4014+
// {
4015+
// auto &tok = input.at (offs);
4016+
// replaced_tokens.push_back (tok->clone_token ());
4017+
// }
4018+
// }
4019+
// i += parsed_toks.size () - 1;
4020+
//
4021+
// }
4022+
// else { replaced_tokens.push_back (tok->clone_token ()); }
39394023
}
39404024

39414025
return replaced_tokens;

‎gcc/rust/expand/rust-macro-expand.h

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,40 @@ struct MacroExpander
183183
size_t &match_amount, size_t lo_bound = 0,
184184
size_t hi_bound = 0);
185185

186+
/**
187+
* Substitute a metavariable by its given fragment in a transcribing context,
188+
* i.e. replacing $var with the associated fragment.
189+
*
190+
* @param input Tokens given to the transcribing context
191+
* @param fragments Fragments given to the macro substitution
192+
* @param metavar Metavariable to try and replace
193+
*
194+
* @return A token containing the associated fragment expanded into tokens if
195+
* any, or the cloned token if no fragment was associated
196+
*/
197+
static std::vector<std::unique_ptr<AST::Token>>
198+
substitute_metavar (std::vector<std::unique_ptr<AST::Token>> &input,
199+
std::map<std::string, MatchedFragment> &fragments,
200+
std::unique_ptr<AST::Token> &metavar);
201+
202+
/**
203+
* Substitute a given token by its appropriate representation
204+
*
205+
* @param input Tokens given to the transcribing context
206+
* @param fragments Fragments given to the macro substitution
207+
* @param token Current token to try and substitute
208+
*
209+
* @return A token containing the associated fragment expanded into tokens if
210+
* any, or the cloned token if no fragment was associated, as well as the
211+
* amount of tokens that should be skipped before the next invocation. Since
212+
* this function may consume more than just one token, it is important to skip
213+
* ahead of the input to avoid mis-substitutions
214+
*/
215+
static std::pair<std::vector<std::unique_ptr<AST::Token>>, size_t>
216+
substitute_token (std::vector<std::unique_ptr<AST::Token>> &input,
217+
std::map<std::string, MatchedFragment> &fragments,
218+
std::unique_ptr<AST::Token> &token);
219+
186220
static std::vector<std::unique_ptr<AST::Token>>
187221
substitute_tokens (std::vector<std::unique_ptr<AST::Token>> &input,
188222
std::vector<std::unique_ptr<AST::Token>> &macro,

0 commit comments

Comments
 (0)