Add initial support for macro expansion

This is the first pass at implementing macros more testcases are needed.

This does not support repetition matchers but it supports simple
declarative macros and transcribes them. The approach taken here is that
we reuse our existing parser to call the apropriate functions as specified
as part of the MacroFragmentType enum if the parser does not have errors
parsing that item then it must be a match.

Then once we match a rule we have a map of the token begin/end offsets
for each fragment match, this is then used to adjust and create a new token
stream for the macro rule definition so that when we feed it to the parser
the tokens are already substituted. The resulting expression or item is
then attached to the respective macro invocation and this is then name
resolved and used for hir lowering.

Fixes #17 #22
Addresses #573

Author: philberty

gcc/rust/ast/rust-ast-full-test.cc

@@ -4390,7 +4390,8 @@ DelimTokenTree::to_token_stream () const
 
   // simulate presence of delimiters
   const_TokenPtr left_paren
-    = Rust::Token::make (LEFT_PAREN, Linemap::unknown_location ());
+    = Rust::Token::make (left_delim_type_tok_token_id (delim_type),
+			 Linemap::unknown_location ());
   tokens.push_back (
     std::unique_ptr<Token> (new Token (std::move (left_paren))));
 
@@ -4403,7 +4404,8 @@ DelimTokenTree::to_token_stream () const
     }
 
   const_TokenPtr right_paren
-    = Rust::Token::make (RIGHT_PAREN, Linemap::unknown_location ());
+    = Rust::Token::make (right_delim_type_tok_token_id (delim_type),
+			 Linemap::unknown_location ());
   tokens.push_back (
     std::unique_ptr<Token> (new Token (std::move (right_paren))));
 

gcc/rust/ast/rust-ast.h

@@ -22,17 +22,13 @@
 
 #include "rust-system.h"
 #include "rust-hir-map.h"
-
-// gccrs imports
-// required for AST::Token
 #include "rust-token.h"
 #include "rust-location.h"
 
 namespace Rust {
 // TODO: remove typedefs and make actual types for these
 typedef std::string Identifier;
 typedef int TupleIndex;
-
 struct Session;
 
 namespace AST {
@@ -48,34 +44,6 @@ enum DelimType
   CURLY
 };
 
-// Base AST node object - TODO is this really required or useful? Where to draw
-// line?
-/*class Node {
-  public:
-    // Gets node's Location.
-    Location get_locus() const {
-	return loc;
-    }
-
-    // Sets node's Location.
-    void set_locus(Location loc_) {
-	loc = loc_;
-    }
-
-    // Get node output as a string. Pure virtual.
-    virtual std::string as_string() const = 0;
-
-    virtual ~Node() {}
-
-    // TODO: constructor including Location? Make all derived classes have
-Location?
-
-  private:
-    // The node's location.
-    Location loc;
-};*/
-// decided to not have node as a "node" would never need to be stored
-
 // forward decl for use in token tree method
 class Token;
 
@@ -108,6 +76,14 @@ class TokenTree
 class MacroMatch
 {
 public:
+  enum MacroMatchType
+  {
+    Fragment,
+    Repetition,
+    Matcher,
+    Tok
+  };
+
   virtual ~MacroMatch () {}
 
   virtual std::string as_string () const = 0;
@@ -121,6 +97,8 @@ class MacroMatch
 
   virtual void accept_vis (ASTVisitor &vis) = 0;
 
+  virtual MacroMatchType get_macro_match_type () const = 0;
+
 protected:
   // pure virtual clone implementation
   virtual MacroMatch *clone_macro_match_impl () const = 0;
@@ -234,6 +212,11 @@ class Token : public TokenTree, public MacroMatch
   // Get a new token pointer copy.
   const_TokenPtr get_tok_ptr () const { return tok_ref; }
 
+  MacroMatchType get_macro_match_type () const override
+  {
+    return MacroMatchType::Tok;
+  }
+
 protected:
   // No virtual for now as not polymorphic but can be in future
   /*virtual*/ Token *clone_token_impl () const { return new Token (*this); }
@@ -788,6 +771,43 @@ class DelimTokenTree : public TokenTree, public AttrInput
   {
     return AttrInput::AttrInputType::TOKEN_TREE;
   }
+
+  std::vector<std::unique_ptr<TokenTree> > &get_token_trees ()
+  {
+    return token_trees;
+  }
+
+  DelimType get_delim_type () const { return delim_type; }
+
+  static TokenId left_delim_type_tok_token_id (DelimType delim_type)
+  {
+    switch (delim_type)
+      {
+      case PARENS:
+	return LEFT_PAREN;
+      case SQUARE:
+	return LEFT_SQUARE;
+      case CURLY:
+	return LEFT_CURLY;
+      default:
+	gcc_unreachable ();
+      }
+  }
+
+  static TokenId right_delim_type_tok_token_id (DelimType delim_type)
+  {
+    switch (delim_type)
+      {
+      case PARENS:
+	return RIGHT_PAREN;
+      case SQUARE:
+	return RIGHT_SQUARE;
+      case CURLY:
+	return RIGHT_CURLY;
+      default:
+	gcc_unreachable ();
+      }
+  }
 };
 
 /* Forward decl - definition moved to rust-expr.h as it requires LiteralExpr to