Lexer; subtly wrong; no makefile

Author: emberian

src/grammar/README.md

@@ -0,0 +1,19 @@
+Reference grammar.
+
+Uses [antlr4](http://www.antlr.org/) and a custom Rust tool to compare
+ASTs/token streams generated.
+
+To use:
+
+```
+antlr4 RustLexer.g4
+javac *.java
+rustc -O verify.rs
+for file in ../*/**.rs; do
+    echo $file;
+    grun RustLexer tokens -tokens < $file | ./verify $file || break
+done
+```
+
+Note That the `../*/**.rs` glob will match every `*.rs` file in the above
+directory and all of its recursive children. This is a zsh extension.

src/grammar/RustLexer.g4

@@ -0,0 +1,165 @@
+lexer grammar RustLexer;
+
+/* Note: due to antlr limitations, we can't represent XID_start and
+ * XID_continue properly. ASCII-only substitute. */
+
+fragment XID_start : [_a-zA-Z] ;
+fragment XID_continue : [_a-zA-Z0-9] ;
+
+/* Expression-operator symbols */
+
+EQ      : '=' ;
+LT      : '<' ;
+LE      : '<=' ;
+EQEQ    : '==' ;
+NE      : '!=' ;
+GE      : '>=' ;
+GT      : '>' ;
+ANDAND  : '&&' ;
+OROR    : '||' ;
+NOT     : '!' ;
+TILDE   : '~' ;
+PLUS    : '+' ;
+MINUS   : '-' ;
+STAR    : '*' ;
+SLASH   : '/' ;
+PERCENT : '%' ;
+CARET   : '^' ;
+AND     : '&' ;
+OR      : '|' ;
+SHL     : '<<' ;
+SHR     : '>>' ;
+
+BINOP
+    : PLUS
+    | MINUS
+    | STAR
+    | PERCENT
+    | CARET
+    | AND
+    | OR
+    | SHL
+    | SHR
+    ;
+
+BINOPEQ : BINOP EQ ;
+
+/* "Structural symbols" */
+
+AT         : '@' ;
+DOT        : '.' ;
+DOTDOT     : '..' ;
+DOTDOTDOT  : '...' ;
+COMMA      : ',' ;
+SEMI       : ';' ;
+COLON      : ':' ;
+MOD_SEP    : '::' ;
+RARROW     : '->' ;
+FAT_ARROW  : '=>' ;
+LPAREN     : '(' ;
+RPAREN     : ')' ;
+LBRACKET   : '[' ;
+RBRACKET   : ']' ;
+LBRACE     : '{' ;
+RBRACE     : '}' ;
+POUND      : '#';
+DOLLAR     : '$' ;
+UNDERSCORE : '_' ;
+
+// Literals
+
+fragment HEXIT
+  : [0-9a-fA-F]
+  ;
+
+fragment CHAR_ESCAPE
+  : [nrt\\'"0]
+  | [xX] HEXIT HEXIT
+  | 'u' HEXIT HEXIT HEXIT HEXIT
+  | 'U' HEXIT HEXIT HEXIT HEXIT HEXIT HEXIT HEXIT HEXIT
+  ;
+
+LIT_CHAR
+  : '\'' ( '\\' CHAR_ESCAPE | ~[\\'\n\t\r] ) '\''
+  ;
+
+INT_SUFFIX
+  : 'i'
+  | 'i8'
+  | 'i16'
+  | 'i32'
+  | 'i64'
+  | 'u'
+  | 'u8'
+  | 'u16'
+  | 'u32'
+  | 'u64'
+  ;
+
+LIT_INTEGER
+  : [0-9][0-9_]* INT_SUFFIX?
+  | '0b' [01][01_]* INT_SUFFIX?
+  | '0o' [0-7][0-7_]* INT_SUFFIX?
+  | '0x' [0-9a-fA-F][0-9a-fA-F_]* INT_SUFFIX?
+  ;
+
+FLOAT_SUFFIX
+  : 'f32'
+  | 'f64'
+  | 'f128'
+  ;
+
+LIT_FLOAT
+  : [0-9][0-9_]* ('.' | ('.' [0-9][0-9_]*)? ([eE] [-+]? [0-9][0-9_]*)? FLOAT_SUFFIX?)
+  ;
+
+LIT_STR
+  : '"' ('\\\n' | '\\\r\n' | '\\' CHAR_ESCAPE | .)*? '"'
+  ;
+
+LIT_BINARY : 'b' LIT_STR ;
+LIT_BINARY_RAW : 'b' LIT_STR_RAW ;
+
+/* this is a bit messy */
+
+fragment LIT_STR_RAW_INNER
+  : '"' .*? '"'
+  | LIT_STR_RAW_INNER2
+  ;
+
+fragment LIT_STR_RAW_INNER2
+  : POUND LIT_STR_RAW_INNER POUND
+  ;
+
+LIT_STR_RAW
+  : 'r' LIT_STR_RAW_INNER
+  ;
+
+IDENT : XID_start XID_continue* ;
+
+LIFETIME : '\'' IDENT ;
+
+WHITESPACE : [ \r\n\t]+ ;
+
+COMMENT
+  : '//' ~[\r\n]*
+  | '////' ~[\r\n]*
+  | BLOCK_COMMENT
+  ;
+
+mode DOCCOMMENT;
+
+fragment DOC_BLOCK_COMMENT
+  : ('/**' | '/*!') (DOC_BLOCK_COMMENT | .)*? '*/'
+  ;
+
+DOC_COMMENT
+  : '///' ~[\r\n]*
+  | '//!' ~[\r\n]*
+  | DOC_BLOCK_COMMENT
+  ;
+
+fragment BLOCK_COMMENT
+  : '/*' (BLOCK_COMMENT | .)*? '*/'
+  ;
+