use externals for comments

done for the sake of differentiating the next character after '//', which requires forward lookup, and could not be achieved via regular expressions

Author: resolritter

corpus/source_files.txt

@@ -59,20 +59,17 @@ Line comments
 Doc comments
 ============================================
 
-/// Foo
-///
-/// # Description
-///
-/// * Bar
+/// Multi
+/// Line
+/// Doc
+/// Comment
+// Line comment
 
 ----
 
 (source_file
   (doc_comment)
-  (doc_comment)
-  (doc_comment)
-  (doc_comment)
-  (doc_comment)) 
+  (line_comment)) 
 
 =====================================
 Greek letters in identifiers

grammar.js

@@ -45,6 +45,8 @@ module.exports = grammar({
     $.raw_string_literal,
     $.float_literal,
     $.block_comment,
+    $.line_comment,
+    $.doc_comment
   ],
 
   supertypes: $ => [
@@ -1410,14 +1412,6 @@ module.exports = grammar({
 
     boolean_literal: $ => choice('true', 'false'),
 
-    doc_comment: $ => token(seq(
-      '///', /.*/
-    )),
-
-    line_comment: $ => token(seq(
-      new RegExp('\/\/[^\/]'), /.*/
-    )),
-
     _path: $ => choice(
       $.self,
       alias(choice(...primitive_types), $.identifier),

src/scanner.c

@@ -6,6 +6,8 @@ enum TokenType {
   RAW_STRING_LITERAL,
   FLOAT_LITERAL,
   BLOCK_COMMENT,
+  LINE_COMMENT,
+  DOC_COMMENT,
 };
 
 void *tree_sitter_rust_external_scanner_create() { return NULL; }
@@ -143,7 +145,49 @@ bool tree_sitter_rust_external_scanner_scan(void *payload, TSLexer *lexer,
 
   if (lexer->lookahead == '/') {
     advance(lexer);
+
+    if ((valid_symbols[LINE_COMMENT] || valid_symbols[DOC_COMMENT]) && lexer->lookahead == '/') {
+      advance(lexer);
+      if (lexer->lookahead == '/') {
+        lexer->result_symbol = DOC_COMMENT;
+        while (true) {
+          while (lexer->lookahead != '\n') {
+            advance(lexer);
+          }
+          if (lexer->lookahead == 0) {
+            break;
+          }
+
+          lexer->mark_end(lexer);
+          advance(lexer);
+          if (lexer->lookahead == '/') {
+            advance(lexer);
+            if (lexer->lookahead == '/') {
+              advance(lexer);
+              if (lexer->lookahead == '/') {
+                advance(lexer);
+              } else {
+                break;
+              }
+            } else {
+              break;
+            }
+          } else {
+            break;
+          }
+        }
+      } else {
+        lexer->result_symbol = LINE_COMMENT;
+        while (lexer->lookahead != '\n' && lexer->lookahead != 0) {
+          advance(lexer);
+        }
+      }
+
+      return true;
+    }
+
     if (lexer->lookahead != '*') return false;
+
     advance(lexer);
 
     bool after_star = false;