use racket-lexer* when there is no #lang line

closes #13

Author: rfindler

syntax-color-lib/syntax-color/module-lexer.rkt

@@ -10,24 +10,26 @@
 
 #|
 
-mode : (or/c #f 'before-lang-line
-             'no-lang-line
+mode : (or/c #f
+             before-lang-line?
+             no-lang-line?
              (cons lexer mode)
              lexer)
 
 the module lexer tracks any white-space and comments before
 the #lang line (if any) explicitly by wrapping calls to the 
-scheme-lexer (in #f or 'before-lang-line mode). 
+racket-lexer (in #f or before-lang-line mode).
 Once it finds a non-white-space and non-comment
 token, it checks to see if there is a #lang line and, if so
 changes the mode to be the lexer that the #lang indicates,
 delegating to it (the last two modes listed above).
 If there is no #lang line, then it continues
-to delegate to the scheme-lexer (in the 'no-lang-line mode).
+to delegate to the racket-lexer (in the no-lang-line mode).
 
 |#
 
 (struct before-lang-line (racket-lexer-mode) #:prefab)
+(struct no-lang-line (racket-lexer-mode) #:prefab)
 (define (do-module-lexer* in offset mode can-return-attribs-hash? filter-lexer)
   (cond
     [(or (not mode) (before-lang-line? mode))
@@ -114,17 +116,17 @@ to delegate to the scheme-lexer (in the 'no-lang-line mode).
            ;; sync ports
            (for ([i (in-range (file-position in) (file-position p))])
              (read-byte-or-special in))
-           (values lexeme 'error data new-token-start end-pos 0 'no-lang-line)]
+           (values lexeme 'error data new-token-start end-pos 0 (no-lang-line #f))]
           [else 
            (for ([i (in-range (file-position in) (file-position lexer-port))])
              (read-byte-or-special in))
            (values lexeme
                    (if can-return-attribs-hash? type (attribs->symbol type))
-                   data new-token-start new-token-end 0 'no-lang-line)])])]
-    [(eq? mode 'no-lang-line)
-     (let-values ([(lexeme type data new-token-start new-token-end) 
-                   (racket-lexer in)])
-       (values lexeme type data new-token-start new-token-end 0 'no-lang-line))]
+                   data new-token-start new-token-end 0 (no-lang-line #f))])])]
+    [(no-lang-line? mode)
+     (define-values (lexeme type data new-token-start new-token-end backup new-mode)
+       (racket-lexer* in offset (no-lang-line-racket-lexer-mode mode)))
+     (values lexeme type data new-token-start new-token-end backup (no-lang-line new-mode))]
     [(pair? mode)
      ;; #lang-selected language consumes and produces a mode:
      (let-values ([(lexeme type data new-token-start new-token-end backup-delta new-mode)

syntax-color-test/tests/syntax-color/module-lexer.rkt

@@ -5,6 +5,7 @@
          rackunit)
 
 (struct before-lang-line (racket-lexer-mode) #:prefab)
+(struct no-lang-line (racket-lexer-mode) #:prefab)
 (define (lex input count? #:modes [modes (list module-lexer module-lexer*)])
   (define results
     (for/list ([lexer (in-list modes)])
@@ -34,6 +35,7 @@
                             [(procedure? mode)
                              `(proc ,(object-name mode))]
                             [(before-lang-line? mode) 'before-lang-line]
+                            [(no-lang-line? mode) 'no-lang-line]
                             [(and (pair? mode)
                                   (procedure? (car mode)))
                              ;; a hack: translate 'racket-lexer* shape to 'racket-lexer
@@ -162,6 +164,25 @@
                 (")" #hash((comment? . #t) (type . parenthesis)) 27 28 (proc racket-lexer))
                 (,eof eof #f #f (proc racket-lexer))))
 
+;; check sexp comment handling when in before-lang-line mode and in there-is-no-lang-line mode
+(check-equal? (lex "#;(a b)\n1\n#;(a b)" #t #:modes (list module-lexer*))
+              `(("#;" sexp-comment 1 3 #f)
+                ("(" #hash((comment? . #t) (type . parenthesis)) 3 4 before-lang-line)
+                ("a" #hash((comment? . #t) (type . symbol)) 4 5 before-lang-line)
+                (" " #hash((comment? . #t) (type . white-space)) 5 6 before-lang-line)
+                ("b" #hash((comment? . #t) (type . symbol)) 6 7 before-lang-line)
+                (")" #hash((comment? . #t) (type . parenthesis)) 7 8 before-lang-line)
+                ("\n" white-space 8 9 before-lang-line)
+                ("1" constant 9 10 before-lang-line)
+                ("\n" white-space 10 11 no-lang-line)
+                ("#;" sexp-comment 11 13 no-lang-line)
+                ("(" #hash((comment? . #t) (type . parenthesis)) 13 14 no-lang-line)
+                ("a" #hash((comment? . #t) (type . symbol)) 14 15 no-lang-line)
+                (" " #hash((comment? . #t) (type . white-space)) 15 16 no-lang-line)
+                ("b" #hash((comment? . #t) (type . symbol)) 16 17 no-lang-line)
+                (")" #hash((comment? . #t) (type . parenthesis)) 17 18 no-lang-line)
+                (,eof eof #f #f no-lang-line)))
+
 (check same?
        (lex "#lang at-exp racket/base\n1\n" #t)
        `(("#lang at-exp racket/base" other 1 25 #f)