@@ -38,7 +38,7 @@ export class Lexer {
38
38
lineStart : number ;
39
39
40
40
constructor ( source : Source ) {
41
- const startOfFileToken = new Token ( TokenKind . SOF , 0 , 0 , 0 , 0 ) ;
41
+ const startOfFileToken = new Token ( TokenKind . SOF , 0 , 0 , 0 , 0 , undefined ) ;
42
42
43
43
this . source = source ;
44
44
this . lastToken = startOfFileToken ;
@@ -174,11 +174,10 @@ function createToken(
174
174
kind : TokenKind ,
175
175
start : number ,
176
176
end : number ,
177
- value ? : string ,
177
+ value : string | undefined ,
178
178
) : Token {
179
- const line = lexer . line ;
180
- const col = 1 + start - lexer . lineStart ;
181
- return new Token ( kind , start , end , line , col , value ) ;
179
+ const col = start - ( lexer . lineStart - 1 ) ;
180
+ return new Token ( kind , start , end , lexer . line , col , value ) ;
182
181
}
183
182
184
183
/**
@@ -248,39 +247,123 @@ function readNextToken(lexer: Lexer, start: number): Token {
248
247
//
249
248
// Punctuator :: one of ! $ & ( ) ... : = @ [ ] { | }
250
249
case 0x0021 : // !
251
- return createToken ( lexer , TokenKind . BANG , position , position + 1 ) ;
250
+ return createToken (
251
+ lexer ,
252
+ TokenKind . BANG ,
253
+ position ,
254
+ position + 1 ,
255
+ undefined ,
256
+ ) ;
252
257
case 0x0024 : // $
253
- return createToken ( lexer , TokenKind . DOLLAR , position , position + 1 ) ;
258
+ return createToken (
259
+ lexer ,
260
+ TokenKind . DOLLAR ,
261
+ position ,
262
+ position + 1 ,
263
+ undefined ,
264
+ ) ;
254
265
case 0x0026 : // &
255
- return createToken ( lexer , TokenKind . AMP , position , position + 1 ) ;
266
+ return createToken (
267
+ lexer ,
268
+ TokenKind . AMP ,
269
+ position ,
270
+ position + 1 ,
271
+ undefined ,
272
+ ) ;
256
273
case 0x0028 : // (
257
- return createToken ( lexer , TokenKind . PAREN_L , position , position + 1 ) ;
274
+ return createToken (
275
+ lexer ,
276
+ TokenKind . PAREN_L ,
277
+ position ,
278
+ position + 1 ,
279
+ undefined ,
280
+ ) ;
258
281
case 0x0029 : // )
259
- return createToken ( lexer , TokenKind . PAREN_R , position , position + 1 ) ;
282
+ return createToken (
283
+ lexer ,
284
+ TokenKind . PAREN_R ,
285
+ position ,
286
+ position + 1 ,
287
+ undefined ,
288
+ ) ;
260
289
case 0x002e : // .
261
290
if (
262
291
body . charCodeAt ( position + 1 ) === 0x002e &&
263
292
body . charCodeAt ( position + 2 ) === 0x002e
264
293
) {
265
- return createToken ( lexer , TokenKind . SPREAD , position , position + 3 ) ;
294
+ return createToken (
295
+ lexer ,
296
+ TokenKind . SPREAD ,
297
+ position ,
298
+ position + 3 ,
299
+ undefined ,
300
+ ) ;
266
301
}
267
302
break ;
268
303
case 0x003a : // :
269
- return createToken ( lexer , TokenKind . COLON , position , position + 1 ) ;
304
+ return createToken (
305
+ lexer ,
306
+ TokenKind . COLON ,
307
+ position ,
308
+ position + 1 ,
309
+ undefined ,
310
+ ) ;
270
311
case 0x003d : // =
271
- return createToken ( lexer , TokenKind . EQUALS , position , position + 1 ) ;
312
+ return createToken (
313
+ lexer ,
314
+ TokenKind . EQUALS ,
315
+ position ,
316
+ position + 1 ,
317
+ undefined ,
318
+ ) ;
272
319
case 0x0040 : // @
273
- return createToken ( lexer , TokenKind . AT , position , position + 1 ) ;
320
+ return createToken (
321
+ lexer ,
322
+ TokenKind . AT ,
323
+ position ,
324
+ position + 1 ,
325
+ undefined ,
326
+ ) ;
274
327
case 0x005b : // [
275
- return createToken ( lexer , TokenKind . BRACKET_L , position , position + 1 ) ;
328
+ return createToken (
329
+ lexer ,
330
+ TokenKind . BRACKET_L ,
331
+ position ,
332
+ position + 1 ,
333
+ undefined ,
334
+ ) ;
276
335
case 0x005d : // ]
277
- return createToken ( lexer , TokenKind . BRACKET_R , position , position + 1 ) ;
336
+ return createToken (
337
+ lexer ,
338
+ TokenKind . BRACKET_R ,
339
+ position ,
340
+ position + 1 ,
341
+ undefined ,
342
+ ) ;
278
343
case 0x007b : // {
279
- return createToken ( lexer , TokenKind . BRACE_L , position , position + 1 ) ;
344
+ return createToken (
345
+ lexer ,
346
+ TokenKind . BRACE_L ,
347
+ position ,
348
+ position + 1 ,
349
+ undefined ,
350
+ ) ;
280
351
case 0x007c : // |
281
- return createToken ( lexer , TokenKind . PIPE , position , position + 1 ) ;
352
+ return createToken (
353
+ lexer ,
354
+ TokenKind . PIPE ,
355
+ position ,
356
+ position + 1 ,
357
+ undefined ,
358
+ ) ;
282
359
case 0x007d : // }
283
- return createToken ( lexer , TokenKind . BRACE_R , position , position + 1 ) ;
360
+ return createToken (
361
+ lexer ,
362
+ TokenKind . BRACE_R ,
363
+ position ,
364
+ position + 1 ,
365
+ undefined ,
366
+ ) ;
284
367
// StringValue
285
368
case 0x0022 : // "
286
369
if (
@@ -313,7 +396,7 @@ function readNextToken(lexer: Lexer, start: number): Token {
313
396
) ;
314
397
}
315
398
316
- return createToken ( lexer , TokenKind . EOF , bodyLength , bodyLength ) ;
399
+ return createToken ( lexer , TokenKind . EOF , bodyLength , bodyLength , undefined ) ;
317
400
}
318
401
319
402
/**
@@ -353,7 +436,7 @@ function readComment(lexer: Lexer, start: number): Token {
353
436
TokenKind . COMMENT ,
354
437
start ,
355
438
position ,
356
- body . slice ( start + 1 , position ) ,
439
+ body . substring ( start + 1 , position ) ,
357
440
) ;
358
441
}
359
442
@@ -454,7 +537,7 @@ function readNumber(lexer: Lexer, start: number, firstCode: number): Token {
454
537
isFloat ? TokenKind . FLOAT : TokenKind . INT ,
455
538
start ,
456
539
position ,
457
- body . slice ( start , position ) ,
540
+ body . substring ( start , position ) ,
458
541
) ;
459
542
}
460
543
@@ -515,13 +598,13 @@ function readString(lexer: Lexer, start: number): Token {
515
598
516
599
// Closing Quote (")
517
600
if ( code === 0x0022 ) {
518
- value += body . slice ( chunkStart , position ) ;
601
+ value += body . substring ( chunkStart , position ) ;
519
602
return createToken ( lexer , TokenKind . STRING , start , position + 1 , value ) ;
520
603
}
521
604
522
605
// Escape Sequence (\)
523
606
if ( code === 0x005c ) {
524
- value += body . slice ( chunkStart , position ) ;
607
+ value += body . substring ( chunkStart , position ) ;
525
608
const escape =
526
609
body . charCodeAt ( position + 1 ) === 0x0075 // u
527
610
? body . charCodeAt ( position + 2 ) === 0x007b // {
@@ -593,7 +676,7 @@ function readEscapedUnicodeVariableWidth(
593
676
throw syntaxError (
594
677
lexer . source ,
595
678
position ,
596
- `Invalid Unicode escape sequence: "${ body . slice (
679
+ `Invalid Unicode escape sequence: "${ body . substring (
597
680
position ,
598
681
position + size ,
599
682
) } ".`,
@@ -635,7 +718,10 @@ function readEscapedUnicodeFixedWidth(
635
718
throw syntaxError (
636
719
lexer . source ,
637
720
position ,
638
- `Invalid Unicode escape sequence: "${ body . slice ( position , position + 6 ) } ".` ,
721
+ `Invalid Unicode escape sequence: "${ body . substring (
722
+ position ,
723
+ position + 6 ,
724
+ ) } ".`,
639
725
) ;
640
726
}
641
727
@@ -717,7 +803,7 @@ function readEscapedCharacter(lexer: Lexer, position: number): EscapeSequence {
717
803
throw syntaxError (
718
804
lexer . source ,
719
805
position ,
720
- `Invalid character escape sequence: "${ body . slice (
806
+ `Invalid character escape sequence: "${ body . substring (
721
807
position ,
722
808
position + 2 ,
723
809
) } ".`,
@@ -755,7 +841,7 @@ function readBlockString(lexer: Lexer, start: number): Token {
755
841
body . charCodeAt ( position + 1 ) === 0x0022 &&
756
842
body . charCodeAt ( position + 2 ) === 0x0022
757
843
) {
758
- currentLine += body . slice ( chunkStart , position ) ;
844
+ currentLine += body . substring ( chunkStart , position ) ;
759
845
blockLines . push ( currentLine ) ;
760
846
761
847
const token = createToken (
@@ -779,15 +865,15 @@ function readBlockString(lexer: Lexer, start: number): Token {
779
865
body . charCodeAt ( position + 2 ) === 0x0022 &&
780
866
body . charCodeAt ( position + 3 ) === 0x0022
781
867
) {
782
- currentLine += body . slice ( chunkStart , position ) ;
868
+ currentLine += body . substring ( chunkStart , position ) ;
783
869
chunkStart = position + 1 ; // skip only slash
784
870
position += 4 ;
785
871
continue ;
786
872
}
787
873
788
874
// LineTerminator
789
875
if ( code === 0x000a || code === 0x000d ) {
790
- currentLine += body . slice ( chunkStart , position ) ;
876
+ currentLine += body . substring ( chunkStart , position ) ;
791
877
blockLines . push ( currentLine ) ;
792
878
793
879
if ( code === 0x000d && body . charCodeAt ( position + 1 ) === 0x000a ) {
@@ -849,6 +935,6 @@ function readName(lexer: Lexer, start: number): Token {
849
935
TokenKind . NAME ,
850
936
start ,
851
937
position ,
852
- body . slice ( start , position ) ,
938
+ body . substring ( start , position ) ,
853
939
) ;
854
940
}
0 commit comments