GraphQL 2020 Lexer Updates

src/Core/Language.Tests/Language.Tests.csproj

@@ -5,6 +5,7 @@
     <ChilliImport>$([System.IO.Path]::Combine($(ChilliCurrentDirectory), '..', '..', '..', 'tools'))</ChilliImport>
     <ChilliFramework>$([System.IO.Path]::Combine($(ChilliImport), 'CoreTestFramework.props'))</ChilliFramework>
     <ChilliBuild>$([System.IO.Path]::Combine($(ChilliImport), 'TestSettings.props'))</ChilliBuild>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>
 
   <Import Project="$(ChilliFramework)" />

src/Core/Language.Tests/Parser/SyntaxTokenInfo.cs

@@ -1,3 +1,6 @@
+using System;
+using System.Text;
+
 namespace HotChocolate.Language
 {
     public sealed class SyntaxTokenInfo
@@ -7,13 +10,15 @@ public SyntaxTokenInfo(
             int start,
             int end,
             int line,
-            int column)
+            int column,
+            string value)
         {
             Kind = kind;
             Start = start;
             End = end;
             Line = line;
             Column = column;
+            Value = value;
         }
 
         /// <summary>
@@ -43,15 +48,27 @@ public SyntaxTokenInfo(
         /// </summary>
         public int Column { get; }
 
-        public static SyntaxTokenInfo FromReader(
-            in Utf8GraphQLReader reader)
+        public string Value { get; }
+
+        public unsafe static SyntaxTokenInfo FromReader(in Utf8GraphQLReader reader)
         {
+            string value = null;
+
+            ReadOnlySpan<byte> token = reader.GraphQLData.Slice(
+                reader.Start, reader.End - reader.Start);
+
+            fixed (byte* b = token)
+            {
+                value = Encoding.UTF8.GetString(b, token.Length);
+            }
+
             return new SyntaxTokenInfo(
                 reader.Kind,
                 reader.Start,
                 reader.End,
                 reader.Line,
-                reader.Column);
+                reader.Column,
+                value);
         }
     }
 }

src/Core/Language.Tests/Parser/Utf8ReaderTests.cs

@@ -12,21 +12,18 @@ public class Utf8ReaderTests
         [Fact]
         public void Read_Two_NameTokens()
         {
-            var source = new ReadOnlySpan<byte>(
-                Encoding.UTF8.GetBytes("type foo"));
+            var source = new ReadOnlySpan<byte>(Encoding.UTF8.GetBytes("type foo"));
             var lexer = new Utf8GraphQLReader(source);
 
             Assert.Equal(TokenKind.StartOfFile, lexer.Kind);
 
             Assert.True(lexer.Read());
             Assert.Equal(TokenKind.Name, lexer.Kind);
-            Assert.Equal("type",
-                Encoding.UTF8.GetString(lexer.Value.ToArray()));
+            Assert.Equal("type", Encoding.UTF8.GetString(lexer.Value.ToArray()));
 
             Assert.True(lexer.Read());
             Assert.Equal(TokenKind.Name, lexer.Kind);
-            Assert.Equal("foo",
-                Encoding.UTF8.GetString(lexer.Value.ToArray()));
+            Assert.Equal("foo", Encoding.UTF8.GetString(lexer.Value.ToArray()));
 
             Assert.False(lexer.Read());
             Assert.Equal(TokenKind.EndOfFile, lexer.Kind);

src/Core/Language.Tests/Parser/__snapshots__/Utf8GraphQLRequestParserTests.Parse_Id_As_Name.snap

@@ -42,7 +42,7 @@ subscription StoryLikeSubscription($input: StoryLikeSubscribeInput) {
 
 fragment frag on Friend {
   foo(size: $size, bar: $b, obj: { key: "value", block: """
-      block string uses \"""
+       block string uses \"""
   """ })
 }
 

src/Core/Language.Tests/Parser/__snapshots__/Utf8GraphQLRequestParserTests.Parse_Kitchen_Sink_Query_AllProps_No_Cache_Query.snap

@@ -42,7 +42,7 @@ subscription StoryLikeSubscription($input: StoryLikeSubscribeInput) {
 
 fragment frag on Friend {
   foo(size: $size, bar: $b, obj: { key: "value", block: """
-      block string uses \"""
+       block string uses \"""
   """ })
 }
 

src/Core/Language.Tests/Parser/__snapshots__/Utf8GraphQLRequestParserTests.Parse_Kitchen_Sink_Query_No_Cache.snap

@@ -42,7 +42,7 @@ subscription StoryLikeSubscription($input: StoryLikeSubscribeInput) {
 
 fragment frag on Friend {
   foo(size: $size, bar: $b, obj: { key: "value", block: """
-      block string uses \"""
+       block string uses \"""
   """ })
 }
 

src/Core/Language.Tests/Parser/__snapshots__/Utf8GraphQLRequestParserTests.Parse_Kitchen_Sink_Query_With_Cache.snap

@@ -42,7 +42,7 @@ subscription StoryLikeSubscription($input: StoryLikeSubscribeInput) {
 
 fragment frag on Friend {
   foo(size: $size, bar: $b, obj: { key: "value", block: """
-      block string uses \"""
+       block string uses \"""
   """ })
 }
 

src/Core/Language.Tests/Parser/__snapshots__/Utf8GraphQLRequestParserTests.Parse_Skip_Custom_Property.snap

@@ -42,7 +42,7 @@ subscription StoryLikeSubscription($input: StoryLikeSubscribeInput) {
 
 fragment frag on Friend {
   foo(size: $size, bar: $b, obj: { key: "value", block: """
-      block string uses \"""
+       block string uses \"""
   """ })
 }
 

src/Core/Language.Tests/Parser/__snapshots__/Utf8KitchenSinkParserTests.ParseFacebookKitchenSinkQuery.snap

@@ -1420,7 +1420,7 @@
                           "Line": 49,
                           "Column": 56
                         },
-                        "Value": "    block string uses \"\"\"",
+                        "Value": "     block string uses \"\"\"",
                         "Block": true
                       }
                     }

src/Core/Language.Tests/Parser/__snapshots__/Utf8KitchenSinkParserTests.ParseFacebookKitchenSinkQuery_sdl.snap

@@ -42,7 +42,7 @@ subscription StoryLikeSubscription($input: StoryLikeSubscribeInput) {
 
 fragment frag on Friend {
   foo(size: $size, bar: $b, obj: { key: "value", block: """
-      block string uses \"""
+       block string uses \"""
   """ })
 }
 

src/Core/Language.Tests/Parser/__snapshots__/Utf8KitchenSinkParserTests.ParseFacebookKitchenSinkSchema.snap

@@ -183,7 +183,7 @@
               "Line": 17,
               "Column": 3
             },
-            "Value": "This is a description of the `two` field.",
+            "Value": " This is a description of the `two` field.",
             "Block": true
           },
           "Arguments": [
@@ -197,7 +197,7 @@
                   "Line": 21,
                   "Column": 5
                 },
-                "Value": "This is a description of the `argument` argument.",
+                "Value": " This is a description of the `argument` argument.",
                 "Block": true
               },
               "Type": {

src/Core/Language.Tests/Parser/__snapshots__/Utf8KitchenSinkParserTests.ParseFacebookKitchenSinkSchema_sdl.snap

@@ -10,10 +10,10 @@ of the `Foo` type.
 type Foo implements Bar & Baz {
   one: Type
   """
-  This is a description of the `two` field.
+   This is a description of the `two` field.
   """
   two("""
-  This is a description of the `argument` argument.
+   This is a description of the `argument` argument.
   """ argument: InputType!): Type
   three(argument: InputType other: String): Int
   four(argument: String = "string"): String

src/Core/Language.Tests/Parser/__snapshots__/Utf8QueryParserTests.KitchenSinkQueryQuery.snap

@@ -1420,7 +1420,7 @@
                           "Line": 49,
                           "Column": 56
                         },
-                        "Value": "    block string uses \"\"\"",
+                        "Value": "     block string uses \"\"\"",
                         "Block": true
                       }
                     }

src/Core/Language.Tests/Parser/__snapshots__/Utf8ReaderTests.Read_BlockStringValue.snap

@@ -4,55 +4,63 @@
     "Start": 0,
     "End": 1,
     "Line": 1,
-    "Column": 1
+    "Column": 1,
+    "Value": "{"
   },
   {
     "Kind": "Name",
     "Start": 2,
     "End": 4,
     "Line": 1,
-    "Column": 3
+    "Column": 3,
+    "Value": "me"
   },
   {
     "Kind": "LeftParenthesis",
     "Start": 4,
     "End": 5,
     "Line": 1,
-    "Column": 5
+    "Column": 5,
+    "Value": "("
   },
   {
     "Kind": "Name",
     "Start": 5,
     "End": 6,
     "Line": 1,
-    "Column": 6
+    "Column": 6,
+    "Value": "a"
   },
   {
     "Kind": "Colon",
     "Start": 6,
     "End": 7,
     "Line": 1,
-    "Column": 7
+    "Column": 7,
+    "Value": ":"
   },
   {
     "Kind": "BlockString",
     "Start": 8,
     "End": 26,
     "Line": 1,
-    "Column": 9
+    "Column": 9,
+    "Value": "\"\"\"\n     Abcdef\n\"\""
   },
   {
     "Kind": "RightParenthesis",
     "Start": 27,
     "End": 28,
     "Line": 3,
-    "Column": 1
+    "Column": 1,
+    "Value": ")"
   },
   {
     "Kind": "RightBrace",
     "Start": 29,
     "End": 30,
     "Line": 3,
-    "Column": 3
+    "Column": 3,
+    "Value": "}"
   }
 ]

src/Core/Language.Tests/Parser/__snapshots__/Utf8ReaderTests.Read_BlockString_SkipEscapes.snap

@@ -4,20 +4,23 @@
     "Start": 0,
     "End": 3,
     "Line": 1,
-    "Column": 1
+    "Column": 1,
+    "Value": "abc"
   },
   {
     "Kind": "BlockString",
     "Start": 4,
     "End": 16,
     "Line": 1,
-    "Column": 5
+    "Column": 5,
+    "Value": "\"\"\"def\\\"\"\"\"\""
   },
   {
     "Kind": "Name",
     "Start": 18,
     "End": 21,
     "Line": 1,
-    "Column": 19
+    "Column": 19,
+    "Value": "ghi"
   }
 ]

src/Core/Language.Tests/Parser/__snapshots__/Utf8ReaderTests.Read_Comment.snap

@@ -4,27 +4,31 @@
     "Start": 0,
     "End": 1,
     "Line": 1,
-    "Column": 1
+    "Column": 1,
+    "Value": "{"
   },
   {
     "Kind": "Comment",
     "Start": 2,
     "End": 19,
     "Line": 1,
-    "Column": 3
+    "Column": 3,
+    "Value": "#test me foo bar "
   },
   {
     "Kind": "Name",
     "Start": 21,
     "End": 23,
     "Line": 2,
-    "Column": 2
+    "Column": 2,
+    "Value": "me"
   },
   {
     "Kind": "RightBrace",
     "Start": 24,
     "End": 25,
     "Line": 2,
-    "Column": 5
+    "Column": 5,
+    "Value": "}"
   }
 ]