Упрощение создания экземпляра лексической структуры (#17)

* удаление лишних пакетов

* миграция

* new test

* version

Author: Stepami

Interpreter.Lib/BackEnd/Instructions/AsString.cs

@@ -1,9 +1,8 @@
 using System;
-using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
+using System.Text.Json;
+using System.Text.Json.Serialization;
 using Interpreter.Lib.BackEnd.Values;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Serialization;
 
 namespace Interpreter.Lib.BackEnd.Instructions
 {
@@ -17,18 +16,15 @@ public AsString(string left, IValue right, int number) :
         public override int Execute(VirtualMachine vm)
         {
             var frame = vm.Frames.Peek();
-            frame[Left] = JsonConvert.SerializeObject(
+            frame[Left] = JsonSerializer.Serialize(
                 right.right.Get(frame),
-                new JsonSerializerSettings
+                new JsonSerializerOptions
                 {
-                    Formatting = Formatting.Indented,
-                    FloatFormatHandling = FloatFormatHandling.Symbol,
-                    ReferenceLoopHandling = ReferenceLoopHandling.Ignore,
-                    ContractResolver = new CamelCasePropertyNamesContractResolver(),
-                    Converters = new List<JsonConverter>
-                    {
-                        new DoubleValueConverter()
-                    }
+                    WriteIndented = true,
+                    PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
+                    ReferenceHandler = ReferenceHandler.IgnoreCycles,
+                    Converters = { new DoubleValueWriteConverter() },
+                    NumberHandling = JsonNumberHandling.AllowNamedFloatingPointLiterals
                 }
             );
 
@@ -38,20 +34,21 @@ public override int Execute(VirtualMachine vm)
         protected override string ToStringRepresentation() => $"{Left} = {right.right} as string";
 
         [ExcludeFromCodeCoverage]
-        private class DoubleValueConverter : JsonConverter<double>
+        private class DoubleValueWriteConverter : JsonConverter<double>
         {
-            public override bool CanRead => false;
+            public override double Read(ref Utf8JsonReader reader,
+                Type typeToConvert, JsonSerializerOptions options) =>
+                throw new NotImplementedException();
 
-            public override void WriteJson(JsonWriter writer, double value, JsonSerializer serializer) =>
+            public override void Write(Utf8JsonWriter writer, 
+                double value, JsonSerializerOptions options)
+            {
                 // ReSharper disable once CompareOfFloatsByEqualityOperator
-                writer.WriteRawValue(value == Math.Truncate(value)
-                    ? JsonConvert.ToString(Convert.ToInt64(value))
-                    : JsonConvert.ToString(value));
-
-            public override double ReadJson(JsonReader reader, Type objectType,
-                double existingValue, bool hasExistingValue,
-                JsonSerializer serializer) =>
-                throw new NotImplementedException("CanRead is false, so reading is unnecessary");
+                if (value == Math.Truncate(value))
+                    writer.WriteNumberValue(Convert.ToInt64(value));
+                else
+                    writer.WriteNumberValue(value);
+            }
         }
     }
 }

Interpreter.Lib/FrontEnd/GetTokens/Data/Token.cs

@@ -20,7 +20,7 @@ public override string ToString()
         {
             var displayValue = Value;
             if (displayValue != null) displayValue = Regex.Replace(Value, "\"", "\\\"");
-            if (Type.WhiteSpace()) displayValue = "";
+            if (Type.CanIgnore()) displayValue = "";
             return $"{Type} {Segment}: {displayValue}";
         }
     }

Interpreter.Lib/FrontEnd/GetTokens/Data/TokenTypes/IgnorableType.cs

@@ -0,0 +1,8 @@
+namespace Interpreter.Lib.FrontEnd.GetTokens.Data.TokenTypes
+{
+    public record IgnorableType(string Tag = null, string Pattern = null, int Priority = 0)
+        : TokenType(Tag, Pattern, Priority)
+    {
+        public override bool CanIgnore() => true;
+    }
+}

Interpreter.Lib/FrontEnd/GetTokens/Data/TokenTypes/TokenType.cs

@@ -2,11 +2,7 @@
 {
     public record TokenType(string Tag, string Pattern, int Priority)
     {
-        public TokenType() : this(null, null, 0)
-        {
-        }
-
-        public virtual bool WhiteSpace() => false;
+        public virtual bool CanIgnore() => false;
 
         public virtual bool EndOfProgram() => false;
 

Interpreter.Lib/FrontEnd/GetTokens/Data/TokenTypes/WhiteSpaceType.cs

@@ -33,7 +33,7 @@ public List<Token> GetTokens(string text)
                     _lines.Add(match.Groups["NEWLINE"].Index);
             }
 
-            return this.Where(t => !t.Type.WhiteSpace()).ToList();
+            return this.Where(t => !t.Type.CanIgnore()).ToList();
         }
 
         public IEnumerator<Token> GetEnumerator()

Interpreter.Lib/FrontEnd/GetTokens/Impl/Lexer.cs

@@ -5,7 +5,6 @@
     </PropertyGroup>
 
     <ItemGroup>
-      <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
       <PackageReference Include="Visitor.NET" Version="1.0.0" />
     </ItemGroup>
 

Interpreter.Lib/Interpreter.Lib.csproj

@@ -1,9 +1,7 @@
 using System.Linq;
 using Interpreter.Lib.FrontEnd.GetTokens;
-using Interpreter.Lib.FrontEnd.GetTokens.Data;
 using Interpreter.Lib.FrontEnd.GetTokens.Impl;
-using Interpreter.Models;
-using Interpreter.Tests.Stubs;
+using Interpreter.Services.Providers.Impl.StructureProvider;
 using Interpreter.Tests.TestData;
 using Xunit;
 
@@ -15,8 +13,7 @@ public class LexerTests
 
         public LexerTests()
         {
-            var mapper = new MapperStub();
-            _lexer = new Lexer(mapper.Map<StructureModel, Structure>(new()));
+            _lexer = new Lexer(new StructureProvider().CreateStructure());
         }
 
         [Theory]
@@ -41,5 +38,15 @@ public void LexerToStringCorrectTest()
         [Fact]
         public void EmptyTextTest() => 
             Assert.NotEmpty(_lexer.GetTokens(""));
+
+        [Fact]
+        public void GetTokensSkipIgnorableTypesTest()
+        {
+            const string text = @"
+                let x = 1 // int
+            ";
+            var tokens = _lexer.GetTokens(text);
+            Assert.DoesNotContain(_lexer.Structure.FindByTag("Comment"), tokens.Select(x => x.Type));
+        }
     }
 }

Interpreter.Tests/Stubs/MapperStub.cs

@@ -1,9 +1,7 @@
-using Interpreter.Lib.FrontEnd.GetTokens.Data;
 using Interpreter.Lib.FrontEnd.GetTokens.Impl;
 using Interpreter.Lib.FrontEnd.TopDownParse;
 using Interpreter.Lib.FrontEnd.TopDownParse.Impl;
-using Interpreter.Models;
-using Interpreter.Tests.Stubs;
+using Interpreter.Services.Providers.Impl.StructureProvider;
 using Interpreter.Tests.TestData;
 using Xunit;
 
@@ -15,10 +13,9 @@ public class ParserTests
 
         public ParserTests()
         {
-            var mapper = new MapperStub();
-
             _parser = new Parser(new Lexer(
-                mapper.Map<StructureModel, Structure>(new())
+                new StructureProvider()
+                    .CreateStructure()
             ));
         }
 

Interpreter.Tests/Unit/FrontEnd/LexerTests.cs

@@ -1,11 +1,13 @@
 using System;
+using System.Collections.Generic;
 using Interpreter.Lib.FrontEnd.GetTokens;
+using Interpreter.Lib.FrontEnd.GetTokens.Data;
+using Interpreter.Lib.FrontEnd.GetTokens.Data.TokenTypes;
 using Interpreter.Lib.FrontEnd.GetTokens.Impl;
 using Interpreter.Lib.FrontEnd.TopDownParse.Impl;
 using Interpreter.Services.Providers;
 using Interpreter.Services.Providers.Impl.LexerProvider;
 using Interpreter.Services.Providers.Impl.ParserProvider;
-using Interpreter.Tests.Stubs;
 using Microsoft.Extensions.Options;
 using Moq;
 using Xunit;
@@ -19,6 +21,10 @@ public class ProvidersTests
         [InlineData(typeof(LoggingLexer), true)]
         public void CertainLexerProvidedTest(Type lexerType, bool dump)
         {
+            var structureProvider = new Mock<IStructureProvider>();
+            structureProvider.Setup(x => x.CreateStructure())
+                .Returns(new Structure(new List<TokenType>()));
+
             var options = new Mock<IOptions<CommandLineSettings>>();
             options.Setup(x => x.Value)
                 .Returns(new CommandLineSettings
@@ -27,7 +33,7 @@ public void CertainLexerProvidedTest(Type lexerType, bool dump)
                     InputFilePath = "file.js"
                 });
 
-            var lexerProvider = new LexerProvider(new MapperStub(), options.Object);
+            var lexerProvider = new LexerProvider(structureProvider.Object, options.Object);
             var lexer = lexerProvider.CreateLexer();
 
             Assert.IsType(lexerType, lexer);

Interpreter.Tests/Unit/FrontEnd/ParserTests.cs

@@ -3,7 +3,6 @@
 using System.IO;
 using CommandLine;
 using CommandLine.Text;
-using Interpreter.Models;
 
 namespace Interpreter
 {
@@ -31,9 +30,8 @@ public static IEnumerable<Example> Examples
             }
         }
 
-        public string GetInputFileName() => InputFilePath.Split(".js")[0];
-
-        public StructureModel StructureModel { get; } = new();
+        public string GetInputFileName() =>
+            InputFilePath.Split(".js")[0];
 
         public virtual string GetText() =>
             File.ReadAllText(InputFilePath);

Interpreter.Tests/Unit/Infrastructure/ProvidersTests.cs

@@ -3,18 +3,17 @@
     <PropertyGroup>
         <OutputType>Exe</OutputType>
         <TargetFramework>net6.0</TargetFramework>
-        <Version>1.1.3</Version>
+        <Version>1.1.4</Version>
     </PropertyGroup>
 
     <ItemGroup>
       <ProjectReference Include="..\Interpreter.Lib\Interpreter.Lib.csproj" />
     </ItemGroup>
 
     <ItemGroup>
-      <PackageReference Include="AutoMapper" Version="11.0.1" />
-      <PackageReference Include="AutoMapper.Extensions.Microsoft.DependencyInjection" Version="11.0.0" />
       <PackageReference Include="CommandLineParser" Version="2.9.1" />
       <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="6.0.0" />
+      <PackageReference Include="Microsoft.Extensions.Options" Version="6.0.0" />
       <PackageReference Include="System.IO.Abstractions" Version="17.2.1" />
     </ItemGroup>
 

Interpreter/CommandLineSettings.cs

@@ -2,14 +2,14 @@
 using System.Diagnostics.CodeAnalysis;
 using CommandLine;
 using Microsoft.Extensions.DependencyInjection;
-using Interpreter.MappingProfiles;
 using Interpreter.Services.Executor;
 using Interpreter.Services.Executor.Impl;
 using Interpreter.Services.Parsing;
 using Interpreter.Services.Parsing.Impl;
 using Interpreter.Services.Providers;
 using Interpreter.Services.Providers.Impl.LexerProvider;
 using Interpreter.Services.Providers.Impl.ParserProvider;
+using Interpreter.Services.Providers.Impl.StructureProvider;
 using Microsoft.Extensions.Options;
 
 namespace Interpreter
@@ -34,12 +34,11 @@ private static void Main(string[] args) =>
 
         private static void ConfigureServices(CommandLineSettings settings)
         {
+            ServiceCollection.AddSingleton<IStructureProvider, StructureProvider>();
             ServiceCollection.AddSingleton<ILexerProvider, LexerProvider>();
             ServiceCollection.AddSingleton<IParserProvider, ParserProvider>();
             ServiceCollection.AddSingleton<IParsingService, ParsingService>();
 
-            ServiceCollection.AddAutoMapper(typeof(TokenTypeProfile), typeof(StructureProfile));
-
             ServiceCollection.AddSingleton<IExecutor, Executor>();
 
             ServiceCollection.AddSingleton(_ => Options.Create(settings));