From 4841f047f66644e45b5a9dcfd626ddacb7780a08 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 25 Jul 2024 15:35:19 +0000 Subject: [PATCH 01/58] Create draft PR for #13 [skip ci] From edc0a054b89fde79fd973b792d7def1ba9293299 Mon Sep 17 00:00:00 2001 From: "annette.findley" Date: Tue, 6 Aug 2024 10:28:03 -0400 Subject: [PATCH 02/58] Got the buffer and inMemory WIP with foreach loop --- .../Hyperbee.Templating.csproj | 1 + .../Text/TemplateParser.cs | 183 ++++++++++++++---- src/Hyperbee.Templating/Text/TokenParser.cs | 49 ++++- .../Text/TemplateParser.ExpressionTests.cs | 29 ++- 4 files changed, 224 insertions(+), 38 deletions(-) diff --git a/src/Hyperbee.Templating/Hyperbee.Templating.csproj b/src/Hyperbee.Templating/Hyperbee.Templating.csproj index 7746141..14fb230 100644 --- a/src/Hyperbee.Templating/Hyperbee.Templating.csproj +++ b/src/Hyperbee.Templating/Hyperbee.Templating.csproj @@ -29,6 +29,7 @@ + diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index be0c434..5d1df10 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -104,7 +104,6 @@ public TemplateParser( TokenStyle style, TemplateDictionary source ) } // Render - all the ways - public void Render( string templateFile, string outputFile ) { using var reader = new StreamReader( templateFile ); @@ -127,7 +126,8 @@ public string Render( ReadOnlySpan template ) return template.ToString(); using var writer = new StringWriter(); - ParseTemplate( template, writer, pos ); + + ParseTemplate( template, writer ); return writer.ToString(); } @@ -168,24 +168,30 @@ public string Resolve( string identifier ) return result; } + // Minimal frame management for flow control + private abstract record Frame( TokenType TokenType ); + private record ConditionalFrame( TokenType TokenType, bool Truthy ) : Frame( TokenType ); + private record IterationFrame( TokenType TokenType, string[] LoopResult ) : Frame( TokenType ) + { + public int Index { get; set; } + public int sourcePos { get; set; } + }; private sealed class TemplateStack { - private record Frame( TokenType TokenType, bool Truthy ); - private readonly Stack _stack = new(); - - public void Push( TokenType tokenType, bool truthy ) => _stack.Push( new Frame( tokenType, truthy ) ); + public readonly Stack _stack = new(); + public void Push( TokenType tokenType, bool truthy ) => _stack.Push( new ConditionalFrame( tokenType, truthy ) ); + public void Push( TokenType tokenType, string[] loopResult ) => _stack.Push( new IterationFrame( tokenType, loopResult ) { Index = 0, sourcePos = 0 } ); public void Pop() => _stack.Pop(); public int Depth => _stack.Count; - public bool IsTokenType( TokenType compare ) => _stack.Count > 0 && _stack.Peek().TokenType == compare; - public bool IsTruthy => _stack.Count == 0 || _stack.Peek().Truthy; + public bool IsTruthy => _stack.Count > 0 && _stack.Peek() is ConditionalFrame { Truthy: true }; public bool IsFalsy => !IsTruthy; + public bool IsIterationFrame => _stack.Count > 0 && _stack.Peek() is IterationFrame; } // Parse template - private enum TemplateScanner { Text, @@ -198,8 +204,9 @@ private sealed class TemplateState public int NextTokenId { get; set; } = 1; } + // parse template that spans multiple read buffers - private void ParseTemplate( TextReader reader, TextWriter writer ) + private void ParseTemplate( TextReader reader, TextWriter writer, TemplateState state = null ) { try { @@ -214,12 +221,19 @@ private void ParseTemplate( TextReader reader, TextWriter writer ) var scanner = TemplateScanner.Text; IndexOfState indexOfState = default; // index-of for right token delimiter could span buffer reads - var state = new TemplateState(); // template state for this parsing session + state ??= new TemplateState(); // template state for this parsing session + + var read = reader.Read( buffer, padding, BlockSize ); + var content = buffer.AsSpan( start, read + (padding - start) ); + + var sourceContent = content; + var sourcePos = 0; + var iterationCount = 0; while ( true ) { - var read = reader.Read( buffer, padding, BlockSize ); - var content = buffer.AsSpan( start, read + (padding - start) ); + //var read = reader.Read( buffer, padding, BlockSize ); + //var content = buffer.AsSpan( start, read + (padding - start) ); if ( content.IsEmpty ) break; @@ -243,6 +257,9 @@ private void ParseTemplate( TextReader reader, TextWriter writer ) content = content[(pos + TokenLeft.Length)..]; + if ( !state.Frame.IsIterationFrame ) + sourcePos = pos + sourcePos + TokenLeft.Length; + // transition state scanner = TemplateScanner.Token; start = padding; @@ -252,13 +269,13 @@ private void ParseTemplate( TextReader reader, TextWriter writer ) // no-match eof: write final content if ( read < BlockSize ) { - if ( !ignore ) + if ( !ignore || state.Frame._stack.Count == 0 ) writer.Write( content ); // write final content return; } // no-match: write content less remainder - if ( !ignore ) + if ( !ignore || state.Frame._stack.Count == 0 ) { var writeLength = content.Length - TokenLeft.Length; @@ -288,14 +305,43 @@ private void ParseTemplate( TextReader reader, TextWriter writer ) tokenWriter.Write( content[..pos] ); content = content[(pos + TokenRight.Length)..]; + if ( !state.Frame.IsIterationFrame ) + sourcePos = pos + sourcePos + TokenRight.Length; + // process token var token = TokenParser.ParseToken( tokenWriter.WrittenSpan, state.NextTokenId++ ); var tokenAction = ProcessTokenKind( token, state.Frame, out var tokenValue ); + if ( state.Frame._stack.Count > 0 && state.Frame._stack.Peek() is IterationFrame iterationFrame ) + { + if ( iterationCount != iterationFrame.LoopResult.Length ) + { + iterationFrame.sourcePos = sourcePos; + tokenValue = sourceContent[iterationFrame.sourcePos..].ToString(); + Tokens.Add( "i", iterationFrame.LoopResult[iterationFrame.Index] ); + iterationFrame.Index++; + iterationCount++; + } + } + if ( tokenAction != TokenAction.Ignore ) ProcessTokenValue( writer, tokenValue, tokenAction, state ); - ignore = state.Frame.IsFalsy; + if ( state.Frame._stack.Count > 0 && state.Frame._stack.Peek() is IterationFrame iterationFrame2 ) + { + content = sourceContent[iterationFrame2.sourcePos..]; + + if ( iterationCount == iterationFrame2.LoopResult.Length ) + { + state.Frame.Pop(); + } + + ignore = true; + } + else + { + ignore = state.Frame.IsFalsy; + } tokenWriter.Clear(); @@ -344,10 +390,11 @@ private void ParseTemplate( TextReader reader, TextWriter writer ) } // parse template that is in memory - private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int pos = int.MinValue ) + private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int pos = int.MinValue, TemplateState state = null ) { try { + // find first token starting position if ( pos == int.MinValue ) pos = content.IndexOf( TokenLeft ); @@ -364,9 +411,12 @@ private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int p var tokenWriter = new ArrayBufferWriter(); // defaults to 256 var scanner = TemplateScanner.Text; var ignore = false; + var iterationCount = 0; IndexOfState indexOfState = default; // index-of for right token delimiter could span buffer reads - var state = new TemplateState(); // template state for this parsing session + state ??= new TemplateState(); // template state for this parsing session + var sourceContent = content; + var sourcePos = 0; while ( true ) { @@ -387,29 +437,31 @@ private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int p // match: write to start of token if ( pos >= 0 ) { - // write content + // write content if ( !ignore ) writer.Write( content[..pos] ); content = content[(pos + TokenLeft.Length)..]; + if ( !state.Frame.IsIterationFrame ) + sourcePos = pos + sourcePos + TokenLeft.Length; // transition state scanner = TemplateScanner.Token; continue; } // no-match eof: write final content - if ( !ignore ) + if ( !ignore || state.Frame._stack.Count == 0 ) writer.Write( content ); // write final content return; } - case TemplateScanner.Token: { // scan: find closing token pattern // token may span multiple reads so track search state pos = IndexOfIgnoreContent( content, TokenRight, ref indexOfState ); + // no-match eof: incomplete token if ( pos < 0 ) throw new TemplateException( "Missing right token delimiter." ); @@ -420,14 +472,43 @@ private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int p tokenWriter.Write( content[..pos] ); content = content[(pos + TokenRight.Length)..]; + if ( !state.Frame.IsIterationFrame ) + sourcePos = pos + sourcePos + TokenRight.Length; + // process token var token = TokenParser.ParseToken( tokenWriter.WrittenSpan, state.NextTokenId++ ); - var tokenAction = ProcessTokenKind( token, state.Frame, out var tokenValue ); + var tokenAction = ProcessTokenKind( token, state.Frame, out var tokenValue ); //TODO Error here + + if ( state.Frame._stack.Count > 0 && state.Frame._stack.Peek() is IterationFrame iterationFrame ) + { + if ( iterationCount != iterationFrame.LoopResult.Length ) + { + iterationFrame.sourcePos = sourcePos; + tokenValue = sourceContent[iterationFrame.sourcePos..].ToString(); + Tokens.Add( "i", iterationFrame.LoopResult[iterationFrame.Index] ); + iterationFrame.Index++; + iterationCount++; + } + } if ( tokenAction != TokenAction.Ignore ) ProcessTokenValue( writer, tokenValue, tokenAction, state ); - ignore = state.Frame.IsFalsy; + if ( state.Frame._stack.Count > 0 && state.Frame._stack.Peek() is IterationFrame iterationFrame2 ) + { + content = sourceContent[iterationFrame2.sourcePos..]; + + if ( iterationCount == iterationFrame2.LoopResult.Length ) + { + state.Frame.Pop(); + } + + ignore = true; + } + else + { + ignore = state.Frame.IsFalsy; + } tokenWriter.Clear(); @@ -454,9 +535,9 @@ private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int p writer.Flush(); } } - // Process Token Kind - private TokenAction ProcessTokenKind( TokenDefinition token, TemplateStack frame, out string value ) + // Process Token Kind + private TokenAction ProcessTokenKind( TokenDefinition token, TemplateStack frame, out string value )//ReadOnlySpan body ) { value = default; @@ -464,8 +545,8 @@ private TokenAction ProcessTokenKind( TokenDefinition token, TemplateStack frame switch ( token.TokenType ) { - case TokenType.Value: - if ( frame.IsFalsy ) + case TokenType.Value: //TODO AF Here + if ( frame.IsFalsy && !frame.IsIterationFrame ) return TokenAction.Ignore; break; @@ -495,6 +576,19 @@ private TokenAction ProcessTokenKind( TokenDefinition token, TemplateStack frame Tokens.Add( token.Name, token.TokenExpression ); return TokenAction.Ignore; + case TokenType.Each: + //ToDo AF + + //Delay processing until we can evaluate the token value. + break; + case TokenType.EndEach: + //ToDo AF + //var iterationFrame = (IterationFrame) frame._stack.Peek(); + //if ( iterationFrame.Index == iterationFrame.LoopResult.Length ) + // frame.Pop(); + + return TokenAction.Ignore; + case TokenType.None: default: throw new NotSupportedException( $"{nameof( ProcessTokenKind )}: Invalid {nameof( TokenType )} {token.TokenType}." ); @@ -505,6 +599,7 @@ private TokenAction ProcessTokenKind( TokenDefinition token, TemplateStack frame var defined = false; var ifResult = false; var expressionError = default( string ); + string[] loopResult = null; switch ( token.TokenType ) { @@ -548,6 +643,26 @@ private TokenAction ProcessTokenKind( TokenDefinition token, TemplateStack frame throw new TemplateException( $"{TokenLeft}Error ({token.Id}):{error ?? "Error in if condition."}{TokenRight}" ); break; } + case TokenType.Each when token.TokenEvaluation == TokenEvaluation.Expression: + { + //TODO: AF + // resolves expression result + if ( TryInvokeTokenExpression( token, out var expressionResult, out expressionError ) ) + { + //This gets the tokens "1,2,3" + var loopString = Convert.ToString( expressionResult, CultureInfo.InvariantCulture ); + if ( loopString != null ) + { + loopResult = loopString.Split( ',' ); + } + + defined = true; + // value = body.ToString(); + frame._stack.Push( new IterationFrame( token.TokenType, loopResult ) { Index = 0 } ); + } + + break; + } } // `if` frame handling @@ -613,7 +728,7 @@ private bool TryInvokeTokenExpression( TokenDefinition token, out object result, { try { - var tokenExpression = TokenExpressionProvider.GetTokenExpression( token.TokenExpression ); + var tokenExpression = TokenExpressionProvider.GetTokenExpression( token.TokenExpression ); //TODO AF each errors var dynamicReadOnlyTokens = new ReadOnlyDynamicDictionary( Tokens, (IReadOnlyDictionary) Methods ); result = tokenExpression( dynamicReadOnlyTokens ); @@ -634,7 +749,7 @@ private bool TryInvokeTokenExpression( TokenDefinition token, out object result, private void ProcessTokenValue( TextWriter writer, ReadOnlySpan value, TokenAction tokenAction, TemplateState state, int recursionCount = 0 ) { - // infinite recursion guard + // infinite recursion guard+- if ( recursionCount++ == MaxTokenDepth ) throw new TemplateException( "Recursion depth exceeded." ); @@ -660,14 +775,15 @@ private void ProcessTokenValue( TextWriter writer, ReadOnlySpan value, Tok } // nested token processing - do { - // write any leading literal - - if ( start > 0 && state.Frame.IsTruthy ) + if ( start > 0 && (state.Frame.IsIterationFrame || state.Frame.IsTruthy) ) + { writer.Write( value[..start] ); + } + // write any leading literal + value = value[(start + TokenLeft.Length)..]; // find token end @@ -681,7 +797,6 @@ private void ProcessTokenValue( TextWriter writer, ReadOnlySpan value, Tok value = value[(stop + TokenRight.Length)..]; // process token - var innerToken = TokenParser.ParseToken( innerValue, state.NextTokenId++ ); tokenAction = ProcessTokenKind( innerToken, state.Frame, out var tokenValue ); diff --git a/src/Hyperbee.Templating/Text/TokenParser.cs b/src/Hyperbee.Templating/Text/TokenParser.cs index 90ce70f..d291971 100644 --- a/src/Hyperbee.Templating/Text/TokenParser.cs +++ b/src/Hyperbee.Templating/Text/TokenParser.cs @@ -3,14 +3,16 @@ namespace Hyperbee.Templating.Text; -internal enum TokenType +public enum TokenType { None, Define, Value, If, Else, - Endif + Endif, + Each, + EndEach//TODO: AF } internal enum TokenEvaluation @@ -44,13 +46,15 @@ public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) // // {{else} // {{/if}} + // + // {{each}} + // {{/each}} var content = token.Trim(); var tokenType = TokenType.None; var tokenConditional = TokenEvaluation.None; var tokenExpression = ReadOnlySpan.Empty; - var name = ReadOnlySpan.Empty; // if handling @@ -121,6 +125,45 @@ public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) tokenType = TokenType.Endif; } + else if ( content.StartsWith( "each", StringComparison.OrdinalIgnoreCase ) ) + { + //TODO: AF + + tokenType = TokenType.Each; + content = content[4..].Trim(); // eat the 'each' + + if ( content.Length >= 4 ) + { + // detect expression syntax + var isFatArrow = content.IndexOfIgnoreDelimitedRanges( "=>", "\"" ) != -1; + + // validate + if ( content.IsEmpty ) + throw new TemplateException( "Invalid `each` statement. Missing identifier." ); + + if ( !isFatArrow && !ValidateKey( content ) ) + throw new TemplateException( "Invalid `each` statement. Invalid identifier in truthy expression." ); + + // results + if ( isFatArrow ) + { + tokenConditional = TokenEvaluation.Expression; + tokenExpression = content; //x=>x.list + } + else + { + tokenConditional = TokenEvaluation.Falsy; + name = content; + } + } + } + else if ( content.StartsWith( "/each", StringComparison.OrdinalIgnoreCase ) ) + { + + tokenType = TokenType.EndEach; + + } + // value handling diff --git a/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs b/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs index cc9f6cd..9e41927 100644 --- a/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs +++ b/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; + +using System.Collections.Generic; using Hyperbee.Collections; using Hyperbee.Templating.Tests.TestSupport; using Hyperbee.Templating.Text; @@ -217,6 +218,32 @@ public void Should_honor_if_expression( ParseTemplateMethod parseMethod ) Assert.AreEqual( expected, result ); } + [DataTestMethod] + + [DataRow( ParseTemplateMethod.Buffered )] + //[DataRow( ParseTemplateMethod.InMemory )] + public void Should_honor_each_expression( ParseTemplateMethod parseMethod ) + { + // arrange + const string expression = "{{each x=>x.list}}World {{i}},{{/each}}"; + const string template = $"hello {expression}."; + + var parser = new TemplateParser + { + Tokens = { + ["list"] = "1,2,3" + } + }; + + // act + var result = parser.Render( template, parseMethod ); + + // assert + var expected = "hello World 1,World 2,World 3,."; + + Assert.AreEqual( expected, result ); + } + [DataTestMethod] [DataRow( ParseTemplateMethod.Buffered )] [DataRow( ParseTemplateMethod.InMemory )] From d844904a694f72efadc9e8ce7b1ae9cd0e94da8a Mon Sep 17 00:00:00 2001 From: "annette.findley" Date: Wed, 7 Aug 2024 08:39:27 -0400 Subject: [PATCH 03/58] wip fixing tests --- .../Text/TemplateParser.cs | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index 5d1df10..ee7bab7 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -187,8 +187,8 @@ private sealed class TemplateStack public int Depth => _stack.Count; public bool IsTokenType( TokenType compare ) => _stack.Count > 0 && _stack.Peek().TokenType == compare; public bool IsTruthy => _stack.Count > 0 && _stack.Peek() is ConditionalFrame { Truthy: true }; - public bool IsFalsy => !IsTruthy; public bool IsIterationFrame => _stack.Count > 0 && _stack.Peek() is IterationFrame; + public bool IsConditionalFrame => _stack.Count > 0 && _stack.Peek() is ConditionalFrame; } // Parse template @@ -338,9 +338,14 @@ private void ParseTemplate( TextReader reader, TextWriter writer, TemplateState ignore = true; } + else if ( state.Frame._stack.Count > 0 && state.Frame._stack.Peek() is ConditionalFrame conditionalFrame ) + { + + ignore = state.Frame.IsTruthy; + } else { - ignore = state.Frame.IsFalsy; + ignore = state.Frame.IsTruthy; } tokenWriter.Clear(); @@ -505,9 +510,14 @@ private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int p ignore = true; } + else if ( state.Frame._stack.Count > 0 && state.Frame._stack.Peek() is ConditionalFrame conditionalFrame ) + { + + ignore = state.Frame.IsTruthy; + } else { - ignore = state.Frame.IsFalsy; + ignore = state.Frame.IsTruthy; } tokenWriter.Clear(); @@ -537,7 +547,7 @@ private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int p } // Process Token Kind - private TokenAction ProcessTokenKind( TokenDefinition token, TemplateStack frame, out string value )//ReadOnlySpan body ) + private TokenAction ProcessTokenKind( TokenDefinition token, TemplateStack frame, out string value ) { value = default; @@ -546,7 +556,7 @@ private TokenAction ProcessTokenKind( TokenDefinition token, TemplateStack frame switch ( token.TokenType ) { case TokenType.Value: //TODO AF Here - if ( frame.IsFalsy && !frame.IsIterationFrame ) + if ( frame.IsTruthy && !frame.IsIterationFrame ) return TokenAction.Ignore; break; @@ -728,7 +738,7 @@ private bool TryInvokeTokenExpression( TokenDefinition token, out object result, { try { - var tokenExpression = TokenExpressionProvider.GetTokenExpression( token.TokenExpression ); //TODO AF each errors + var tokenExpression = TokenExpressionProvider.GetTokenExpression( token.TokenExpression ); var dynamicReadOnlyTokens = new ReadOnlyDynamicDictionary( Tokens, (IReadOnlyDictionary) Methods ); result = tokenExpression( dynamicReadOnlyTokens ); From d3e697745d548b5c0cbe5421f285bbdddff96dad Mon Sep 17 00:00:00 2001 From: "annette.findley" Date: Wed, 7 Aug 2024 11:48:01 -0400 Subject: [PATCH 04/58] wip tests --- .../Text/TemplateParser.cs | 32 +++++++------------ .../Text/TemplateParser.ExpressionTests.cs | 2 +- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index ee7bab7..9368bf3 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -336,17 +336,13 @@ private void ParseTemplate( TextReader reader, TextWriter writer, TemplateState state.Frame.Pop(); } - ignore = true; + ignore = !state.Frame.IsTruthy; } - else if ( state.Frame._stack.Count > 0 && state.Frame._stack.Peek() is ConditionalFrame conditionalFrame ) - { + //else if ( state.Frame._stack.Count > 0 && state.Frame._stack.Peek() is ConditionalFrame conditionalFrame ) + //{ - ignore = state.Frame.IsTruthy; - } - else - { - ignore = state.Frame.IsTruthy; - } + // ignore = state.Frame.IsTruthy; + //} tokenWriter.Clear(); @@ -450,6 +446,7 @@ private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int p if ( !state.Frame.IsIterationFrame ) sourcePos = pos + sourcePos + TokenLeft.Length; + // transition state scanner = TemplateScanner.Token; continue; @@ -508,17 +505,13 @@ private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int p state.Frame.Pop(); } - ignore = true; + ignore = !state.Frame.IsTruthy; } - else if ( state.Frame._stack.Count > 0 && state.Frame._stack.Peek() is ConditionalFrame conditionalFrame ) - { + //else if ( state.Frame._stack.Count > 0 && state.Frame._stack.Peek() is ConditionalFrame conditionalFrame ) + //{ + // ignore = !state.Frame.IsTruthy; + //} - ignore = state.Frame.IsTruthy; - } - else - { - ignore = state.Frame.IsTruthy; - } tokenWriter.Clear(); @@ -556,7 +549,7 @@ private TokenAction ProcessTokenKind( TokenDefinition token, TemplateStack frame switch ( token.TokenType ) { case TokenType.Value: //TODO AF Here - if ( frame.IsTruthy && !frame.IsIterationFrame ) + if ( frame._stack.Count > 0 && !frame.IsIterationFrame || (frame.IsConditionalFrame && !frame.IsTruthy) ) return TokenAction.Ignore; break; @@ -667,7 +660,6 @@ private TokenAction ProcessTokenKind( TokenDefinition token, TemplateStack frame } defined = true; - // value = body.ToString(); frame._stack.Push( new IterationFrame( token.TokenType, loopResult ) { Index = 0 } ); } diff --git a/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs b/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs index 9e41927..0fc47cb 100644 --- a/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs +++ b/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs @@ -221,7 +221,7 @@ public void Should_honor_if_expression( ParseTemplateMethod parseMethod ) [DataTestMethod] [DataRow( ParseTemplateMethod.Buffered )] - //[DataRow( ParseTemplateMethod.InMemory )] + [DataRow( ParseTemplateMethod.InMemory )] public void Should_honor_each_expression( ParseTemplateMethod parseMethod ) { // arrange From 5f25cd40922e81086e95eb72c275c531694f2b14 Mon Sep 17 00:00:00 2001 From: "annette.findley" Date: Thu, 8 Aug 2024 09:31:30 -0400 Subject: [PATCH 05/58] WIP with tests --- .../Text/TemplateParser.cs | 74 ++++++++++++++----- 1 file changed, 57 insertions(+), 17 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index 9368bf3..c9007ab 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -172,23 +172,48 @@ public string Resolve( string identifier ) // Minimal frame management for flow control private abstract record Frame( TokenType TokenType ); private record ConditionalFrame( TokenType TokenType, bool Truthy ) : Frame( TokenType ); + private record IterationFrame( TokenType TokenType, string[] LoopResult ) : Frame( TokenType ) { public int Index { get; set; } - public int sourcePos { get; set; } + public int SourcePos { get; set; } }; private sealed class TemplateStack { public readonly Stack _stack = new(); public void Push( TokenType tokenType, bool truthy ) => _stack.Push( new ConditionalFrame( tokenType, truthy ) ); - public void Push( TokenType tokenType, string[] loopResult ) => _stack.Push( new IterationFrame( tokenType, loopResult ) { Index = 0, sourcePos = 0 } ); + public void Push( TokenType tokenType, string[] loopResult ) => _stack.Push( new IterationFrame( tokenType, loopResult ) { Index = 0, SourcePos = 0 } ); public void Pop() => _stack.Pop(); public int Depth => _stack.Count; public bool IsTokenType( TokenType compare ) => _stack.Count > 0 && _stack.Peek().TokenType == compare; public bool IsTruthy => _stack.Count > 0 && _stack.Peek() is ConditionalFrame { Truthy: true }; public bool IsIterationFrame => _stack.Count > 0 && _stack.Peek() is IterationFrame; public bool IsConditionalFrame => _stack.Count > 0 && _stack.Peek() is ConditionalFrame; + + public bool? IsFalsy() + { + if ( _stack == null ) + { + return null; + } + + foreach ( var frame in _stack.Reverse() ) + { + if ( frame is ConditionalFrame conditionalFrame ) + { + return conditionalFrame.Truthy; + } + } + + return null; + } + + public bool IsComplete + { + get; + set; + } } // Parse template @@ -316,8 +341,8 @@ private void ParseTemplate( TextReader reader, TextWriter writer, TemplateState { if ( iterationCount != iterationFrame.LoopResult.Length ) { - iterationFrame.sourcePos = sourcePos; - tokenValue = sourceContent[iterationFrame.sourcePos..].ToString(); + iterationFrame.SourcePos = sourcePos; + tokenValue = sourceContent[iterationFrame.SourcePos..].ToString(); Tokens.Add( "i", iterationFrame.LoopResult[iterationFrame.Index] ); iterationFrame.Index++; iterationCount++; @@ -329,20 +354,25 @@ private void ParseTemplate( TextReader reader, TextWriter writer, TemplateState if ( state.Frame._stack.Count > 0 && state.Frame._stack.Peek() is IterationFrame iterationFrame2 ) { - content = sourceContent[iterationFrame2.sourcePos..]; + content = sourceContent[iterationFrame2.SourcePos..]; if ( iterationCount == iterationFrame2.LoopResult.Length ) { state.Frame.Pop(); + state.Frame.IsComplete = true; } ignore = !state.Frame.IsTruthy; } - //else if ( state.Frame._stack.Count > 0 && state.Frame._stack.Peek() is ConditionalFrame conditionalFrame ) - //{ + else if ( state.Frame._stack.Count == 0 ) + { - // ignore = state.Frame.IsTruthy; - //} + ignore = state.Frame.IsTruthy; + } + else + { + ignore = !state.Frame.IsTruthy; + } tokenWriter.Clear(); @@ -485,8 +515,8 @@ private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int p { if ( iterationCount != iterationFrame.LoopResult.Length ) { - iterationFrame.sourcePos = sourcePos; - tokenValue = sourceContent[iterationFrame.sourcePos..].ToString(); + iterationFrame.SourcePos = sourcePos; + tokenValue = sourceContent[iterationFrame.SourcePos..].ToString(); Tokens.Add( "i", iterationFrame.LoopResult[iterationFrame.Index] ); iterationFrame.Index++; iterationCount++; @@ -498,19 +528,25 @@ private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int p if ( state.Frame._stack.Count > 0 && state.Frame._stack.Peek() is IterationFrame iterationFrame2 ) { - content = sourceContent[iterationFrame2.sourcePos..]; + content = sourceContent[iterationFrame2.SourcePos..]; if ( iterationCount == iterationFrame2.LoopResult.Length ) { state.Frame.Pop(); + state.Frame.IsComplete = true; } + else + ignore = !state.Frame.IsTruthy; + } + else if ( state.Frame._stack.Count == 0 ) + { + ignore = state.Frame.IsTruthy; + } + else + { ignore = !state.Frame.IsTruthy; } - //else if ( state.Frame._stack.Count > 0 && state.Frame._stack.Peek() is ConditionalFrame conditionalFrame ) - //{ - // ignore = !state.Frame.IsTruthy; - //} tokenWriter.Clear(); @@ -549,7 +585,11 @@ private TokenAction ProcessTokenKind( TokenDefinition token, TemplateStack frame switch ( token.TokenType ) { case TokenType.Value: //TODO AF Here - if ( frame._stack.Count > 0 && !frame.IsIterationFrame || (frame.IsConditionalFrame && !frame.IsTruthy) ) + if ( (frame._stack.Count == 0 && frame.IsTruthy) || (frame._stack.Count == 0 && frame.IsComplete) ) + return TokenAction.Ignore; + if ( frame._stack.Count > 0 && frame.IsIterationFrame && frame.IsComplete ) + return TokenAction.Ignore; + if ( (frame._stack.Count > 0 && frame.IsConditionalFrame && !frame.IsTruthy) || (frame._stack.Count > 0 && frame.IsIterationFrame && frame.IsComplete) ) return TokenAction.Ignore; break; From 7752cf0f041f7be6fe862a83a69b8fadbfd24149 Mon Sep 17 00:00:00 2001 From: "annette.findley" Date: Thu, 8 Aug 2024 11:41:30 -0400 Subject: [PATCH 06/58] wip Tests --- src/Hyperbee.Templating/Text/TemplateParser.cs | 10 ++++++++++ .../Text/TemplateParser.ExpressionTests.cs | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index c9007ab..eb12a27 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -364,6 +364,11 @@ private void ParseTemplate( TextReader reader, TextWriter writer, TemplateState ignore = !state.Frame.IsTruthy; } + else if ( state.Frame._stack.Count == 0 && state.Frame.IsComplete ) + { + + ignore = !state.Frame.IsTruthy; + } else if ( state.Frame._stack.Count == 0 ) { @@ -538,6 +543,11 @@ private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int p else ignore = !state.Frame.IsTruthy; } + else if ( state.Frame._stack.Count == 0 && state.Frame.IsComplete ) + { + + ignore = !state.Frame.IsTruthy; + } else if ( state.Frame._stack.Count == 0 ) { diff --git a/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs b/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs index 0fc47cb..f65d166 100644 --- a/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs +++ b/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs @@ -259,7 +259,7 @@ public void Should_honor_conditional_nested_tokens( ParseTemplateMethod parseMet { ["name"] = "{{first}} {{last_condition}}", ["first"] = "hari", - ["last"] = "seldon", + ["last"] = " seldon", ["last_condition"] = "{{if upper}}{{last_upper}}{{else}}{{last}}{{/if}}", ["last_upper"] = "{{x=>x.last.ToUpper()}}", @@ -293,7 +293,7 @@ public void Should_resolve_conditional_nested_tokens_with_custom_source( ParseTe { ["name"] = "{{first}} {{last_condition}}", ["first"] = "not-hari", - ["last"] = "seldon", + ["last"] = " seldon", ["last_upper"] = "{{x=>x.last.ToUpper()}}", } ); From 2f7283d470505d56d621ccca6d7b8fb3731e93d6 Mon Sep 17 00:00:00 2001 From: "annette.findley" Date: Thu, 8 Aug 2024 13:48:41 -0400 Subject: [PATCH 07/58] WIP Tests Are working --- src/Hyperbee.Templating/Text/TemplateParser.cs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index eb12a27..c7a232d 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -236,7 +236,6 @@ private void ParseTemplate( TextReader reader, TextWriter writer, TemplateState try { var ignore = false; - var padding = Math.Max( TokenLeft.Length, TokenRight.Length ); var start = padding; @@ -248,17 +247,15 @@ private void ParseTemplate( TextReader reader, TextWriter writer, TemplateState IndexOfState indexOfState = default; // index-of for right token delimiter could span buffer reads state ??= new TemplateState(); // template state for this parsing session - var read = reader.Read( buffer, padding, BlockSize ); - var content = buffer.AsSpan( start, read + (padding - start) ); - - var sourceContent = content; var sourcePos = 0; var iterationCount = 0; while ( true ) { - //var read = reader.Read( buffer, padding, BlockSize ); - //var content = buffer.AsSpan( start, read + (padding - start) ); + var read = reader.Read( buffer, padding, BlockSize ); + var content = buffer.AsSpan( start, read + (padding - start) ); + + var sourceContent = content; if ( content.IsEmpty ) break; From e51d9ea12cc31316363d2e69ab5ccf78090c3004 Mon Sep 17 00:00:00 2001 From: "annette.findley" Date: Fri, 9 Aug 2024 12:15:36 -0400 Subject: [PATCH 08/58] Fixed all the tests --- .../Text/TemplateParser.cs | 131 ++++++++---------- .../Text/TemplateParserTests.cs | 2 +- 2 files changed, 55 insertions(+), 78 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index c7a232d..bae8c01 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -190,30 +190,7 @@ private sealed class TemplateStack public bool IsTruthy => _stack.Count > 0 && _stack.Peek() is ConditionalFrame { Truthy: true }; public bool IsIterationFrame => _stack.Count > 0 && _stack.Peek() is IterationFrame; public bool IsConditionalFrame => _stack.Count > 0 && _stack.Peek() is ConditionalFrame; - - public bool? IsFalsy() - { - if ( _stack == null ) - { - return null; - } - - foreach ( var frame in _stack.Reverse() ) - { - if ( frame is ConditionalFrame conditionalFrame ) - { - return conditionalFrame.Truthy; - } - } - - return null; - } - - public bool IsComplete - { - get; - set; - } + public bool IsComplete { get; set; } } // Parse template @@ -349,31 +326,30 @@ private void ParseTemplate( TextReader reader, TextWriter writer, TemplateState if ( tokenAction != TokenAction.Ignore ) ProcessTokenValue( writer, tokenValue, tokenAction, state ); - if ( state.Frame._stack.Count > 0 && state.Frame._stack.Peek() is IterationFrame iterationFrame2 ) + switch ( state.Frame._stack.Count ) { - content = sourceContent[iterationFrame2.SourcePos..]; - - if ( iterationCount == iterationFrame2.LoopResult.Length ) - { - state.Frame.Pop(); - state.Frame.IsComplete = true; - } - - ignore = !state.Frame.IsTruthy; - } - else if ( state.Frame._stack.Count == 0 && state.Frame.IsComplete ) - { - - ignore = !state.Frame.IsTruthy; - } - else if ( state.Frame._stack.Count == 0 ) - { - - ignore = state.Frame.IsTruthy; - } - else - { - ignore = !state.Frame.IsTruthy; + case > 0 when state.Frame._stack.Peek() is IterationFrame iterationFrame2: + { + content = sourceContent[iterationFrame2.SourcePos..]; + + if ( iterationCount == iterationFrame2.LoopResult.Length ) + { + state.Frame.Pop(); + state.Frame.IsComplete = true; + } + + ignore = !state.Frame.IsTruthy; + break; + } + case 0 when state.Frame.IsComplete: + ignore = !state.Frame.IsTruthy; + break; + case 0: + ignore = state.Frame.IsTruthy; + break; + default: + ignore = !state.Frame.IsTruthy; + break; } tokenWriter.Clear(); @@ -528,31 +504,31 @@ private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int p if ( tokenAction != TokenAction.Ignore ) ProcessTokenValue( writer, tokenValue, tokenAction, state ); - if ( state.Frame._stack.Count > 0 && state.Frame._stack.Peek() is IterationFrame iterationFrame2 ) - { - content = sourceContent[iterationFrame2.SourcePos..]; - - if ( iterationCount == iterationFrame2.LoopResult.Length ) - { - state.Frame.Pop(); - state.Frame.IsComplete = true; - } - else - ignore = !state.Frame.IsTruthy; - } - else if ( state.Frame._stack.Count == 0 && state.Frame.IsComplete ) + switch ( state.Frame._stack.Count ) { + case > 0 when state.Frame._stack.Peek() is IterationFrame iterationFrame2: + { + content = sourceContent[iterationFrame2.SourcePos..]; - ignore = !state.Frame.IsTruthy; - } - else if ( state.Frame._stack.Count == 0 ) - { + if ( iterationCount == iterationFrame2.LoopResult.Length ) + { + state.Frame.Pop(); + state.Frame.IsComplete = true; + } + else + ignore = !state.Frame.IsTruthy; - ignore = state.Frame.IsTruthy; - } - else - { - ignore = !state.Frame.IsTruthy; + break; + } + case 0 when state.Frame.IsComplete: + ignore = !state.Frame.IsTruthy; + break; + case 0: + ignore = state.Frame.IsTruthy; + break; + default: + ignore = !state.Frame.IsTruthy; + break; } @@ -592,12 +568,13 @@ private TokenAction ProcessTokenKind( TokenDefinition token, TemplateStack frame switch ( token.TokenType ) { case TokenType.Value: //TODO AF Here - if ( (frame._stack.Count == 0 && frame.IsTruthy) || (frame._stack.Count == 0 && frame.IsComplete) ) - return TokenAction.Ignore; - if ( frame._stack.Count > 0 && frame.IsIterationFrame && frame.IsComplete ) - return TokenAction.Ignore; - if ( (frame._stack.Count > 0 && frame.IsConditionalFrame && !frame.IsTruthy) || (frame._stack.Count > 0 && frame.IsIterationFrame && frame.IsComplete) ) - return TokenAction.Ignore; + switch ( frame._stack.Count ) + { + case 0 when (frame.IsTruthy || frame.IsComplete): + case > 0 when (frame.IsConditionalFrame && !frame.IsTruthy || frame.IsIterationFrame && frame.IsComplete): + return TokenAction.Ignore; + } + break; case TokenType.If: @@ -628,7 +605,6 @@ private TokenAction ProcessTokenKind( TokenDefinition token, TemplateStack frame case TokenType.Each: //ToDo AF - //Delay processing until we can evaluate the token value. break; case TokenType.EndEach: @@ -862,6 +838,7 @@ private void ProcessTokenValue( TextWriter writer, ReadOnlySpan value, Tok } while ( start != -1 ); } + // IndexOf helper private record struct IndexOfState() diff --git a/test/Hyperbee.Templating.Tests/Text/TemplateParserTests.cs b/test/Hyperbee.Templating.Tests/Text/TemplateParserTests.cs index f0ac563..eda5773 100644 --- a/test/Hyperbee.Templating.Tests/Text/TemplateParserTests.cs +++ b/test/Hyperbee.Templating.Tests/Text/TemplateParserTests.cs @@ -258,7 +258,7 @@ public void Should_render_nested_tokens( ParseTemplateMethod parseMethod ) { ["name"] = "{{first}} {{last_expression}}", ["first"] = "hari", - ["last"] = "seldon", + ["last"] = " seldon", ["last_expression"] = "{{last}}" } }; From 88177fc6dcd99302e797bbad7f4e74d3171db7c0 Mon Sep 17 00:00:00 2001 From: Brenton Farmer Date: Sat, 10 Aug 2024 19:48:45 -0700 Subject: [PATCH 09/58] WIP Add 'while' and refactor to use TokenProcessor --- .../Text/TemplateParser.cs | 282 +++++------------- src/Hyperbee.Templating/Text/TokenAction.cs | 3 +- .../Text/TokenDefinition.cs | 1 + src/Hyperbee.Templating/Text/TokenParser.cs | 134 +++++++-- .../Text/TokenProcessor.cs | 267 +++++++++++++++++ .../Text/TemplateParser.ExpressionTests.cs | 21 ++ .../Text/TemplateParser.ParsingTests.cs | 2 +- 7 files changed, 467 insertions(+), 243 deletions(-) create mode 100644 src/Hyperbee.Templating/Text/TokenProcessor.cs diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index be0c434..8ea2a2e 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -1,5 +1,4 @@ using System.Buffers; -using System.Globalization; using Hyperbee.Templating.Collections; using Hyperbee.Templating.Compiler; using Hyperbee.Templating.Extensions; @@ -40,12 +39,11 @@ public class TemplateParser private string TokenLeft { get; } private string TokenRight { get; } - private TokenParser _tokenParser; internal TokenParser TokenParser { - get { return _tokenParser ??= new TokenParser( Tokens.Validator ); } + get { return _tokenParser ??= new TokenParser( Tokens.Validator, TokenLeft, TokenRight ); } } public TemplateParser() @@ -168,22 +166,6 @@ public string Resolve( string identifier ) return result; } - // Minimal frame management for flow control - - private sealed class TemplateStack - { - private record Frame( TokenType TokenType, bool Truthy ); - private readonly Stack _stack = new(); - - public void Push( TokenType tokenType, bool truthy ) => _stack.Push( new Frame( tokenType, truthy ) ); - public void Pop() => _stack.Pop(); - public int Depth => _stack.Count; - - public bool IsTokenType( TokenType compare ) => _stack.Count > 0 && _stack.Peek().TokenType == compare; - public bool IsTruthy => _stack.Count == 0 || _stack.Peek().Truthy; - public bool IsFalsy => !IsTruthy; - } - // Parse template private enum TemplateScanner @@ -192,15 +174,20 @@ private enum TemplateScanner Token } - private sealed class TemplateState - { - public TemplateStack Frame { get; } = new(); - public int NextTokenId { get; set; } = 1; - } - // parse template that spans multiple read buffers private void ParseTemplate( TextReader reader, TextWriter writer ) { + var tokenProcessor = new TokenProcessor( + Tokens, + Methods, + TokenHandler, + TokenExpressionProvider, + IgnoreMissingTokens, + SubstituteEnvironmentVariables, + TokenLeft, + TokenRight + ); + try { var ignore = false; @@ -290,10 +277,10 @@ private void ParseTemplate( TextReader reader, TextWriter writer ) // process token var token = TokenParser.ParseToken( tokenWriter.WrittenSpan, state.NextTokenId++ ); - var tokenAction = ProcessTokenKind( token, state.Frame, out var tokenValue ); + var tokenAction = tokenProcessor.ProcessTokenType( token, state, out var tokenValue ); if ( tokenAction != TokenAction.Ignore ) - ProcessTokenValue( writer, tokenValue, tokenAction, state ); + WriteTokenValue( writer, tokenProcessor, tokenValue, tokenAction, state ); ignore = state.Frame.IsFalsy; @@ -346,6 +333,17 @@ private void ParseTemplate( TextReader reader, TextWriter writer ) // parse template that is in memory private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int pos = int.MinValue ) { + var tokenProcessor = new TokenProcessor( + Tokens, + Methods, + TokenHandler, + TokenExpressionProvider, + IgnoreMissingTokens, + SubstituteEnvironmentVariables, + TokenLeft, + TokenRight + ); + try { // find first token starting position @@ -368,6 +366,8 @@ private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int p IndexOfState indexOfState = default; // index-of for right token delimiter could span buffer reads var state = new TemplateState(); // template state for this parsing session + var originalSpan = content; // Keep the original content span for resetting the position + while ( true ) { if ( content.IsEmpty ) @@ -375,6 +375,8 @@ private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int p while ( !content.IsEmpty ) { + state.CurrentPos = originalSpan.Length - content.Length; // Track the current position + switch ( scanner ) { case TemplateScanner.Text: @@ -416,16 +418,28 @@ private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int p // match: process completed token + // update CurrentPos to point to the first character after the token + state.CurrentPos = originalSpan.Length - content.Length + pos + TokenRight.Length; + // save token chars tokenWriter.Write( content[..pos] ); content = content[(pos + TokenRight.Length)..]; // process token var token = TokenParser.ParseToken( tokenWriter.WrittenSpan, state.NextTokenId++ ); - var tokenAction = ProcessTokenKind( token, state.Frame, out var tokenValue ); + var tokenAction = tokenProcessor.ProcessTokenType( token, state, out var tokenValue ); + + if ( tokenAction == TokenAction.Replay ) + { + // Reset the position to start of while block + content = originalSpan[state.Frame.Peek().StartPos..]; // Reset position to StartPos + scanner = TemplateScanner.Text; + tokenWriter.Clear(); + continue; + } if ( tokenAction != TokenAction.Ignore ) - ProcessTokenValue( writer, tokenValue, tokenAction, state ); + WriteTokenValue( writer, tokenProcessor, tokenValue, tokenAction, state ); ignore = state.Frame.IsFalsy; @@ -454,185 +468,8 @@ private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int p writer.Flush(); } } - // Process Token Kind - - private TokenAction ProcessTokenKind( TokenDefinition token, TemplateStack frame, out string value ) - { - value = default; - - // flow control - - switch ( token.TokenType ) - { - case TokenType.Value: - if ( frame.IsFalsy ) - return TokenAction.Ignore; - break; - - case TokenType.If: - // ifs are truthy. delay processing until we can evaluate the token value. - break; - - case TokenType.Else: - if ( !frame.IsTokenType( TokenType.If ) ) - throw new TemplateException( "Syntax error. Invalid `else` without matching `if`." ); - - frame.Push( TokenType.Else, !frame.IsTruthy ); - return TokenAction.Ignore; - - case TokenType.Endif: - if ( frame.Depth == 0 || !frame.IsTokenType( TokenType.If ) && !frame.IsTokenType( TokenType.Else ) ) - throw new TemplateException( "Syntax error. Invalid `/if` without matching `if`." ); - - if ( frame.IsTokenType( TokenType.Else ) ) - frame.Pop(); // pop the else - - frame.Pop(); // pop the if - - return TokenAction.Ignore; - - case TokenType.Define: - Tokens.Add( token.Name, token.TokenExpression ); - return TokenAction.Ignore; - - case TokenType.None: - default: - throw new NotSupportedException( $"{nameof( ProcessTokenKind )}: Invalid {nameof( TokenType )} {token.TokenType}." ); - } - - // resolve value - - var defined = false; - var ifResult = false; - var expressionError = default( string ); - - switch ( token.TokenType ) - { - case TokenType.Value when token.TokenEvaluation != TokenEvaluation.Expression: - case TokenType.If when token.TokenEvaluation != TokenEvaluation.Expression: - { - // resolve variable value - defined = Tokens.TryGetValue( token.Name, out value ); - - if ( !defined && SubstituteEnvironmentVariables ) - { - // optionally try and replace value from environment variable - // otherwise set token value to null and behavior to error - - value = Environment.GetEnvironmentVariable( token.Name ); - defined = value != null; - } - - // resolve if truthy result - if ( token.TokenType == TokenType.If ) - ifResult = defined && TemplateHelper.Truthy( value ); - break; - } - case TokenType.Value when token.TokenEvaluation == TokenEvaluation.Expression: - { - // resolve variable expression - if ( TryInvokeTokenExpression( token, out var expressionResult, out expressionError ) ) - { - value = Convert.ToString( expressionResult, CultureInfo.InvariantCulture ); - defined = true; - } - - break; - } - case TokenType.If when token.TokenEvaluation == TokenEvaluation.Expression: - { - // resolve if expression result - if ( TryInvokeTokenExpression( token, out var expressionResult, out var error ) ) - ifResult = Convert.ToBoolean( expressionResult ); - else - throw new TemplateException( $"{TokenLeft}Error ({token.Id}):{error ?? "Error in if condition."}{TokenRight}" ); - break; - } - } - - // `if` frame handling - - if ( token.TokenType == TokenType.If ) - { - var frameIsTruthy = token.TokenEvaluation == TokenEvaluation.Falsy ? !ifResult : ifResult; - - frame.Push( token.TokenType, frameIsTruthy ); - return TokenAction.Ignore; - } - - // set token action - - var tokenAction = defined - ? TokenAction.Replace - : IgnoreMissingTokens - ? TokenAction.Ignore - : TokenAction.Error; - - // invoke any token handler - - if ( TokenHandler != null ) - { - var eventArgs = new TemplateEventArgs - { - Id = token.Id, - Name = token.Name, - Value = value, - Action = tokenAction, - UnknownToken = !defined - }; - - TokenHandler( this, eventArgs ); - - // the token handler may have modified token properties - // get any potentially updated values - - value = eventArgs.Value; - tokenAction = eventArgs.Action; - } - - // handle token action - - switch ( tokenAction ) - { - case TokenAction.Ignore: - return TokenAction.Ignore; - - case TokenAction.Error: - value = $"{TokenLeft}Error ({token.Id}):{expressionError ?? token.Name}{TokenRight}"; - return TokenAction.Error; - - case TokenAction.Replace: - return TokenAction.Replace; - - default: - throw new NotSupportedException( $"{nameof( ProcessTokenKind )}: Invalid {nameof( TokenAction )} {tokenAction}." ); - } - } - - private bool TryInvokeTokenExpression( TokenDefinition token, out object result, out string error ) - { - try - { - var tokenExpression = TokenExpressionProvider.GetTokenExpression( token.TokenExpression ); - var dynamicReadOnlyTokens = new ReadOnlyDynamicDictionary( Tokens, (IReadOnlyDictionary) Methods ); - - result = tokenExpression( dynamicReadOnlyTokens ); - error = default; - - return true; - } - catch ( Exception ex ) - { - error = ex.Message; - } - - result = default; - return false; - } - // Process Template Value (recursive) - - private void ProcessTokenValue( TextWriter writer, ReadOnlySpan value, TokenAction tokenAction, TemplateState state, int recursionCount = 0 ) + private void WriteTokenValue( TextWriter writer, TokenProcessor tokenProcessor, ReadOnlySpan value, TokenAction tokenAction, TemplateState state, int recursionCount = 0 ) { // infinite recursion guard @@ -683,10 +520,10 @@ private void ProcessTokenValue( TextWriter writer, ReadOnlySpan value, Tok // process token var innerToken = TokenParser.ParseToken( innerValue, state.NextTokenId++ ); - tokenAction = ProcessTokenKind( innerToken, state.Frame, out var tokenValue ); + tokenAction = tokenProcessor.ProcessTokenType( innerToken, state, out var tokenValue ); if ( tokenAction != TokenAction.Ignore ) - ProcessTokenValue( writer, tokenValue, tokenAction, state, recursionCount ); + WriteTokenValue( writer, tokenProcessor, tokenValue, tokenAction, state, recursionCount ); // find next token start @@ -754,3 +591,30 @@ private static int IndexOfIgnoreContent( ReadOnlySpan span, ReadOnlySpan _stack = new(); + + public void Push( TokenDefinition token, bool truthy, int startPos = -1 ) + => _stack.Push( new Frame( token, truthy, startPos ) ); + + public Frame Peek() => _stack.Peek(); + public void Pop() => _stack.Pop(); + public int Depth => _stack.Count; + + public bool IsTokenType( TokenType compare ) => _stack.Count > 0 && _stack.Peek().Token.TokenType == compare; + public bool IsTruthy => _stack.Count == 0 || _stack.Peek().Truthy; + public bool IsFalsy => !IsTruthy; +} + +internal sealed class TemplateState +{ + public TemplateStack Frame { get; } = new(); + public int NextTokenId { get; set; } = 1; + public int CurrentPos { get; set; } +} diff --git a/src/Hyperbee.Templating/Text/TokenAction.cs b/src/Hyperbee.Templating/Text/TokenAction.cs index 7344ca6..d9a54ba 100644 --- a/src/Hyperbee.Templating/Text/TokenAction.cs +++ b/src/Hyperbee.Templating/Text/TokenAction.cs @@ -5,5 +5,6 @@ public enum TokenAction { Replace, Error, - Ignore + Ignore, + Replay } diff --git a/src/Hyperbee.Templating/Text/TokenDefinition.cs b/src/Hyperbee.Templating/Text/TokenDefinition.cs index 7c80234..f91ead2 100644 --- a/src/Hyperbee.Templating/Text/TokenDefinition.cs +++ b/src/Hyperbee.Templating/Text/TokenDefinition.cs @@ -7,4 +7,5 @@ internal record TokenDefinition public TokenType TokenType { get; init; } public TokenEvaluation TokenEvaluation { get; init; } public string TokenExpression { get; init; } + public int TokenLength { get; init; } } diff --git a/src/Hyperbee.Templating/Text/TokenParser.cs b/src/Hyperbee.Templating/Text/TokenParser.cs index 90ce70f..457a6f5 100644 --- a/src/Hyperbee.Templating/Text/TokenParser.cs +++ b/src/Hyperbee.Templating/Text/TokenParser.cs @@ -10,7 +10,9 @@ internal enum TokenType Value, If, Else, - Endif + Endif, + While, + EndWhile } internal enum TokenEvaluation @@ -24,10 +26,14 @@ internal enum TokenEvaluation internal class TokenParser { private KeyValidator ValidateKey { get; } + private string TokenLeft { get; } + private string TokenRight { get; } - internal TokenParser( KeyValidator validator ) + internal TokenParser( KeyValidator validator, string tokenLeft, string tokenRight ) { - ValidateKey = validator ?? throw new ArgumentNullException( nameof( validator ) ); + ValidateKey = validator ?? throw new ArgumentNullException( nameof(validator) ); + TokenLeft = tokenLeft ?? throw new ArgumentNullException( nameof(tokenLeft) ); + TokenRight = tokenRight ?? throw new ArgumentNullException( nameof(tokenRight) ); } public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) @@ -45,42 +51,42 @@ public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) // {{else} // {{/if}} - var content = token.Trim(); + var span = token.Trim(); var tokenType = TokenType.None; - var tokenConditional = TokenEvaluation.None; + var tokenEvaluation = TokenEvaluation.None; var tokenExpression = ReadOnlySpan.Empty; var name = ReadOnlySpan.Empty; // if handling - if ( content.StartsWith( "if", StringComparison.OrdinalIgnoreCase ) ) + if ( span.StartsWith( "if", StringComparison.OrdinalIgnoreCase ) ) { - if ( content.Length == 2 || char.IsWhiteSpace( content[2] ) ) + if ( span.Length == 2 || char.IsWhiteSpace( span[2] ) ) { tokenType = TokenType.If; - content = content[2..].Trim(); // eat the 'if' + span = span[2..].Trim(); // eat the 'if' // parse for bang var bang = false; - if ( content[0] == '!' ) + if ( span[0] == '!' ) { bang = true; - content = content[1..].Trim(); // eat the '!' + span = span[1..].Trim(); // eat the '!' } // detect expression syntax - var isFatArrow = content.IndexOfIgnoreDelimitedRanges( "=>", "\"" ) != -1; + var isFatArrow = span.IndexOfIgnoreDelimitedRanges( "=>", "\"" ) != -1; // validate - if ( content.IsEmpty ) + if ( span.IsEmpty ) throw new TemplateException( "Invalid `if` statement. Missing identifier." ); - if ( !isFatArrow && !ValidateKey( content ) ) + if ( !isFatArrow && !ValidateKey( span ) ) throw new TemplateException( "Invalid `if` statement. Invalid identifier in truthy expression." ); if ( bang && isFatArrow ) @@ -90,70 +96,133 @@ public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) if ( isFatArrow ) { - tokenConditional = TokenEvaluation.Expression; - tokenExpression = content; + tokenEvaluation = TokenEvaluation.Expression; + tokenExpression = span; } else { - tokenConditional = bang ? TokenEvaluation.Falsy : TokenEvaluation.Truthy; - name = content; + tokenEvaluation = bang ? TokenEvaluation.Falsy : TokenEvaluation.Truthy; + name = span; } } } - else if ( content.StartsWith( "else", StringComparison.OrdinalIgnoreCase ) ) + else if ( span.StartsWith( "else", StringComparison.OrdinalIgnoreCase ) ) { - if ( content.Length == 4 ) + if ( span.Length == 4 ) { tokenType = TokenType.Else; } else { - if ( char.IsWhiteSpace( content[4] ) ) + if ( char.IsWhiteSpace( span[4] ) ) throw new TemplateException( "Invalid `else` statement. Invalid trailing characters." ); // this is just a token name starting with `else*` } } - else if ( content.StartsWith( "/if", StringComparison.OrdinalIgnoreCase ) ) + else if ( span.StartsWith( "/if", StringComparison.OrdinalIgnoreCase ) ) { - if ( content.Length != 3 ) + if ( span.Length != 3 ) throw new TemplateException( "Invalid `/if` statement. Invalid characters." ); tokenType = TokenType.Endif; } + // while handling + + if ( span.StartsWith( "while", StringComparison.OrdinalIgnoreCase ) ) + { + if ( span.Length == 5 || char.IsWhiteSpace( span[5] ) ) + { + tokenType = TokenType.While; + span = span[5..].Trim(); // eat the 'while' + + // parse for bang + var bang = false; + + if ( span[0] == '!' ) + { + bang = true; + span = span[1..].Trim(); // eat the '!' + } + + // detect expression syntax + var isFatArrow = span.IndexOfIgnoreDelimitedRanges( "=>", "\"" ) != -1; + + // validate + if ( span.IsEmpty ) + throw new TemplateException( "Invalid `while` statement. Missing identifier." ); + + if ( !isFatArrow && !ValidateKey( span ) ) + throw new TemplateException( "Invalid `while` statement. Invalid identifier in truthy expression." ); + + if ( bang && isFatArrow ) + throw new TemplateException( "Invalid `while` statement. The '!' operator is not supported for token expressions." ); + + // results + if ( isFatArrow ) + { + tokenEvaluation = TokenEvaluation.Expression; + tokenExpression = span; + } + else + { + tokenEvaluation = bang ? TokenEvaluation.Falsy : TokenEvaluation.Truthy; + name = span; + } + } + } + else if ( span.StartsWith( "/while", StringComparison.OrdinalIgnoreCase ) ) + { + if ( span.Length != 6 ) + throw new TemplateException( "Invalid `/while` statement. Invalid characters." ); + + tokenType = TokenType.EndWhile; + } + // value handling if ( tokenType == TokenType.None ) { - var defineTokenPos = content.IndexOfIgnoreDelimitedRanges( ":", "\"" ); - var fatArrowPos = content.IndexOfIgnoreDelimitedRanges( "=>", "\"" ); + var defineTokenPos = span.IndexOfIgnoreDelimitedRanges( ":", "\"" ); + var fatArrowPos = span.IndexOfIgnoreDelimitedRanges( "=>", "\"" ); if ( defineTokenPos > -1 && (fatArrowPos == -1 || defineTokenPos < fatArrowPos) ) { - // define value + // Define value tokenType = TokenType.Define; - name = content[..defineTokenPos].Trim(); - tokenExpression = UnQuote( content[(defineTokenPos + 1)..] ); + name = span[..defineTokenPos].Trim(); + tokenExpression = UnQuote( span[(defineTokenPos + 1)..] ); + + if ( fatArrowPos > 0 ) + { + tokenEvaluation = TokenEvaluation.Expression; + + // Check and remove surrounding token delimiters (e.g., {{ and }}) + if ( tokenExpression.StartsWith( TokenLeft ) && tokenExpression.EndsWith( TokenRight ) ) + { + tokenExpression = tokenExpression[TokenLeft.Length..^TokenRight.Length].Trim(); + } + } } else if ( fatArrowPos > -1 && (defineTokenPos == -1 || fatArrowPos < defineTokenPos) ) { // fat arrow value tokenType = TokenType.Value; - tokenConditional = TokenEvaluation.Expression; - tokenExpression = content; + tokenEvaluation = TokenEvaluation.Expression; + tokenExpression = span; } else { // identifier value - if ( !ValidateKey( content ) ) + if ( !ValidateKey( span ) ) throw new TemplateException( "Invalid token name." ); tokenType = TokenType.Value; - name = content; + name = span; } } @@ -164,7 +233,8 @@ public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) Id = tokenId.ToString(), Name = name.ToString(), TokenType = tokenType, - TokenEvaluation = tokenConditional, + TokenLength = token.Length, + TokenEvaluation = tokenEvaluation, TokenExpression = tokenExpression.ToString() }; } diff --git a/src/Hyperbee.Templating/Text/TokenProcessor.cs b/src/Hyperbee.Templating/Text/TokenProcessor.cs new file mode 100644 index 0000000..dba070e --- /dev/null +++ b/src/Hyperbee.Templating/Text/TokenProcessor.cs @@ -0,0 +1,267 @@ +using System.Globalization; +using Hyperbee.Templating.Compiler; + +namespace Hyperbee.Templating.Text; + +internal class TokenProcessor +{ + private readonly TemplateDictionary _tokens; + private readonly IDictionary _methods; + private readonly Action _tokenHandler; + private readonly ITokenExpressionProvider _tokenExpressionProvider; + private readonly bool _ignoreMissingTokens; + private readonly bool _substituteEnvironmentVariables; + private readonly string _tokenLeft; + private readonly string _tokenRight; + + public TokenProcessor( + TemplateDictionary tokens, + IDictionary methods, + Action tokenHandler, + ITokenExpressionProvider tokenExpressionProvider, + bool ignoreMissingTokens, + bool substituteEnvironmentVariables, + string tokenLeft, + string tokenRight ) + { + _tokens = tokens ?? throw new ArgumentNullException( nameof(tokens) ); + _methods = methods ?? throw new ArgumentNullException( nameof(methods) ); + _tokenHandler = tokenHandler; + _tokenExpressionProvider = tokenExpressionProvider ?? throw new ArgumentNullException( nameof(tokenExpressionProvider) ); + _ignoreMissingTokens = ignoreMissingTokens; + _substituteEnvironmentVariables = substituteEnvironmentVariables; + _tokenLeft = tokenLeft; + _tokenRight = tokenRight; + } + + public TokenAction ProcessTokenType( TokenDefinition token, TemplateState state, out string value ) + { + value = default; + var frame = state.Frame; + + // Frame handling: pre-value processing + switch ( token.TokenType ) + { + case TokenType.Value: + if ( frame.IsFalsy ) + return TokenAction.Ignore; + break; + + case TokenType.If: + // Fall through to resolve value. + break; + + case TokenType.Else: + return ProcessElseToken( frame, token ); + + case TokenType.Endif: + return ProcessEndIfToken( frame ); + + case TokenType.While: + // Fall through to resolve value. + break; + + case TokenType.EndWhile: + return ProcessEndWhileToken( frame ); + + case TokenType.Define: + return ProcessDefineToken( token ); + + case TokenType.None: + default: + throw new NotSupportedException( $"{nameof(ProcessTokenType)}: Invalid {nameof(TokenType)} {token.TokenType}." ); + } + + // Resolve value + + ResolveValue( token, out value, out var defined, out var ifResult, out var expressionError ); + + // Frame handling: post-value processing + + switch ( token.TokenType ) + { + case TokenType.If: + case TokenType.While: + { + var frameIsTruthy = token.TokenEvaluation == TokenEvaluation.Falsy ? !ifResult : ifResult; + var startPos = token.TokenType == TokenType.While ? state.CurrentPos : -1; + + frame.Push( token, frameIsTruthy, startPos ); + + return TokenAction.Ignore; + } + } + + // Token handling: user-defined token action + + _ = TryInvokeTokenHandler( token, defined, ref value, out var tokenAction ); + + // Handle final token action + + switch ( tokenAction ) + { + case TokenAction.Ignore: + case TokenAction.Replace: + break; + + case TokenAction.Error: + value = $"{_tokenLeft}Error ({token.Id}):{expressionError ?? token.Name}{_tokenRight}"; + break; + + default: + throw new NotSupportedException( $"{nameof(ProcessTokenType)}: Invalid {nameof(TokenAction)} {tokenAction}." ); + } + + return tokenAction; + } + + private static TokenAction ProcessElseToken( TemplateStack frame, TokenDefinition token ) + { + if ( !frame.IsTokenType( TokenType.If ) ) + throw new TemplateException( "Syntax error. Invalid `else` without matching `if`." ); + + frame.Push( token, !frame.IsTruthy ); + return TokenAction.Ignore; + } + + private static TokenAction ProcessEndIfToken( TemplateStack frame ) + { + if ( frame.Depth == 0 || !frame.IsTokenType( TokenType.If ) && !frame.IsTokenType( TokenType.Else ) ) + throw new TemplateException( "Syntax error. Invalid `/if` without matching `if`." ); + + if ( frame.IsTokenType( TokenType.Else ) ) + frame.Pop(); // pop the else + + frame.Pop(); // pop the if + + return TokenAction.Ignore; + } + + private TokenAction ProcessEndWhileToken( TemplateStack frame ) + { + if ( frame.Depth == 0 || !frame.IsTokenType( TokenType.While ) ) + throw new TemplateException( "Syntax error. Invalid `/while` without matching `while`." ); + + var whileFrame = frame.Peek(); + var whileToken = whileFrame.Token; + + string expressionError = null; + + var conditionIsTrue = whileToken.TokenEvaluation switch + { + TokenEvaluation.Expression when TryInvokeTokenExpression( whileToken, out var expressionResult, out expressionError ) => Convert.ToBoolean( expressionResult ), + TokenEvaluation.Expression => throw new TemplateException( $"{_tokenLeft}Error ({whileToken.Id}):{expressionError ?? "Error in while condition."}{_tokenRight}" ), + _ => TemplateHelper.Truthy( _tokens[whileToken.Name] ) // Re-evaluate the condition + }; + + if ( conditionIsTrue ) // If the condition is true, replay the while block + return TokenAction.Replay; + + // Otherwise, pop the frame and exit the loop + frame.Pop(); + return TokenAction.Ignore; + } + + private TokenAction ProcessDefineToken( TokenDefinition token ) + { + string expressionError = null; + + _tokens[token.Name] = token.TokenEvaluation switch + { + TokenEvaluation.Expression when TryInvokeTokenExpression( token, out var expressionResult, out expressionError ) => Convert.ToString( expressionResult, CultureInfo.InvariantCulture ), + TokenEvaluation.Expression => throw new TemplateException( $"Error evaluating define expression for {token.Name}: {expressionError}" ), + _ => token.TokenExpression + }; + return TokenAction.Ignore; + } + + private void ResolveValue( TokenDefinition token, out string value, out bool defined, out bool ifResult, out string expressionError ) + { + value = default; + defined = false; + ifResult = false; + expressionError = null; + + switch ( token.TokenType ) + { + case TokenType.Value when token.TokenEvaluation != TokenEvaluation.Expression: + case TokenType.If when token.TokenEvaluation != TokenEvaluation.Expression: + case TokenType.While when token.TokenEvaluation != TokenEvaluation.Expression: + defined = _tokens.TryGetValue( token.Name, out value ); + + if ( !defined && _substituteEnvironmentVariables ) + { + value = Environment.GetEnvironmentVariable( token.Name ); + defined = value != null; + } + + if ( token.TokenType == TokenType.If || token.TokenType == TokenType.While ) + ifResult = defined && TemplateHelper.Truthy( value ); + break; + + case TokenType.Value when token.TokenEvaluation == TokenEvaluation.Expression: + if ( TryInvokeTokenExpression( token, out var valueExprResult, out expressionError ) ) + { + value = Convert.ToString( valueExprResult, CultureInfo.InvariantCulture ); + defined = true; + } + + break; + + case TokenType.If when token.TokenEvaluation == TokenEvaluation.Expression: + case TokenType.While when token.TokenEvaluation == TokenEvaluation.Expression: + if ( TryInvokeTokenExpression( token, out var condExprResult, out var error ) ) + ifResult = Convert.ToBoolean( condExprResult ); + else + throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{error ?? "Error in if condition."}{_tokenRight}" ); + break; + } + } + + private bool TryInvokeTokenHandler( TokenDefinition token, bool defined, ref string value, out TokenAction tokenAction ) + { + tokenAction = defined ? TokenAction.Replace : (_ignoreMissingTokens ? TokenAction.Ignore : TokenAction.Error); + + // Invoke any token handler + if ( _tokenHandler == null ) + return false; + + var eventArgs = new TemplateEventArgs + { + Id = token.Id, + Name = token.Name, + Value = value, + Action = tokenAction, + UnknownToken = !defined + }; + + _tokenHandler.Invoke( null, eventArgs ); + + // The token handler may have modified token properties + value = eventArgs.Value; + tokenAction = eventArgs.Action; + + return true; + } + + private bool TryInvokeTokenExpression( TokenDefinition token, out object result, out string error ) + { + try + { + var tokenExpression = _tokenExpressionProvider.GetTokenExpression( token.TokenExpression ); + var dynamicReadOnlyTokens = new ReadOnlyDynamicDictionary( _tokens, (IReadOnlyDictionary) _methods ); + + result = tokenExpression( dynamicReadOnlyTokens ); + error = default; + + return true; + } + catch ( Exception ex ) + { + error = ex.Message; + } + + result = default; + return false; + } +} diff --git a/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs b/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs index cc9f6cd..2d23363 100644 --- a/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs +++ b/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs @@ -9,6 +9,27 @@ namespace Hyperbee.Templating.Tests.Text; [TestClass] public class TemplateParserExpressionTests { + [DataTestMethod] + //[DataRow( ParseTemplateMethod.Buffered )] + [DataRow( ParseTemplateMethod.InMemory )] + public void Should_honor_while_condition( ParseTemplateMethod parseMethod ) + { + // arrange + const string expression = "{{while x => int.Parse(x.counter) < 3}}{{counter}}{{counter:{{x => int.Parse(x.counter) + 1}}}}{{/while}}"; + const string template = $"count: {expression}."; + + var parser = new TemplateParser { Tokens = { ["counter"] = "0" } }; + + // act + var result = parser.Render( template, parseMethod ); + + // assert + var expected = "count: 012."; + + Assert.AreEqual( expected, result ); + } + + [DataTestMethod] [DataRow( ParseTemplateMethod.Buffered )] [DataRow( ParseTemplateMethod.InMemory )] diff --git a/test/Hyperbee.Templating.Tests/Text/TemplateParser.ParsingTests.cs b/test/Hyperbee.Templating.Tests/Text/TemplateParser.ParsingTests.cs index d5f1352..4c4fc43 100644 --- a/test/Hyperbee.Templating.Tests/Text/TemplateParser.ParsingTests.cs +++ b/test/Hyperbee.Templating.Tests/Text/TemplateParser.ParsingTests.cs @@ -13,7 +13,7 @@ public class TemplateParserParsingTests [DataRow( " token ", nameof( TokenType.Value ), nameof( TokenEvaluation.None ) )] [DataRow( "x=>x.token", nameof( TokenType.Value ), nameof( TokenEvaluation.Expression ) )] [DataRow( "x => x.token", nameof( TokenType.Value ), nameof( TokenEvaluation.Expression ) )] - [DataRow( "token:x => x.token", nameof( TokenType.Define ), nameof( TokenEvaluation.None ) )] + [DataRow( "token:x => x.token", nameof( TokenType.Define ), nameof( TokenEvaluation.Expression ) )] [DataRow( "token: \"x => x.token\" ", nameof( TokenType.Define ), nameof( TokenEvaluation.None ) )] public void Should_parse_token( string token, string expectedTokenType, string expectedTokenEvaluation ) { From 0b5c3cb79cda3db90683fbe88a1527132b65e742 Mon Sep 17 00:00:00 2001 From: Brenton Farmer Date: Sat, 10 Aug 2024 20:04:13 -0700 Subject: [PATCH 10/58] lazy is better --- .../Text/TemplateParser.cs | 56 ++++++++----------- 1 file changed, 22 insertions(+), 34 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index 8ea2a2e..f8d3221 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -40,12 +40,11 @@ public class TemplateParser private string TokenRight { get; } private TokenParser _tokenParser; + internal TokenParser TokenParser => _tokenParser ??= new TokenParser( Tokens.Validator, TokenLeft, TokenRight ); - internal TokenParser TokenParser - { - get { return _tokenParser ??= new TokenParser( Tokens.Validator, TokenLeft, TokenRight ); } - } - + private readonly Lazy _lazyTokenProcessor; + private TokenProcessor TokenProcessor => _lazyTokenProcessor.Value; + public TemplateParser() : this( TokenStyle.Default ) { @@ -77,6 +76,17 @@ public TemplateParser( TokenStyle style, TemplateDictionary source ) Tokens = source; + _lazyTokenProcessor = new Lazy( () => new TokenProcessor( + Tokens, + Methods, + TokenHandler, + TokenExpressionProvider, + IgnoreMissingTokens, + SubstituteEnvironmentVariables, + TokenLeft, + TokenRight + ) ); + switch ( style ) { case TokenStyle.Default: @@ -177,17 +187,6 @@ private enum TemplateScanner // parse template that spans multiple read buffers private void ParseTemplate( TextReader reader, TextWriter writer ) { - var tokenProcessor = new TokenProcessor( - Tokens, - Methods, - TokenHandler, - TokenExpressionProvider, - IgnoreMissingTokens, - SubstituteEnvironmentVariables, - TokenLeft, - TokenRight - ); - try { var ignore = false; @@ -277,10 +276,10 @@ private void ParseTemplate( TextReader reader, TextWriter writer ) // process token var token = TokenParser.ParseToken( tokenWriter.WrittenSpan, state.NextTokenId++ ); - var tokenAction = tokenProcessor.ProcessTokenType( token, state, out var tokenValue ); + var tokenAction = TokenProcessor.ProcessTokenType( token, state, out var tokenValue ); if ( tokenAction != TokenAction.Ignore ) - WriteTokenValue( writer, tokenProcessor, tokenValue, tokenAction, state ); + WriteTokenValue( writer, tokenValue, tokenAction, state ); ignore = state.Frame.IsFalsy; @@ -333,17 +332,6 @@ private void ParseTemplate( TextReader reader, TextWriter writer ) // parse template that is in memory private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int pos = int.MinValue ) { - var tokenProcessor = new TokenProcessor( - Tokens, - Methods, - TokenHandler, - TokenExpressionProvider, - IgnoreMissingTokens, - SubstituteEnvironmentVariables, - TokenLeft, - TokenRight - ); - try { // find first token starting position @@ -427,7 +415,7 @@ private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int p // process token var token = TokenParser.ParseToken( tokenWriter.WrittenSpan, state.NextTokenId++ ); - var tokenAction = tokenProcessor.ProcessTokenType( token, state, out var tokenValue ); + var tokenAction = TokenProcessor.ProcessTokenType( token, state, out var tokenValue ); if ( tokenAction == TokenAction.Replay ) { @@ -439,7 +427,7 @@ private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int p } if ( tokenAction != TokenAction.Ignore ) - WriteTokenValue( writer, tokenProcessor, tokenValue, tokenAction, state ); + WriteTokenValue( writer, tokenValue, tokenAction, state ); ignore = state.Frame.IsFalsy; @@ -469,7 +457,7 @@ private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int p } } - private void WriteTokenValue( TextWriter writer, TokenProcessor tokenProcessor, ReadOnlySpan value, TokenAction tokenAction, TemplateState state, int recursionCount = 0 ) + private void WriteTokenValue( TextWriter writer, ReadOnlySpan value, TokenAction tokenAction, TemplateState state, int recursionCount = 0 ) { // infinite recursion guard @@ -520,10 +508,10 @@ private void WriteTokenValue( TextWriter writer, TokenProcessor tokenProcessor, // process token var innerToken = TokenParser.ParseToken( innerValue, state.NextTokenId++ ); - tokenAction = tokenProcessor.ProcessTokenType( innerToken, state, out var tokenValue ); + tokenAction = TokenProcessor.ProcessTokenType( innerToken, state, out var tokenValue ); if ( tokenAction != TokenAction.Ignore ) - WriteTokenValue( writer, tokenProcessor, tokenValue, tokenAction, state, recursionCount ); + WriteTokenValue( writer, tokenValue, tokenAction, state, recursionCount ); // find next token start From 5290f6ab8c4cdd252bbeddcee10ff3c80837ea9d Mon Sep 17 00:00:00 2001 From: Brenton Farmer Date: Sun, 11 Aug 2024 15:59:02 -0700 Subject: [PATCH 11/58] create a buffermanager and template incremental parser to use it --- src/Hyperbee.Templating/Text/BufferManager.cs | 202 ++++++++++++++++++ .../Text/TemplateParser.cs | 202 +++++++++--------- src/Hyperbee.Templating/Text/TokenAction.cs | 2 +- .../Text/TokenProcessor.cs | 8 +- .../Text/TemplateParser.ExpressionTests.cs | 4 +- 5 files changed, 308 insertions(+), 110 deletions(-) create mode 100644 src/Hyperbee.Templating/Text/BufferManager.cs diff --git a/src/Hyperbee.Templating/Text/BufferManager.cs b/src/Hyperbee.Templating/Text/BufferManager.cs new file mode 100644 index 0000000..6e3be1c --- /dev/null +++ b/src/Hyperbee.Templating/Text/BufferManager.cs @@ -0,0 +1,202 @@ +using System.Buffers; + +namespace Hyperbee.Templating.Text; + +internal sealed class BufferManager : IDisposable +{ + /* + * BufferManager Class Overview: + * + * The BufferManager class manages a series of buffers to facilitate reading and processing + * data in chunks. It supports both growing the buffer list (to handle large or complex data + * streams) and efficiently managing data within a single buffer when growth is not needed. + * + * Key Concepts and Variables: + * + * 1. _padding: + * - Represents additional space at the start of each buffer. + * - Used to manage scenarios where data spans across buffer boundaries. + * - When new data is read into a buffer, any unprocessed data from the previous read + * is moved (slid) into the padding area to maintain data continuity. + * + * 2. _currentBufferPos: + * - Tracks the user's current position within the buffer. + * - Indicates where the next read or operation should occur within the buffer. + * - If padding is included, _currentBufferPos starts at 0; otherwise, it starts after the padding. + * + * 3. TotalCharacters: + * - Represents the total number of characters read into a buffer, including padding if relevant. + * - Reflects the full extent of usable data within the buffer. + * - Used to calculate the correct span to return to the user. + * + * 4. IncludePadding (within BufferState): + * - Indicates whether the padding area is part of the usable span. + * - If true, _currentBufferPos is set to 0 and TotalCharacters includes padding. + * - If false, _currentBufferPos starts after the padding, and TotalCharacters excludes it. + * + * 5. Buffer Management: + * - Buffers are managed as a list of BufferState objects. + * - Buffers can grow as needed or be reused if growth is disabled. + * - Sliding the remainder of a buffer ensures that unprocessed data is carried over to the next read. + * + * Purpose: + * The BufferManager is designed to handle both simple and complex data streams efficiently. + * It ensures that data spanning multiple buffers is managed without loss or corruption, + * making it suitable for scenarios where templates or other data streams are processed in chunks. + */ + + private readonly ArrayPool _arrayPool; + private readonly List _buffers = []; + private int _currentBufferIndex; + private int _currentBufferPos; + private readonly int _bufferSize; + private readonly int _padding; + private bool _grow; + + public BufferManager( int bufferSize, int padding ) + { + _arrayPool = ArrayPool.Shared; + _bufferSize = bufferSize; + _padding = padding; + } + + public void SetGrow( bool grow ) => _grow = grow; + + public Span GetCurrentSpan() + { + var bufferState = _buffers[_currentBufferIndex]; + var length = bufferState.TotalCharacters - _currentBufferPos + (bufferState.IncludePadding ? 0 : _padding); + return bufferState.Buffer.AsSpan( _currentBufferPos, length ); + } + + public Span GetCurrentSpan( int moveBy ) + { + _currentBufferPos += moveBy; + return GetCurrentSpan(); + } + + public Span ReadSpan( TextReader reader ) + { + // Return an existing buffer if we have it + if ( _currentBufferIndex < _buffers.Count - 1 ) + { + _currentBufferIndex++; + _currentBufferPos = _padding; + return GetCurrentSpan(); + } + + // Determine if we need to rent a new buffer + var rent = _grow || _buffers.Count == 0; + BufferState bufferState; + + if ( rent ) + { + // Rent a new buffer and add to the list + var buffer = _arrayPool.Rent( _bufferSize + _padding ); + + bufferState = new BufferState( buffer ) + { + IncludePadding = _buffers.Count != 0 + }; + + _buffers.Add( bufferState ); + _currentBufferIndex = _buffers.Count - 1; + _currentBufferPos = _padding; + } + else + { + // Use the existing buffer and adjust position based on padding + bufferState = _buffers[_currentBufferIndex]; + _currentBufferPos = bufferState.IncludePadding ? 0 : _padding; + } + + // Slide the remainder of the current buffer in place if necessary + if ( _buffers.Count > 0 ) + { + SlideRemainderToFront(); + } + + // Read from the reader + var span = bufferState.Buffer.AsSpan( _padding, _bufferSize ); + var read = reader.Read( span ); + + // Calculate total characters based on whether padding is included + bufferState.TotalCharacters = read + (bufferState.IncludePadding ? _padding : 0); + + return read == 0 + ? [] + : bufferState.Buffer.AsSpan( _currentBufferPos, bufferState.TotalCharacters ); + } + + private void SlideRemainderToFront() + { + if ( _currentBufferPos >= _bufferSize ) + return; + + var remainingSize = _bufferSize - _currentBufferPos; + Array.Copy( _buffers[_currentBufferIndex].Buffer, _padding, _buffers[_currentBufferIndex].Buffer, 0, remainingSize ); + } + + public int CurrentPosition => _currentBufferIndex * _bufferSize + (_currentBufferPos - _padding); + + public void Position( int position ) + { + var remainingPosition = position; + + // Iterate over buffers to find the correct position + for ( var i = 0; i < _buffers.Count; i++ ) + { + if ( remainingPosition < _bufferSize ) + { + _currentBufferIndex = i; + _currentBufferPos = remainingPosition + _padding; + return; + } + + remainingPosition -= _bufferSize; + } + + throw new InvalidOperationException( "Position exceeds buffered content." ); + } + + public void TrimBuffers() + { + while ( _buffers.Count > 1 ) + { + _arrayPool.Return( _buffers[0].Buffer ); + _buffers.RemoveAt( 0 ); + } + + _currentBufferPos = _padding; + _currentBufferIndex = 0; + } + + public void ReleaseBuffers() + { + foreach ( var buffer in _buffers ) + { + _arrayPool.Return( buffer.Buffer ); + } + + _buffers.Clear(); + } + + public void Dispose() + { + ReleaseBuffers(); + } + + private class BufferState + { + public char[] Buffer { get; } + public int TotalCharacters { get; set; } + public bool IncludePadding { get; set; } + + public BufferState( char[] buffer ) + { + Buffer = buffer; + TotalCharacters = 0; + IncludePadding = false; + } + } +} diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index f8d3221..09f7430 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -184,80 +184,61 @@ private enum TemplateScanner Token } - // parse template that spans multiple read buffers + // parse incremental template private void ParseTemplate( TextReader reader, TextWriter writer ) { - try - { - var ignore = false; - - var padding = Math.Max( TokenLeft.Length, TokenRight.Length ); - var start = padding; + var padding = Math.Max( TokenLeft.Length, TokenRight.Length ); + var bufferManager = new BufferManager( BlockSize, padding ); // Instantiate BufferManager here - var buffer = new char[padding + BlockSize]; // padding is used to manage delimiters that `span` reads - var tokenWriter = new ArrayBufferWriter(); // defaults to 256 + var tokenWriter = new ArrayBufferWriter(); // defaults to 256 + var scanner = TemplateScanner.Text; + var ignore = false; + var whileDepth = 0; - var scanner = TemplateScanner.Text; - - IndexOfState indexOfState = default; // index-of for right token delimiter could span buffer reads - var state = new TemplateState(); // template state for this parsing session + IndexOfState indexOfState = default; // index-of for right token delimiter could span buffer reads + var state = new TemplateState(); // template state for this parsing session + try + { while ( true ) { - var read = reader.Read( buffer, padding, BlockSize ); - var content = buffer.AsSpan( start, read + (padding - start) ); + var span = bufferManager.ReadSpan( reader ); - if ( content.IsEmpty ) + if ( span.IsEmpty ) break; - while ( !content.IsEmpty ) + while ( !span.IsEmpty ) { + state.CurrentPos = bufferManager.CurrentPosition; int pos; switch ( scanner ) { case TemplateScanner.Text: { - pos = content.IndexOf( TokenLeft ); + pos = span.IndexOf( TokenLeft ); // match: write to start of token if ( pos >= 0 ) { // write content if ( !ignore ) - writer.Write( content[..pos] ); + writer.Write( span[..pos] ); + + //span = span[(pos + TokenLeft.Length)..]; - content = content[(pos + TokenLeft.Length)..]; + span = bufferManager.GetCurrentSpan( pos + TokenLeft.Length ); // transition state scanner = TemplateScanner.Token; - start = padding; continue; } - // no-match eof: write final content - if ( read < BlockSize ) - { - if ( !ignore ) - writer.Write( content ); // write final content - return; - } - - // no-match: write content less remainder + // no-match and eof: write final content if ( !ignore ) - { - var writeLength = content.Length - TokenLeft.Length; - - if ( writeLength > 0 ) - writer.Write( content[..writeLength] ); - } - - // slide remainder - var remainderLength = Math.Min( TokenLeft.Length, content.Length ); - start = padding - remainderLength; - content[^remainderLength..].CopyTo( buffer.AsSpan( start ) ); - content = []; + writer.Write( span ); + span = []; break; } @@ -265,19 +246,49 @@ private void ParseTemplate( TextReader reader, TextWriter writer ) { // scan: find closing token pattern // token may span multiple reads so track search state - pos = IndexOfIgnoreContent( content, TokenRight, ref indexOfState ); + pos = IndexOfIgnoreContent( span, TokenRight, ref indexOfState ); // match: process completed token if ( pos >= 0 ) { + // update CurrentPos to point to the first character after the token + state.CurrentPos += pos + TokenRight.Length; + // save token chars - tokenWriter.Write( content[..pos] ); - content = content[(pos + TokenRight.Length)..]; + tokenWriter.Write( span[..pos] ); + span = bufferManager.GetCurrentSpan( pos + TokenRight.Length ); // process token var token = TokenParser.ParseToken( tokenWriter.WrittenSpan, state.NextTokenId++ ); - var tokenAction = TokenProcessor.ProcessTokenType( token, state, out var tokenValue ); - + var tokenAction = TokenProcessor.ProcessToken( token, state, out var tokenValue ); + + // loop handling + if ( tokenAction == TokenAction.Loop ) + { + // Reset the position to the start of the while block + bufferManager.Position( state.Frame.Peek().StartPos ); + span = bufferManager.GetCurrentSpan(); + scanner = TemplateScanner.Text; + tokenWriter.Clear(); + continue; + } + + // loop buffer management + if ( token.TokenType == TokenType.While ) + { + if ( whileDepth++ == 0 ) + bufferManager.SetGrow( true ); + } + else if ( token.TokenType == TokenType.EndWhile ) + { + if ( --whileDepth == 0 ) + { + bufferManager.SetGrow( false ); + bufferManager.TrimBuffers(); + } + } + + // write value if ( tokenAction != TokenAction.Ignore ) WriteTokenValue( writer, tokenValue, tokenAction, state ); @@ -287,37 +298,21 @@ private void ParseTemplate( TextReader reader, TextWriter writer ) // transition state scanner = TemplateScanner.Text; - start = padding; continue; } - // no-match eof: incomplete token - if ( read < BlockSize ) - throw new TemplateException( "Missing right token delimiter." ); - - // no-match: save partial token less remainder - var writeLength = content.Length - TokenRight.Length; - - if ( writeLength > 0 ) - tokenWriter.Write( content[..writeLength] ); - - // slide remainder - var remainderLength = Math.Min( TokenRight.Length, content.Length ); - start = padding - remainderLength; - content[^remainderLength..].CopyTo( buffer.AsSpan( start ) ); - content = []; - + span = []; break; } default: - throw new ArgumentOutOfRangeException( scanner.ToString(), "Is" ); + throw new ArgumentOutOfRangeException( scanner.ToString(), $"Invalid scanner state: {scanner}." ); } } } if ( state.Frame.Depth != 0 ) - throw new TemplateException( "Mismatched if else /if." ); + throw new TemplateException( "Missing end if, or end while." ); } catch ( Exception ex ) { @@ -326,44 +321,45 @@ private void ParseTemplate( TextReader reader, TextWriter writer ) finally { writer.Flush(); + bufferManager.ReleaseBuffers(); } } - // parse template that is in memory - private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int pos = int.MinValue ) + // parse in-memory template + private void ParseTemplate( ReadOnlySpan templateSpan, TextWriter writer, int pos = int.MinValue ) { - try - { - // find first token starting position - if ( pos == int.MinValue ) - pos = content.IndexOf( TokenLeft ); + // find: first token start - if ( pos < 0 ) // no-match eof: write final content - { - writer.Write( content ); - return; - } + if ( pos == int.MinValue ) + pos = templateSpan.IndexOf( TokenLeft ); - var skipIndexOf = true; + if ( pos < 0 ) // no-match: quick out + { + writer.Write( templateSpan ); + return; + } - // set up for token processing - var tokenWriter = new ArrayBufferWriter(); // defaults to 256 - var scanner = TemplateScanner.Text; - var ignore = false; + // match: process template - IndexOfState indexOfState = default; // index-of for right token delimiter could span buffer reads - var state = new TemplateState(); // template state for this parsing session + var skipIndexOf = true; + var tokenWriter = new ArrayBufferWriter(); // defaults to 256 + var scanner = TemplateScanner.Text; + var ignore = false; - var originalSpan = content; // Keep the original content span for resetting the position + IndexOfState indexOfState = default; // index-of for right token delimiter could span buffer reads + var state = new TemplateState(); // template state for this parsing session + var span = templateSpan; // Keep the original template span for resetting the position + try + { while ( true ) { - if ( content.IsEmpty ) + if ( span.IsEmpty ) break; - while ( !content.IsEmpty ) + while ( !span.IsEmpty ) { - state.CurrentPos = originalSpan.Length - content.Length; // Track the current position + state.CurrentPos = templateSpan.Length - span.Length; // Track the current position switch ( scanner ) { @@ -372,16 +368,16 @@ private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int p if ( skipIndexOf ) skipIndexOf = false; else - pos = content.IndexOf( TokenLeft ); + pos = span.IndexOf( TokenLeft ); // match: write to start of token if ( pos >= 0 ) { // write content if ( !ignore ) - writer.Write( content[..pos] ); + writer.Write( span[..pos] ); - content = content[(pos + TokenLeft.Length)..]; + span = span[(pos + TokenLeft.Length)..]; // transition state scanner = TemplateScanner.Token; @@ -390,7 +386,7 @@ private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int p // no-match eof: write final content if ( !ignore ) - writer.Write( content ); // write final content + writer.Write( span ); // write final content return; } @@ -398,7 +394,7 @@ private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int p { // scan: find closing token pattern // token may span multiple reads so track search state - pos = IndexOfIgnoreContent( content, TokenRight, ref indexOfState ); + pos = IndexOfIgnoreContent( span, TokenRight, ref indexOfState ); // no-match eof: incomplete token if ( pos < 0 ) @@ -407,20 +403,20 @@ private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int p // match: process completed token // update CurrentPos to point to the first character after the token - state.CurrentPos = originalSpan.Length - content.Length + pos + TokenRight.Length; + state.CurrentPos = templateSpan.Length - span.Length + pos + TokenRight.Length; // save token chars - tokenWriter.Write( content[..pos] ); - content = content[(pos + TokenRight.Length)..]; + tokenWriter.Write( span[..pos] ); + span = span[(pos + TokenRight.Length)..]; // process token var token = TokenParser.ParseToken( tokenWriter.WrittenSpan, state.NextTokenId++ ); - var tokenAction = TokenProcessor.ProcessTokenType( token, state, out var tokenValue ); + var tokenAction = TokenProcessor.ProcessToken( token, state, out var tokenValue ); - if ( tokenAction == TokenAction.Replay ) + if ( tokenAction == TokenAction.Loop ) { // Reset the position to start of while block - content = originalSpan[state.Frame.Peek().StartPos..]; // Reset position to StartPos + span = templateSpan[state.Frame.Peek().StartPos..]; // Reset position to StartPos scanner = TemplateScanner.Text; tokenWriter.Clear(); continue; @@ -439,13 +435,13 @@ private void ParseTemplate( ReadOnlySpan content, TextWriter writer, int p } default: - throw new ArgumentOutOfRangeException( scanner.ToString(), "Is" ); + throw new ArgumentOutOfRangeException( scanner.ToString(), $"Invalid scanner state: {scanner}." ); } } } if ( state.Frame.Depth != 0 ) - throw new TemplateException( "Mismatched if else /if." ); + throw new TemplateException( "Missing end if or end while." ); } catch ( Exception ex ) { @@ -508,7 +504,7 @@ private void WriteTokenValue( TextWriter writer, ReadOnlySpan value, Token // process token var innerToken = TokenParser.ParseToken( innerValue, state.NextTokenId++ ); - tokenAction = TokenProcessor.ProcessTokenType( innerToken, state, out var tokenValue ); + tokenAction = TokenProcessor.ProcessToken( innerToken, state, out var tokenValue ); if ( tokenAction != TokenAction.Ignore ) WriteTokenValue( writer, tokenValue, tokenAction, state, recursionCount ); diff --git a/src/Hyperbee.Templating/Text/TokenAction.cs b/src/Hyperbee.Templating/Text/TokenAction.cs index d9a54ba..e73c3f2 100644 --- a/src/Hyperbee.Templating/Text/TokenAction.cs +++ b/src/Hyperbee.Templating/Text/TokenAction.cs @@ -6,5 +6,5 @@ public enum TokenAction Replace, Error, Ignore, - Replay + Loop } diff --git a/src/Hyperbee.Templating/Text/TokenProcessor.cs b/src/Hyperbee.Templating/Text/TokenProcessor.cs index dba070e..bf8c08e 100644 --- a/src/Hyperbee.Templating/Text/TokenProcessor.cs +++ b/src/Hyperbee.Templating/Text/TokenProcessor.cs @@ -34,7 +34,7 @@ public TokenProcessor( _tokenRight = tokenRight; } - public TokenAction ProcessTokenType( TokenDefinition token, TemplateState state, out string value ) + public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out string value ) { value = default; var frame = state.Frame; @@ -69,7 +69,7 @@ public TokenAction ProcessTokenType( TokenDefinition token, TemplateState state, case TokenType.None: default: - throw new NotSupportedException( $"{nameof(ProcessTokenType)}: Invalid {nameof(TokenType)} {token.TokenType}." ); + throw new NotSupportedException( $"{nameof(ProcessToken)}: Invalid {nameof(TokenType)} {token.TokenType}." ); } // Resolve value @@ -109,7 +109,7 @@ public TokenAction ProcessTokenType( TokenDefinition token, TemplateState state, break; default: - throw new NotSupportedException( $"{nameof(ProcessTokenType)}: Invalid {nameof(TokenAction)} {tokenAction}." ); + throw new NotSupportedException( $"{nameof(ProcessToken)}: Invalid {nameof(TokenAction)} {tokenAction}." ); } return tokenAction; @@ -155,7 +155,7 @@ TokenEvaluation.Expression when TryInvokeTokenExpression( whileToken, out var ex }; if ( conditionIsTrue ) // If the condition is true, replay the while block - return TokenAction.Replay; + return TokenAction.Loop; // Otherwise, pop the frame and exit the loop frame.Pop(); diff --git a/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs b/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs index 2d23363..c20d5cc 100644 --- a/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs +++ b/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs @@ -10,8 +10,8 @@ namespace Hyperbee.Templating.Tests.Text; public class TemplateParserExpressionTests { [DataTestMethod] - //[DataRow( ParseTemplateMethod.Buffered )] - [DataRow( ParseTemplateMethod.InMemory )] + [DataRow( ParseTemplateMethod.Buffered )] + //[DataRow( ParseTemplateMethod.InMemory )] public void Should_honor_while_condition( ParseTemplateMethod parseMethod ) { // arrange From 0b1092d9877b2d5964f2d96bdceca6d9d471532d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 11 Aug 2024 23:08:31 +0000 Subject: [PATCH 12/58] Create draft PR for #15 [skip ci] From 860dda45da61846926f926905c12b95c57cf1821 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 11 Aug 2024 23:09:55 +0000 Subject: [PATCH 13/58] Updated code formatting to match rules in .editorconfig --- src/Hyperbee.Templating/Text/BufferManager.cs | 8 +++--- .../Text/TemplateParser.cs | 4 +-- src/Hyperbee.Templating/Text/TokenParser.cs | 6 ++--- .../Text/TokenProcessor.cs | 26 +++++++++---------- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/Hyperbee.Templating/Text/BufferManager.cs b/src/Hyperbee.Templating/Text/BufferManager.cs index 6e3be1c..6f0f922 100644 --- a/src/Hyperbee.Templating/Text/BufferManager.cs +++ b/src/Hyperbee.Templating/Text/BufferManager.cs @@ -93,10 +93,10 @@ public Span ReadSpan( TextReader reader ) { // Rent a new buffer and add to the list var buffer = _arrayPool.Rent( _bufferSize + _padding ); - - bufferState = new BufferState( buffer ) - { - IncludePadding = _buffers.Count != 0 + + bufferState = new BufferState( buffer ) + { + IncludePadding = _buffers.Count != 0 }; _buffers.Add( bufferState ); diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index 09f7430..1840227 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -44,7 +44,7 @@ public class TemplateParser private readonly Lazy _lazyTokenProcessor; private TokenProcessor TokenProcessor => _lazyTokenProcessor.Value; - + public TemplateParser() : this( TokenStyle.Default ) { @@ -86,7 +86,7 @@ public TemplateParser( TokenStyle style, TemplateDictionary source ) TokenLeft, TokenRight ) ); - + switch ( style ) { case TokenStyle.Default: diff --git a/src/Hyperbee.Templating/Text/TokenParser.cs b/src/Hyperbee.Templating/Text/TokenParser.cs index 457a6f5..40253bf 100644 --- a/src/Hyperbee.Templating/Text/TokenParser.cs +++ b/src/Hyperbee.Templating/Text/TokenParser.cs @@ -31,9 +31,9 @@ internal class TokenParser internal TokenParser( KeyValidator validator, string tokenLeft, string tokenRight ) { - ValidateKey = validator ?? throw new ArgumentNullException( nameof(validator) ); - TokenLeft = tokenLeft ?? throw new ArgumentNullException( nameof(tokenLeft) ); - TokenRight = tokenRight ?? throw new ArgumentNullException( nameof(tokenRight) ); + ValidateKey = validator ?? throw new ArgumentNullException( nameof( validator ) ); + TokenLeft = tokenLeft ?? throw new ArgumentNullException( nameof( tokenLeft ) ); + TokenRight = tokenRight ?? throw new ArgumentNullException( nameof( tokenRight ) ); } public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) diff --git a/src/Hyperbee.Templating/Text/TokenProcessor.cs b/src/Hyperbee.Templating/Text/TokenProcessor.cs index bf8c08e..05813e8 100644 --- a/src/Hyperbee.Templating/Text/TokenProcessor.cs +++ b/src/Hyperbee.Templating/Text/TokenProcessor.cs @@ -24,10 +24,10 @@ public TokenProcessor( string tokenLeft, string tokenRight ) { - _tokens = tokens ?? throw new ArgumentNullException( nameof(tokens) ); - _methods = methods ?? throw new ArgumentNullException( nameof(methods) ); + _tokens = tokens ?? throw new ArgumentNullException( nameof( tokens ) ); + _methods = methods ?? throw new ArgumentNullException( nameof( methods ) ); _tokenHandler = tokenHandler; - _tokenExpressionProvider = tokenExpressionProvider ?? throw new ArgumentNullException( nameof(tokenExpressionProvider) ); + _tokenExpressionProvider = tokenExpressionProvider ?? throw new ArgumentNullException( nameof( tokenExpressionProvider ) ); _ignoreMissingTokens = ignoreMissingTokens; _substituteEnvironmentVariables = substituteEnvironmentVariables; _tokenLeft = tokenLeft; @@ -69,27 +69,27 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out case TokenType.None: default: - throw new NotSupportedException( $"{nameof(ProcessToken)}: Invalid {nameof(TokenType)} {token.TokenType}." ); + throw new NotSupportedException( $"{nameof( ProcessToken )}: Invalid {nameof( TokenType )} {token.TokenType}." ); } // Resolve value - + ResolveValue( token, out value, out var defined, out var ifResult, out var expressionError ); // Frame handling: post-value processing - + switch ( token.TokenType ) { case TokenType.If: case TokenType.While: - { - var frameIsTruthy = token.TokenEvaluation == TokenEvaluation.Falsy ? !ifResult : ifResult; - var startPos = token.TokenType == TokenType.While ? state.CurrentPos : -1; + { + var frameIsTruthy = token.TokenEvaluation == TokenEvaluation.Falsy ? !ifResult : ifResult; + var startPos = token.TokenType == TokenType.While ? state.CurrentPos : -1; - frame.Push( token, frameIsTruthy, startPos ); + frame.Push( token, frameIsTruthy, startPos ); - return TokenAction.Ignore; - } + return TokenAction.Ignore; + } } // Token handling: user-defined token action @@ -109,7 +109,7 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out break; default: - throw new NotSupportedException( $"{nameof(ProcessToken)}: Invalid {nameof(TokenAction)} {tokenAction}." ); + throw new NotSupportedException( $"{nameof( ProcessToken )}: Invalid {nameof( TokenAction )} {tokenAction}." ); } return tokenAction; From f2e353d916f156eec384ee37cd9567e8dc532baf Mon Sep 17 00:00:00 2001 From: Brenton Farmer Date: Mon, 12 Aug 2024 11:40:05 -0700 Subject: [PATCH 14/58] corrected buffermanager corner cases --- src/Hyperbee.Templating/Text/BufferManager.cs | 129 ++++-------------- .../Text/TemplateParser.cs | 53 ++++++- .../Text/TemplateParser.ParsingTests.cs | 24 ++-- 3 files changed, 87 insertions(+), 119 deletions(-) diff --git a/src/Hyperbee.Templating/Text/BufferManager.cs b/src/Hyperbee.Templating/Text/BufferManager.cs index 6f0f922..56b494c 100644 --- a/src/Hyperbee.Templating/Text/BufferManager.cs +++ b/src/Hyperbee.Templating/Text/BufferManager.cs @@ -4,152 +4,89 @@ namespace Hyperbee.Templating.Text; internal sealed class BufferManager : IDisposable { - /* - * BufferManager Class Overview: - * - * The BufferManager class manages a series of buffers to facilitate reading and processing - * data in chunks. It supports both growing the buffer list (to handle large or complex data - * streams) and efficiently managing data within a single buffer when growth is not needed. - * - * Key Concepts and Variables: - * - * 1. _padding: - * - Represents additional space at the start of each buffer. - * - Used to manage scenarios where data spans across buffer boundaries. - * - When new data is read into a buffer, any unprocessed data from the previous read - * is moved (slid) into the padding area to maintain data continuity. - * - * 2. _currentBufferPos: - * - Tracks the user's current position within the buffer. - * - Indicates where the next read or operation should occur within the buffer. - * - If padding is included, _currentBufferPos starts at 0; otherwise, it starts after the padding. - * - * 3. TotalCharacters: - * - Represents the total number of characters read into a buffer, including padding if relevant. - * - Reflects the full extent of usable data within the buffer. - * - Used to calculate the correct span to return to the user. - * - * 4. IncludePadding (within BufferState): - * - Indicates whether the padding area is part of the usable span. - * - If true, _currentBufferPos is set to 0 and TotalCharacters includes padding. - * - If false, _currentBufferPos starts after the padding, and TotalCharacters excludes it. - * - * 5. Buffer Management: - * - Buffers are managed as a list of BufferState objects. - * - Buffers can grow as needed or be reused if growth is disabled. - * - Sliding the remainder of a buffer ensures that unprocessed data is carried over to the next read. - * - * Purpose: - * The BufferManager is designed to handle both simple and complex data streams efficiently. - * It ensures that data spanning multiple buffers is managed without loss or corruption, - * making it suitable for scenarios where templates or other data streams are processed in chunks. - */ - private readonly ArrayPool _arrayPool; private readonly List _buffers = []; private int _currentBufferIndex; private int _currentBufferPos; private readonly int _bufferSize; - private readonly int _padding; private bool _grow; - public BufferManager( int bufferSize, int padding ) + public BufferManager( int bufferSize ) { _arrayPool = ArrayPool.Shared; _bufferSize = bufferSize; - _padding = padding; } public void SetGrow( bool grow ) => _grow = grow; + + public void AdvanceCurrentSpan( int advanceBy ) => _currentBufferPos += advanceBy; public Span GetCurrentSpan() { var bufferState = _buffers[_currentBufferIndex]; - var length = bufferState.TotalCharacters - _currentBufferPos + (bufferState.IncludePadding ? 0 : _padding); - return bufferState.Buffer.AsSpan( _currentBufferPos, length ); + return bufferState.Buffer.AsSpan( _currentBufferPos, bufferState.TotalCharacters - _currentBufferPos ); } - public Span GetCurrentSpan( int moveBy ) + public Span GetCurrentSpan( int advanceBy ) { - _currentBufferPos += moveBy; + AdvanceCurrentSpan( advanceBy ); return GetCurrentSpan(); } public Span ReadSpan( TextReader reader ) { - // Return an existing buffer if we have it - if ( _currentBufferIndex < _buffers.Count - 1 ) - { - _currentBufferIndex++; - _currentBufferPos = _padding; - return GetCurrentSpan(); - } + var first = _buffers.Count == 0; + var rent = _grow || first; - // Determine if we need to rent a new buffer - var rent = _grow || _buffers.Count == 0; BufferState bufferState; if ( rent ) { // Rent a new buffer and add to the list - var buffer = _arrayPool.Rent( _bufferSize + _padding ); - - bufferState = new BufferState( buffer ) - { - IncludePadding = _buffers.Count != 0 - }; - + bufferState = new BufferState( _arrayPool.Rent( _bufferSize ) ); _buffers.Add( bufferState ); _currentBufferIndex = _buffers.Count - 1; - _currentBufferPos = _padding; } else { - // Use the existing buffer and adjust position based on padding + // Use the existing buffer bufferState = _buffers[_currentBufferIndex]; - _currentBufferPos = bufferState.IncludePadding ? 0 : _padding; } - // Slide the remainder of the current buffer in place if necessary - if ( _buffers.Count > 0 ) + // Slide remainder of the current buffer if necessary + var remainder = 0; + + if ( !first ) { - SlideRemainderToFront(); - } + remainder = bufferState.TotalCharacters - _currentBufferPos; - // Read from the reader - var span = bufferState.Buffer.AsSpan( _padding, _bufferSize ); - var read = reader.Read( span ); + if ( remainder > 0 ) + Array.Copy( bufferState.Buffer, _currentBufferPos, bufferState.Buffer, 0, remainder ); + } - // Calculate total characters based on whether padding is included - bufferState.TotalCharacters = read + (bufferState.IncludePadding ? _padding : 0); + // Read new data into the buffer + _currentBufferPos = 0; - return read == 0 - ? [] - : bufferState.Buffer.AsSpan( _currentBufferPos, bufferState.TotalCharacters ); - } + var span = bufferState.Buffer.AsSpan( remainder, _bufferSize - remainder ); + var read = reader.Read( span ); - private void SlideRemainderToFront() - { - if ( _currentBufferPos >= _bufferSize ) - return; + bufferState.TotalCharacters = read + remainder; - var remainingSize = _bufferSize - _currentBufferPos; - Array.Copy( _buffers[_currentBufferIndex].Buffer, _padding, _buffers[_currentBufferIndex].Buffer, 0, remainingSize ); + return bufferState.TotalCharacters == 0 ? [] : bufferState.Buffer.AsSpan( 0, bufferState.TotalCharacters ); } - public int CurrentPosition => _currentBufferIndex * _bufferSize + (_currentBufferPos - _padding); + public int CurrentPosition => _currentBufferIndex * _bufferSize + _currentBufferPos; public void Position( int position ) { var remainingPosition = position; - // Iterate over buffers to find the correct position for ( var i = 0; i < _buffers.Count; i++ ) { if ( remainingPosition < _bufferSize ) { _currentBufferIndex = i; - _currentBufferPos = remainingPosition + _padding; + _currentBufferPos = remainingPosition; return; } @@ -167,7 +104,7 @@ public void TrimBuffers() _buffers.RemoveAt( 0 ); } - _currentBufferPos = _padding; + _currentBufferPos = 0; _currentBufferIndex = 0; } @@ -186,17 +123,9 @@ public void Dispose() ReleaseBuffers(); } - private class BufferState + private class BufferState( char[] buffer ) { - public char[] Buffer { get; } + public char[] Buffer { get; } = buffer; public int TotalCharacters { get; set; } - public bool IncludePadding { get; set; } - - public BufferState( char[] buffer ) - { - Buffer = buffer; - TotalCharacters = 0; - IncludePadding = false; - } } } diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index 1840227..d76e286 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -24,7 +24,7 @@ public enum TokenStyle public class TemplateParser { - internal static int BlockSize = 1024; + internal static int BufferSize = 1024; public bool IgnoreMissingTokens { get; init; } = false; public bool SubstituteEnvironmentVariables { get; init; } = false; @@ -187,8 +187,8 @@ private enum TemplateScanner // parse incremental template private void ParseTemplate( TextReader reader, TextWriter writer ) { - var padding = Math.Max( TokenLeft.Length, TokenRight.Length ); - var bufferManager = new BufferManager( BlockSize, padding ); // Instantiate BufferManager here + var bufferSize = GetScopedBufferSize( BufferSize, TokenLeft.Length, TokenRight.Length ); // min buffer size is max token delimiter plus 1 + var bufferManager = new BufferManager( bufferSize ); var tokenWriter = new ArrayBufferWriter(); // defaults to 256 var scanner = TemplateScanner.Text; @@ -203,6 +203,7 @@ private void ParseTemplate( TextReader reader, TextWriter writer ) while ( true ) { var span = bufferManager.ReadSpan( reader ); + var lastReadBytes = span.Length; if ( span.IsEmpty ) break; @@ -225,8 +226,6 @@ private void ParseTemplate( TextReader reader, TextWriter writer ) if ( !ignore ) writer.Write( span[..pos] ); - //span = span[(pos + TokenLeft.Length)..]; - span = bufferManager.GetCurrentSpan( pos + TokenLeft.Length ); // transition state @@ -234,9 +233,25 @@ private void ParseTemplate( TextReader reader, TextWriter writer ) continue; } - // no-match and eof: write final content + // no-match eof: write final content + if ( lastReadBytes < bufferSize ) + { + if ( !ignore ) + writer.Write( span ); // write final content + return; + } + + // no-match: write content less remainder if ( !ignore ) - writer.Write( span ); + { + var writeLength = span.Length - TokenLeft.Length; + + if ( writeLength > 0 ) + { + writer.Write( span[..writeLength] ); + bufferManager.AdvanceCurrentSpan( writeLength ); + } + } span = []; break; @@ -301,6 +316,19 @@ private void ParseTemplate( TextReader reader, TextWriter writer ) continue; } + // no-match eof: incomplete token + if ( lastReadBytes < bufferSize ) + throw new TemplateException( "Missing right token delimiter." ); + + // no-match: save partial token less remainder + var writeLength = span.Length - TokenRight.Length; + + if ( writeLength > 0 ) + { + tokenWriter.Write( span[..writeLength] ); + bufferManager.AdvanceCurrentSpan( writeLength ); + } + span = []; break; } @@ -323,6 +351,17 @@ private void ParseTemplate( TextReader reader, TextWriter writer ) writer.Flush(); bufferManager.ReleaseBuffers(); } + + return; + + static int GetScopedBufferSize( int bufferSize, int tokenLeftSize, int tokenRightSize ) + { + // because of the way we read the buffer, we need to ensure that the buffer size + // is at least the size of the longest token delimiter plus one character. + + var maxDelimiter = Math.Max( tokenLeftSize, tokenRightSize ); + return Math.Max( bufferSize, maxDelimiter + 1 ); + } } // parse in-memory template diff --git a/test/Hyperbee.Templating.Tests/Text/TemplateParser.ParsingTests.cs b/test/Hyperbee.Templating.Tests/Text/TemplateParser.ParsingTests.cs index 4c4fc43..9ad704b 100644 --- a/test/Hyperbee.Templating.Tests/Text/TemplateParser.ParsingTests.cs +++ b/test/Hyperbee.Templating.Tests/Text/TemplateParser.ParsingTests.cs @@ -30,17 +30,17 @@ public void Should_parse_token( string token, string expectedTokenType, string e } [DataTestMethod] - [DataRow( 1 )] - [DataRow( 9 )] - [DataRow( 10 )] - [DataRow( 11 )] - [DataRow( 12 )] - [DataRow( 15 )] - [DataRow( 16 )] - [DataRow( 17 )] - [DataRow( 18 )] - [DataRow( 19 )] - [DataRow( 50 )] + [DataRow( 2 )] //f + //[DataRow( 9 )] + //[DataRow( 10 )] + //[DataRow( 11 )] + //[DataRow( 12 )] + //[DataRow( 15 )] + //[DataRow( 16 )] + //[DataRow( 17 )] + //[DataRow( 18 )] + //[DataRow( 19 )] + //[DataRow( 50 )] public void Should_parse_tokens_with_buffer_wraps( int size ) { // arrange @@ -48,7 +48,7 @@ public void Should_parse_tokens_with_buffer_wraps( int size ) // 123456789+123456789+123456789+123456789+123456789+ const string template = "all your {{thing}} are belong to {{who}}."; - TemplateParser.BlockSize = size; + TemplateParser.BufferSize = size; var parser = new TemplateParser { From cda53529e546b8261c469b08e57b27f7bfcec9d6 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 12 Aug 2024 18:40:45 +0000 Subject: [PATCH 15/58] Updated code formatting to match rules in .editorconfig --- src/Hyperbee.Templating/Text/BufferManager.cs | 2 +- src/Hyperbee.Templating/Text/TemplateParser.cs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Hyperbee.Templating/Text/BufferManager.cs b/src/Hyperbee.Templating/Text/BufferManager.cs index 56b494c..0b5d91d 100644 --- a/src/Hyperbee.Templating/Text/BufferManager.cs +++ b/src/Hyperbee.Templating/Text/BufferManager.cs @@ -18,7 +18,7 @@ public BufferManager( int bufferSize ) } public void SetGrow( bool grow ) => _grow = grow; - + public void AdvanceCurrentSpan( int advanceBy ) => _currentBufferPos += advanceBy; public Span GetCurrentSpan() diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index d76e286..22d1c98 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -188,7 +188,7 @@ private enum TemplateScanner private void ParseTemplate( TextReader reader, TextWriter writer ) { var bufferSize = GetScopedBufferSize( BufferSize, TokenLeft.Length, TokenRight.Length ); // min buffer size is max token delimiter plus 1 - var bufferManager = new BufferManager( bufferSize ); + var bufferManager = new BufferManager( bufferSize ); var tokenWriter = new ArrayBufferWriter(); // defaults to 256 var scanner = TemplateScanner.Text; @@ -317,7 +317,7 @@ private void ParseTemplate( TextReader reader, TextWriter writer ) } // no-match eof: incomplete token - if ( lastReadBytes < bufferSize ) + if ( lastReadBytes < bufferSize ) throw new TemplateException( "Missing right token delimiter." ); // no-match: save partial token less remainder @@ -360,7 +360,7 @@ static int GetScopedBufferSize( int bufferSize, int tokenLeftSize, int tokenRigh // is at least the size of the longest token delimiter plus one character. var maxDelimiter = Math.Max( tokenLeftSize, tokenRightSize ); - return Math.Max( bufferSize, maxDelimiter + 1 ); + return Math.Max( bufferSize, maxDelimiter + 1 ); } } From 98917f406783841f59cf88e5ecfc8e823b590cf0 Mon Sep 17 00:00:00 2001 From: Brenton Farmer Date: Mon, 12 Aug 2024 14:12:06 -0700 Subject: [PATCH 16/58] Remove in-memory parser implementation --- src/Hyperbee.Templating/Text/BufferManager.cs | 75 +++++-- .../Text/TemplateParser.cs | 206 ++++-------------- 2 files changed, 106 insertions(+), 175 deletions(-) diff --git a/src/Hyperbee.Templating/Text/BufferManager.cs b/src/Hyperbee.Templating/Text/BufferManager.cs index 0b5d91d..b791c5e 100644 --- a/src/Hyperbee.Templating/Text/BufferManager.cs +++ b/src/Hyperbee.Templating/Text/BufferManager.cs @@ -1,40 +1,71 @@ -using System.Buffers; +using System.Buffers; +using System.Runtime.CompilerServices; namespace Hyperbee.Templating.Text; -internal sealed class BufferManager : IDisposable +internal ref struct BufferManager { private readonly ArrayPool _arrayPool; - private readonly List _buffers = []; + private readonly List _buffers; private int _currentBufferIndex; private int _currentBufferPos; private readonly int _bufferSize; private bool _grow; + private readonly ReadOnlySpan _fixedSpan; + public BufferManager( int bufferSize ) { _arrayPool = ArrayPool.Shared; + _buffers = []; _bufferSize = bufferSize; } - public void SetGrow( bool grow ) => _grow = grow; + public BufferManager( ReadOnlySpan span ) + { + _bufferSize = span.Length; + _fixedSpan = span; + } + + public readonly int BufferSize => _bufferSize; + public readonly bool IsFixed => _fixedSpan.Length > 0; + [MethodImpl( MethodImplOptions.AggressiveInlining )] public void AdvanceCurrentSpan( int advanceBy ) => _currentBufferPos += advanceBy; - public Span GetCurrentSpan() + [MethodImpl( MethodImplOptions.AggressiveInlining )] + public readonly ReadOnlySpan GetCurrentSpan() { + if ( IsFixed ) + return _fixedSpan[_currentBufferPos..]; + var bufferState = _buffers[_currentBufferIndex]; return bufferState.Buffer.AsSpan( _currentBufferPos, bufferState.TotalCharacters - _currentBufferPos ); } - public Span GetCurrentSpan( int advanceBy ) + [MethodImpl( MethodImplOptions.AggressiveInlining )] + public ReadOnlySpan GetCurrentSpan( int advanceBy ) { AdvanceCurrentSpan( advanceBy ); return GetCurrentSpan(); } - public Span ReadSpan( TextReader reader ) + public void SetGrow( bool grow ) { + if ( IsFixed ) + throw new InvalidOperationException( "Cannot set grow on a fixed span." ); + + _grow = grow; + } + + public ReadOnlySpan ReadSpan( TextReader reader ) + { + if ( IsFixed && reader == null ) + { + _currentBufferPos = 0; + return _fixedSpan; + } + var first = _buffers.Count == 0; var rent = _grow || first; @@ -75,10 +106,18 @@ public Span ReadSpan( TextReader reader ) return bufferState.TotalCharacters == 0 ? [] : bufferState.Buffer.AsSpan( 0, bufferState.TotalCharacters ); } - public int CurrentPosition => _currentBufferIndex * _bufferSize + _currentBufferPos; + public readonly int CurrentPosition => IsFixed ? _currentBufferPos : _currentBufferIndex * _bufferSize + _currentBufferPos; public void Position( int position ) { + ArgumentOutOfRangeException.ThrowIfNegative( position, nameof( position ) ); + + if ( IsFixed ) + { + _currentBufferPos = position; + return; + } + var remainingPosition = position; for ( var i = 0; i < _buffers.Count; i++ ) @@ -98,18 +137,27 @@ public void Position( int position ) public void TrimBuffers() { + _currentBufferPos = 0; + _currentBufferIndex = 0; + + if ( IsFixed ) + return; + while ( _buffers.Count > 1 ) { _arrayPool.Return( _buffers[0].Buffer ); _buffers.RemoveAt( 0 ); } - - _currentBufferPos = 0; - _currentBufferIndex = 0; } public void ReleaseBuffers() { + _currentBufferPos = 0; + _currentBufferIndex = 0; + + if ( IsFixed ) + return; + foreach ( var buffer in _buffers ) { _arrayPool.Return( buffer.Buffer ); @@ -118,11 +166,6 @@ public void ReleaseBuffers() _buffers.Clear(); } - public void Dispose() - { - ReleaseBuffers(); - } - private class BufferState( char[] buffer ) { public char[] Buffer { get; } = buffer; diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index 22d1c98..5b01a84 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -1,12 +1,10 @@ -using System.Buffers; +using System.Buffers; using Hyperbee.Templating.Collections; using Hyperbee.Templating.Compiler; using Hyperbee.Templating.Extensions; namespace Hyperbee.Templating.Text; -//public delegate string TemplateMethod( string value, params string[] args ); - public enum TokenStyle { // if you are considering implementing a new token style make sure the @@ -25,14 +23,13 @@ public enum TokenStyle public class TemplateParser { internal static int BufferSize = 1024; + public bool IgnoreMissingTokens { get; init; } = false; public bool SubstituteEnvironmentVariables { get; init; } = false; - public int MaxTokenDepth { get; init; } = 20; public ITokenExpressionProvider TokenExpressionProvider { get; init; } = new RoslynTokenExpressionProvider(); public IDictionary Methods { get; init; } = new Dictionary( StringComparer.OrdinalIgnoreCase ); - public TemplateDictionary Tokens { get; init; } public Action TokenHandler { get; init; } @@ -135,7 +132,9 @@ public string Render( ReadOnlySpan template ) return template.ToString(); using var writer = new StringWriter(); - ParseTemplate( template, writer, pos ); + writer.Write( template[..pos] ); + + ParseTemplate( template[pos..], writer ); return writer.ToString(); } @@ -184,13 +183,35 @@ private enum TemplateScanner Token } - // parse incremental template + private void ParseTemplate( ReadOnlySpan templateSpan, TextWriter writer ) + { + var bufferManager = new BufferManager( templateSpan ); + ParseTemplate( ref bufferManager, null, writer ); + } + private void ParseTemplate( TextReader reader, TextWriter writer ) { - var bufferSize = GetScopedBufferSize( BufferSize, TokenLeft.Length, TokenRight.Length ); // min buffer size is max token delimiter plus 1 + var bufferSize = GetScopedBufferSize( BufferSize, TokenLeft.Length, TokenRight.Length ); var bufferManager = new BufferManager( bufferSize ); - var tokenWriter = new ArrayBufferWriter(); // defaults to 256 + ParseTemplate( ref bufferManager, reader, writer ); + + return; + + static int GetScopedBufferSize( int bufferSize, int tokenLeftSize, int tokenRightSize ) + { + // because of the way we read the buffer, we need to ensure that the buffer size + // is at least the size of the longest token delimiter plus one character. + + var maxDelimiter = Math.Max( tokenLeftSize, tokenRightSize ); + return Math.Max( bufferSize, maxDelimiter + 1 ); + } + } + + // parse incremental template + private void ParseTemplate( ref BufferManager bufferManager, TextReader reader, TextWriter writer ) + { + var tokenWriter = new ArrayBufferWriter(); // defaults to 256 var scanner = TemplateScanner.Text; var ignore = false; var whileDepth = 0; @@ -234,7 +255,7 @@ private void ParseTemplate( TextReader reader, TextWriter writer ) } // no-match eof: write final content - if ( lastReadBytes < bufferSize ) + if ( bufferManager.IsFixed || lastReadBytes < bufferManager.BufferSize ) { if ( !ignore ) writer.Write( span ); // write final content @@ -289,17 +310,20 @@ private void ParseTemplate( TextReader reader, TextWriter writer ) } // loop buffer management - if ( token.TokenType == TokenType.While ) - { - if ( whileDepth++ == 0 ) - bufferManager.SetGrow( true ); - } - else if ( token.TokenType == TokenType.EndWhile ) + if ( !bufferManager.IsFixed ) { - if ( --whileDepth == 0 ) + if ( token.TokenType == TokenType.While ) + { + if ( whileDepth++ == 0 ) + bufferManager.SetGrow( true ); + } + else if ( token.TokenType == TokenType.EndWhile ) { - bufferManager.SetGrow( false ); - bufferManager.TrimBuffers(); + if ( --whileDepth == 0 ) + { + bufferManager.SetGrow( false ); + bufferManager.TrimBuffers(); + } } } @@ -317,7 +341,7 @@ private void ParseTemplate( TextReader reader, TextWriter writer ) } // no-match eof: incomplete token - if ( lastReadBytes < bufferSize ) + if ( bufferManager.IsFixed || lastReadBytes < bufferManager.BufferSize ) throw new TemplateException( "Missing right token delimiter." ); // no-match: save partial token less remainder @@ -337,150 +361,13 @@ private void ParseTemplate( TextReader reader, TextWriter writer ) throw new ArgumentOutOfRangeException( scanner.ToString(), $"Invalid scanner state: {scanner}." ); } } - } - - if ( state.Frame.Depth != 0 ) - throw new TemplateException( "Missing end if, or end while." ); - } - catch ( Exception ex ) - { - throw new TemplateException( "Error processing template.", ex ); - } - finally - { - writer.Flush(); - bufferManager.ReleaseBuffers(); - } - - return; - - static int GetScopedBufferSize( int bufferSize, int tokenLeftSize, int tokenRightSize ) - { - // because of the way we read the buffer, we need to ensure that the buffer size - // is at least the size of the longest token delimiter plus one character. - - var maxDelimiter = Math.Max( tokenLeftSize, tokenRightSize ); - return Math.Max( bufferSize, maxDelimiter + 1 ); - } - } - - // parse in-memory template - private void ParseTemplate( ReadOnlySpan templateSpan, TextWriter writer, int pos = int.MinValue ) - { - // find: first token start - - if ( pos == int.MinValue ) - pos = templateSpan.IndexOf( TokenLeft ); - - if ( pos < 0 ) // no-match: quick out - { - writer.Write( templateSpan ); - return; - } - - // match: process template - - var skipIndexOf = true; - var tokenWriter = new ArrayBufferWriter(); // defaults to 256 - var scanner = TemplateScanner.Text; - var ignore = false; - IndexOfState indexOfState = default; // index-of for right token delimiter could span buffer reads - var state = new TemplateState(); // template state for this parsing session - var span = templateSpan; // Keep the original template span for resetting the position - - try - { - while ( true ) - { - if ( span.IsEmpty ) + if ( bufferManager.IsFixed || lastReadBytes < bufferManager.BufferSize ) break; - - while ( !span.IsEmpty ) - { - state.CurrentPos = templateSpan.Length - span.Length; // Track the current position - - switch ( scanner ) - { - case TemplateScanner.Text: - { - if ( skipIndexOf ) - skipIndexOf = false; - else - pos = span.IndexOf( TokenLeft ); - - // match: write to start of token - if ( pos >= 0 ) - { - // write content - if ( !ignore ) - writer.Write( span[..pos] ); - - span = span[(pos + TokenLeft.Length)..]; - - // transition state - scanner = TemplateScanner.Token; - continue; - } - - // no-match eof: write final content - if ( !ignore ) - writer.Write( span ); // write final content - return; - } - - case TemplateScanner.Token: - { - // scan: find closing token pattern - // token may span multiple reads so track search state - pos = IndexOfIgnoreContent( span, TokenRight, ref indexOfState ); - - // no-match eof: incomplete token - if ( pos < 0 ) - throw new TemplateException( "Missing right token delimiter." ); - - // match: process completed token - - // update CurrentPos to point to the first character after the token - state.CurrentPos = templateSpan.Length - span.Length + pos + TokenRight.Length; - - // save token chars - tokenWriter.Write( span[..pos] ); - span = span[(pos + TokenRight.Length)..]; - - // process token - var token = TokenParser.ParseToken( tokenWriter.WrittenSpan, state.NextTokenId++ ); - var tokenAction = TokenProcessor.ProcessToken( token, state, out var tokenValue ); - - if ( tokenAction == TokenAction.Loop ) - { - // Reset the position to start of while block - span = templateSpan[state.Frame.Peek().StartPos..]; // Reset position to StartPos - scanner = TemplateScanner.Text; - tokenWriter.Clear(); - continue; - } - - if ( tokenAction != TokenAction.Ignore ) - WriteTokenValue( writer, tokenValue, tokenAction, state ); - - ignore = state.Frame.IsFalsy; - - tokenWriter.Clear(); - - // transition state - scanner = TemplateScanner.Text; - continue; - } - - default: - throw new ArgumentOutOfRangeException( scanner.ToString(), $"Invalid scanner state: {scanner}." ); - } - } } if ( state.Frame.Depth != 0 ) - throw new TemplateException( "Missing end if or end while." ); + throw new TemplateException( "Missing end if, or end while." ); } catch ( Exception ex ) { @@ -489,6 +376,7 @@ private void ParseTemplate( ReadOnlySpan templateSpan, TextWriter writer, finally { writer.Flush(); + bufferManager.ReleaseBuffers(); } } From d2b76a37f5ead8783958de1315e35f3ce7132955 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 12 Aug 2024 21:12:37 +0000 Subject: [PATCH 17/58] Updated code formatting to match rules in .editorconfig --- src/Hyperbee.Templating/Text/BufferManager.cs | 2 +- src/Hyperbee.Templating/Text/TemplateParser.cs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Hyperbee.Templating/Text/BufferManager.cs b/src/Hyperbee.Templating/Text/BufferManager.cs index b791c5e..c58d12e 100644 --- a/src/Hyperbee.Templating/Text/BufferManager.cs +++ b/src/Hyperbee.Templating/Text/BufferManager.cs @@ -1,4 +1,4 @@ -using System.Buffers; +using System.Buffers; using System.Runtime.CompilerServices; namespace Hyperbee.Templating.Text; diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index 5b01a84..419ef7d 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -1,4 +1,4 @@ -using System.Buffers; +using System.Buffers; using Hyperbee.Templating.Collections; using Hyperbee.Templating.Compiler; using Hyperbee.Templating.Extensions; @@ -191,7 +191,7 @@ private void ParseTemplate( ReadOnlySpan templateSpan, TextWriter writer ) private void ParseTemplate( TextReader reader, TextWriter writer ) { - var bufferSize = GetScopedBufferSize( BufferSize, TokenLeft.Length, TokenRight.Length ); + var bufferSize = GetScopedBufferSize( BufferSize, TokenLeft.Length, TokenRight.Length ); var bufferManager = new BufferManager( bufferSize ); ParseTemplate( ref bufferManager, reader, writer ); @@ -341,7 +341,7 @@ private void ParseTemplate( ref BufferManager bufferManager, TextReader reader, } // no-match eof: incomplete token - if ( bufferManager.IsFixed || lastReadBytes < bufferManager.BufferSize ) + if ( bufferManager.IsFixed || lastReadBytes < bufferManager.BufferSize ) throw new TemplateException( "Missing right token delimiter." ); // no-match: save partial token less remainder From 8150dc7c76606017429b57f620cfcffbde6156b8 Mon Sep 17 00:00:00 2001 From: Brenton Farmer Date: Mon, 12 Aug 2024 16:56:18 -0700 Subject: [PATCH 18/58] cleanup --- .../Text/TemplateParser.cs | 87 ++++++++++--------- src/Hyperbee.Templating/Text/TokenParser.cs | 23 +++-- .../Text/TokenProcessor.cs | 37 ++++---- .../Text/TemplateParser.ParsingTests.cs | 22 ++--- 4 files changed, 92 insertions(+), 77 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index 419ef7d..dbceaee 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -214,7 +214,7 @@ private void ParseTemplate( ref BufferManager bufferManager, TextReader reader, var tokenWriter = new ArrayBufferWriter(); // defaults to 256 var scanner = TemplateScanner.Text; var ignore = false; - var whileDepth = 0; + var loopDepth = 0; IndexOfState indexOfState = default; // index-of for right token delimiter could span buffer reads var state = new TemplateState(); // template state for this parsing session @@ -290,53 +290,25 @@ private void ParseTemplate( ref BufferManager bufferManager, TextReader reader, // update CurrentPos to point to the first character after the token state.CurrentPos += pos + TokenRight.Length; - // save token chars + // process token tokenWriter.Write( span[..pos] ); span = bufferManager.GetCurrentSpan( pos + TokenRight.Length ); - // process token var token = TokenParser.ParseToken( tokenWriter.WrittenSpan, state.NextTokenId++ ); var tokenAction = TokenProcessor.ProcessToken( token, state, out var tokenValue ); + + tokenWriter.Clear(); + scanner = TemplateScanner.Text; // loop handling - if ( tokenAction == TokenAction.Loop ) - { - // Reset the position to the start of the while block - bufferManager.Position( state.Frame.Peek().StartPos ); - span = bufferManager.GetCurrentSpan(); - scanner = TemplateScanner.Text; - tokenWriter.Clear(); + if ( ProcessLoop( tokenAction, token, state, ref span, ref bufferManager, ref loopDepth ) ) continue; - } - - // loop buffer management - if ( !bufferManager.IsFixed ) - { - if ( token.TokenType == TokenType.While ) - { - if ( whileDepth++ == 0 ) - bufferManager.SetGrow( true ); - } - else if ( token.TokenType == TokenType.EndWhile ) - { - if ( --whileDepth == 0 ) - { - bufferManager.SetGrow( false ); - bufferManager.TrimBuffers(); - } - } - } // write value if ( tokenAction != TokenAction.Ignore ) WriteTokenValue( writer, tokenValue, tokenAction, state ); - ignore = state.Frame.IsFalsy; - - tokenWriter.Clear(); - - // transition state - scanner = TemplateScanner.Text; + ignore = state.Frames.IsFalsy; continue; } @@ -366,7 +338,7 @@ private void ParseTemplate( ref BufferManager bufferManager, TextReader reader, break; } - if ( state.Frame.Depth != 0 ) + if ( state.Frames.Depth != 0 ) throw new TemplateException( "Missing end if, or end while." ); } catch ( Exception ex ) @@ -378,6 +350,42 @@ private void ParseTemplate( ref BufferManager bufferManager, TextReader reader, writer.Flush(); bufferManager.ReleaseBuffers(); } + + return; + + static bool ProcessLoop( TokenAction tokenAction, TokenDefinition token, TemplateState state, ref ReadOnlySpan span, ref BufferManager bufferManager, ref int loopDepth ) + { + // loop handling + + if ( tokenAction == TokenAction.Loop ) + { + // Reset position to the start of the loop block + bufferManager.Position( state.CurrentFrame().StartPos ); + span = bufferManager.GetCurrentSpan(); + return true; + } + + // loop buffer management + + if ( bufferManager.IsFixed ) + return false; + + if ( token.TokenType.HasFlag( TokenType.LoopStart ) ) + { + if ( loopDepth++ == 0 ) + bufferManager.SetGrow( true ); + } + else if ( token.TokenType.HasFlag( TokenType.LoopEnd ) ) + { + if ( --loopDepth != 0 ) + return false; + + bufferManager.SetGrow( false ); + bufferManager.TrimBuffers(); + } + + return false; + } } private void WriteTokenValue( TextWriter writer, ReadOnlySpan value, TokenAction tokenAction, TemplateState state, int recursionCount = 0 ) @@ -413,7 +421,7 @@ private void WriteTokenValue( TextWriter writer, ReadOnlySpan value, Token { // write any leading literal - if ( start > 0 && state.Frame.IsTruthy ) + if ( start > 0 && state.Frames.IsTruthy ) writer.Write( value[..start] ); value = value[(start + TokenLeft.Length)..]; @@ -440,7 +448,7 @@ private void WriteTokenValue( TextWriter writer, ReadOnlySpan value, Token start = !value.IsEmpty ? value.IndexOfIgnoreEscaped( TokenLeft ) : -1; - if ( start == -1 && !value.IsEmpty && state.Frame.IsTruthy ) + if ( start == -1 && !value.IsEmpty && state.Frames.IsTruthy ) writer.Write( value ); } while ( start != -1 ); @@ -525,7 +533,8 @@ public void Push( TokenDefinition token, bool truthy, int startPos = -1 ) internal sealed class TemplateState { - public TemplateStack Frame { get; } = new(); + public TemplateStack Frames { get; } = new(); public int NextTokenId { get; set; } = 1; public int CurrentPos { get; set; } + public TemplateStack.Frame CurrentFrame() => Frames.Peek(); } diff --git a/src/Hyperbee.Templating/Text/TokenParser.cs b/src/Hyperbee.Templating/Text/TokenParser.cs index 40253bf..e795c74 100644 --- a/src/Hyperbee.Templating/Text/TokenParser.cs +++ b/src/Hyperbee.Templating/Text/TokenParser.cs @@ -3,16 +3,23 @@ namespace Hyperbee.Templating.Text; +[Flags] internal enum TokenType { - None, - Define, - Value, - If, - Else, - Endif, - While, - EndWhile + None = 0x00, + Define = 0x01, + Value = 0x02, + If = 0x03, + Else = 0x04, + Endif = 0x05, + + LoopStart = 0x10, // loop category + LoopEnd = 0x20, + + While = 0x11, + EndWhile = 0x21, + Each = 0x12, + EndEach = 0x22 } internal enum TokenEvaluation diff --git a/src/Hyperbee.Templating/Text/TokenProcessor.cs b/src/Hyperbee.Templating/Text/TokenProcessor.cs index 05813e8..3101719 100644 --- a/src/Hyperbee.Templating/Text/TokenProcessor.cs +++ b/src/Hyperbee.Templating/Text/TokenProcessor.cs @@ -37,13 +37,13 @@ public TokenProcessor( public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out string value ) { value = default; - var frame = state.Frame; + var frames = state.Frames; // Frame handling: pre-value processing switch ( token.TokenType ) { case TokenType.Value: - if ( frame.IsFalsy ) + if ( frames.IsFalsy ) return TokenAction.Ignore; break; @@ -52,17 +52,17 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out break; case TokenType.Else: - return ProcessElseToken( frame, token ); + return ProcessElseToken( frames, token ); case TokenType.Endif: - return ProcessEndIfToken( frame ); + return ProcessEndIfToken( frames ); case TokenType.While: // Fall through to resolve value. break; case TokenType.EndWhile: - return ProcessEndWhileToken( frame ); + return ProcessEndWhileToken( frames ); case TokenType.Define: return ProcessDefineToken( token ); @@ -86,7 +86,7 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out var frameIsTruthy = token.TokenEvaluation == TokenEvaluation.Falsy ? !ifResult : ifResult; var startPos = token.TokenType == TokenType.While ? state.CurrentPos : -1; - frame.Push( token, frameIsTruthy, startPos ); + frames.Push( token, frameIsTruthy, startPos ); return TokenAction.Ignore; } @@ -115,35 +115,34 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out return tokenAction; } - private static TokenAction ProcessElseToken( TemplateStack frame, TokenDefinition token ) + private static TokenAction ProcessElseToken( TemplateStack frames, TokenDefinition token ) { - if ( !frame.IsTokenType( TokenType.If ) ) + if ( !frames.IsTokenType( TokenType.If ) ) throw new TemplateException( "Syntax error. Invalid `else` without matching `if`." ); - frame.Push( token, !frame.IsTruthy ); + frames.Push( token, !frames.IsTruthy ); return TokenAction.Ignore; } - private static TokenAction ProcessEndIfToken( TemplateStack frame ) + private static TokenAction ProcessEndIfToken( TemplateStack frames ) { - if ( frame.Depth == 0 || !frame.IsTokenType( TokenType.If ) && !frame.IsTokenType( TokenType.Else ) ) + if ( frames.Depth == 0 || !frames.IsTokenType( TokenType.If ) && !frames.IsTokenType( TokenType.Else ) ) throw new TemplateException( "Syntax error. Invalid `/if` without matching `if`." ); - if ( frame.IsTokenType( TokenType.Else ) ) - frame.Pop(); // pop the else + if ( frames.IsTokenType( TokenType.Else ) ) + frames.Pop(); // pop the else - frame.Pop(); // pop the if + frames.Pop(); // pop the if return TokenAction.Ignore; } - private TokenAction ProcessEndWhileToken( TemplateStack frame ) + private TokenAction ProcessEndWhileToken( TemplateStack frames ) { - if ( frame.Depth == 0 || !frame.IsTokenType( TokenType.While ) ) + if ( frames.Depth == 0 || !frames.IsTokenType( TokenType.While ) ) throw new TemplateException( "Syntax error. Invalid `/while` without matching `while`." ); - var whileFrame = frame.Peek(); - var whileToken = whileFrame.Token; + var whileToken = frames.Peek().Token; string expressionError = null; @@ -158,7 +157,7 @@ TokenEvaluation.Expression when TryInvokeTokenExpression( whileToken, out var ex return TokenAction.Loop; // Otherwise, pop the frame and exit the loop - frame.Pop(); + frames.Pop(); return TokenAction.Ignore; } diff --git a/test/Hyperbee.Templating.Tests/Text/TemplateParser.ParsingTests.cs b/test/Hyperbee.Templating.Tests/Text/TemplateParser.ParsingTests.cs index 9ad704b..b4e3e5a 100644 --- a/test/Hyperbee.Templating.Tests/Text/TemplateParser.ParsingTests.cs +++ b/test/Hyperbee.Templating.Tests/Text/TemplateParser.ParsingTests.cs @@ -30,17 +30,17 @@ public void Should_parse_token( string token, string expectedTokenType, string e } [DataTestMethod] - [DataRow( 2 )] //f - //[DataRow( 9 )] - //[DataRow( 10 )] - //[DataRow( 11 )] - //[DataRow( 12 )] - //[DataRow( 15 )] - //[DataRow( 16 )] - //[DataRow( 17 )] - //[DataRow( 18 )] - //[DataRow( 19 )] - //[DataRow( 50 )] + [DataRow( 2 )] + [DataRow( 9 )] + [DataRow( 10 )] + [DataRow( 11 )] + [DataRow( 12 )] + [DataRow( 15 )] + [DataRow( 16 )] + [DataRow( 17 )] + [DataRow( 18 )] + [DataRow( 19 )] + [DataRow( 50 )] public void Should_parse_tokens_with_buffer_wraps( int size ) { // arrange From c6e1807c23c7b7fee031dcb1abece88030cc4305 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 12 Aug 2024 23:56:48 +0000 Subject: [PATCH 19/58] Updated code formatting to match rules in .editorconfig --- src/Hyperbee.Templating/Text/TemplateParser.cs | 2 +- .../Text/TemplateParser.ParsingTests.cs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index dbceaee..ca1ced8 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -296,7 +296,7 @@ private void ParseTemplate( ref BufferManager bufferManager, TextReader reader, var token = TokenParser.ParseToken( tokenWriter.WrittenSpan, state.NextTokenId++ ); var tokenAction = TokenProcessor.ProcessToken( token, state, out var tokenValue ); - + tokenWriter.Clear(); scanner = TemplateScanner.Text; diff --git a/test/Hyperbee.Templating.Tests/Text/TemplateParser.ParsingTests.cs b/test/Hyperbee.Templating.Tests/Text/TemplateParser.ParsingTests.cs index b4e3e5a..14986c9 100644 --- a/test/Hyperbee.Templating.Tests/Text/TemplateParser.ParsingTests.cs +++ b/test/Hyperbee.Templating.Tests/Text/TemplateParser.ParsingTests.cs @@ -31,13 +31,13 @@ public void Should_parse_token( string token, string expectedTokenType, string e [DataTestMethod] [DataRow( 2 )] - [DataRow( 9 )] - [DataRow( 10 )] + [DataRow( 9 )] + [DataRow( 10 )] [DataRow( 11 )] [DataRow( 12 )] - [DataRow( 15 )] + [DataRow( 15 )] [DataRow( 16 )] - [DataRow( 17 )] + [DataRow( 17 )] [DataRow( 18 )] [DataRow( 19 )] [DataRow( 50 )] From 96fe0e74b22334d2e447e0a543a741e5c394823c Mon Sep 17 00:00:00 2001 From: Brenton Farmer Date: Mon, 12 Aug 2024 20:47:21 -0700 Subject: [PATCH 20/58] Don't package benchmark --- .../Hyperbee.Templating.Benchmark.csproj | 4 +++- .../Hyperbee.Templating.Tests.csproj | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/test/Hyperbee.Templating.Benchmark/Hyperbee.Templating.Benchmark.csproj b/test/Hyperbee.Templating.Benchmark/Hyperbee.Templating.Benchmark.csproj index 3cf885c..5d8b9a2 100644 --- a/test/Hyperbee.Templating.Benchmark/Hyperbee.Templating.Benchmark.csproj +++ b/test/Hyperbee.Templating.Benchmark/Hyperbee.Templating.Benchmark.csproj @@ -5,10 +5,12 @@ net8.0 enable enable + false + true - + diff --git a/test/Hyperbee.Templating.Tests/Hyperbee.Templating.Tests.csproj b/test/Hyperbee.Templating.Tests/Hyperbee.Templating.Tests.csproj index 2601684..9e93127 100644 --- a/test/Hyperbee.Templating.Tests/Hyperbee.Templating.Tests.csproj +++ b/test/Hyperbee.Templating.Tests/Hyperbee.Templating.Tests.csproj @@ -13,8 +13,8 @@ - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive From 81482aa9852d8c40146908dff5bc33ef69d59765 Mon Sep 17 00:00:00 2001 From: Brenton Farmer Date: Mon, 12 Aug 2024 21:05:31 -0700 Subject: [PATCH 21/58] Adjusted readme and project structure to follow patterns in our other projects --- Hyperbee.Templating.sln | 1 - .../Hyperbee.Templating.csproj | 55 +++++------- src/Hyperbee.Templating/README.md | 90 ------------------- 3 files changed, 24 insertions(+), 122 deletions(-) delete mode 100644 src/Hyperbee.Templating/README.md diff --git a/Hyperbee.Templating.sln b/Hyperbee.Templating.sln index ea39cdb..295b7b5 100644 --- a/Hyperbee.Templating.sln +++ b/Hyperbee.Templating.sln @@ -7,7 +7,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution ProjectSection(SolutionItems) = preProject Directory.Build.props = Directory.Build.props Directory.Build.targets = Directory.Build.targets - README.md = README.md solution-helper.psm1 = solution-helper.psm1 EndProjectSection EndProject diff --git a/src/Hyperbee.Templating/Hyperbee.Templating.csproj b/src/Hyperbee.Templating/Hyperbee.Templating.csproj index 7746141..1b7e280 100644 --- a/src/Hyperbee.Templating/Hyperbee.Templating.csproj +++ b/src/Hyperbee.Templating/Hyperbee.Templating.csproj @@ -2,51 +2,44 @@ net8.0 enable - true - - Stillpoint Software, Inc. - README.md - templating;NET;template engine - icon.png - https://github.com/Stillpoint-Software/Hyperbee.Templating/ - net8.0 - LICENSE - Stillpoint Software, Inc. - Hyperbee Templating - - A simple templating engine supporting value replacements, code expressions, token nesting, - in-line definitions, and `if` `else` conditions. - - https://github.com/Stillpoint-Software/Hyperbee.Templating - git - https://github.com/Stillpoint-Software/hyperbee.templating/releases/latest - true - + true + Stillpoint Software, Inc. + README.md + templating;template engine + icon.png + https://github.com/Stillpoint-Software/Hyperbee.Templating/ + net8.0 + LICENSE + Stillpoint Software, Inc. + Hyperbee Templating + + A simple templating engine supporting value replacements, code expressions, token nesting, + in-line definitions, conditions, and looping. + + https://github.com/Stillpoint-Software/Hyperbee.Templating + git + https://github.com/Stillpoint-Software/hyperbee.templating/releases/latest + true + <_Parameter1>$(AssemblyName).Tests - - - - + - - PreserveNewest - - - True - \ - + + all runtime; build; native; contentfiles; analyzers; buildtransitive + + \ No newline at end of file diff --git a/src/Hyperbee.Templating/README.md b/src/Hyperbee.Templating/README.md deleted file mode 100644 index 8c9cfc0..0000000 --- a/src/Hyperbee.Templating/README.md +++ /dev/null @@ -1,90 +0,0 @@ -# Hyperbee.Templating - -## Syntax -The templating engine supports a simple syntax. - -### Token Values -Token values are either simple substitutions or expression results: - -* `{{token}}` -* `{{context => context.token}}` -* `{{context => context.token.ToUpper()}}` - -### Token Conditions -Token conditions are either simple truthy tokens or expression results: - -* `{{if token}} _truthy_content_ {{/if}}` -* `{{if !token}} _falsy_content_ {{/if}}` -* `{{if token}} _true_content_ {{else}} _false_content_ {{/if}}` -* `{{if context => context.token == "test"}} _true_content_ {{/if}}` -* `{{if context => context.token == "test"}} _true_content_ {{else}} _false_content_ {{/if}}` - -## Methods -The templating engine supports two kinds of method evaluations; strongly typed CLR class -methods through the Roslyn compiler, and dynamic methods in the form of Func expressions -that are runtime bound to the expression argument context. - -```csharp -\\ example of CLR String.ToUpper - -var parser = new TemplateParser -{ - Tokens = - { - ["name"] = "me" - } -}; - -var result = parser.Render( $"hello {{x => x.name.ToUpper()}}." ); -``` - -```csharp -\\ example of a Func expression MyUpper - -var parser = new TemplateParser -{ - Methods = - { - ["MyUpper"] = args => ((string)args[0]).ToUpper() - }, - Tokens = - { - ["name"] = "me" - } -}; - -var result = parser.Render( $"hello {{x => x.MyUpper( x.name )}}." ); -``` - -## Token Nesting -Token values can contain tokens. - -```csharp -\\ example of token nesting - -var parser = new TemplateParser -{ - Tokens = - { - ["fullname"] = "{{first}} {{last}}", - ["first"] = "Hari", - ["last"] = "Seldon" - } -}; - -var result = parser.Render( $"hello {{fullname}}." ); -``` - -## Inline Token Definitions -You can define tokens, inline, within a template. Inline tokens must be defined before they are referenced. - -```csharp -{{identity:"me"}} -hello {{identity}}. -``` - -```csharp -{{identity:{{x=> "me"}} }} -hello {{identity}}. -``` - From 64638157f4c0d34d32cc1b6b3a8f4edb07734983 Mon Sep 17 00:00:00 2001 From: "annette.findley" Date: Tue, 13 Aug 2024 12:07:09 -0400 Subject: [PATCH 22/58] additiona --- .../Text/TemplateParser.cs | 20 +++----- src/Hyperbee.Templating/Text/TokenParser.cs | 23 +++++++-- .../Text/TokenProcessor.cs | 51 +++++++++++++++++++ 3 files changed, 77 insertions(+), 17 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index ab5c3e3..6de251a 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -1,4 +1,4 @@ -using System.Buffers; +using System.Buffers; using Hyperbee.Templating.Collections; using Hyperbee.Templating.Compiler; using Hyperbee.Templating.Extensions; @@ -247,9 +247,6 @@ private void ParseTemplate( ref BufferManager bufferManager, TextReader reader, span = bufferManager.GetCurrentSpan( pos + TokenLeft.Length ); - if ( !state.Frame.IsIterationFrame ) - sourcePos = pos + sourcePos + TokenLeft.Length; - // transition state scanner = TemplateScanner.Token; continue; @@ -264,7 +261,7 @@ private void ParseTemplate( ref BufferManager bufferManager, TextReader reader, } // no-match: write content less remainder - if ( !ignore || state.Frame._stack.Count == 0 ) + if ( !ignore ) { var writeLength = span.Length - TokenLeft.Length; @@ -291,9 +288,6 @@ private void ParseTemplate( ref BufferManager bufferManager, TextReader reader, // update CurrentPos to point to the first character after the token state.CurrentPos += pos + TokenRight.Length; - if ( !state.Frame.IsIterationFrame ) - sourcePos = pos + sourcePos + TokenRight.Length; - // process token tokenWriter.Write( span[..pos] ); span = bufferManager.GetCurrentSpan( pos + TokenRight.Length ); @@ -422,11 +416,11 @@ private void WriteTokenValue( TextWriter writer, ReadOnlySpan value, Token // nested token processing do { - if ( start > 0 && (state.Frame.IsIterationFrame || state.Frame.IsTruthy) ) - { - writer.Write( value[..start] ); + //if ( start > 0 && (state.Frame.IsIterationFrame || state.Frame.IsTruthy) ) + //{ + // writer.Write( value[..start] ); - } + //} // write any leading literal if ( start > 0 && state.Frames.IsTruthy ) @@ -537,6 +531,8 @@ public void Push( TokenDefinition token, bool truthy, int startPos = -1 ) public bool IsTokenType( TokenType compare ) => _stack.Count > 0 && _stack.Peek().Token.TokenType == compare; public bool IsTruthy => _stack.Count == 0 || _stack.Peek().Truthy; public bool IsFalsy => !IsTruthy; + public IEnumerable Enumerable { get; set; } + } internal sealed class TemplateState diff --git a/src/Hyperbee.Templating/Text/TokenParser.cs b/src/Hyperbee.Templating/Text/TokenParser.cs index 94c0635..3379f23 100644 --- a/src/Hyperbee.Templating/Text/TokenParser.cs +++ b/src/Hyperbee.Templating/Text/TokenParser.cs @@ -1,4 +1,4 @@ -using Hyperbee.Templating.Collections; +using Hyperbee.Templating.Collections; using Hyperbee.Templating.Extensions; namespace Hyperbee.Templating.Text; @@ -136,9 +136,7 @@ public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) tokenType = TokenType.Endif; } - else if ( content.StartsWith( "each", StringComparison.OrdinalIgnoreCase ) ) - { - //TODO: AF + // while handling @@ -192,7 +190,22 @@ public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) tokenType = TokenType.EndWhile; } - // value handling + // each handling + if ( span.StartsWith( "each", StringComparison.OrdinalIgnoreCase ) ) + { + //TODO: AF + + // value handling + } + else if ( span.StartsWith( "/each", StringComparison.OrdinalIgnoreCase ) ) + { + if ( span.Length != 5 ) + throw new TemplateException( "Invalid `/each` statement. Invalid characters." ); + + tokenType = TokenType.EndEach; + } + + if ( tokenType == TokenType.None ) { diff --git a/src/Hyperbee.Templating/Text/TokenProcessor.cs b/src/Hyperbee.Templating/Text/TokenProcessor.cs index 3101719..2e7236d 100644 --- a/src/Hyperbee.Templating/Text/TokenProcessor.cs +++ b/src/Hyperbee.Templating/Text/TokenProcessor.cs @@ -64,6 +64,13 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out case TokenType.EndWhile: return ProcessEndWhileToken( frames ); + case TokenType.Each: + // Fall through to resolve value. + break; + + case TokenType.EndEach: + return ProcessEndEachToken( frames ); + case TokenType.Define: return ProcessDefineToken( token ); @@ -161,6 +168,39 @@ TokenEvaluation.Expression when TryInvokeTokenExpression( whileToken, out var ex return TokenAction.Ignore; } + private TokenAction ProcessEndEachToken( TemplateStack frames ) + { + if ( frames.Depth == 0 || !frames.IsTokenType( TokenType.Each ) ) + throw new TemplateException( "Syntax error. Invalid `/each` without matching `each`." ); + + var eachToken = frames.Peek().Token; + + string expressionError = null; + + var conditionIsTrue = eachToken.TokenEvaluation switch + { + TokenEvaluation.Expression when TryInvokeTokenExpression( eachToken, out var expressionResult, out expressionError ) => + frames.Enumerable = expressionResult.GetType().GetProperties().Select( prop => prop.GetValue( expressionResult )?.ToString() ).ToList(), + + + // frames.Enumerable = expressionResult.GetType() + // .GetProperties() + // .Select( prop => prop.GetValue( expressionResult )?.ToString() ) , + + ////Convert.ToBoolean( expressionResult ), + + TokenEvaluation.Expression => throw new TemplateException( $"{_tokenLeft}Error ({eachToken.Id}):{expressionError ?? "Error in each condition."}{_tokenRight}" ), + _ => TemplateHelper.Truthy( _tokens[eachToken.Name] ) // Re-evaluate the condition + }; + + if ( conditionIsTrue ) // If the condition is true, replay the while block + return TokenAction.Loop; + + // Otherwise, pop the frame and exit the loop + frames.Pop(); + return TokenAction.Ignore; + } + private TokenAction ProcessDefineToken( TokenDefinition token ) { string expressionError = null; @@ -186,6 +226,7 @@ private void ResolveValue( TokenDefinition token, out string value, out bool def case TokenType.Value when token.TokenEvaluation != TokenEvaluation.Expression: case TokenType.If when token.TokenEvaluation != TokenEvaluation.Expression: case TokenType.While when token.TokenEvaluation != TokenEvaluation.Expression: + case TokenType.Each when token.TokenEvaluation != TokenEvaluation.Expression: defined = _tokens.TryGetValue( token.Name, out value ); if ( !defined && _substituteEnvironmentVariables ) @@ -214,6 +255,15 @@ private void ResolveValue( TokenDefinition token, out string value, out bool def else throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{error ?? "Error in if condition."}{_tokenRight}" ); break; + case TokenType.Each when token.TokenEvaluation == TokenEvaluation.Expression: + if ( TryInvokeTokenExpression( token, out var eachExprResult, out var errorEach ) ) + { + var eachResult = eachExprResult; + ifResult = true; + } + else + throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{errorEach ?? "Error in each condition."}{_tokenRight}" ); + break; } } @@ -263,4 +313,5 @@ private bool TryInvokeTokenExpression( TokenDefinition token, out object result, result = default; return false; } + } From 90c8c11a71fa77d85b4adb6935eb7b27893c6c38 Mon Sep 17 00:00:00 2001 From: "annette.findley" Date: Tue, 13 Aug 2024 15:28:43 -0400 Subject: [PATCH 23/58] WIp --- .../Text/TemplateParser.cs | 15 +++------- src/Hyperbee.Templating/Text/TokenParser.cs | 28 ++++++++++++++++++- .../Text/TokenProcessor.cs | 25 ++++++++--------- .../Hyperbee.Templating.Tests.csproj | 1 + 4 files changed, 43 insertions(+), 26 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index 6de251a..ee514c6 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -416,13 +416,6 @@ private void WriteTokenValue( TextWriter writer, ReadOnlySpan value, Token // nested token processing do { - //if ( start > 0 && (state.Frame.IsIterationFrame || state.Frame.IsTruthy) ) - //{ - // writer.Write( value[..start] ); - - //} - // write any leading literal - if ( start > 0 && state.Frames.IsTruthy ) writer.Write( value[..start] ); @@ -517,12 +510,12 @@ private static int IndexOfIgnoreContent( ReadOnlySpan span, ReadOnlySpan Iterator = null, int StartPos = -1 ); private readonly Stack _stack = new(); - public void Push( TokenDefinition token, bool truthy, int startPos = -1 ) - => _stack.Push( new Frame( token, truthy, startPos ) ); + public void Push( TokenDefinition token, bool truthy, IEnumerable iterator = null, int startPos = -1 ) + => _stack.Push( new Frame( token, truthy, iterator, startPos ) ); public Frame Peek() => _stack.Peek(); public void Pop() => _stack.Pop(); @@ -531,7 +524,7 @@ public void Push( TokenDefinition token, bool truthy, int startPos = -1 ) public bool IsTokenType( TokenType compare ) => _stack.Count > 0 && _stack.Peek().Token.TokenType == compare; public bool IsTruthy => _stack.Count == 0 || _stack.Peek().Truthy; public bool IsFalsy => !IsTruthy; - public IEnumerable Enumerable { get; set; } + public IEnumerable Iterator { get; set; } } diff --git a/src/Hyperbee.Templating/Text/TokenParser.cs b/src/Hyperbee.Templating/Text/TokenParser.cs index 3379f23..1e76abe 100644 --- a/src/Hyperbee.Templating/Text/TokenParser.cs +++ b/src/Hyperbee.Templating/Text/TokenParser.cs @@ -195,7 +195,33 @@ public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) { //TODO: AF - // value handling + tokenType = TokenType.Each; + span = span[4..].Trim(); // eat the 'each' + + if ( span.Length >= 4 ) + { + // detect expression syntax + var isFatArrow = span.IndexOfIgnoreDelimitedRanges( "=>", "\"" ) != -1; + + // validate + if ( span.IsEmpty ) + throw new TemplateException( "Invalid `each` statement. Missing identifier." ); + + if ( !isFatArrow && !ValidateKey( span ) ) + throw new TemplateException( "Invalid `each` statement. Invalid identifier in truthy expression." ); + + // results + if ( isFatArrow ) + { + tokenEvaluation = TokenEvaluation.Expression; + tokenExpression = span; //x=>x.list + } + else + { + tokenEvaluation = TokenEvaluation.Falsy; + name = span; + } + } } else if ( span.StartsWith( "/each", StringComparison.OrdinalIgnoreCase ) ) { diff --git a/src/Hyperbee.Templating/Text/TokenProcessor.cs b/src/Hyperbee.Templating/Text/TokenProcessor.cs index 2e7236d..1a4b284 100644 --- a/src/Hyperbee.Templating/Text/TokenProcessor.cs +++ b/src/Hyperbee.Templating/Text/TokenProcessor.cs @@ -81,7 +81,7 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out // Resolve value - ResolveValue( token, out value, out var defined, out var ifResult, out var expressionError ); + ResolveValue( token, out value, out var defined, out var ifResult, out var iterator, out var expressionError ); // Frame handling: post-value processing @@ -93,7 +93,7 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out var frameIsTruthy = token.TokenEvaluation == TokenEvaluation.Falsy ? !ifResult : ifResult; var startPos = token.TokenType == TokenType.While ? state.CurrentPos : -1; - frames.Push( token, frameIsTruthy, startPos ); + frames.Push( token, frameIsTruthy, null, startPos ); return TokenAction.Ignore; } @@ -101,7 +101,7 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out // Token handling: user-defined token action - _ = TryInvokeTokenHandler( token, defined, ref value, out var tokenAction ); + _ = TryInvokeTokenHandler( token, defined, ref value, ref iterator, out var tokenAction ); // Handle final token action @@ -180,25 +180,21 @@ private TokenAction ProcessEndEachToken( TemplateStack frames ) var conditionIsTrue = eachToken.TokenEvaluation switch { TokenEvaluation.Expression when TryInvokeTokenExpression( eachToken, out var expressionResult, out expressionError ) => - frames.Enumerable = expressionResult.GetType().GetProperties().Select( prop => prop.GetValue( expressionResult )?.ToString() ).ToList(), + frames.Iterator = expressionResult.GetType().GetProperties().Select( prop => prop.GetValue( expressionResult )?.ToString() ).ToList(), - // frames.Enumerable = expressionResult.GetType() - // .GetProperties() - // .Select( prop => prop.GetValue( expressionResult )?.ToString() ) , - - ////Convert.ToBoolean( expressionResult ), - TokenEvaluation.Expression => throw new TemplateException( $"{_tokenLeft}Error ({eachToken.Id}):{expressionError ?? "Error in each condition."}{_tokenRight}" ), _ => TemplateHelper.Truthy( _tokens[eachToken.Name] ) // Re-evaluate the condition }; + if ( conditionIsTrue ) // If the condition is true, replay the while block return TokenAction.Loop; // Otherwise, pop the frame and exit the loop frames.Pop(); return TokenAction.Ignore; + } private TokenAction ProcessDefineToken( TokenDefinition token ) @@ -214,12 +210,13 @@ TokenEvaluation.Expression when TryInvokeTokenExpression( token, out var express return TokenAction.Ignore; } - private void ResolveValue( TokenDefinition token, out string value, out bool defined, out bool ifResult, out string expressionError ) + private void ResolveValue( TokenDefinition token, out string value, out bool defined, out bool ifResult, out IEnumerable iterator, out string expressionError ) { value = default; defined = false; ifResult = false; expressionError = null; + iterator = null; switch ( token.TokenType ) { @@ -258,8 +255,8 @@ private void ResolveValue( TokenDefinition token, out string value, out bool def case TokenType.Each when token.TokenEvaluation == TokenEvaluation.Expression: if ( TryInvokeTokenExpression( token, out var eachExprResult, out var errorEach ) ) { - var eachResult = eachExprResult; - ifResult = true; + var results = (string) eachExprResult; + iterator = results.Split( ',' ).Select( v => v.Trim() ); } else throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{errorEach ?? "Error in each condition."}{_tokenRight}" ); @@ -267,7 +264,7 @@ private void ResolveValue( TokenDefinition token, out string value, out bool def } } - private bool TryInvokeTokenHandler( TokenDefinition token, bool defined, ref string value, out TokenAction tokenAction ) + private bool TryInvokeTokenHandler( TokenDefinition token, bool defined, ref string value, ref IEnumerable iterator, out TokenAction tokenAction ) { tokenAction = defined ? TokenAction.Replace : (_ignoreMissingTokens ? TokenAction.Ignore : TokenAction.Error); diff --git a/test/Hyperbee.Templating.Tests/Hyperbee.Templating.Tests.csproj b/test/Hyperbee.Templating.Tests/Hyperbee.Templating.Tests.csproj index 9e93127..e9d9908 100644 --- a/test/Hyperbee.Templating.Tests/Hyperbee.Templating.Tests.csproj +++ b/test/Hyperbee.Templating.Tests/Hyperbee.Templating.Tests.csproj @@ -1,6 +1,7 @@  net8.0 + false false From 4a66af3443d4e495da6e4f539e0d8b75320e0fcb Mon Sep 17 00:00:00 2001 From: "annette.findley" Date: Wed, 28 Aug 2024 12:43:05 -0400 Subject: [PATCH 24/58] merge conflicts --- src/Hyperbee.Templating/Text/TokenParser.cs | 6 +-- .../Text/TokenProcessor.cs | 53 ++++++++++--------- .../Text/TemplateParser.ExpressionTests.cs | 38 +++++-------- 3 files changed, 42 insertions(+), 55 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TokenParser.cs b/src/Hyperbee.Templating/Text/TokenParser.cs index 0cc6026..ece9f4e 100644 --- a/src/Hyperbee.Templating/Text/TokenParser.cs +++ b/src/Hyperbee.Templating/Text/TokenParser.cs @@ -1,4 +1,4 @@ -using Hyperbee.Templating.Configure; +using Hyperbee.Templating.Configure; using Hyperbee.Templating.Core; namespace Hyperbee.Templating.Text; @@ -189,7 +189,7 @@ public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) if ( span.IsEmpty ) throw new TemplateException( "Invalid `while` statement. Missing identifier." ); - if ( !isFatArrow && !ValidateKey( span ) ) + if ( !isFatArrow && !_validateKey( span ) ) throw new TemplateException( "Invalid `while` statement. Invalid identifier in truthy expression." ); if ( bang && isFatArrow ) @@ -233,7 +233,7 @@ public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) if ( span.IsEmpty ) throw new TemplateException( "Invalid `each` statement. Missing identifier." ); - if ( !isFatArrow && !ValidateKey( span ) ) + if ( !isFatArrow && !_validateKey( span ) ) throw new TemplateException( "Invalid `each` statement. Invalid identifier in truthy expression." ); // results diff --git a/src/Hyperbee.Templating/Text/TokenProcessor.cs b/src/Hyperbee.Templating/Text/TokenProcessor.cs index 241a564..341ca74 100644 --- a/src/Hyperbee.Templating/Text/TokenProcessor.cs +++ b/src/Hyperbee.Templating/Text/TokenProcessor.cs @@ -1,4 +1,4 @@ -using System.Globalization; +using System.Globalization; using Hyperbee.Templating.Compiler; using Hyperbee.Templating.Configure; @@ -64,12 +64,13 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out case TokenType.EndWhile: return ProcessEndWhileToken( frames ); - + case TokenType.Each: - //Fall through to resolve value. - break; + //Fall through to resolve value. + break; + case TokenType.EndEach: - return ProcessEndEachToken(frames); + return ProcessEndEachToken( frames ); case TokenType.Define: return ProcessDefineToken( token ); @@ -81,7 +82,7 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out // Resolve value - ResolveValue( token, out value, out var defined, out var ifResult, out var expressionError ); + ResolveValue( token, out value, out var defined, out var ifResult, out IEnumerable iterator, out var expressionError ); // Frame handling: post-value processing @@ -93,7 +94,7 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out var frameIsTruthy = token.TokenEvaluation == TokenEvaluation.Falsy ? !ifResult : ifResult; var startPos = token.TokenType == TokenType.While ? state.CurrentPos : -1; - frames.Push( token, frameIsTruthy, startPos ); + frames.Push( token, frameIsTruthy, null, startPos ); return TokenAction.Ignore; } @@ -169,26 +170,24 @@ TokenEvaluation.Expression when TryInvokeTokenExpression( whileToken, out var ex return TokenAction.Ignore; } - private TokenAction ProcessEndEachToken(TemplateStack frames) + private TokenAction ProcessEndEachToken( FrameStack frames ) { - if( frames.Depth ==0 || !frames.IsTokenType(TokenType.Each)) - throw new TemplateException( "Syntax error. Invalie `/endEach` without matching `each`. "); + if ( frames.Depth == 0 || !frames.IsTokenType( TokenType.Each ) ) + throw new TemplateException( "Syntax error. Invalid `/each` without matching `each`. " ); var eachToken = frames.Peek().Token; string expressionError = null; var conditionIsTrue = eachToken.TokenEvaluation switch { - TokenEvaluation.Expression when TryInvokeTokenExpression( eachToken, out var expressionResult, out expressionError) => - frames.Iterator = expressionResult.GetType().GetProperties().Select(prop => prop.GetValue(expressionResult)?.ToString).ToList, + TokenEvaluation.Expression when TryInvokeTokenExpression( eachToken, out var expressionResult, out expressionError ) => Convert.ToBoolean( expressionResult ), + TokenEvaluation.Expression => throw new TemplateException( $"{_tokenLeft}Error ({eachToken.Id}):{expressionError ?? "Error in each condition."}{_tokenRight}" ), + _ => Truthy( _members[eachToken.Name] ) // Re-evaluate the condition - TokenEvaluation.Expression => throw new TemplateException( $"{_tokenLeft}Error ({eachToken.Id}):{expressionError ?? "Error in each condition."}{_tokenRight}"), - _=> TemplateHelper.Truthy(_tokens[eachToken.Name]) // re-evaluated the condition - }; - if(conditionIsTrue) // If the condition is true, replay the each block - return TokenAction.Loop; + if ( conditionIsTrue ) // If the condition is true, replay the each block + return TokenAction.ContinueLoop; //Otherwise, pop the frame and exit the loop frames.Pop(); @@ -211,11 +210,12 @@ TokenEvaluation.Expression when TryInvokeTokenExpression( token, out var express return TokenAction.Ignore; } - private void ResolveValue( TokenDefinition token, out string value, out bool defined, out bool ifResult, out string expressionError ) + private void ResolveValue( TokenDefinition token, out string value, out bool defined, out bool ifResult, out IEnumerable iterator, out string expressionError ) { value = default; defined = false; ifResult = false; + iterator = null; expressionError = null; switch ( token.TokenType ) @@ -223,7 +223,7 @@ private void ResolveValue( TokenDefinition token, out string value, out bool def case TokenType.Value when token.TokenEvaluation != TokenEvaluation.Expression: case TokenType.If when token.TokenEvaluation != TokenEvaluation.Expression: case TokenType.While when token.TokenEvaluation != TokenEvaluation.Expression: - case TokenType.Each When token.TokenEvaluation != TokenEvaluation.Expression: + case TokenType.Each when token.TokenEvaluation != TokenEvaluation.Expression: defined = _members.TryGetValue( token.Name, out value ); if ( !defined && _substituteEnvironmentVariables ) @@ -253,14 +253,15 @@ private void ResolveValue( TokenDefinition token, out string value, out bool def throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{error ?? "Error in if condition."}{_tokenRight}" ); break; case TokenType.Each when token.TokenEvaluation == TokenEvaluation.Expression: - if( TryInvokeTokenExpression (token, out var eachExprResult, out var errorEach)) - { - var results = (string) eachExprResult; - iterator = results.Split(',').Select(v=> v.Trim()); - } - else - throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{errorEach ?? "Error in each condition."}{_tokenRight}"); + if ( TryInvokeTokenExpression( token, out var eachExprResult, out var errorEach ) ) + { + var results = (string) eachExprResult; + iterator = results.Split( ',' ).Select( v => v.Trim() ); + } + else + throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{errorEach ?? "Error in each condition."}{_tokenRight}" ); break; + } } diff --git a/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs b/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs index 7c25c27..11c693c 100644 --- a/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs +++ b/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs @@ -1,4 +1,4 @@ -using Hyperbee.Templating.Tests.TestSupport; +using Hyperbee.Templating.Tests.TestSupport; using Hyperbee.Templating.Text; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -16,7 +16,7 @@ public void Should_honor_while_condition( ParseTemplateMethod parseMethod ) const string expression = "{{while x => int.Parse(x.counter) < 3}}{{counter}}{{counter:{{x => int.Parse(x.counter) + 1}}}}{{/while}}"; const string template = $"count: {expression}."; - var parser = new TemplateParser { Tokens = { ["counter"] = "0" } }; + var parser = new TemplateParser { Variables = { ["counter"] = "0" } }; // act var result = parser.Render( template, parseMethod ); @@ -48,13 +48,7 @@ public void Should_honor_block_expression( ParseTemplateMethod parseMethod ) const string template = $"hello {expression}."; - var parser = new TemplateParser - { - Variables = - { - ["choice"] = "2" - } - }; + var parser = new TemplateParser { Variables = { ["choice"] = "2" } }; // act @@ -111,13 +105,7 @@ public void Should_honor_inline_block_expression( ParseTemplateMethod parseMetho const string template = $"{definition}hello {expression}."; - var parser = new TemplateParser - { - Variables = - { - ["choice"] = "2" - } - }; + var parser = new TemplateParser { Variables = { ["choice"] = "2" } }; // act var result = parser.Render( template, parseMethod ); @@ -129,6 +117,7 @@ public void Should_honor_inline_block_expression( ParseTemplateMethod parseMetho Assert.AreEqual( expected, result ); } + [DataTestMethod] [DataRow( ParseTemplateMethod.Buffered )] @@ -140,18 +129,15 @@ public void Should_honor_each_expression( ParseTemplateMethod parseMethod ) const string template = $"hello {expression}."; { - var parser = new TemplateParser - Tokens = { - } - ["list"] = "1,2,3" - }; + var parser = new TemplateParser { Variables = { ["list"] = "1,2,3" } }; - var result = parser.Render( template, parseMethod ); - // act + var result = parser.Render( template, parseMethod ); + // act - // assert - var expected = "hello World 1,World 2,World 3,."; + // assert + var expected = "hello World 1,World 2,World 3,."; - Assert.AreEqual( expected, result ); + Assert.AreEqual( expected, result ); + } } } From e8a7fd104b645d99590e844b8cea0696a40da085 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 28 Aug 2024 16:43:55 +0000 Subject: [PATCH 25/58] Updated code formatting to match rules in .editorconfig --- src/Hyperbee.Templating/Text/TemplateParser.cs | 4 ++-- test/Hyperbee.Templating.Tests/Text/TemplateParserTests.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index 6e277eb..7671946 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -1,4 +1,4 @@ -using System.Buffers; +using System.Buffers; using Hyperbee.Templating.Compiler; using Hyperbee.Templating.Configure; using Hyperbee.Templating.Core; @@ -511,5 +511,5 @@ public bool IsTokenType( TokenType compare ) public bool IsTruthy => _stack.Count == 0 || _stack.Peek().Truthy; public bool IsFalsy => !IsTruthy; - public IEnumerable Iterator {get;set;} + public IEnumerable Iterator { get; set; } } diff --git a/test/Hyperbee.Templating.Tests/Text/TemplateParserTests.cs b/test/Hyperbee.Templating.Tests/Text/TemplateParserTests.cs index a22864a..ecd39b0 100644 --- a/test/Hyperbee.Templating.Tests/Text/TemplateParserTests.cs +++ b/test/Hyperbee.Templating.Tests/Text/TemplateParserTests.cs @@ -1,4 +1,4 @@ -using Hyperbee.Templating.Tests.TestSupport; +using Hyperbee.Templating.Tests.TestSupport; using Hyperbee.Templating.Text; using Microsoft.VisualStudio.TestTools.UnitTesting; From 4193ff2b51b9ebb31b8eb14992b3da174971f213 Mon Sep 17 00:00:00 2001 From: "annette.findley" Date: Thu, 29 Aug 2024 12:31:04 -0400 Subject: [PATCH 26/58] Added each process --- .../Text/TemplateParser.cs | 3 +- src/Hyperbee.Templating/Text/TokenParser.cs | 62 ++----------------- .../Text/TokenProcessor.cs | 14 ++++- .../Text/TemplateParser.ExpressionTests.cs | 22 ------- .../Text/TemplateParser.LoopTests.cs | 22 +++++++ 5 files changed, 38 insertions(+), 85 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index 6e277eb..8ea31a1 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -1,4 +1,4 @@ -using System.Buffers; +using System.Buffers; using Hyperbee.Templating.Compiler; using Hyperbee.Templating.Configure; using Hyperbee.Templating.Core; @@ -511,5 +511,4 @@ public bool IsTokenType( TokenType compare ) public bool IsTruthy => _stack.Count == 0 || _stack.Peek().Truthy; public bool IsFalsy => !IsTruthy; - public IEnumerable Iterator {get;set;} } diff --git a/src/Hyperbee.Templating/Text/TokenParser.cs b/src/Hyperbee.Templating/Text/TokenParser.cs index ece9f4e..4a34f23 100644 --- a/src/Hyperbee.Templating/Text/TokenParser.cs +++ b/src/Hyperbee.Templating/Text/TokenParser.cs @@ -33,6 +33,10 @@ public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) // {{while [!]token}} // {{while x => x.token}} // {{/while}} + // + // {{each [!]token}} + // {{each x => x.token}} + // {{/each}} var span = token.Trim(); @@ -162,65 +166,9 @@ public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) tokenType = TokenType.EndWhile; } - // value handling - - // while handling - - if ( span.StartsWith( "while", StringComparison.OrdinalIgnoreCase ) ) - { - if ( span.Length == 5 || char.IsWhiteSpace( span[5] ) ) - { - tokenType = TokenType.While; - span = span[5..].Trim(); // eat the 'while' - - // parse for bang - var bang = false; - - if ( span[0] == '!' ) - { - bang = true; - span = span[1..].Trim(); // eat the '!' - } - - // detect expression syntax - var isFatArrow = span.IndexOfIgnoreDelimitedRanges( "=>", "\"" ) != -1; - - // validate - if ( span.IsEmpty ) - throw new TemplateException( "Invalid `while` statement. Missing identifier." ); - - if ( !isFatArrow && !_validateKey( span ) ) - throw new TemplateException( "Invalid `while` statement. Invalid identifier in truthy expression." ); - - if ( bang && isFatArrow ) - throw new TemplateException( "Invalid `while` statement. The '!' operator is not supported for token expressions." ); - - // results - if ( isFatArrow ) - { - tokenEvaluation = TokenEvaluation.Expression; - tokenExpression = span; - } - else - { - tokenEvaluation = bang ? TokenEvaluation.Falsy : TokenEvaluation.Truthy; - name = span; - } - } - } - else if ( span.StartsWith( "/while", StringComparison.OrdinalIgnoreCase ) ) - { - if ( span.Length != 6 ) - throw new TemplateException( "Invalid `/while` statement. Invalid characters." ); - - tokenType = TokenType.EndWhile; - } - // each handling if ( span.StartsWith( "each", StringComparison.OrdinalIgnoreCase ) ) { - //TODO: AF - tokenType = TokenType.Each; span = span[4..].Trim(); // eat the 'each' @@ -257,8 +205,6 @@ public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) tokenType = TokenType.EndEach; } - - if ( tokenType == TokenType.None ) { var defineTokenPos = span.IndexOfIgnoreDelimitedRanges( ":", "\"" ); diff --git a/src/Hyperbee.Templating/Text/TokenProcessor.cs b/src/Hyperbee.Templating/Text/TokenProcessor.cs index 341ca74..ac36fe2 100644 --- a/src/Hyperbee.Templating/Text/TokenProcessor.cs +++ b/src/Hyperbee.Templating/Text/TokenProcessor.cs @@ -98,6 +98,14 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out return TokenAction.Ignore; } + case TokenType.Each: + { + var startPos = token.TokenType == TokenType.Each ? state.CurrentPos : -1; + + frames.Push( token, false, iterator, startPos ); + + return TokenAction.ContinueLoop; + } } // Token handling: user-defined token action @@ -180,14 +188,14 @@ private TokenAction ProcessEndEachToken( FrameStack frames ) var conditionIsTrue = eachToken.TokenEvaluation switch { - TokenEvaluation.Expression when TryInvokeTokenExpression( eachToken, out var expressionResult, out expressionError ) => Convert.ToBoolean( expressionResult ), + TokenEvaluation.Expression when TryInvokeTokenExpression( eachToken, out var expressionResult, out expressionError ) => expressionResult, TokenEvaluation.Expression => throw new TemplateException( $"{_tokenLeft}Error ({eachToken.Id}):{expressionError ?? "Error in each condition."}{_tokenRight}" ), _ => Truthy( _members[eachToken.Name] ) // Re-evaluate the condition }; - if ( conditionIsTrue ) // If the condition is true, replay the each block - return TokenAction.ContinueLoop; + if ( conditionIsTrue != null ) // If the condition is true, replay the each block + return TokenAction.Ignore; //Otherwise, pop the frame and exit the loop frames.Pop(); diff --git a/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs b/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs index 11c693c..50171b7 100644 --- a/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs +++ b/test/Hyperbee.Templating.Tests/Text/TemplateParser.ExpressionTests.cs @@ -118,26 +118,4 @@ public void Should_honor_inline_block_expression( ParseTemplateMethod parseMetho Assert.AreEqual( expected, result ); } - [DataTestMethod] - - [DataRow( ParseTemplateMethod.Buffered )] - [DataRow( ParseTemplateMethod.InMemory )] - public void Should_honor_each_expression( ParseTemplateMethod parseMethod ) - { - // arrange - const string expression = "{{each x=>x.list}}World {{i}},{{/each}}"; - - const string template = $"hello {expression}."; - { - var parser = new TemplateParser { Variables = { ["list"] = "1,2,3" } }; - - var result = parser.Render( template, parseMethod ); - // act - - // assert - var expected = "hello World 1,World 2,World 3,."; - - Assert.AreEqual( expected, result ); - } - } } diff --git a/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs b/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs index a57d169..e97cf9a 100644 --- a/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs +++ b/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs @@ -32,4 +32,26 @@ public void Should_honor_while_condition( ParseTemplateMethod parseMethod ) Assert.AreEqual( expected, result ); } + + [DataTestMethod] + [DataRow( ParseTemplateMethod.Buffered )] + [DataRow( ParseTemplateMethod.InMemory )] + public void Should_honor_each_expression( ParseTemplateMethod parseMethod ) + { + // arrange + const string expression = "{{each x=>x.list}}World {{i}},{{/each}}"; + + const string template = $"hello {expression}."; + { + var parser = new TemplateParser { Variables = { ["list"] = "1,2,3" } }; + + var result = parser.Render( template, parseMethod ); + // act + + // assert + var expected = "hello World 1,World 2,World 3,."; + + Assert.AreEqual( expected, result ); + } + } } From a459170b7fa2ef0a4624044572f2361aa940a5fd Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 29 Aug 2024 16:32:05 +0000 Subject: [PATCH 27/58] Updated code formatting to match rules in .editorconfig --- src/Hyperbee.Templating/Text/TemplateParser.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index c3028c2..8ea31a1 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -1,4 +1,4 @@ -using System.Buffers; +using System.Buffers; using Hyperbee.Templating.Compiler; using Hyperbee.Templating.Configure; using Hyperbee.Templating.Core; From 9b698bf895e7254649f500de71afb07167dff4d9 Mon Sep 17 00:00:00 2001 From: "annette.findley" Date: Wed, 30 Oct 2024 10:48:50 -0400 Subject: [PATCH 28/58] Wip foreach --- .../Text/TemplateParser.cs | 7 ++- src/Hyperbee.Templating/Text/TokenParser.cs | 2 +- .../Text/TokenProcessor.cs | 44 +++++++++++++------ .../Hyperbee.Templating.Tests.csproj | 8 ++-- .../Text/TemplateParser.LoopTests.cs | 4 +- 5 files changed, 40 insertions(+), 25 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index 8ea31a1..fcdd0d9 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -488,19 +488,18 @@ internal sealed class TemplateState public FrameStack Frames { get; } = new(); public int NextTokenId { get; set; } = 1; public int CurrentPos { get; set; } - public FrameStack.Frame CurrentFrame() => Frames.Depth > 0 ? Frames.Peek() : default; } internal sealed class FrameStack { - public record Frame( TokenDefinition Token, bool Truthy, IEnumerable Iterator, int StartPos = -1 ); + public record Frame( TokenDefinition Token, bool Truthy, IEnumerator Enumerator, int StartPos = -1 ); private readonly Stack _stack = new(); - public void Push( TokenDefinition token, bool truthy, IEnumerable iterator = null, int startPos = -1 ) - => _stack.Push( new Frame( token, truthy, iterator, startPos ) ); + public void Push( TokenDefinition token, bool truthy, IEnumerator enumerator, int startPos = -1 ) + => _stack.Push( new Frame( token, truthy, enumerator, startPos ) ); public Frame Peek() => _stack.Peek(); public void Pop() => _stack.Pop(); diff --git a/src/Hyperbee.Templating/Text/TokenParser.cs b/src/Hyperbee.Templating/Text/TokenParser.cs index 4a34f23..af5a0ca 100644 --- a/src/Hyperbee.Templating/Text/TokenParser.cs +++ b/src/Hyperbee.Templating/Text/TokenParser.cs @@ -182,7 +182,7 @@ public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) throw new TemplateException( "Invalid `each` statement. Missing identifier." ); if ( !isFatArrow && !_validateKey( span ) ) - throw new TemplateException( "Invalid `each` statement. Invalid identifier in truthy expression." ); + throw new TemplateException( "Invalid `each` statement. Invalid identifier in expression." ); // results if ( isFatArrow ) diff --git a/src/Hyperbee.Templating/Text/TokenProcessor.cs b/src/Hyperbee.Templating/Text/TokenProcessor.cs index ac36fe2..f230e27 100644 --- a/src/Hyperbee.Templating/Text/TokenProcessor.cs +++ b/src/Hyperbee.Templating/Text/TokenProcessor.cs @@ -101,10 +101,8 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out case TokenType.Each: { var startPos = token.TokenType == TokenType.Each ? state.CurrentPos : -1; - - frames.Push( token, false, iterator, startPos ); - - return TokenAction.ContinueLoop; + frames.Push( token, false, null, startPos ); + return TokenAction.Ignore; } } @@ -136,7 +134,7 @@ private static TokenAction ProcessElseToken( FrameStack frames, TokenDefinition if ( !frames.IsTokenType( TokenType.If ) ) throw new TemplateException( "Syntax error. Invalid `else` without matching `if`." ); - frames.Push( token, !frames.IsTruthy ); + frames.Push( token, !frames.IsTruthy, null ); return TokenAction.Ignore; } @@ -184,24 +182,42 @@ private TokenAction ProcessEndEachToken( FrameStack frames ) throw new TemplateException( "Syntax error. Invalid `/each` without matching `each`. " ); var eachToken = frames.Peek().Token; - string expressionError = null; - var conditionIsTrue = eachToken.TokenEvaluation switch + var currentFrame = frames.Peek(); + if ( currentFrame.Enumerator == null ) { - TokenEvaluation.Expression when TryInvokeTokenExpression( eachToken, out var expressionResult, out expressionError ) => expressionResult, - TokenEvaluation.Expression => throw new TemplateException( $"{_tokenLeft}Error ({eachToken.Id}):{expressionError ?? "Error in each condition."}{_tokenRight}" ), - _ => Truthy( _members[eachToken.Name] ) // Re-evaluate the condition + string expressionError = null; + var values = eachToken.TokenEvaluation switch + { + TokenEvaluation.Expression when TryInvokeTokenExpression( eachToken, out var expressionResult, out expressionError ) => + expressionResult.ToString()!.Split( ',' ).Select( v => v.Trim() ), + TokenEvaluation.Expression => throw new TemplateException( $"{_tokenLeft}Error ({eachToken.Id}):{expressionError ?? "Error in each condition."}{_tokenRight}" ), + _ => _members[eachToken.Name].Split( ',' ).Select( v => v.Trim() ) + }; + + currentFrame = currentFrame with { Enumerator = values.GetEnumerator() }; + frames.Push( currentFrame.Token, currentFrame.Truthy, currentFrame.Enumerator, currentFrame.StartPos ); + return TokenAction.ContinueLoop; - }; + } - if ( conditionIsTrue != null ) // If the condition is true, replay the each block - return TokenAction.Ignore; + // Iterate over the collection + var items = currentFrame.Enumerator; + while ( items.MoveNext() ) + { + var item = items.Current; + _members.Add( "x", item ); + frames.Push( currentFrame.Token, true, currentFrame.Enumerator, currentFrame.StartPos ); + return TokenAction.ContinueLoop; + } //Otherwise, pop the frame and exit the loop frames.Pop(); return TokenAction.Ignore; } + + private TokenAction ProcessDefineToken( TokenDefinition token ) { // ReSharper disable once RedundantAssignment @@ -240,7 +256,7 @@ private void ResolveValue( TokenDefinition token, out string value, out bool def defined = value != null; } - if ( token.TokenType == TokenType.If || token.TokenType == TokenType.While ) + if ( token.TokenType == TokenType.If || token.TokenType == TokenType.While || token.TokenType == TokenType.Each ) ifResult = defined && Truthy( value ); break; diff --git a/test/Hyperbee.Templating.Tests/Hyperbee.Templating.Tests.csproj b/test/Hyperbee.Templating.Tests/Hyperbee.Templating.Tests.csproj index cf99c5a..03d6561 100644 --- a/test/Hyperbee.Templating.Tests/Hyperbee.Templating.Tests.csproj +++ b/test/Hyperbee.Templating.Tests/Hyperbee.Templating.Tests.csproj @@ -13,14 +13,14 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs b/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs index e97cf9a..d031978 100644 --- a/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs +++ b/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs @@ -39,14 +39,14 @@ public void Should_honor_while_condition( ParseTemplateMethod parseMethod ) public void Should_honor_each_expression( ParseTemplateMethod parseMethod ) { // arrange - const string expression = "{{each x=>x.list}}World {{i}},{{/each}}"; + const string expression = "{{each x=>x.list}}World {{x}},{{/each}}"; const string template = $"hello {expression}."; { var parser = new TemplateParser { Variables = { ["list"] = "1,2,3" } }; - var result = parser.Render( template, parseMethod ); // act + var result = parser.Render( template, parseMethod ); // assert var expected = "hello World 1,World 2,World 3,."; From 245e3d209003bec2660c7c5ec7f0a8d2dc2c5790 Mon Sep 17 00:00:00 2001 From: "annette.findley" Date: Wed, 30 Oct 2024 14:55:05 -0400 Subject: [PATCH 29/58] updated processor --- src/Hyperbee.Templating/Text/TokenProcessor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Hyperbee.Templating/Text/TokenProcessor.cs b/src/Hyperbee.Templating/Text/TokenProcessor.cs index f230e27..1801614 100644 --- a/src/Hyperbee.Templating/Text/TokenProcessor.cs +++ b/src/Hyperbee.Templating/Text/TokenProcessor.cs @@ -196,7 +196,7 @@ TokenEvaluation.Expression when TryInvokeTokenExpression( eachToken, out var exp }; currentFrame = currentFrame with { Enumerator = values.GetEnumerator() }; - frames.Push( currentFrame.Token, currentFrame.Truthy, currentFrame.Enumerator, currentFrame.StartPos ); + frames.Push( currentFrame.Token, true, currentFrame.Enumerator, currentFrame.StartPos ); return TokenAction.ContinueLoop; } From 5554bd0ab59afb8dc0f4e7eadaf8963827d30a6b Mon Sep 17 00:00:00 2001 From: "annette.findley" Date: Mon, 4 Nov 2024 14:34:54 -0500 Subject: [PATCH 30/58] Second try. --- .../Text/TemplateParser.cs | 6 ++- .../Text/TokenProcessor.cs | 51 ++++++++----------- 2 files changed, 26 insertions(+), 31 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index fcdd0d9..b581836 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -494,11 +494,13 @@ public FrameStack.Frame CurrentFrame() => internal sealed class FrameStack { - public record Frame( TokenDefinition Token, bool Truthy, IEnumerator Enumerator, int StartPos = -1 ); + // Updated Frame to include an IEnumerator + public record Frame( TokenDefinition Token, bool Truthy, IEnumerator Enumerator = null, int StartPos = -1 ); private readonly Stack _stack = new(); - public void Push( TokenDefinition token, bool truthy, IEnumerator enumerator, int startPos = -1 ) + // Updated Push method to accept an IEnumerator parameter + public void Push( TokenDefinition token, bool truthy, IEnumerator enumerator = null, int startPos = -1 ) => _stack.Push( new Frame( token, truthy, enumerator, startPos ) ); public Frame Peek() => _stack.Peek(); diff --git a/src/Hyperbee.Templating/Text/TokenProcessor.cs b/src/Hyperbee.Templating/Text/TokenProcessor.cs index 1801614..ab7797e 100644 --- a/src/Hyperbee.Templating/Text/TokenProcessor.cs +++ b/src/Hyperbee.Templating/Text/TokenProcessor.cs @@ -82,7 +82,7 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out // Resolve value - ResolveValue( token, out value, out var defined, out var ifResult, out IEnumerable iterator, out var expressionError ); + ResolveValue( token, out value, out var defined, out var ifResult, out IEnumerable enumerator, out var expressionError ); // Frame handling: post-value processing @@ -100,8 +100,11 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out } case TokenType.Each: { + var result = enumerator.GetEnumerator(); var startPos = token.TokenType == TokenType.Each ? state.CurrentPos : -1; - frames.Push( token, false, null, startPos ); + + frames.Push( token, true, result, startPos ); + return TokenAction.Ignore; } } @@ -134,7 +137,7 @@ private static TokenAction ProcessElseToken( FrameStack frames, TokenDefinition if ( !frames.IsTokenType( TokenType.If ) ) throw new TemplateException( "Syntax error. Invalid `else` without matching `if`." ); - frames.Push( token, !frames.IsTruthy, null ); + frames.Push( token, !frames.IsTruthy ); return TokenAction.Ignore; } @@ -182,42 +185,32 @@ private TokenAction ProcessEndEachToken( FrameStack frames ) throw new TemplateException( "Syntax error. Invalid `/each` without matching `each`. " ); var eachToken = frames.Peek().Token; - var currentFrame = frames.Peek(); - if ( currentFrame.Enumerator == null ) - { - string expressionError = null; - var values = eachToken.TokenEvaluation switch - { - TokenEvaluation.Expression when TryInvokeTokenExpression( eachToken, out var expressionResult, out expressionError ) => - expressionResult.ToString()!.Split( ',' ).Select( v => v.Trim() ), - TokenEvaluation.Expression => throw new TemplateException( $"{_tokenLeft}Error ({eachToken.Id}):{expressionError ?? "Error in each condition."}{_tokenRight}" ), - _ => _members[eachToken.Name].Split( ',' ).Select( v => v.Trim() ) - }; - - currentFrame = currentFrame with { Enumerator = values.GetEnumerator() }; - frames.Push( currentFrame.Token, true, currentFrame.Enumerator, currentFrame.StartPos ); - return TokenAction.ContinueLoop; - } + string expressionError = null; - // Iterate over the collection - var items = currentFrame.Enumerator; - while ( items.MoveNext() ) + var conditionIsTrue = eachToken.TokenEvaluation switch { - var item = items.Current; - _members.Add( "x", item ); - frames.Push( currentFrame.Token, true, currentFrame.Enumerator, currentFrame.StartPos ); + TokenEvaluation.Expression when TryInvokeTokenExpression( eachToken, out var expressionResult, out expressionError ) => + expressionResult.ToString()!.Split( ',' ).Select( v => v.Trim() ).Any(), + TokenEvaluation.Expression => throw new TemplateException( $"{_tokenLeft}Error ({eachToken.Id}):{expressionError ?? "Error in while condition."}{_tokenRight}" ), + _ => Truthy( _members[eachToken.Name] ) // Re-evaluate the condition + }; + + if ( conditionIsTrue && currentFrame.Enumerator.MoveNext() && currentFrame.Enumerator.Current != null ) + { + _members["x"] = currentFrame.Enumerator.Current; return TokenAction.ContinueLoop; } + + //Otherwise, pop the frame and exit the loop frames.Pop(); return TokenAction.Ignore; } - private TokenAction ProcessDefineToken( TokenDefinition token ) { // ReSharper disable once RedundantAssignment @@ -234,12 +227,12 @@ TokenEvaluation.Expression when TryInvokeTokenExpression( token, out var express return TokenAction.Ignore; } - private void ResolveValue( TokenDefinition token, out string value, out bool defined, out bool ifResult, out IEnumerable iterator, out string expressionError ) + private void ResolveValue( TokenDefinition token, out string value, out bool defined, out bool ifResult, out IEnumerable enumerator, out string expressionError ) { value = default; defined = false; ifResult = false; - iterator = null; + enumerator = null; expressionError = null; switch ( token.TokenType ) @@ -280,7 +273,7 @@ private void ResolveValue( TokenDefinition token, out string value, out bool def if ( TryInvokeTokenExpression( token, out var eachExprResult, out var errorEach ) ) { var results = (string) eachExprResult; - iterator = results.Split( ',' ).Select( v => v.Trim() ); + enumerator = results.Split( ',' ).Select( v => v.Trim() ); } else throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{errorEach ?? "Error in each condition."}{_tokenRight}" ); From 041ae5bec71849c7ba3776a79eab1f7996e9b3bf Mon Sep 17 00:00:00 2001 From: "annette.findley" Date: Fri, 8 Nov 2024 16:19:03 -0500 Subject: [PATCH 31/58] WIP Each changes --- .../Text/TemplateParser.cs | 12 ++--- src/Hyperbee.Templating/Text/TokenParser.cs | 53 +++++++++++++------ .../Text/TokenProcessor.cs | 32 +++++------ .../Text/TemplateParser.LoopTests.cs | 4 +- 4 files changed, 62 insertions(+), 39 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index b581836..d75b537 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -19,7 +19,6 @@ public class TemplateParser private readonly int _maxTokenDepth; private readonly string _tokenLeft; private readonly string _tokenRight; - private enum TemplateScanner { Text, @@ -253,6 +252,7 @@ private void ParseTemplate( ref BufferManager bufferManager, TextReader reader, WriteTokenValue( writer, tokenValue, tokenAction, state ); ignore = state.Frames.IsFalsy; + continue; } @@ -494,14 +494,14 @@ public FrameStack.Frame CurrentFrame() => internal sealed class FrameStack { - // Updated Frame to include an IEnumerator - public record Frame( TokenDefinition Token, bool Truthy, IEnumerator Enumerator = null, int StartPos = -1 ); + public record Frame( TokenDefinition Token, bool Truthy, EnumeratorDefinition EnumeratorDefinition = null, int StartPos = -1 ); + + public record EnumeratorDefinition( string Name, IEnumerator Enumerator ); private readonly Stack _stack = new(); - // Updated Push method to accept an IEnumerator parameter - public void Push( TokenDefinition token, bool truthy, IEnumerator enumerator = null, int startPos = -1 ) - => _stack.Push( new Frame( token, truthy, enumerator, startPos ) ); + public void Push( TokenDefinition token, bool truthy, EnumeratorDefinition enumeratorDefinition = null, int startPos = -1 ) + => _stack.Push( new Frame( token, truthy, enumeratorDefinition, startPos ) ); public Frame Peek() => _stack.Peek(); public void Pop() => _stack.Pop(); diff --git a/src/Hyperbee.Templating/Text/TokenParser.cs b/src/Hyperbee.Templating/Text/TokenParser.cs index af5a0ca..7a44d7f 100644 --- a/src/Hyperbee.Templating/Text/TokenParser.cs +++ b/src/Hyperbee.Templating/Text/TokenParser.cs @@ -34,7 +34,7 @@ public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) // {{while x => x.token}} // {{/while}} // - // {{each [!]token}} + // {{each [!]token}} -- not necessary??? // {{each x => x.token}} // {{/each}} @@ -169,30 +169,51 @@ public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) // each handling if ( span.StartsWith( "each", StringComparison.OrdinalIgnoreCase ) ) { - tokenType = TokenType.Each; - span = span[4..].Trim(); // eat the 'each' - if ( span.Length >= 4 ) + //AF Put this separate from Define because I needed to eat the 'each' before checking for the rest of the syntax + //Planned on cleaning up with Define later + if ( span.Length == 4 || char.IsWhiteSpace( span[4] ) ) { - // detect expression syntax - var isFatArrow = span.IndexOfIgnoreDelimitedRanges( "=>", "\"" ) != -1; + tokenType = TokenType.Define; + span = span[4..].Trim(); // eat the 'each' + // Define value + var defineTokenPos = span.IndexOfIgnoreDelimitedRanges( ":", "\"" ); + var fatArrowPos = span.IndexOfIgnoreDelimitedRanges( "=>", "\"" ); - // validate - if ( span.IsEmpty ) - throw new TemplateException( "Invalid `each` statement. Missing identifier." ); + if ( defineTokenPos > -1 && (fatArrowPos == -1 || defineTokenPos < fatArrowPos) ) + { - if ( !isFatArrow && !_validateKey( span ) ) - throw new TemplateException( "Invalid `each` statement. Invalid identifier in expression." ); + tokenType = TokenType.Each; //HERE + name = span[..defineTokenPos].Trim(); + tokenExpression = UnQuote( span[(defineTokenPos + 1)..] ); - // results - if ( isFatArrow ) + if ( fatArrowPos > 0 ) + { + tokenEvaluation = TokenEvaluation.Expression; + + // Check and remove surrounding token delimiters (e.g., {{ and }}) + if ( tokenExpression.StartsWith( _tokenLeft ) && tokenExpression.EndsWith( _tokenRight ) ) + { + tokenExpression = tokenExpression[_tokenLeft.Length..^_tokenRight.Length].Trim(); + } + } + } + else if ( fatArrowPos > -1 && (defineTokenPos == -1 || fatArrowPos < defineTokenPos) ) { + // fat arrow value + + tokenType = TokenType.Each; tokenEvaluation = TokenEvaluation.Expression; - tokenExpression = span; //x=>x.list + tokenExpression = span; } else { - tokenEvaluation = TokenEvaluation.Falsy; + // identifier value + + if ( !_validateKey( span ) ) + throw new TemplateException( "Invalid token name." ); + + tokenType = TokenType.Each; name = span; } } @@ -212,8 +233,6 @@ public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) if ( defineTokenPos > -1 && (fatArrowPos == -1 || defineTokenPos < fatArrowPos) ) { - // Define value - tokenType = TokenType.Define; name = span[..defineTokenPos].Trim(); tokenExpression = UnQuote( span[(defineTokenPos + 1)..] ); diff --git a/src/Hyperbee.Templating/Text/TokenProcessor.cs b/src/Hyperbee.Templating/Text/TokenProcessor.cs index ab7797e..0dbf0c1 100644 --- a/src/Hyperbee.Templating/Text/TokenProcessor.cs +++ b/src/Hyperbee.Templating/Text/TokenProcessor.cs @@ -66,14 +66,14 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out return ProcessEndWhileToken( frames ); case TokenType.Each: - //Fall through to resolve value. + // Fall through to resolve value. break; case TokenType.EndEach: return ProcessEndEachToken( frames ); case TokenType.Define: - return ProcessDefineToken( token ); + return ProcessDefineToken( token ); //inserts value case TokenType.None: default: @@ -82,7 +82,7 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out // Resolve value - ResolveValue( token, out value, out var defined, out var ifResult, out IEnumerable enumerator, out var expressionError ); + ResolveValue( token, out value, out var defined, out var ifResult, out var expressionError ); // Frame handling: post-value processing @@ -100,10 +100,13 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out } case TokenType.Each: { - var result = enumerator.GetEnumerator(); + //AF + var enumerator = value.Split( "," ).AsEnumerable().GetEnumerator(); + var enumeratorDefinition = new FrameStack.EnumeratorDefinition( Name: token.Name, Enumerator: enumerator ); + var startPos = token.TokenType == TokenType.Each ? state.CurrentPos : -1; - frames.Push( token, true, result, startPos ); + frames.Push( token, false, enumeratorDefinition, state.CurrentPos ); //AF set value to name return TokenAction.Ignore; } @@ -197,20 +200,18 @@ TokenEvaluation.Expression when TryInvokeTokenExpression( eachToken, out var exp _ => Truthy( _members[eachToken.Name] ) // Re-evaluate the condition }; - if ( conditionIsTrue && currentFrame.Enumerator.MoveNext() && currentFrame.Enumerator.Current != null ) + //AF Adding enumerator to the members + if ( conditionIsTrue && currentFrame.EnumeratorDefinition.Enumerator.MoveNext() && currentFrame.EnumeratorDefinition.Enumerator.Current != null ) { - _members["x"] = currentFrame.Enumerator.Current; + _members[currentFrame.EnumeratorDefinition.Name] = currentFrame.EnumeratorDefinition.Enumerator.Current; return TokenAction.ContinueLoop; } - - //Otherwise, pop the frame and exit the loop frames.Pop(); return TokenAction.Ignore; } - private TokenAction ProcessDefineToken( TokenDefinition token ) { // ReSharper disable once RedundantAssignment @@ -224,15 +225,16 @@ TokenEvaluation.Expression when TryInvokeTokenExpression( token, out var express => throw new TemplateException( $"Error evaluating define expression for {token.Name}: {expressionError}" ), _ => token.TokenExpression }; + + return TokenAction.Ignore; } - private void ResolveValue( TokenDefinition token, out string value, out bool defined, out bool ifResult, out IEnumerable enumerator, out string expressionError ) + private void ResolveValue( TokenDefinition token, out string value, out bool defined, out bool ifResult, out string expressionError ) { value = default; defined = false; ifResult = false; - enumerator = null; expressionError = null; switch ( token.TokenType ) @@ -272,18 +274,18 @@ private void ResolveValue( TokenDefinition token, out string value, out bool def case TokenType.Each when token.TokenEvaluation == TokenEvaluation.Expression: if ( TryInvokeTokenExpression( token, out var eachExprResult, out var errorEach ) ) { - var results = (string) eachExprResult; - enumerator = results.Split( ',' ).Select( v => v.Trim() ); + value = (string) eachExprResult; + //enumerator = results.Split( ',' ).Select( v => v.Trim() ); } else throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{errorEach ?? "Error in each condition."}{_tokenRight}" ); break; - } } private bool TryInvokeTokenHandler( TokenDefinition token, bool defined, ref string value, out TokenAction tokenAction ) { + tokenAction = defined ? TokenAction.Replace : (_ignoreMissingTokens ? TokenAction.Ignore : TokenAction.Error); // Invoke any token handler diff --git a/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs b/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs index d031978..4bcadf8 100644 --- a/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs +++ b/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs @@ -39,7 +39,9 @@ public void Should_honor_while_condition( ParseTemplateMethod parseMethod ) public void Should_honor_each_expression( ParseTemplateMethod parseMethod ) { // arrange - const string expression = "{{each x=>x.list}}World {{x}},{{/each}}"; + //AF original expression + //const string expression = "{{each x=>x.list}}World {{x}},{{/each}}"; + const string expression = "{{each n:x => x.list}}World {{n}},{{/each}}"; const string template = $"hello {expression}."; { From 19c029b652e1a9ca1bed7c936a36a7f894f96657 Mon Sep 17 00:00:00 2001 From: "annette.findley" Date: Mon, 11 Nov 2024 14:33:24 -0500 Subject: [PATCH 32/58] Had to update the truthyness when going through the enumerator. --- src/Hyperbee.Templating/Text/TemplateParser.cs | 3 +++ src/Hyperbee.Templating/Text/TokenProcessor.cs | 2 ++ 2 files changed, 5 insertions(+) diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index d75b537..af93ded 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -245,7 +245,10 @@ private void ParseTemplate( ref BufferManager bufferManager, TextReader reader, ProcessFrame( state.CurrentFrame(), tokenAction, token.TokenType, ref span, ref bufferManager, ref loopDepth ); if ( tokenAction == TokenAction.ContinueLoop ) + { + ignore = false; continue; + } // write value if ( tokenAction != TokenAction.Ignore ) diff --git a/src/Hyperbee.Templating/Text/TokenProcessor.cs b/src/Hyperbee.Templating/Text/TokenProcessor.cs index 0dbf0c1..0d1390c 100644 --- a/src/Hyperbee.Templating/Text/TokenProcessor.cs +++ b/src/Hyperbee.Templating/Text/TokenProcessor.cs @@ -204,6 +204,8 @@ TokenEvaluation.Expression when TryInvokeTokenExpression( eachToken, out var exp if ( conditionIsTrue && currentFrame.EnumeratorDefinition.Enumerator.MoveNext() && currentFrame.EnumeratorDefinition.Enumerator.Current != null ) { _members[currentFrame.EnumeratorDefinition.Name] = currentFrame.EnumeratorDefinition.Enumerator.Current; + frames.Pop(); + frames.Push( eachToken, true, currentFrame.EnumeratorDefinition, currentFrame.StartPos ); return TokenAction.ContinueLoop; } From 0c896dd0b7301be5610fa7cf7c3f74298553eb91 Mon Sep 17 00:00:00 2001 From: "annette.findley" Date: Tue, 12 Nov 2024 10:07:59 -0500 Subject: [PATCH 33/58] Preliminary completion of Each --- src/Hyperbee.Templating/Text/TemplateParser.cs | 16 ++++++++++++++-- src/Hyperbee.Templating/Text/TokenParser.cs | 2 +- src/Hyperbee.Templating/Text/TokenProcessor.cs | 8 ++------ 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index af93ded..df7ba8a 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -246,7 +246,7 @@ private void ParseTemplate( ref BufferManager bufferManager, TextReader reader, if ( tokenAction == TokenAction.ContinueLoop ) { - ignore = false; + ignore = state.Frames.IsFalsy; continue; } @@ -513,6 +513,18 @@ public void Push( TokenDefinition token, bool truthy, EnumeratorDefinition enume public bool IsTokenType( TokenType compare ) => _stack.Count > 0 && _stack.Peek().Token.TokenType == compare; - public bool IsTruthy => _stack.Count == 0 || _stack.Peek().Truthy; + public bool IsTruthy + { + get => _stack.Count == 0 || _stack.Peek().Truthy; + set + { + if ( _stack.Count > 0 ) + { + // Modify the Truthy value of the top frame + var topFrame = _stack.Pop(); + _stack.Push( topFrame with { Truthy = value } ); + } + } + } public bool IsFalsy => !IsTruthy; } diff --git a/src/Hyperbee.Templating/Text/TokenParser.cs b/src/Hyperbee.Templating/Text/TokenParser.cs index 7a44d7f..0c21d6c 100644 --- a/src/Hyperbee.Templating/Text/TokenParser.cs +++ b/src/Hyperbee.Templating/Text/TokenParser.cs @@ -174,7 +174,7 @@ public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) //Planned on cleaning up with Define later if ( span.Length == 4 || char.IsWhiteSpace( span[4] ) ) { - tokenType = TokenType.Define; + tokenType = TokenType.Each; span = span[4..].Trim(); // eat the 'each' // Define value var defineTokenPos = span.IndexOfIgnoreDelimitedRanges( ":", "\"" ); diff --git a/src/Hyperbee.Templating/Text/TokenProcessor.cs b/src/Hyperbee.Templating/Text/TokenProcessor.cs index 0d1390c..b62099e 100644 --- a/src/Hyperbee.Templating/Text/TokenProcessor.cs +++ b/src/Hyperbee.Templating/Text/TokenProcessor.cs @@ -200,12 +200,11 @@ TokenEvaluation.Expression when TryInvokeTokenExpression( eachToken, out var exp _ => Truthy( _members[eachToken.Name] ) // Re-evaluate the condition }; - //AF Adding enumerator to the members + //AF Adding enumerator to the members; needed to update truthy to true so the value gets replaced. if ( conditionIsTrue && currentFrame.EnumeratorDefinition.Enumerator.MoveNext() && currentFrame.EnumeratorDefinition.Enumerator.Current != null ) { _members[currentFrame.EnumeratorDefinition.Name] = currentFrame.EnumeratorDefinition.Enumerator.Current; - frames.Pop(); - frames.Push( eachToken, true, currentFrame.EnumeratorDefinition, currentFrame.StartPos ); + frames.IsTruthy = true; return TokenAction.ContinueLoop; } @@ -275,10 +274,7 @@ private void ResolveValue( TokenDefinition token, out string value, out bool def break; case TokenType.Each when token.TokenEvaluation == TokenEvaluation.Expression: if ( TryInvokeTokenExpression( token, out var eachExprResult, out var errorEach ) ) - { value = (string) eachExprResult; - //enumerator = results.Split( ',' ).Select( v => v.Trim() ); - } else throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{errorEach ?? "Error in each condition."}{_tokenRight}" ); break; From 1bbbbe020b211b2db905ab1174412b6ff5fc7b4b Mon Sep 17 00:00:00 2001 From: Brenton Farmer Date: Wed, 13 Nov 2024 19:52:08 -0800 Subject: [PATCH 34/58] Changes to enumerator handling --- .../Text/TemplateParser.cs | 4 +- src/Hyperbee.Templating/Text/TokenParser.cs | 144 +++++++--------- .../Text/TokenProcessor.cs | 161 +++++++++++------- .../Text/TemplateParser.LoopTests.cs | 22 ++- 4 files changed, 172 insertions(+), 159 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index df7ba8a..1018244 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -495,12 +495,12 @@ public FrameStack.Frame CurrentFrame() => Frames.Depth > 0 ? Frames.Peek() : default; } +internal record EnumeratorDefinition( string Name, IEnumerator Enumerator ); + internal sealed class FrameStack { public record Frame( TokenDefinition Token, bool Truthy, EnumeratorDefinition EnumeratorDefinition = null, int StartPos = -1 ); - public record EnumeratorDefinition( string Name, IEnumerator Enumerator ); - private readonly Stack _stack = new(); public void Push( TokenDefinition token, bool truthy, EnumeratorDefinition enumeratorDefinition = null, int startPos = -1 ) diff --git a/src/Hyperbee.Templating/Text/TokenParser.cs b/src/Hyperbee.Templating/Text/TokenParser.cs index 0c21d6c..51998e3 100644 --- a/src/Hyperbee.Templating/Text/TokenParser.cs +++ b/src/Hyperbee.Templating/Text/TokenParser.cs @@ -11,7 +11,7 @@ internal class TokenParser internal TokenParser( TemplateOptions options ) { - _validateKey = options.Validator ?? throw new ArgumentNullException( nameof( options.Validator ) ); + _validateKey = options.Validator ?? throw new ArgumentNullException( nameof( options ), $"{nameof(options.Validator)} cannot be null." ); (_tokenLeft, _tokenRight) = options.TokenDelimiters(); } @@ -34,7 +34,6 @@ public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) // {{while x => x.token}} // {{/while}} // - // {{each [!]token}} -- not necessary??? // {{each x => x.token}} // {{/each}} @@ -169,53 +168,10 @@ public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) // each handling if ( span.StartsWith( "each", StringComparison.OrdinalIgnoreCase ) ) { - - //AF Put this separate from Define because I needed to eat the 'each' before checking for the rest of the syntax - //Planned on cleaning up with Define later if ( span.Length == 4 || char.IsWhiteSpace( span[4] ) ) { tokenType = TokenType.Each; span = span[4..].Trim(); // eat the 'each' - // Define value - var defineTokenPos = span.IndexOfIgnoreDelimitedRanges( ":", "\"" ); - var fatArrowPos = span.IndexOfIgnoreDelimitedRanges( "=>", "\"" ); - - if ( defineTokenPos > -1 && (fatArrowPos == -1 || defineTokenPos < fatArrowPos) ) - { - - tokenType = TokenType.Each; //HERE - name = span[..defineTokenPos].Trim(); - tokenExpression = UnQuote( span[(defineTokenPos + 1)..] ); - - if ( fatArrowPos > 0 ) - { - tokenEvaluation = TokenEvaluation.Expression; - - // Check and remove surrounding token delimiters (e.g., {{ and }}) - if ( tokenExpression.StartsWith( _tokenLeft ) && tokenExpression.EndsWith( _tokenRight ) ) - { - tokenExpression = tokenExpression[_tokenLeft.Length..^_tokenRight.Length].Trim(); - } - } - } - else if ( fatArrowPos > -1 && (defineTokenPos == -1 || fatArrowPos < defineTokenPos) ) - { - // fat arrow value - - tokenType = TokenType.Each; - tokenEvaluation = TokenEvaluation.Expression; - tokenExpression = span; - } - else - { - // identifier value - - if ( !_validateKey( span ) ) - throw new TemplateException( "Invalid token name." ); - - tokenType = TokenType.Each; - name = span; - } } } else if ( span.StartsWith( "/each", StringComparison.OrdinalIgnoreCase ) ) @@ -226,46 +182,9 @@ public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) tokenType = TokenType.EndEach; } - if ( tokenType == TokenType.None ) + if ( tokenType == TokenType.None || tokenType == TokenType.Each ) { - var defineTokenPos = span.IndexOfIgnoreDelimitedRanges( ":", "\"" ); - var fatArrowPos = span.IndexOfIgnoreDelimitedRanges( "=>", "\"" ); - - if ( defineTokenPos > -1 && (fatArrowPos == -1 || defineTokenPos < fatArrowPos) ) - { - tokenType = TokenType.Define; - name = span[..defineTokenPos].Trim(); - tokenExpression = UnQuote( span[(defineTokenPos + 1)..] ); - - if ( fatArrowPos > 0 ) - { - tokenEvaluation = TokenEvaluation.Expression; - - // Check and remove surrounding token delimiters (e.g., {{ and }}) - if ( tokenExpression.StartsWith( _tokenLeft ) && tokenExpression.EndsWith( _tokenRight ) ) - { - tokenExpression = tokenExpression[_tokenLeft.Length..^_tokenRight.Length].Trim(); - } - } - } - else if ( fatArrowPos > -1 && (defineTokenPos == -1 || fatArrowPos < defineTokenPos) ) - { - // fat arrow value - - tokenType = TokenType.Value; - tokenEvaluation = TokenEvaluation.Expression; - tokenExpression = span; - } - else - { - // identifier value - - if ( !_validateKey( span ) ) - throw new TemplateException( "Invalid token name." ); - - tokenType = TokenType.Value; - name = span; - } + tokenType = GetTokenNameAndExpression( tokenType, span, ref name, ref tokenExpression, ref tokenEvaluation ); } // return the definition @@ -281,6 +200,63 @@ public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) }; } + private TokenType GetTokenNameAndExpression( TokenType tokenType, ReadOnlySpan span, ref ReadOnlySpan name, ref ReadOnlySpan tokenExpression, ref TokenEvaluation tokenEvaluation ) + { + if ( tokenType != TokenType.None && tokenType != TokenType.Each ) + { + return tokenType; + } + + var defineTokenPos = span.IndexOfIgnoreDelimitedRanges( ":", "\"" ); + var fatArrowPos = span.IndexOfIgnoreDelimitedRanges( "=>", "\"" ); + + if ( defineTokenPos > -1 && (fatArrowPos == -1 || defineTokenPos < fatArrowPos) ) + { + if ( tokenType == TokenType.None ) + tokenType = TokenType.Define; + + name = span[..defineTokenPos].Trim(); + tokenExpression = UnQuote( span[(defineTokenPos + 1)..] ); + + if ( fatArrowPos <= 0 ) + { + return tokenType; + } + + tokenEvaluation = TokenEvaluation.Expression; + + // Check and remove surrounding token delimiters (e.g., {{ and }}) + if ( tokenExpression.StartsWith( _tokenLeft ) && tokenExpression.EndsWith( _tokenRight ) ) + { + tokenExpression = tokenExpression[_tokenLeft.Length..^_tokenRight.Length].Trim(); + } + } + else if ( fatArrowPos > -1 && (defineTokenPos == -1 || fatArrowPos < defineTokenPos) ) + { + // fat arrow value + + if ( tokenType == TokenType.None ) + tokenType = TokenType.Value; + + tokenEvaluation = TokenEvaluation.Expression; + tokenExpression = span; + } + else + { + // identifier value + + if ( !_validateKey( span ) ) + throw new TemplateException( "Invalid token name." ); + + if ( tokenType == TokenType.None ) + tokenType = TokenType.Value; + + name = span; + } + + return tokenType; + } + private static ReadOnlySpan UnQuote( ReadOnlySpan span ) { var found = false; diff --git a/src/Hyperbee.Templating/Text/TokenProcessor.cs b/src/Hyperbee.Templating/Text/TokenProcessor.cs index b62099e..f800f58 100644 --- a/src/Hyperbee.Templating/Text/TokenProcessor.cs +++ b/src/Hyperbee.Templating/Text/TokenProcessor.cs @@ -1,4 +1,5 @@ -using System.Globalization; +using System.Collections; +using System.Globalization; using Hyperbee.Templating.Compiler; using Hyperbee.Templating.Configure; @@ -20,10 +21,10 @@ public TokenProcessor( MemberDictionary members, TemplateOptions options ) ArgumentNullException.ThrowIfNull( members ); if ( options.Methods == null ) - throw new ArgumentNullException( nameof( options ), $"{nameof( options.Methods )} cannot be null." ); + throw new ArgumentNullException( nameof(options), $"{nameof(options.Methods)} cannot be null." ); if ( options.TokenExpressionProvider == null ) - throw new ArgumentNullException( nameof( options ), $"{nameof( options.TokenExpressionProvider )} cannot be null." ); + throw new ArgumentNullException( nameof(options), $"{nameof(options.TokenExpressionProvider)} cannot be null." ); _tokenExpressionProvider = options.TokenExpressionProvider; _tokenHandler = options.TokenHandler; @@ -49,6 +50,8 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out break; case TokenType.If: + case TokenType.While: + case TokenType.Each: // Fall through to resolve value. break; @@ -58,17 +61,9 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out case TokenType.Endif: return ProcessEndIfToken( frames ); - case TokenType.While: - // Fall through to resolve value. - break; - case TokenType.EndWhile: return ProcessEndWhileToken( frames ); - case TokenType.Each: - // Fall through to resolve value. - break; - case TokenType.EndEach: return ProcessEndEachToken( frames ); @@ -77,12 +72,12 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out case TokenType.None: default: - throw new NotSupportedException( $"{nameof( ProcessToken )}: Invalid {nameof( TokenType )} {token.TokenType}." ); + throw new NotSupportedException( $"{nameof(ProcessToken)}: Invalid {nameof(TokenType)} {token.TokenType}." ); } // Resolve value - ResolveValue( token, out value, out var defined, out var ifResult, out var expressionError ); + ResolveValue( token, out var resolvedValue, out var defined, out var ifResult, out var expressionError ); // Frame handling: post-value processing @@ -90,28 +85,37 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out { case TokenType.If: case TokenType.While: - { - var frameIsTruthy = token.TokenEvaluation == TokenEvaluation.Falsy ? !ifResult : ifResult; - var startPos = token.TokenType == TokenType.While ? state.CurrentPos : -1; + { + var frameIsTruthy = token.TokenEvaluation == TokenEvaluation.Falsy ? !ifResult : ifResult; + var startPos = token.TokenType == TokenType.While ? state.CurrentPos : -1; - frames.Push( token, frameIsTruthy, null, startPos ); + frames.Push( token, frameIsTruthy, null, startPos ); - return TokenAction.Ignore; - } + return TokenAction.Ignore; + } case TokenType.Each: - { - //AF - var enumerator = value.Split( "," ).AsEnumerable().GetEnumerator(); - var enumeratorDefinition = new FrameStack.EnumeratorDefinition( Name: token.Name, Enumerator: enumerator ); + { + var enumerator = resolvedValue as IEnumerator; + var frameIsTruthy = enumerator!.MoveNext(); - var startPos = token.TokenType == TokenType.Each ? state.CurrentPos : -1; + if ( frameIsTruthy ) + { + value = enumerator.Current; + _members[token.Name] = value; + } + + var enumeratorDefinition = new EnumeratorDefinition( Name: token.Name, Enumerator: enumerator ); - frames.Push( token, false, enumeratorDefinition, state.CurrentPos ); //AF set value to name + frames.Push( token, frameIsTruthy, enumeratorDefinition, state.CurrentPos ); - return TokenAction.Ignore; - } + return TokenAction.Ignore; + } } + // make sure we have a string value + + value = (string) resolvedValue; + // Token handling: user-defined token action _ = TryInvokeTokenHandler( token, defined, ref value, out var tokenAction ); @@ -129,7 +133,7 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out break; default: - throw new NotSupportedException( $"{nameof( ProcessToken )}: Invalid {nameof( TokenAction )} {tokenAction}." ); + throw new NotSupportedException( $"{nameof(ProcessToken)}: Invalid {nameof(TokenAction)} {tokenAction}." ); } return tokenAction; @@ -185,38 +189,29 @@ TokenEvaluation.Expression when TryInvokeTokenExpression( whileToken, out var ex private TokenAction ProcessEndEachToken( FrameStack frames ) { if ( frames.Depth == 0 || !frames.IsTokenType( TokenType.Each ) ) - throw new TemplateException( "Syntax error. Invalid `/each` without matching `each`. " ); + throw new TemplateException( "Syntax error. Invalid /each without matching each." ); - var eachToken = frames.Peek().Token; - var currentFrame = frames.Peek(); + // Evaluate the condition and move to the next item in the enumerator - string expressionError = null; + var frame = frames.Peek(); + var (currentName, enumerator) = frame.EnumeratorDefinition; - var conditionIsTrue = eachToken.TokenEvaluation switch + if ( enumerator!.MoveNext() ) { - TokenEvaluation.Expression when TryInvokeTokenExpression( eachToken, out var expressionResult, out expressionError ) => - expressionResult.ToString()!.Split( ',' ).Select( v => v.Trim() ).Any(), - TokenEvaluation.Expression => throw new TemplateException( $"{_tokenLeft}Error ({eachToken.Id}):{expressionError ?? "Error in while condition."}{_tokenRight}" ), - _ => Truthy( _members[eachToken.Name] ) // Re-evaluate the condition - }; - - //AF Adding enumerator to the members; needed to update truthy to true so the value gets replaced. - if ( conditionIsTrue && currentFrame.EnumeratorDefinition.Enumerator.MoveNext() && currentFrame.EnumeratorDefinition.Enumerator.Current != null ) - { - _members[currentFrame.EnumeratorDefinition.Name] = currentFrame.EnumeratorDefinition.Enumerator.Current; + _members[currentName] = enumerator.Current; frames.IsTruthy = true; return TokenAction.ContinueLoop; } - - //Otherwise, pop the frame and exit the loop + + // Otherwise, pop the frame and exit the loop + _members[currentName] = default; frames.Pop(); return TokenAction.Ignore; } private TokenAction ProcessDefineToken( TokenDefinition token ) { - // ReSharper disable once RedundantAssignment - string expressionError = null; // assign to avoid compiler complaint + string expressionError = null; _members[token.Name] = token.TokenEvaluation switch { @@ -227,11 +222,10 @@ TokenEvaluation.Expression when TryInvokeTokenExpression( token, out var express _ => token.TokenExpression }; - return TokenAction.Ignore; } - private void ResolveValue( TokenDefinition token, out string value, out bool defined, out bool ifResult, out string expressionError ) + private void ResolveValue( TokenDefinition token, out object value, out bool defined, out bool ifResult, out string expressionError ) { value = default; defined = false; @@ -243,20 +237,26 @@ private void ResolveValue( TokenDefinition token, out string value, out bool def case TokenType.Value when token.TokenEvaluation != TokenEvaluation.Expression: case TokenType.If when token.TokenEvaluation != TokenEvaluation.Expression: case TokenType.While when token.TokenEvaluation != TokenEvaluation.Expression: - case TokenType.Each when token.TokenEvaluation != TokenEvaluation.Expression: - defined = _members.TryGetValue( token.Name, out value ); + { + defined = _members.TryGetValue( token.Name, out var valueMember ); if ( !defined && _substituteEnvironmentVariables ) { - value = Environment.GetEnvironmentVariable( token.Name ); + valueMember = Environment.GetEnvironmentVariable( token.Name ); defined = value != null; } if ( token.TokenType == TokenType.If || token.TokenType == TokenType.While || token.TokenType == TokenType.Each ) - ifResult = defined && Truthy( value ); + { + ifResult = defined && Truthy( valueMember ); + } + + value = valueMember; break; + } case TokenType.Value when token.TokenEvaluation == TokenEvaluation.Expression: + { if ( TryInvokeTokenExpression( token, out var valueExprResult, out expressionError ) ) { value = Convert.ToString( valueExprResult, CultureInfo.InvariantCulture ); @@ -264,26 +264,34 @@ private void ResolveValue( TokenDefinition token, out string value, out bool def } break; + } case TokenType.If when token.TokenEvaluation == TokenEvaluation.Expression: case TokenType.While when token.TokenEvaluation == TokenEvaluation.Expression: - if ( TryInvokeTokenExpression( token, out var condExprResult, out var error ) ) - ifResult = Convert.ToBoolean( condExprResult ); - else + { + if ( !TryInvokeTokenExpression( token, out var condExprResult, out var error ) ) throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{error ?? "Error in if condition."}{_tokenRight}" ); + + ifResult = Convert.ToBoolean( condExprResult ); break; - case TokenType.Each when token.TokenEvaluation == TokenEvaluation.Expression: - if ( TryInvokeTokenExpression( token, out var eachExprResult, out var errorEach ) ) - value = (string) eachExprResult; - else + } + + case TokenType.Each: + { + if ( token.TokenEvaluation != TokenEvaluation.Expression ) + throw new TemplateException( "Invalid token expression for each. Are you missing a fat arrow?" ); + + if ( !TryInvokeTokenExpression( token, out var eachExprResult, out var errorEach ) ) throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{errorEach ?? "Error in each condition."}{_tokenRight}" ); + + value = new EnumeratorAdapter( (IEnumerable) eachExprResult ); break; + } } } private bool TryInvokeTokenHandler( TokenDefinition token, bool defined, ref string value, out TokenAction tokenAction ) { - tokenAction = defined ? TokenAction.Replace : (_ignoreMissingTokens ? TokenAction.Ignore : TokenAction.Error); // Invoke any token handler @@ -353,3 +361,34 @@ private static bool Truthy( ReadOnlySpan value ) return truthy; } } + +internal sealed class EnumeratorAdapter : IEnumerator +{ + private readonly IEnumerator _inner; + + internal EnumeratorAdapter( IEnumerable enumerable ) + { + if ( enumerable == null ) + throw new ArgumentNullException( nameof(enumerable) ); + + // ReSharper disable once GenericEnumeratorNotDisposed + _inner = enumerable.GetEnumerator(); + } + + public string Current => (string) _inner.Current; + + object IEnumerator.Current => _inner.Current; + + public bool MoveNext() => _inner.MoveNext(); + + public void Reset() => _inner.Reset(); + + public void Dispose() + { + if ( _inner is IDisposable disposable ) + { + disposable.Dispose(); + } + } +} + diff --git a/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs b/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs index 4bcadf8..09ca9f2 100644 --- a/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs +++ b/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs @@ -39,21 +39,19 @@ public void Should_honor_while_condition( ParseTemplateMethod parseMethod ) public void Should_honor_each_expression( ParseTemplateMethod parseMethod ) { // arrange - //AF original expression - //const string expression = "{{each x=>x.list}}World {{x}},{{/each}}"; - const string expression = "{{each n:x => x.list}}World {{n}},{{/each}}"; - const string template = $"hello {expression}."; - { - var parser = new TemplateParser { Variables = { ["list"] = "1,2,3" } }; + const string expression = "{{each n:x => x.list.Split( \",\" )}}World {{n}},{{/each}}"; - // act - var result = parser.Render( template, parseMethod ); + const string template = $"hello {expression}."; + + var parser = new TemplateParser { Variables = { ["list"] = "1,2,3" } }; + + // act + var result = parser.Render( template, parseMethod ); - // assert - var expected = "hello World 1,World 2,World 3,."; + // assert + var expected = "hello World 1,World 2,World 3,."; - Assert.AreEqual( expected, result ); - } + Assert.AreEqual( expected, result ); } } From 968ab55e95b8c07dfb800d55427d3c5fe59ac857 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 14 Nov 2024 03:52:42 +0000 Subject: [PATCH 35/58] Updated code formatting to match rules in .editorconfig --- src/Hyperbee.Templating/Text/TokenParser.cs | 2 +- .../Text/TokenProcessor.cs | 128 +++++++++--------- .../Text/TemplateParser.LoopTests.cs | 4 +- 3 files changed, 67 insertions(+), 67 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TokenParser.cs b/src/Hyperbee.Templating/Text/TokenParser.cs index 51998e3..fc479ce 100644 --- a/src/Hyperbee.Templating/Text/TokenParser.cs +++ b/src/Hyperbee.Templating/Text/TokenParser.cs @@ -11,7 +11,7 @@ internal class TokenParser internal TokenParser( TemplateOptions options ) { - _validateKey = options.Validator ?? throw new ArgumentNullException( nameof( options ), $"{nameof(options.Validator)} cannot be null." ); + _validateKey = options.Validator ?? throw new ArgumentNullException( nameof( options ), $"{nameof( options.Validator )} cannot be null." ); (_tokenLeft, _tokenRight) = options.TokenDelimiters(); } diff --git a/src/Hyperbee.Templating/Text/TokenProcessor.cs b/src/Hyperbee.Templating/Text/TokenProcessor.cs index f800f58..8d61725 100644 --- a/src/Hyperbee.Templating/Text/TokenProcessor.cs +++ b/src/Hyperbee.Templating/Text/TokenProcessor.cs @@ -21,10 +21,10 @@ public TokenProcessor( MemberDictionary members, TemplateOptions options ) ArgumentNullException.ThrowIfNull( members ); if ( options.Methods == null ) - throw new ArgumentNullException( nameof(options), $"{nameof(options.Methods)} cannot be null." ); + throw new ArgumentNullException( nameof( options ), $"{nameof( options.Methods )} cannot be null." ); if ( options.TokenExpressionProvider == null ) - throw new ArgumentNullException( nameof(options), $"{nameof(options.TokenExpressionProvider)} cannot be null." ); + throw new ArgumentNullException( nameof( options ), $"{nameof( options.TokenExpressionProvider )} cannot be null." ); _tokenExpressionProvider = options.TokenExpressionProvider; _tokenHandler = options.TokenHandler; @@ -72,12 +72,12 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out case TokenType.None: default: - throw new NotSupportedException( $"{nameof(ProcessToken)}: Invalid {nameof(TokenType)} {token.TokenType}." ); + throw new NotSupportedException( $"{nameof( ProcessToken )}: Invalid {nameof( TokenType )} {token.TokenType}." ); } // Resolve value - ResolveValue( token, out var resolvedValue, out var defined, out var ifResult, out var expressionError ); + ResolveValue( token, out var resolvedValue, out var defined, out var ifResult, out var expressionError ); // Frame handling: post-value processing @@ -85,31 +85,31 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out { case TokenType.If: case TokenType.While: - { - var frameIsTruthy = token.TokenEvaluation == TokenEvaluation.Falsy ? !ifResult : ifResult; - var startPos = token.TokenType == TokenType.While ? state.CurrentPos : -1; + { + var frameIsTruthy = token.TokenEvaluation == TokenEvaluation.Falsy ? !ifResult : ifResult; + var startPos = token.TokenType == TokenType.While ? state.CurrentPos : -1; - frames.Push( token, frameIsTruthy, null, startPos ); + frames.Push( token, frameIsTruthy, null, startPos ); - return TokenAction.Ignore; - } + return TokenAction.Ignore; + } case TokenType.Each: - { - var enumerator = resolvedValue as IEnumerator; - var frameIsTruthy = enumerator!.MoveNext(); - - if ( frameIsTruthy ) { - value = enumerator.Current; - _members[token.Name] = value; - } - - var enumeratorDefinition = new EnumeratorDefinition( Name: token.Name, Enumerator: enumerator ); + var enumerator = resolvedValue as IEnumerator; + var frameIsTruthy = enumerator!.MoveNext(); + + if ( frameIsTruthy ) + { + value = enumerator.Current; + _members[token.Name] = value; + } - frames.Push( token, frameIsTruthy, enumeratorDefinition, state.CurrentPos ); + var enumeratorDefinition = new EnumeratorDefinition( Name: token.Name, Enumerator: enumerator ); - return TokenAction.Ignore; - } + frames.Push( token, frameIsTruthy, enumeratorDefinition, state.CurrentPos ); + + return TokenAction.Ignore; + } } // make sure we have a string value @@ -133,7 +133,7 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out break; default: - throw new NotSupportedException( $"{nameof(ProcessToken)}: Invalid {nameof(TokenAction)} {tokenAction}." ); + throw new NotSupportedException( $"{nameof( ProcessToken )}: Invalid {nameof( TokenAction )} {tokenAction}." ); } return tokenAction; @@ -202,7 +202,7 @@ private TokenAction ProcessEndEachToken( FrameStack frames ) frames.IsTruthy = true; return TokenAction.ContinueLoop; } - + // Otherwise, pop the frame and exit the loop _members[currentName] = default; frames.Pop(); @@ -237,56 +237,56 @@ private void ResolveValue( TokenDefinition token, out object value, out bool def case TokenType.Value when token.TokenEvaluation != TokenEvaluation.Expression: case TokenType.If when token.TokenEvaluation != TokenEvaluation.Expression: case TokenType.While when token.TokenEvaluation != TokenEvaluation.Expression: - { - defined = _members.TryGetValue( token.Name, out var valueMember ); - - if ( !defined && _substituteEnvironmentVariables ) { - valueMember = Environment.GetEnvironmentVariable( token.Name ); - defined = value != null; - } + defined = _members.TryGetValue( token.Name, out var valueMember ); - if ( token.TokenType == TokenType.If || token.TokenType == TokenType.While || token.TokenType == TokenType.Each ) - { - ifResult = defined && Truthy( valueMember ); - } + if ( !defined && _substituteEnvironmentVariables ) + { + valueMember = Environment.GetEnvironmentVariable( token.Name ); + defined = value != null; + } - value = valueMember; - break; - } + if ( token.TokenType == TokenType.If || token.TokenType == TokenType.While || token.TokenType == TokenType.Each ) + { + ifResult = defined && Truthy( valueMember ); + } + + value = valueMember; + break; + } case TokenType.Value when token.TokenEvaluation == TokenEvaluation.Expression: - { - if ( TryInvokeTokenExpression( token, out var valueExprResult, out expressionError ) ) { - value = Convert.ToString( valueExprResult, CultureInfo.InvariantCulture ); - defined = true; - } + if ( TryInvokeTokenExpression( token, out var valueExprResult, out expressionError ) ) + { + value = Convert.ToString( valueExprResult, CultureInfo.InvariantCulture ); + defined = true; + } - break; - } + break; + } case TokenType.If when token.TokenEvaluation == TokenEvaluation.Expression: case TokenType.While when token.TokenEvaluation == TokenEvaluation.Expression: - { - if ( !TryInvokeTokenExpression( token, out var condExprResult, out var error ) ) - throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{error ?? "Error in if condition."}{_tokenRight}" ); - - ifResult = Convert.ToBoolean( condExprResult ); - break; - } + { + if ( !TryInvokeTokenExpression( token, out var condExprResult, out var error ) ) + throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{error ?? "Error in if condition."}{_tokenRight}" ); + + ifResult = Convert.ToBoolean( condExprResult ); + break; + } case TokenType.Each: - { - if ( token.TokenEvaluation != TokenEvaluation.Expression ) - throw new TemplateException( "Invalid token expression for each. Are you missing a fat arrow?" ); - - if ( !TryInvokeTokenExpression( token, out var eachExprResult, out var errorEach ) ) - throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{errorEach ?? "Error in each condition."}{_tokenRight}" ); - - value = new EnumeratorAdapter( (IEnumerable) eachExprResult ); - break; - } + { + if ( token.TokenEvaluation != TokenEvaluation.Expression ) + throw new TemplateException( "Invalid token expression for each. Are you missing a fat arrow?" ); + + if ( !TryInvokeTokenExpression( token, out var eachExprResult, out var errorEach ) ) + throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{errorEach ?? "Error in each condition."}{_tokenRight}" ); + + value = new EnumeratorAdapter( (IEnumerable) eachExprResult ); + break; + } } } @@ -369,7 +369,7 @@ internal sealed class EnumeratorAdapter : IEnumerator internal EnumeratorAdapter( IEnumerable enumerable ) { if ( enumerable == null ) - throw new ArgumentNullException( nameof(enumerable) ); + throw new ArgumentNullException( nameof( enumerable ) ); // ReSharper disable once GenericEnumeratorNotDisposed _inner = enumerable.GetEnumerator(); diff --git a/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs b/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs index 09ca9f2..381d98b 100644 --- a/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs +++ b/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs @@ -43,9 +43,9 @@ public void Should_honor_each_expression( ParseTemplateMethod parseMethod ) const string expression = "{{each n:x => x.list.Split( \",\" )}}World {{n}},{{/each}}"; const string template = $"hello {expression}."; - + var parser = new TemplateParser { Variables = { ["list"] = "1,2,3" } }; - + // act var result = parser.Render( template, parseMethod ); From 184a45669087f0cd27cf3f171d84bb9d213950bb Mon Sep 17 00:00:00 2001 From: "annette.findley" Date: Thu, 14 Nov 2024 08:24:22 -0500 Subject: [PATCH 36/58] A little cleanup. --- .../Text/TemplateParser.cs | 19 +++---------------- .../Text/TokenProcessor.cs | 1 - 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index 1018244..a8130b2 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -245,10 +245,8 @@ private void ParseTemplate( ref BufferManager bufferManager, TextReader reader, ProcessFrame( state.CurrentFrame(), tokenAction, token.TokenType, ref span, ref bufferManager, ref loopDepth ); if ( tokenAction == TokenAction.ContinueLoop ) - { - ignore = state.Frames.IsFalsy; continue; - } + // write value if ( tokenAction != TokenAction.Ignore ) @@ -513,18 +511,7 @@ public void Push( TokenDefinition token, bool truthy, EnumeratorDefinition enume public bool IsTokenType( TokenType compare ) => _stack.Count > 0 && _stack.Peek().Token.TokenType == compare; - public bool IsTruthy - { - get => _stack.Count == 0 || _stack.Peek().Truthy; - set - { - if ( _stack.Count > 0 ) - { - // Modify the Truthy value of the top frame - var topFrame = _stack.Pop(); - _stack.Push( topFrame with { Truthy = value } ); - } - } - } + public bool IsTruthy => _stack.Count == 0 || _stack.Peek().Truthy; + public bool IsFalsy => !IsTruthy; } diff --git a/src/Hyperbee.Templating/Text/TokenProcessor.cs b/src/Hyperbee.Templating/Text/TokenProcessor.cs index 8d61725..ca7ece2 100644 --- a/src/Hyperbee.Templating/Text/TokenProcessor.cs +++ b/src/Hyperbee.Templating/Text/TokenProcessor.cs @@ -199,7 +199,6 @@ private TokenAction ProcessEndEachToken( FrameStack frames ) if ( enumerator!.MoveNext() ) { _members[currentName] = enumerator.Current; - frames.IsTruthy = true; return TokenAction.ContinueLoop; } From 5b3c0ed64c9437ab8477c3625c31ba17fd275132 Mon Sep 17 00:00:00 2001 From: Brenton Farmer Date: Thu, 14 Nov 2024 11:09:01 -0800 Subject: [PATCH 37/58] Clean up token processor --- .../Text/TokenProcessor.cs | 319 ++++++++---------- 1 file changed, 135 insertions(+), 184 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TokenProcessor.cs b/src/Hyperbee.Templating/Text/TokenProcessor.cs index ca7ece2..19edbf2 100644 --- a/src/Hyperbee.Templating/Text/TokenProcessor.cs +++ b/src/Hyperbee.Templating/Text/TokenProcessor.cs @@ -2,6 +2,7 @@ using System.Globalization; using Hyperbee.Templating.Compiler; using Hyperbee.Templating.Configure; +// ReSharper disable RedundantAssignment namespace Hyperbee.Templating.Text; @@ -13,26 +14,23 @@ internal class TokenProcessor private readonly bool _substituteEnvironmentVariables; private readonly string _tokenLeft; private readonly string _tokenRight; - private readonly MemberDictionary _members; - public TokenProcessor( MemberDictionary members, TemplateOptions options ) + public TokenProcessor(MemberDictionary members, TemplateOptions options) { - ArgumentNullException.ThrowIfNull( members ); + ArgumentNullException.ThrowIfNull(members); - if ( options.Methods == null ) - throw new ArgumentNullException( nameof( options ), $"{nameof( options.Methods )} cannot be null." ); + if (options.Methods == null) + throw new ArgumentNullException(nameof(options), $"{nameof(options.Methods)} cannot be null."); - if ( options.TokenExpressionProvider == null ) - throw new ArgumentNullException( nameof( options ), $"{nameof( options.TokenExpressionProvider )} cannot be null." ); + if (options.TokenExpressionProvider == null) + throw new ArgumentNullException(nameof(options), $"{nameof(options.TokenExpressionProvider)} cannot be null."); _tokenExpressionProvider = options.TokenExpressionProvider; _tokenHandler = options.TokenHandler; _ignoreMissingTokens = options.IgnoreMissingTokens; _substituteEnvironmentVariables = options.SubstituteEnvironmentVariables; - _members = members; - (_tokenLeft, _tokenRight) = options.TokenDelimiters(); } @@ -41,12 +39,11 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out value = default; var frames = state.Frames; - // Frame handling: pre-value processing + // Initial handling based on token type switch ( token.TokenType ) { case TokenType.Value: - if ( frames.IsFalsy ) - return TokenAction.Ignore; + if ( frames.IsFalsy ) return TokenAction.Ignore; break; case TokenType.If: @@ -68,75 +65,69 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out return ProcessEndEachToken( frames ); case TokenType.Define: - return ProcessDefineToken( token ); //inserts value - - case TokenType.None: - default: - throw new NotSupportedException( $"{nameof( ProcessToken )}: Invalid {nameof( TokenType )} {token.TokenType}." ); + return ProcessDefineToken( token ); } - // Resolve value - - ResolveValue( token, out var resolvedValue, out var defined, out var ifResult, out var expressionError ); - - // Frame handling: post-value processing + // Resolve value (called only once) + ResolveValue( token, out var resolvedValue, out var defined, out var conditionalResult, out var expressionError ); + // Conditional frame handling based on token type after value resolution switch ( token.TokenType ) { case TokenType.If: - case TokenType.While: - { - var frameIsTruthy = token.TokenEvaluation == TokenEvaluation.Falsy ? !ifResult : ifResult; - var startPos = token.TokenType == TokenType.While ? state.CurrentPos : -1; + return ProcessIfToken( token, frames, conditionalResult ); - frames.Push( token, frameIsTruthy, null, startPos ); + case TokenType.While: + return ProcessWhileToken( token, frames, conditionalResult, state ); - return TokenAction.Ignore; - } case TokenType.Each: - { - var enumerator = resolvedValue as IEnumerator; - var frameIsTruthy = enumerator!.MoveNext(); - - if ( frameIsTruthy ) - { - value = enumerator.Current; - _members[token.Name] = value; - } - - var enumeratorDefinition = new EnumeratorDefinition( Name: token.Name, Enumerator: enumerator ); - - frames.Push( token, frameIsTruthy, enumeratorDefinition, state.CurrentPos ); + return ProcessEachToken( token, frames, resolvedValue, state, out value ); - return TokenAction.Ignore; - } + default: + value = (string) resolvedValue; + break; } - // make sure we have a string value + // Final action determination for all tokens + return ProcessTokenHandler( token, defined, ref value, expressionError ); + } - value = (string) resolvedValue; - // Token handling: user-defined token action + private TokenAction ProcessTokenHandler( TokenDefinition token, bool defined, ref string value, string expressionError ) + { + if ( !TryInvokeTokenHandler( token, defined, ref value, out var tokenAction ) ) + { + tokenAction = defined ? TokenAction.Replace : (_ignoreMissingTokens ? TokenAction.Ignore : TokenAction.Error); + } - _ = TryInvokeTokenHandler( token, defined, ref value, out var tokenAction ); + // Determine final action based on token handler and missing tokens + if ( tokenAction == TokenAction.Error && !defined ) + { + value = $"{_tokenLeft}Error ({token.Id}):{expressionError ?? token.Name}{_tokenRight}"; + } - // Handle final token action + return tokenAction; + } - switch ( tokenAction ) + private TokenAction ProcessDefineToken( TokenDefinition token ) + { + string expressionError = null; + var value = token.TokenEvaluation switch { - case TokenAction.Ignore: - case TokenAction.Replace: - break; - - case TokenAction.Error: - value = $"{_tokenLeft}Error ({token.Id}):{expressionError ?? token.Name}{_tokenRight}"; - break; + TokenEvaluation.Expression when TryInvokeTokenExpression( token, out var expressionResult, out expressionError ) => Convert.ToString( expressionResult, CultureInfo.InvariantCulture ), + TokenEvaluation.Expression => throw new TemplateException( $"Error evaluating define expression for {token.Name}: {expressionError}" ), + _ => token.TokenExpression + }; - default: - throw new NotSupportedException( $"{nameof( ProcessToken )}: Invalid {nameof( TokenAction )} {tokenAction}." ); - } + _members[token.Name] = value; + return TokenAction.Ignore; + } - return tokenAction; + private TokenAction ProcessIfToken(TokenDefinition token, FrameStack frames, bool conditionalResult) + { + var frameIsTruthy = token.TokenEvaluation == TokenEvaluation.Falsy ? !conditionalResult : conditionalResult; + frames.Push(token, frameIsTruthy ); + return TokenAction.Ignore; } private static TokenAction ProcessElseToken( FrameStack frames, TokenDefinition token ) @@ -157,7 +148,13 @@ private static TokenAction ProcessEndIfToken( FrameStack frames ) frames.Pop(); // pop the else frames.Pop(); // pop the if + return TokenAction.Ignore; + } + private TokenAction ProcessWhileToken( TokenDefinition token, FrameStack frames, bool conditionalResult, TemplateState state ) + { + var frameIsTruthy = token.TokenEvaluation == TokenEvaluation.Falsy ? !conditionalResult : conditionalResult; + frames.Push( token, frameIsTruthy, null, state.CurrentPos ); return TokenAction.Ignore; } @@ -167,18 +164,18 @@ private TokenAction ProcessEndWhileToken( FrameStack frames ) throw new TemplateException( "Syntax error. Invalid `/while` without matching `while`." ); var whileToken = frames.Peek().Token; - - // ReSharper disable once RedundantAssignment - string expressionError = null; // assign to avoid compiler complaint + string expressionError = null; var conditionIsTrue = whileToken.TokenEvaluation switch { - TokenEvaluation.Expression when TryInvokeTokenExpression( whileToken, out var expressionResult, out expressionError ) => Convert.ToBoolean( expressionResult ), - TokenEvaluation.Expression => throw new TemplateException( $"{_tokenLeft}Error ({whileToken.Id}):{expressionError ?? "Error in while condition."}{_tokenRight}" ), - _ => Truthy( _members[whileToken.Name] ) // Re-evaluate the condition + TokenEvaluation.Expression when TryInvokeTokenExpression( whileToken, out var expressionResult, out expressionError ) + => Convert.ToBoolean( expressionResult ), + TokenEvaluation.Expression + => throw new TemplateException( $"{_tokenLeft}Error ({whileToken.Id}):{expressionError ?? "Error in while condition."}{_tokenRight}" ), + _ => Truthy( _members[whileToken.Name] ) }; - if ( conditionIsTrue ) // If the condition is true, replay the while block + if ( conditionIsTrue ) return TokenAction.ContinueLoop; // Otherwise, pop the frame and exit the loop @@ -186,49 +183,44 @@ TokenEvaluation.Expression when TryInvokeTokenExpression( whileToken, out var ex return TokenAction.Ignore; } - private TokenAction ProcessEndEachToken( FrameStack frames ) + private TokenAction ProcessEachToken(TokenDefinition token, FrameStack frames, object resolvedValue, TemplateState state, out string value) { - if ( frames.Depth == 0 || !frames.IsTokenType( TokenType.Each ) ) - throw new TemplateException( "Syntax error. Invalid /each without matching each." ); + value = default; + + if ( resolvedValue is IEnumerator enumerator && enumerator.MoveNext() ) + { + value = enumerator.Current; + _members[token.Name] = value; + frames.Push( token, true, new EnumeratorDefinition( Name: token.Name, Enumerator: enumerator ), state.CurrentPos ); + } - // Evaluate the condition and move to the next item in the enumerator + return TokenAction.Ignore; + } + + private TokenAction ProcessEndEachToken(FrameStack frames) + { + if (frames.Depth == 0 || !frames.IsTokenType(TokenType.Each)) + throw new TemplateException("Syntax error. Invalid /each without matching each."); var frame = frames.Peek(); var (currentName, enumerator) = frame.EnumeratorDefinition; - if ( enumerator!.MoveNext() ) + if (enumerator!.MoveNext()) { _members[currentName] = enumerator.Current; return TokenAction.ContinueLoop; } - // Otherwise, pop the frame and exit the loop _members[currentName] = default; frames.Pop(); return TokenAction.Ignore; } - private TokenAction ProcessDefineToken( TokenDefinition token ) - { - string expressionError = null; - - _members[token.Name] = token.TokenEvaluation switch - { - TokenEvaluation.Expression when TryInvokeTokenExpression( token, out var expressionResult, out expressionError ) - => Convert.ToString( expressionResult, CultureInfo.InvariantCulture ), - TokenEvaluation.Expression - => throw new TemplateException( $"Error evaluating define expression for {token.Name}: {expressionError}" ), - _ => token.TokenExpression - }; - - return TokenAction.Ignore; - } - - private void ResolveValue( TokenDefinition token, out object value, out bool defined, out bool ifResult, out string expressionError ) + private void ResolveValue( TokenDefinition token, out object value, out bool defined, out bool conditionalResult, out string expressionError ) { value = default; defined = false; - ifResult = false; + conditionalResult = false; expressionError = null; switch ( token.TokenType ) @@ -236,66 +228,61 @@ private void ResolveValue( TokenDefinition token, out object value, out bool def case TokenType.Value when token.TokenEvaluation != TokenEvaluation.Expression: case TokenType.If when token.TokenEvaluation != TokenEvaluation.Expression: case TokenType.While when token.TokenEvaluation != TokenEvaluation.Expression: - { - defined = _members.TryGetValue( token.Name, out var valueMember ); - - if ( !defined && _substituteEnvironmentVariables ) - { - valueMember = Environment.GetEnvironmentVariable( token.Name ); - defined = value != null; - } - - if ( token.TokenType == TokenType.If || token.TokenType == TokenType.While || token.TokenType == TokenType.Each ) - { - ifResult = defined && Truthy( valueMember ); - } + { + defined = _members.TryGetValue( token.Name, out var valueMember ); + value = defined ? valueMember : GetEnvironmentVariableValue( token.Name ); - value = valueMember; - break; - } + if ( token.TokenType == TokenType.If || token.TokenType == TokenType.While || token.TokenType == TokenType.Each ) + conditionalResult = defined && Truthy( valueMember ); + break; + } case TokenType.Value when token.TokenEvaluation == TokenEvaluation.Expression: + { + if ( TryInvokeTokenExpression( token, out var valueExprResult, out expressionError ) ) { - if ( TryInvokeTokenExpression( token, out var valueExprResult, out expressionError ) ) - { - value = Convert.ToString( valueExprResult, CultureInfo.InvariantCulture ); - defined = true; - } - - break; + value = Convert.ToString( valueExprResult, CultureInfo.InvariantCulture ); + defined = true; } + break; + } + case TokenType.If when token.TokenEvaluation == TokenEvaluation.Expression: case TokenType.While when token.TokenEvaluation == TokenEvaluation.Expression: - { - if ( !TryInvokeTokenExpression( token, out var condExprResult, out var error ) ) - throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{error ?? "Error in if condition."}{_tokenRight}" ); - - ifResult = Convert.ToBoolean( condExprResult ); - break; - } + { + if ( TryInvokeTokenExpression( token, out var condExprResult, out var error ) ) + conditionalResult = Convert.ToBoolean( condExprResult ); + else + throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{error ?? "Error in condition."}{_tokenRight}" ); + break; + } case TokenType.Each: - { - if ( token.TokenEvaluation != TokenEvaluation.Expression ) - throw new TemplateException( "Invalid token expression for each. Are you missing a fat arrow?" ); - - if ( !TryInvokeTokenExpression( token, out var eachExprResult, out var errorEach ) ) - throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{errorEach ?? "Error in each condition."}{_tokenRight}" ); + { + if ( token.TokenEvaluation != TokenEvaluation.Expression ) + throw new TemplateException( "Invalid token expression for each. Are you missing a fat arrow?" ); + if ( TryInvokeTokenExpression( token, out var eachExprResult, out var errorEach ) ) value = new EnumeratorAdapter( (IEnumerable) eachExprResult ); - break; - } + else + throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{errorEach ?? "Error in each condition."}{_tokenRight}" ); + break; + } + } + + return; + + string GetEnvironmentVariableValue( string name ) + { + return _substituteEnvironmentVariables ? Environment.GetEnvironmentVariable( token.Name ) : default; } } - private bool TryInvokeTokenHandler( TokenDefinition token, bool defined, ref string value, out TokenAction tokenAction ) + private bool TryInvokeTokenHandler(TokenDefinition token, bool defined, ref string value, out TokenAction tokenAction) { tokenAction = defined ? TokenAction.Replace : (_ignoreMissingTokens ? TokenAction.Ignore : TokenAction.Error); - - // Invoke any token handler - if ( _tokenHandler == null ) - return false; + if (_tokenHandler == null) return false; var eventArgs = new TemplateEventArgs { @@ -306,58 +293,36 @@ private bool TryInvokeTokenHandler( TokenDefinition token, bool defined, ref str UnknownToken = !defined }; - _tokenHandler.Invoke( null, eventArgs ); + _tokenHandler.Invoke(null, eventArgs); - // The token handler may have modified token properties value = eventArgs.Value; tokenAction = eventArgs.Action; - return true; } - private bool TryInvokeTokenExpression( TokenDefinition token, out object result, out string error ) + private bool TryInvokeTokenExpression(TokenDefinition token, out object result, out string error) { try { - var tokenExpression = _tokenExpressionProvider.GetTokenExpression( token.TokenExpression ); - - result = tokenExpression( _members ); - error = default; - + var tokenExpression = _tokenExpressionProvider.GetTokenExpression(token.TokenExpression); + result = tokenExpression(_members); + error = null; return true; } - catch ( Exception ex ) + catch (Exception ex) { error = ex.Message; + result = null; + return false; } - - result = default; - return false; } private static readonly string[] FalsyStrings = ["False", "No", "Off", "0"]; - private static bool Truthy( ReadOnlySpan value ) + private static bool Truthy(ReadOnlySpan value) { - // falsy => null, String.Empty, False, No, Off, 0 - - var truthy = !value.IsEmpty; - - if ( !truthy ) - return false; - - var compare = value.Trim(); - - foreach ( var item in FalsyStrings ) - { - if ( !compare.SequenceEqual( item ) ) - continue; - - truthy = false; - break; - } - - return truthy; + var trimmed = value.Trim(); + return !trimmed.IsEmpty && Array.IndexOf(FalsyStrings, trimmed.ToString()) == -1; } } @@ -367,27 +332,13 @@ internal sealed class EnumeratorAdapter : IEnumerator internal EnumeratorAdapter( IEnumerable enumerable ) { - if ( enumerable == null ) - throw new ArgumentNullException( nameof( enumerable ) ); - - // ReSharper disable once GenericEnumeratorNotDisposed - _inner = enumerable.GetEnumerator(); + _inner = enumerable?.GetEnumerator() ?? throw new ArgumentNullException( nameof(enumerable) ); } public string Current => (string) _inner.Current; - object IEnumerator.Current => _inner.Current; public bool MoveNext() => _inner.MoveNext(); - public void Reset() => _inner.Reset(); - - public void Dispose() - { - if ( _inner is IDisposable disposable ) - { - disposable.Dispose(); - } - } + public void Dispose() => (_inner as IDisposable)?.Dispose(); } - From c2bd9fa79ff3e3f1a9413bb6c04a3d502706b97e Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 14 Nov 2024 19:09:34 +0000 Subject: [PATCH 38/58] Updated code formatting to match rules in .editorconfig --- .../Text/TokenProcessor.cs | 108 +++++++++--------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TokenProcessor.cs b/src/Hyperbee.Templating/Text/TokenProcessor.cs index 19edbf2..3bba0bb 100644 --- a/src/Hyperbee.Templating/Text/TokenProcessor.cs +++ b/src/Hyperbee.Templating/Text/TokenProcessor.cs @@ -16,15 +16,15 @@ internal class TokenProcessor private readonly string _tokenRight; private readonly MemberDictionary _members; - public TokenProcessor(MemberDictionary members, TemplateOptions options) + public TokenProcessor( MemberDictionary members, TemplateOptions options ) { - ArgumentNullException.ThrowIfNull(members); + ArgumentNullException.ThrowIfNull( members ); - if (options.Methods == null) - throw new ArgumentNullException(nameof(options), $"{nameof(options.Methods)} cannot be null."); + if ( options.Methods == null ) + throw new ArgumentNullException( nameof( options ), $"{nameof( options.Methods )} cannot be null." ); - if (options.TokenExpressionProvider == null) - throw new ArgumentNullException(nameof(options), $"{nameof(options.TokenExpressionProvider)} cannot be null."); + if ( options.TokenExpressionProvider == null ) + throw new ArgumentNullException( nameof( options ), $"{nameof( options.TokenExpressionProvider )} cannot be null." ); _tokenExpressionProvider = options.TokenExpressionProvider; _tokenHandler = options.TokenHandler; @@ -123,10 +123,10 @@ TokenEvaluation.Expression when TryInvokeTokenExpression( token, out var express return TokenAction.Ignore; } - private TokenAction ProcessIfToken(TokenDefinition token, FrameStack frames, bool conditionalResult) + private TokenAction ProcessIfToken( TokenDefinition token, FrameStack frames, bool conditionalResult ) { var frameIsTruthy = token.TokenEvaluation == TokenEvaluation.Falsy ? !conditionalResult : conditionalResult; - frames.Push(token, frameIsTruthy ); + frames.Push( token, frameIsTruthy ); return TokenAction.Ignore; } @@ -183,7 +183,7 @@ TokenEvaluation.Expression when TryInvokeTokenExpression( whileToken, out var ex return TokenAction.Ignore; } - private TokenAction ProcessEachToken(TokenDefinition token, FrameStack frames, object resolvedValue, TemplateState state, out string value) + private TokenAction ProcessEachToken( TokenDefinition token, FrameStack frames, object resolvedValue, TemplateState state, out string value ) { value = default; @@ -197,15 +197,15 @@ private TokenAction ProcessEachToken(TokenDefinition token, FrameStack frames, o return TokenAction.Ignore; } - private TokenAction ProcessEndEachToken(FrameStack frames) + private TokenAction ProcessEndEachToken( FrameStack frames ) { - if (frames.Depth == 0 || !frames.IsTokenType(TokenType.Each)) - throw new TemplateException("Syntax error. Invalid /each without matching each."); + if ( frames.Depth == 0 || !frames.IsTokenType( TokenType.Each ) ) + throw new TemplateException( "Syntax error. Invalid /each without matching each." ); var frame = frames.Peek(); var (currentName, enumerator) = frame.EnumeratorDefinition; - if (enumerator!.MoveNext()) + if ( enumerator!.MoveNext() ) { _members[currentName] = enumerator.Current; return TokenAction.ContinueLoop; @@ -228,47 +228,47 @@ private void ResolveValue( TokenDefinition token, out object value, out bool def case TokenType.Value when token.TokenEvaluation != TokenEvaluation.Expression: case TokenType.If when token.TokenEvaluation != TokenEvaluation.Expression: case TokenType.While when token.TokenEvaluation != TokenEvaluation.Expression: - { - defined = _members.TryGetValue( token.Name, out var valueMember ); - value = defined ? valueMember : GetEnvironmentVariableValue( token.Name ); + { + defined = _members.TryGetValue( token.Name, out var valueMember ); + value = defined ? valueMember : GetEnvironmentVariableValue( token.Name ); - if ( token.TokenType == TokenType.If || token.TokenType == TokenType.While || token.TokenType == TokenType.Each ) - conditionalResult = defined && Truthy( valueMember ); - break; - } + if ( token.TokenType == TokenType.If || token.TokenType == TokenType.While || token.TokenType == TokenType.Each ) + conditionalResult = defined && Truthy( valueMember ); + break; + } case TokenType.Value when token.TokenEvaluation == TokenEvaluation.Expression: - { - if ( TryInvokeTokenExpression( token, out var valueExprResult, out expressionError ) ) { - value = Convert.ToString( valueExprResult, CultureInfo.InvariantCulture ); - defined = true; - } + if ( TryInvokeTokenExpression( token, out var valueExprResult, out expressionError ) ) + { + value = Convert.ToString( valueExprResult, CultureInfo.InvariantCulture ); + defined = true; + } - break; - } + break; + } case TokenType.If when token.TokenEvaluation == TokenEvaluation.Expression: case TokenType.While when token.TokenEvaluation == TokenEvaluation.Expression: - { - if ( TryInvokeTokenExpression( token, out var condExprResult, out var error ) ) - conditionalResult = Convert.ToBoolean( condExprResult ); - else - throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{error ?? "Error in condition."}{_tokenRight}" ); - break; - } + { + if ( TryInvokeTokenExpression( token, out var condExprResult, out var error ) ) + conditionalResult = Convert.ToBoolean( condExprResult ); + else + throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{error ?? "Error in condition."}{_tokenRight}" ); + break; + } case TokenType.Each: - { - if ( token.TokenEvaluation != TokenEvaluation.Expression ) - throw new TemplateException( "Invalid token expression for each. Are you missing a fat arrow?" ); - - if ( TryInvokeTokenExpression( token, out var eachExprResult, out var errorEach ) ) - value = new EnumeratorAdapter( (IEnumerable) eachExprResult ); - else - throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{errorEach ?? "Error in each condition."}{_tokenRight}" ); - break; - } + { + if ( token.TokenEvaluation != TokenEvaluation.Expression ) + throw new TemplateException( "Invalid token expression for each. Are you missing a fat arrow?" ); + + if ( TryInvokeTokenExpression( token, out var eachExprResult, out var errorEach ) ) + value = new EnumeratorAdapter( (IEnumerable) eachExprResult ); + else + throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{errorEach ?? "Error in each condition."}{_tokenRight}" ); + break; + } } return; @@ -279,10 +279,10 @@ string GetEnvironmentVariableValue( string name ) } } - private bool TryInvokeTokenHandler(TokenDefinition token, bool defined, ref string value, out TokenAction tokenAction) + private bool TryInvokeTokenHandler( TokenDefinition token, bool defined, ref string value, out TokenAction tokenAction ) { tokenAction = defined ? TokenAction.Replace : (_ignoreMissingTokens ? TokenAction.Ignore : TokenAction.Error); - if (_tokenHandler == null) return false; + if ( _tokenHandler == null ) return false; var eventArgs = new TemplateEventArgs { @@ -293,23 +293,23 @@ private bool TryInvokeTokenHandler(TokenDefinition token, bool defined, ref stri UnknownToken = !defined }; - _tokenHandler.Invoke(null, eventArgs); + _tokenHandler.Invoke( null, eventArgs ); value = eventArgs.Value; tokenAction = eventArgs.Action; return true; } - private bool TryInvokeTokenExpression(TokenDefinition token, out object result, out string error) + private bool TryInvokeTokenExpression( TokenDefinition token, out object result, out string error ) { try { - var tokenExpression = _tokenExpressionProvider.GetTokenExpression(token.TokenExpression); - result = tokenExpression(_members); + var tokenExpression = _tokenExpressionProvider.GetTokenExpression( token.TokenExpression ); + result = tokenExpression( _members ); error = null; return true; } - catch (Exception ex) + catch ( Exception ex ) { error = ex.Message; result = null; @@ -319,10 +319,10 @@ private bool TryInvokeTokenExpression(TokenDefinition token, out object result, private static readonly string[] FalsyStrings = ["False", "No", "Off", "0"]; - private static bool Truthy(ReadOnlySpan value) + private static bool Truthy( ReadOnlySpan value ) { var trimmed = value.Trim(); - return !trimmed.IsEmpty && Array.IndexOf(FalsyStrings, trimmed.ToString()) == -1; + return !trimmed.IsEmpty && Array.IndexOf( FalsyStrings, trimmed.ToString() ) == -1; } } @@ -332,7 +332,7 @@ internal sealed class EnumeratorAdapter : IEnumerator internal EnumeratorAdapter( IEnumerable enumerable ) { - _inner = enumerable?.GetEnumerator() ?? throw new ArgumentNullException( nameof(enumerable) ); + _inner = enumerable?.GetEnumerator() ?? throw new ArgumentNullException( nameof( enumerable ) ); } public string Current => (string) _inner.Current; From f95214ea6d2af84f43be0d5463564c2891269ff1 Mon Sep 17 00:00:00 2001 From: Brenton Farmer Date: Thu, 14 Nov 2024 11:12:52 -0800 Subject: [PATCH 39/58] remove useless comments --- src/Hyperbee.Templating/Text/TokenProcessor.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TokenProcessor.cs b/src/Hyperbee.Templating/Text/TokenProcessor.cs index 19edbf2..019c1b1 100644 --- a/src/Hyperbee.Templating/Text/TokenProcessor.cs +++ b/src/Hyperbee.Templating/Text/TokenProcessor.cs @@ -68,7 +68,7 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out return ProcessDefineToken( token ); } - // Resolve value (called only once) + // Resolve value ResolveValue( token, out var resolvedValue, out var defined, out var conditionalResult, out var expressionError ); // Conditional frame handling based on token type after value resolution @@ -88,7 +88,6 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out break; } - // Final action determination for all tokens return ProcessTokenHandler( token, defined, ref value, expressionError ); } From ccdca8751292a97a819f535977b8f208521f9d0d Mon Sep 17 00:00:00 2001 From: Brenton Farmer Date: Thu, 14 Nov 2024 12:36:33 -0800 Subject: [PATCH 40/58] clean up --- src/Hyperbee.Templating/Text/TokenProcessor.cs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TokenProcessor.cs b/src/Hyperbee.Templating/Text/TokenProcessor.cs index ba78411..2907b92 100644 --- a/src/Hyperbee.Templating/Text/TokenProcessor.cs +++ b/src/Hyperbee.Templating/Text/TokenProcessor.cs @@ -66,6 +66,12 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out case TokenType.Define: return ProcessDefineToken( token ); + + case TokenType.None: + case TokenType.LoopStart: // loop category + case TokenType.LoopEnd: // loop category + default: + throw new NotSupportedException( $"{nameof(ProcessToken)}: Invalid {nameof(TokenType)} {token.TokenType}." ); } // Resolve value @@ -162,9 +168,9 @@ private TokenAction ProcessEndWhileToken( FrameStack frames ) if ( frames.Depth == 0 || !frames.IsTokenType( TokenType.While ) ) throw new TemplateException( "Syntax error. Invalid `/while` without matching `while`." ); - var whileToken = frames.Peek().Token; string expressionError = null; - + var whileToken = frames.Peek().Token; + var conditionIsTrue = whileToken.TokenEvaluation switch { TokenEvaluation.Expression when TryInvokeTokenExpression( whileToken, out var expressionResult, out expressionError ) @@ -177,7 +183,6 @@ TokenEvaluation.Expression when TryInvokeTokenExpression( whileToken, out var ex if ( conditionIsTrue ) return TokenAction.ContinueLoop; - // Otherwise, pop the frame and exit the loop frames.Pop(); return TokenAction.Ignore; } @@ -316,12 +321,12 @@ private bool TryInvokeTokenExpression( TokenDefinition token, out object result, } } - private static readonly string[] FalsyStrings = ["False", "No", "Off", "0"]; + private static readonly HashSet FalsyStrings = new(["False", "No", "Off", "0"], StringComparer.OrdinalIgnoreCase ); private static bool Truthy( ReadOnlySpan value ) { var trimmed = value.Trim(); - return !trimmed.IsEmpty && Array.IndexOf( FalsyStrings, trimmed.ToString() ) == -1; + return !trimmed.IsEmpty && !FalsyStrings.Contains( trimmed.ToString() ); } } @@ -331,6 +336,7 @@ internal sealed class EnumeratorAdapter : IEnumerator internal EnumeratorAdapter( IEnumerable enumerable ) { + // ReSharper disable once GenericEnumeratorNotDisposed _inner = enumerable?.GetEnumerator() ?? throw new ArgumentNullException( nameof( enumerable ) ); } From 484ac90e06a94cc2127dfe9aec3b87910e971808 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 14 Nov 2024 20:37:08 +0000 Subject: [PATCH 41/58] Updated code formatting to match rules in .editorconfig --- src/Hyperbee.Templating/Text/TokenProcessor.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TokenProcessor.cs b/src/Hyperbee.Templating/Text/TokenProcessor.cs index 2907b92..bc47b79 100644 --- a/src/Hyperbee.Templating/Text/TokenProcessor.cs +++ b/src/Hyperbee.Templating/Text/TokenProcessor.cs @@ -71,7 +71,7 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out case TokenType.LoopStart: // loop category case TokenType.LoopEnd: // loop category default: - throw new NotSupportedException( $"{nameof(ProcessToken)}: Invalid {nameof(TokenType)} {token.TokenType}." ); + throw new NotSupportedException( $"{nameof( ProcessToken )}: Invalid {nameof( TokenType )} {token.TokenType}." ); } // Resolve value @@ -170,7 +170,7 @@ private TokenAction ProcessEndWhileToken( FrameStack frames ) string expressionError = null; var whileToken = frames.Peek().Token; - + var conditionIsTrue = whileToken.TokenEvaluation switch { TokenEvaluation.Expression when TryInvokeTokenExpression( whileToken, out var expressionResult, out expressionError ) @@ -321,7 +321,7 @@ private bool TryInvokeTokenExpression( TokenDefinition token, out object result, } } - private static readonly HashSet FalsyStrings = new(["False", "No", "Off", "0"], StringComparer.OrdinalIgnoreCase ); + private static readonly HashSet FalsyStrings = new( ["False", "No", "Off", "0"], StringComparer.OrdinalIgnoreCase ); private static bool Truthy( ReadOnlySpan value ) { From b9ae425d02233e92ea04aafec09fde8e46309788 Mon Sep 17 00:00:00 2001 From: Brenton Farmer Date: Thu, 14 Nov 2024 14:28:21 -0800 Subject: [PATCH 42/58] clean up --- .../Core/EnumeratorAdapter.cs | 21 +++ src/Hyperbee.Templating/Text/BufferManager.cs | 174 ------------------ .../Text/TemplateHelper.cs | 2 +- .../Text/TemplateParser.cs | 40 +--- src/Hyperbee.Templating/Text/TemplateState.cs | 35 ++++ .../Text/TokenProcessor.cs | 20 +- 6 files changed, 62 insertions(+), 230 deletions(-) create mode 100644 src/Hyperbee.Templating/Core/EnumeratorAdapter.cs delete mode 100644 src/Hyperbee.Templating/Text/BufferManager.cs create mode 100644 src/Hyperbee.Templating/Text/TemplateState.cs diff --git a/src/Hyperbee.Templating/Core/EnumeratorAdapter.cs b/src/Hyperbee.Templating/Core/EnumeratorAdapter.cs new file mode 100644 index 0000000..142cd45 --- /dev/null +++ b/src/Hyperbee.Templating/Core/EnumeratorAdapter.cs @@ -0,0 +1,21 @@ +using System.Collections; + +namespace Hyperbee.Templating.Core; + +internal sealed class EnumeratorAdapter : IEnumerator +{ + private readonly IEnumerator _inner; + + internal EnumeratorAdapter( IEnumerable enumerable ) + { + // ReSharper disable once GenericEnumeratorNotDisposed + _inner = enumerable?.GetEnumerator() ?? throw new ArgumentNullException( nameof( enumerable ) ); + } + + public string Current => (string) _inner.Current; + object IEnumerator.Current => _inner.Current; + + public bool MoveNext() => _inner.MoveNext(); + public void Reset() => _inner.Reset(); + public void Dispose() => (_inner as IDisposable)?.Dispose(); +} diff --git a/src/Hyperbee.Templating/Text/BufferManager.cs b/src/Hyperbee.Templating/Text/BufferManager.cs deleted file mode 100644 index c58d12e..0000000 --- a/src/Hyperbee.Templating/Text/BufferManager.cs +++ /dev/null @@ -1,174 +0,0 @@ -using System.Buffers; -using System.Runtime.CompilerServices; - -namespace Hyperbee.Templating.Text; - -internal ref struct BufferManager -{ - private readonly ArrayPool _arrayPool; - private readonly List _buffers; - private int _currentBufferIndex; - private int _currentBufferPos; - private readonly int _bufferSize; - private bool _grow; - - private readonly ReadOnlySpan _fixedSpan; - - public BufferManager( int bufferSize ) - { - _arrayPool = ArrayPool.Shared; - _buffers = []; - _bufferSize = bufferSize; - } - - public BufferManager( ReadOnlySpan span ) - { - _bufferSize = span.Length; - _fixedSpan = span; - } - - public readonly int BufferSize => _bufferSize; - public readonly bool IsFixed => _fixedSpan.Length > 0; - - [MethodImpl( MethodImplOptions.AggressiveInlining )] - public void AdvanceCurrentSpan( int advanceBy ) => _currentBufferPos += advanceBy; - - [MethodImpl( MethodImplOptions.AggressiveInlining )] - public readonly ReadOnlySpan GetCurrentSpan() - { - if ( IsFixed ) - return _fixedSpan[_currentBufferPos..]; - - var bufferState = _buffers[_currentBufferIndex]; - return bufferState.Buffer.AsSpan( _currentBufferPos, bufferState.TotalCharacters - _currentBufferPos ); - } - - [MethodImpl( MethodImplOptions.AggressiveInlining )] - public ReadOnlySpan GetCurrentSpan( int advanceBy ) - { - AdvanceCurrentSpan( advanceBy ); - return GetCurrentSpan(); - } - - public void SetGrow( bool grow ) - { - if ( IsFixed ) - throw new InvalidOperationException( "Cannot set grow on a fixed span." ); - - _grow = grow; - } - - public ReadOnlySpan ReadSpan( TextReader reader ) - { - if ( IsFixed && reader == null ) - { - _currentBufferPos = 0; - return _fixedSpan; - } - - var first = _buffers.Count == 0; - var rent = _grow || first; - - BufferState bufferState; - - if ( rent ) - { - // Rent a new buffer and add to the list - bufferState = new BufferState( _arrayPool.Rent( _bufferSize ) ); - _buffers.Add( bufferState ); - _currentBufferIndex = _buffers.Count - 1; - } - else - { - // Use the existing buffer - bufferState = _buffers[_currentBufferIndex]; - } - - // Slide remainder of the current buffer if necessary - var remainder = 0; - - if ( !first ) - { - remainder = bufferState.TotalCharacters - _currentBufferPos; - - if ( remainder > 0 ) - Array.Copy( bufferState.Buffer, _currentBufferPos, bufferState.Buffer, 0, remainder ); - } - - // Read new data into the buffer - _currentBufferPos = 0; - - var span = bufferState.Buffer.AsSpan( remainder, _bufferSize - remainder ); - var read = reader.Read( span ); - - bufferState.TotalCharacters = read + remainder; - - return bufferState.TotalCharacters == 0 ? [] : bufferState.Buffer.AsSpan( 0, bufferState.TotalCharacters ); - } - - public readonly int CurrentPosition => IsFixed ? _currentBufferPos : _currentBufferIndex * _bufferSize + _currentBufferPos; - - public void Position( int position ) - { - ArgumentOutOfRangeException.ThrowIfNegative( position, nameof( position ) ); - - if ( IsFixed ) - { - _currentBufferPos = position; - return; - } - - var remainingPosition = position; - - for ( var i = 0; i < _buffers.Count; i++ ) - { - if ( remainingPosition < _bufferSize ) - { - _currentBufferIndex = i; - _currentBufferPos = remainingPosition; - return; - } - - remainingPosition -= _bufferSize; - } - - throw new InvalidOperationException( "Position exceeds buffered content." ); - } - - public void TrimBuffers() - { - _currentBufferPos = 0; - _currentBufferIndex = 0; - - if ( IsFixed ) - return; - - while ( _buffers.Count > 1 ) - { - _arrayPool.Return( _buffers[0].Buffer ); - _buffers.RemoveAt( 0 ); - } - } - - public void ReleaseBuffers() - { - _currentBufferPos = 0; - _currentBufferIndex = 0; - - if ( IsFixed ) - return; - - foreach ( var buffer in _buffers ) - { - _arrayPool.Return( buffer.Buffer ); - } - - _buffers.Clear(); - } - - private class BufferState( char[] buffer ) - { - public char[] Buffer { get; } = buffer; - public int TotalCharacters { get; set; } - } -} diff --git a/src/Hyperbee.Templating/Text/TemplateHelper.cs b/src/Hyperbee.Templating/Text/TemplateHelper.cs index d47c0e3..054dc16 100644 --- a/src/Hyperbee.Templating/Text/TemplateHelper.cs +++ b/src/Hyperbee.Templating/Text/TemplateHelper.cs @@ -8,7 +8,7 @@ public static bool ValidateKey( string key ) { // do-not-remove this method. // - // this method is required despite code analysis claiming the method is not referenced. + // this method is required despite code analysis claiming the method isn't referenced. // // this overload is required (and used) by generic delegates which don't support // ReadOnlySpan as a generic argument. diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index a8130b2..2b983e1 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -215,7 +215,6 @@ private void ParseTemplate( ref BufferManager bufferManager, TextReader reader, } } - span = []; // clear span for read break; } @@ -270,13 +269,14 @@ private void ParseTemplate( ref BufferManager bufferManager, TextReader reader, bufferManager.AdvanceCurrentSpan( writeLength ); } - span = []; // clear span for read break; } default: throw new ArgumentOutOfRangeException( scanner.ToString(), $"Invalid scanner state: {scanner}." ); } + + span = []; // clear span for read } if ( bufferManager.IsFixed || bytesRead < bufferManager.BufferSize ) @@ -298,7 +298,7 @@ private void ParseTemplate( ref BufferManager bufferManager, TextReader reader, return; - static void ProcessFrame( FrameStack.Frame frame, TokenAction tokenAction, TokenType tokenType, ref ReadOnlySpan span, ref BufferManager bufferManager, ref int loopDepth ) + static void ProcessFrame( Frame frame, TokenAction tokenAction, TokenType tokenType, ref ReadOnlySpan span, ref BufferManager bufferManager, ref int loopDepth ) { // loop handling @@ -481,37 +481,3 @@ private int IndexOfIgnoreQuotedContent( ReadOnlySpan span, ReadOnlySpan - Frames.Depth > 0 ? Frames.Peek() : default; -} - -internal record EnumeratorDefinition( string Name, IEnumerator Enumerator ); - -internal sealed class FrameStack -{ - public record Frame( TokenDefinition Token, bool Truthy, EnumeratorDefinition EnumeratorDefinition = null, int StartPos = -1 ); - - private readonly Stack _stack = new(); - - public void Push( TokenDefinition token, bool truthy, EnumeratorDefinition enumeratorDefinition = null, int startPos = -1 ) - => _stack.Push( new Frame( token, truthy, enumeratorDefinition, startPos ) ); - - public Frame Peek() => _stack.Peek(); - public void Pop() => _stack.Pop(); - public int Depth => _stack.Count; - - public bool IsTokenType( TokenType compare ) - => _stack.Count > 0 && _stack.Peek().Token.TokenType == compare; - - public bool IsTruthy => _stack.Count == 0 || _stack.Peek().Truthy; - - public bool IsFalsy => !IsTruthy; -} diff --git a/src/Hyperbee.Templating/Text/TemplateState.cs b/src/Hyperbee.Templating/Text/TemplateState.cs new file mode 100644 index 0000000..e9ba8f9 --- /dev/null +++ b/src/Hyperbee.Templating/Text/TemplateState.cs @@ -0,0 +1,35 @@ +namespace Hyperbee.Templating.Text; + +internal sealed class TemplateState +{ + public FrameStack Frames { get; } = new(); + public int NextTokenId { get; set; } = 1; + public int CurrentPos { get; set; } + public Frame CurrentFrame() => + Frames.Depth > 0 ? Frames.Peek() : default; +} + +// Minimal frame management for flow control + +internal record EnumeratorDefinition( string Name, IEnumerator Enumerator ); + +internal record Frame( TokenDefinition Token, bool Truthy, EnumeratorDefinition EnumeratorDefinition = null, int StartPos = -1 ); + +internal sealed class FrameStack +{ + private readonly Stack _stack = new(); + + public void Push( TokenDefinition token, bool truthy, EnumeratorDefinition enumeratorDefinition = null, int startPos = -1 ) + => _stack.Push( new Frame( token, truthy, enumeratorDefinition, startPos ) ); + + public Frame Peek() => _stack.Peek(); + public void Pop() => _stack.Pop(); + public int Depth => _stack.Count; + + public bool IsTokenType( TokenType compare ) + => _stack.Count > 0 && _stack.Peek().Token.TokenType == compare; + + public bool IsTruthy => _stack.Count == 0 || _stack.Peek().Truthy; + + public bool IsFalsy => !IsTruthy; +} diff --git a/src/Hyperbee.Templating/Text/TokenProcessor.cs b/src/Hyperbee.Templating/Text/TokenProcessor.cs index bc47b79..148df1f 100644 --- a/src/Hyperbee.Templating/Text/TokenProcessor.cs +++ b/src/Hyperbee.Templating/Text/TokenProcessor.cs @@ -2,6 +2,8 @@ using System.Globalization; using Hyperbee.Templating.Compiler; using Hyperbee.Templating.Configure; +using Hyperbee.Templating.Core; + // ReSharper disable RedundantAssignment namespace Hyperbee.Templating.Text; @@ -329,21 +331,3 @@ private static bool Truthy( ReadOnlySpan value ) return !trimmed.IsEmpty && !FalsyStrings.Contains( trimmed.ToString() ); } } - -internal sealed class EnumeratorAdapter : IEnumerator -{ - private readonly IEnumerator _inner; - - internal EnumeratorAdapter( IEnumerable enumerable ) - { - // ReSharper disable once GenericEnumeratorNotDisposed - _inner = enumerable?.GetEnumerator() ?? throw new ArgumentNullException( nameof( enumerable ) ); - } - - public string Current => (string) _inner.Current; - object IEnumerator.Current => _inner.Current; - - public bool MoveNext() => _inner.MoveNext(); - public void Reset() => _inner.Reset(); - public void Dispose() => (_inner as IDisposable)?.Dispose(); -} From b839461568a6fd96b2257fabb99acbb095287600 Mon Sep 17 00:00:00 2001 From: Brenton Farmer Date: Thu, 14 Nov 2024 15:02:25 -0800 Subject: [PATCH 43/58] more clean up --- src/Hyperbee.Templating/Configure/TemplateOptions.cs | 3 ++- .../{Text/TemplateHelper.cs => Core/KeyHelper.cs} | 4 ++-- src/Hyperbee.Templating/Text/MemberDictionary.cs | 3 ++- src/Hyperbee.Templating/Text/TemplateState.cs | 3 +-- 4 files changed, 7 insertions(+), 6 deletions(-) rename src/Hyperbee.Templating/{Text/TemplateHelper.cs => Core/KeyHelper.cs} (91%) diff --git a/src/Hyperbee.Templating/Configure/TemplateOptions.cs b/src/Hyperbee.Templating/Configure/TemplateOptions.cs index 51ddd40..8a526bc 100644 --- a/src/Hyperbee.Templating/Configure/TemplateOptions.cs +++ b/src/Hyperbee.Templating/Configure/TemplateOptions.cs @@ -1,6 +1,7 @@ using System.ComponentModel; using System.Reflection; using Hyperbee.Templating.Compiler; +using Hyperbee.Templating.Core; using Hyperbee.Templating.Text; namespace Hyperbee.Templating.Configure; @@ -13,7 +14,7 @@ public class TemplateOptions public IDictionary Variables { get; init; } public TokenStyle TokenStyle { get; set; } = TokenStyle.Default; - public KeyValidator Validator { get; set; } = TemplateHelper.ValidateKey; + public KeyValidator Validator { get; set; } = KeyHelper.ValidateKey; public bool IgnoreMissingTokens { get; set; } public bool SubstituteEnvironmentVariables { get; set; } diff --git a/src/Hyperbee.Templating/Text/TemplateHelper.cs b/src/Hyperbee.Templating/Core/KeyHelper.cs similarity index 91% rename from src/Hyperbee.Templating/Text/TemplateHelper.cs rename to src/Hyperbee.Templating/Core/KeyHelper.cs index 054dc16..8c76867 100644 --- a/src/Hyperbee.Templating/Text/TemplateHelper.cs +++ b/src/Hyperbee.Templating/Core/KeyHelper.cs @@ -1,8 +1,8 @@ -namespace Hyperbee.Templating.Text; +namespace Hyperbee.Templating.Core; public delegate bool KeyValidator( ReadOnlySpan key ); -internal static class TemplateHelper +internal static class KeyHelper { public static bool ValidateKey( string key ) { diff --git a/src/Hyperbee.Templating/Text/MemberDictionary.cs b/src/Hyperbee.Templating/Text/MemberDictionary.cs index 524e218..6ee05ba 100644 --- a/src/Hyperbee.Templating/Text/MemberDictionary.cs +++ b/src/Hyperbee.Templating/Text/MemberDictionary.cs @@ -1,5 +1,6 @@ using System.Collections; using Hyperbee.Templating.Compiler; +using Hyperbee.Templating.Core; namespace Hyperbee.Templating.Text; @@ -17,7 +18,7 @@ public class MemberDictionary : IReadOnlyMemberDictionary public KeyValidator Validator { get; } public MemberDictionary( IDictionary source, IReadOnlyDictionary methods = default ) - : this( TemplateHelper.ValidateKey, source, methods ) + : this( KeyHelper.ValidateKey, source, methods ) { } diff --git a/src/Hyperbee.Templating/Text/TemplateState.cs b/src/Hyperbee.Templating/Text/TemplateState.cs index e9ba8f9..66373d6 100644 --- a/src/Hyperbee.Templating/Text/TemplateState.cs +++ b/src/Hyperbee.Templating/Text/TemplateState.cs @@ -5,8 +5,7 @@ internal sealed class TemplateState public FrameStack Frames { get; } = new(); public int NextTokenId { get; set; } = 1; public int CurrentPos { get; set; } - public Frame CurrentFrame() => - Frames.Depth > 0 ? Frames.Peek() : default; + public Frame CurrentFrame() => Frames.Depth > 0 ? Frames.Peek() : default; } // Minimal frame management for flow control From 73e36392b03ce26dd3c96853c43d5554bf69b963 Mon Sep 17 00:00:00 2001 From: Brenton Farmer Date: Fri, 15 Nov 2024 07:25:09 -0800 Subject: [PATCH 44/58] more clean up --- .../Compiler/RoslynTokenExpressionProvider.cs | 6 +- .../Text/MemberDictionary.cs | 4 +- .../Text/TemplateParser.cs | 4 +- src/Hyperbee.Templating/Text/TokenEnums.cs | 3 +- src/Hyperbee.Templating/Text/TokenParser.cs | 15 ++-- .../Text/TokenProcessor.cs | 69 +++++++++++-------- 6 files changed, 57 insertions(+), 44 deletions(-) diff --git a/src/Hyperbee.Templating/Compiler/RoslynTokenExpressionProvider.cs b/src/Hyperbee.Templating/Compiler/RoslynTokenExpressionProvider.cs index c7d643a..1ce11cf 100644 --- a/src/Hyperbee.Templating/Compiler/RoslynTokenExpressionProvider.cs +++ b/src/Hyperbee.Templating/Compiler/RoslynTokenExpressionProvider.cs @@ -125,7 +125,7 @@ public static object Invoke( {{nameof( IReadOnlyMemberDictionary )}} members ) // // 1. x => x.someProp to x["someProp"] // 2. x => x.someProp to x.GetValueAs("someProp") -// 3. x => x.someMethod(..) to x.InvokeMethod("someMethod", ..) +// 3. x => x.someMethod(..) to x.Invoke("someMethod", ..) internal class TokenExpressionRewriter( string parameterName ) : CSharpSyntaxRewriter { @@ -232,12 +232,12 @@ private InvocationExpressionSyntax RewriteMethodInvocation( MemberAccessExpressi .Select( arg => (ExpressionSyntax) Visit( arg.Expression ) ) .ToArray(); - // Create the InvokeMethod call: x.InvokeMethod("MethodName", arg1, arg2, ...) + // Create the InvokeMethod call: x.Invoke("MethodName", arg1, arg2, ...) var invokeMethodCall = SyntaxFactory.InvocationExpression( SyntaxFactory.MemberAccessExpression( SyntaxKind.SimpleMemberAccessExpression, memberAccess.Expression, // This is `x` - SyntaxFactory.IdentifierName( "InvokeMethod" ) + SyntaxFactory.IdentifierName( "Invoke" ) ), SyntaxFactory.ArgumentList( SyntaxFactory.SeparatedList( diff --git a/src/Hyperbee.Templating/Text/MemberDictionary.cs b/src/Hyperbee.Templating/Text/MemberDictionary.cs index 6ee05ba..fd65a56 100644 --- a/src/Hyperbee.Templating/Text/MemberDictionary.cs +++ b/src/Hyperbee.Templating/Text/MemberDictionary.cs @@ -7,7 +7,7 @@ namespace Hyperbee.Templating.Text; public interface IReadOnlyMemberDictionary : IReadOnlyDictionary { public TType GetValueAs( string name ) where TType : IConvertible; - public object InvokeMethod( string methodName, params object[] args ); + public object Invoke( string methodName, params object[] args ); } public class MemberDictionary : IReadOnlyMemberDictionary @@ -102,7 +102,7 @@ public TType GetValueAs( string name ) where TType : IConvertible return (TType) Convert.ChangeType( this[name], typeof( TType ) ); } - public object InvokeMethod( string methodName, params object[] args ) + public object Invoke( string methodName, params object[] args ) { if ( !Methods.TryGetValue( methodName, out var methodInvoker ) ) throw new MissingMethodException( $"Failed to invoke method '{methodName}'." ); diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index 2b983e1..e71a7da 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -131,14 +131,14 @@ private void ParseTemplate( ReadOnlySpan templateSpan, TextWriter writer ) private void ParseTemplate( TextReader reader, TextWriter writer ) { - var bufferSize = GetScopedBufferSize( BufferSize, _tokenLeft.Length, _tokenRight.Length ); + var bufferSize = GetAdjustedBufferSize( BufferSize, _tokenLeft.Length, _tokenRight.Length ); var bufferManager = new BufferManager( bufferSize ); ParseTemplate( ref bufferManager, reader, writer ); return; - static int GetScopedBufferSize( int bufferSize, int tokenLeftSize, int tokenRightSize ) + static int GetAdjustedBufferSize( int bufferSize, int tokenLeftSize, int tokenRightSize ) { // because of the way we read the buffer, we need to ensure that the buffer size // is at least the size of the longest token delimiter plus one character. diff --git a/src/Hyperbee.Templating/Text/TokenEnums.cs b/src/Hyperbee.Templating/Text/TokenEnums.cs index f54a5f5..8b5578e 100644 --- a/src/Hyperbee.Templating/Text/TokenEnums.cs +++ b/src/Hyperbee.Templating/Text/TokenEnums.cs @@ -34,7 +34,8 @@ internal enum TokenEvaluation [Flags] internal enum TokenType { - None = 0x00, + Undefined = 0x00, + Define = 0x01, Value = 0x02, If = 0x03, diff --git a/src/Hyperbee.Templating/Text/TokenParser.cs b/src/Hyperbee.Templating/Text/TokenParser.cs index fc479ce..81d225c 100644 --- a/src/Hyperbee.Templating/Text/TokenParser.cs +++ b/src/Hyperbee.Templating/Text/TokenParser.cs @@ -34,12 +34,13 @@ public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) // {{while x => x.token}} // {{/while}} // - // {{each x => x.token}} + // {{each i:x => enumerable}} + // {{i}} // {{/each}} var span = token.Trim(); - var tokenType = TokenType.None; + var tokenType = TokenType.Undefined; var tokenEvaluation = TokenEvaluation.None; var tokenExpression = ReadOnlySpan.Empty; var name = ReadOnlySpan.Empty; @@ -182,7 +183,7 @@ public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) tokenType = TokenType.EndEach; } - if ( tokenType == TokenType.None || tokenType == TokenType.Each ) + if ( tokenType == TokenType.Undefined || tokenType == TokenType.Each ) { tokenType = GetTokenNameAndExpression( tokenType, span, ref name, ref tokenExpression, ref tokenEvaluation ); } @@ -202,7 +203,7 @@ public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) private TokenType GetTokenNameAndExpression( TokenType tokenType, ReadOnlySpan span, ref ReadOnlySpan name, ref ReadOnlySpan tokenExpression, ref TokenEvaluation tokenEvaluation ) { - if ( tokenType != TokenType.None && tokenType != TokenType.Each ) + if ( tokenType != TokenType.Undefined && tokenType != TokenType.Each ) { return tokenType; } @@ -212,7 +213,7 @@ private TokenType GetTokenNameAndExpression( TokenType tokenType, ReadOnlySpan -1 && (fatArrowPos == -1 || defineTokenPos < fatArrowPos) ) { - if ( tokenType == TokenType.None ) + if ( tokenType == TokenType.Undefined ) tokenType = TokenType.Define; name = span[..defineTokenPos].Trim(); @@ -235,7 +236,7 @@ private TokenType GetTokenNameAndExpression( TokenType tokenType, ReadOnlySpan Convert.ToString( expressionResult, CultureInfo.InvariantCulture ), - TokenEvaluation.Expression => throw new TemplateException( $"Error evaluating define expression for {token.Name}: {expressionError}" ), - _ => token.TokenExpression - }; + case TokenEvaluation.Expression when TryInvokeTokenExpression( token, out var expressionResult, out expressionError ): + value = Convert.ToString( expressionResult, CultureInfo.InvariantCulture ); + break; + case TokenEvaluation.Expression: + throw new TemplateException( $"Error evaluating define expression for {token.Name}: {expressionError}" ); + default: + value = token.TokenExpression; + break; + } _members[token.Name] = value; return TokenAction.Ignore; } - private TokenAction ProcessIfToken( TokenDefinition token, FrameStack frames, bool conditionalResult ) + private static TokenAction ProcessIfToken( TokenDefinition token, FrameStack frames, bool conditionalResult ) { var frameIsTruthy = token.TokenEvaluation == TokenEvaluation.Falsy ? !conditionalResult : conditionalResult; frames.Push( token, frameIsTruthy ); @@ -158,7 +164,7 @@ private static TokenAction ProcessEndIfToken( FrameStack frames ) return TokenAction.Ignore; } - private TokenAction ProcessWhileToken( TokenDefinition token, FrameStack frames, bool conditionalResult, TemplateState state ) + private static TokenAction ProcessWhileToken( TokenDefinition token, FrameStack frames, bool conditionalResult, TemplateState state ) { var frameIsTruthy = token.TokenEvaluation == TokenEvaluation.Falsy ? !conditionalResult : conditionalResult; frames.Push( token, frameIsTruthy, null, state.CurrentPos ); @@ -170,17 +176,22 @@ private TokenAction ProcessEndWhileToken( FrameStack frames ) if ( frames.Depth == 0 || !frames.IsTokenType( TokenType.While ) ) throw new TemplateException( "Syntax error. Invalid `/while` without matching `while`." ); - string expressionError = null; var whileToken = frames.Peek().Token; - var conditionIsTrue = whileToken.TokenEvaluation switch + bool conditionIsTrue; + string expressionError = null; + + switch ( whileToken.TokenEvaluation ) { - TokenEvaluation.Expression when TryInvokeTokenExpression( whileToken, out var expressionResult, out expressionError ) - => Convert.ToBoolean( expressionResult ), - TokenEvaluation.Expression - => throw new TemplateException( $"{_tokenLeft}Error ({whileToken.Id}):{expressionError ?? "Error in while condition."}{_tokenRight}" ), - _ => Truthy( _members[whileToken.Name] ) - }; + case TokenEvaluation.Expression when TryInvokeTokenExpression( whileToken, out var expressionResult, out expressionError ): + conditionIsTrue = Convert.ToBoolean( expressionResult ); + break; + case TokenEvaluation.Expression: + throw new TemplateException( $"{_tokenLeft}Error ({whileToken.Id}):{expressionError ?? "Error in while condition."}{_tokenRight}" ); + default: + conditionIsTrue = Truthy( _members[whileToken.Name] ); + break; + } if ( conditionIsTrue ) return TokenAction.ContinueLoop; @@ -256,23 +267,23 @@ private void ResolveValue( TokenDefinition token, out object value, out bool def case TokenType.If when token.TokenEvaluation == TokenEvaluation.Expression: case TokenType.While when token.TokenEvaluation == TokenEvaluation.Expression: - { - if ( TryInvokeTokenExpression( token, out var condExprResult, out var error ) ) - conditionalResult = Convert.ToBoolean( condExprResult ); - else - throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{error ?? "Error in condition."}{_tokenRight}" ); - break; - } + { + if ( !TryInvokeTokenExpression( token, out var condExprResult, out var error ) ) + throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{error ?? "Error in condition."}{_tokenRight}" ); + + conditionalResult = Convert.ToBoolean( condExprResult ); + break; + } case TokenType.Each: { if ( token.TokenEvaluation != TokenEvaluation.Expression ) throw new TemplateException( "Invalid token expression for each. Are you missing a fat arrow?" ); - if ( TryInvokeTokenExpression( token, out var eachExprResult, out var errorEach ) ) - value = new EnumeratorAdapter( (IEnumerable) eachExprResult ); - else + if ( !TryInvokeTokenExpression( token, out var eachExprResult, out var errorEach ) ) throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{errorEach ?? "Error in each condition."}{_tokenRight}" ); + + value = new EnumeratorAdapter( (IEnumerable) eachExprResult ); break; } } From de6fb5a2403b8dcfc56f6f90a215509b00596572 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 15 Nov 2024 15:25:43 +0000 Subject: [PATCH 45/58] Updated code formatting to match rules in .editorconfig --- .../Text/TokenProcessor.cs | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TokenProcessor.cs b/src/Hyperbee.Templating/Text/TokenProcessor.cs index c1fdad5..ca24655 100644 --- a/src/Hyperbee.Templating/Text/TokenProcessor.cs +++ b/src/Hyperbee.Templating/Text/TokenProcessor.cs @@ -71,8 +71,8 @@ public TokenAction ProcessToken( TokenDefinition token, TemplateState state, out return ProcessDefineToken( token ); case TokenType.Undefined: - case TokenType.LoopStart: - case TokenType.LoopEnd: + case TokenType.LoopStart: + case TokenType.LoopEnd: default: throw new NotSupportedException( $"{nameof( ProcessToken )}: Invalid {nameof( TokenType )} {token.TokenType}." ); } @@ -267,13 +267,13 @@ private void ResolveValue( TokenDefinition token, out object value, out bool def case TokenType.If when token.TokenEvaluation == TokenEvaluation.Expression: case TokenType.While when token.TokenEvaluation == TokenEvaluation.Expression: - { - if ( !TryInvokeTokenExpression( token, out var condExprResult, out var error ) ) - throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{error ?? "Error in condition."}{_tokenRight}" ); - - conditionalResult = Convert.ToBoolean( condExprResult ); - break; - } + { + if ( !TryInvokeTokenExpression( token, out var condExprResult, out var error ) ) + throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{error ?? "Error in condition."}{_tokenRight}" ); + + conditionalResult = Convert.ToBoolean( condExprResult ); + break; + } case TokenType.Each: { @@ -282,7 +282,7 @@ private void ResolveValue( TokenDefinition token, out object value, out bool def if ( !TryInvokeTokenExpression( token, out var eachExprResult, out var errorEach ) ) throw new TemplateException( $"{_tokenLeft}Error ({token.Id}):{errorEach ?? "Error in each condition."}{_tokenRight}" ); - + value = new EnumeratorAdapter( (IEnumerable) eachExprResult ); break; } From c1811413927fc781f383d3ddd6793784ce296414 Mon Sep 17 00:00:00 2001 From: Brenton Farmer Date: Fri, 15 Nov 2024 10:01:14 -0800 Subject: [PATCH 46/58] comments --- src/Hyperbee.Templating/Text/TokenParser.cs | 24 ++++++++++++++++--- .../Text/TemplateParser.LoopTests.cs | 17 ++++++++++--- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/src/Hyperbee.Templating/Text/TokenParser.cs b/src/Hyperbee.Templating/Text/TokenParser.cs index 81d225c..8b2ba7c 100644 --- a/src/Hyperbee.Templating/Text/TokenParser.cs +++ b/src/Hyperbee.Templating/Text/TokenParser.cs @@ -19,7 +19,7 @@ public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) { // token syntax: // - // {{token:definition}} + // {{token: definition}} // // {{token}} // {{x => x.token}} @@ -34,10 +34,28 @@ public TokenDefinition ParseToken( ReadOnlySpan token, int tokenId ) // {{while x => x.token}} // {{/while}} // - // {{each i:x => enumerable}} - // {{i}} + // {{each n[,i]: x => enumerable}} #1 + // {{n}} // {{/each}} + // {{each n[,i]: Person}} // person values or person[] fallback #2 + // {{n}} + // {{/each}} + + /* + x => x.Person* // rewrites to x => x.Enumerate( "Person[*]" ) #4 + x => x.Enumerate( regex ) #3 + + #5 is nesting + + Person[0].Name + Person[1].Name + + Person = "a,b,c" + Person = { "1", "1", "2" } + */ + + var span = token.Trim(); var tokenType = TokenType.Undefined; diff --git a/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs b/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs index 381d98b..e946175 100644 --- a/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs +++ b/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs @@ -13,7 +13,14 @@ public class TemplateParserLoopTests public void Should_honor_while_condition( ParseTemplateMethod parseMethod ) { // arrange - const string expression = "{{while x => int.Parse(x.counter) < 3}}{{counter}}{{counter:{{x => int.Parse(x.counter) + 1}}}}{{/while}}"; + const string expression = + """ + {{while x => int.Parse(x.counter) < 3}} + {{counter}} + {{counter:{{x => int.Parse(x.counter) + 1}}}} + {{/while}}" + """; + const string template = $"count: {expression}."; var parser = new TemplateParser @@ -39,8 +46,12 @@ public void Should_honor_while_condition( ParseTemplateMethod parseMethod ) public void Should_honor_each_expression( ParseTemplateMethod parseMethod ) { // arrange - - const string expression = "{{each n:x => x.list.Split( \",\" )}}World {{n}},{{/each}}"; + const string expression = + """ + {{each n:x => x.list.Split( \",\" )}} + World {{n}}, + {{/each}} + """; const string template = $"hello {expression}."; From c53377758d9309887132a3e7f0121aaa4d9ac845 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 15 Nov 2024 18:01:59 +0000 Subject: [PATCH 47/58] Updated code formatting to match rules in .editorconfig --- test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs b/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs index e946175..0bb24d8 100644 --- a/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs +++ b/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs @@ -46,7 +46,7 @@ public void Should_honor_while_condition( ParseTemplateMethod parseMethod ) public void Should_honor_each_expression( ParseTemplateMethod parseMethod ) { // arrange - const string expression = + const string expression = """ {{each n:x => x.list.Split( \",\" )}} World {{n}}, From 5b8c1aef2c3504d0bdf61c95bd8838c605254291 Mon Sep 17 00:00:00 2001 From: "annette.findley" Date: Mon, 16 Dec 2024 11:14:36 -0500 Subject: [PATCH 48/58] WIP another for each. --- .../Compiler/RoslynTokenExpressionProvider.cs | 36 ++++++++++-------- .../Text/TemplateParser.cs | 9 ++++- .../Text/TemplateParser.LoopTests.cs | 38 ++++++++++++------- 3 files changed, 53 insertions(+), 30 deletions(-) diff --git a/src/Hyperbee.Templating/Compiler/RoslynTokenExpressionProvider.cs b/src/Hyperbee.Templating/Compiler/RoslynTokenExpressionProvider.cs index 1ce11cf..b2b30e6 100644 --- a/src/Hyperbee.Templating/Compiler/RoslynTokenExpressionProvider.cs +++ b/src/Hyperbee.Templating/Compiler/RoslynTokenExpressionProvider.cs @@ -1,6 +1,5 @@ using System.Collections.Concurrent; using System.Collections.Immutable; -using System.Diagnostics; using System.Reflection; using System.Runtime.CompilerServices; using Hyperbee.Templating.Text; @@ -19,7 +18,8 @@ internal sealed class RoslynTokenExpressionProvider : ITokenExpressionProvider MetadataReference.CreateFromFile( typeof( object ).Assembly.Location.Replace( "System.Private.CoreLib", "System.Runtime" ) ), MetadataReference.CreateFromFile( typeof( RuntimeBinderException ).Assembly.Location ), MetadataReference.CreateFromFile( typeof( DynamicAttribute ).Assembly.Location ), - MetadataReference.CreateFromFile( typeof( RoslynTokenExpressionProvider ).Assembly.Location ) + MetadataReference.CreateFromFile( typeof( RoslynTokenExpressionProvider ).Assembly.Location ), + MetadataReference.CreateFromFile(typeof(System.Text.RegularExpressions.Regex).Assembly.Location) ]; private sealed class RuntimeContext( ImmutableArray metadataReferences ) @@ -49,19 +49,23 @@ private static TokenExpression Compile( string codeExpression ) { // Create a shim to compile the expression var codeShim = - $$""" - using Hyperbee.Templating.Text; - using Hyperbee.Templating.Compiler; - - public static class TokenExpressionInvoker - { - public static object Invoke( {{nameof( IReadOnlyMemberDictionary )}} members ) - { - TokenExpression expr = {{codeExpression}}; - return expr( members ); - } - } - """; + $$$""" + using Hyperbee.Templating.Text; + using Hyperbee.Templating.Compiler; + using System; + using System.Linq; + using System.Text.RegularExpressions; + + + public static class TokenExpressionInvoker + { + public static object Invoke( {{{nameof( IReadOnlyMemberDictionary )}}} members ) + { + TokenExpression expr = {{{codeExpression}}}; + return expr( members ); + } + } + """; // Parse the code expression var syntaxTree = CSharpSyntaxTree.ParseText( codeShim ); @@ -83,7 +87,7 @@ public static object Invoke( {{nameof( IReadOnlyMemberDictionary )}} members ) var rewriter = new TokenExpressionRewriter( parameterName ); var rewrittenSyntaxTree = rewriter.Visit( root ); - //var rewrittenCode = rewrittenSyntaxTree.ToFullString(); // Keep for debugging + var rewrittenCode = rewrittenSyntaxTree.ToFullString(); // Keep for debugging // Compile the rewritten code var counter = Interlocked.Increment( ref __counter ); diff --git a/src/Hyperbee.Templating/Text/TemplateParser.cs b/src/Hyperbee.Templating/Text/TemplateParser.cs index e71a7da..72c1064 100644 --- a/src/Hyperbee.Templating/Text/TemplateParser.cs +++ b/src/Hyperbee.Templating/Text/TemplateParser.cs @@ -418,6 +418,8 @@ private int IndexOfIgnoreQuotedContent( ReadOnlySpan span, ReadOnlySpan span, ReadOnlySpan span, ReadOnlySpan int.Parse(x.counter) < 3}} - {{counter}} - {{counter:{{x => int.Parse(x.counter) + 1}}}} - {{/while}}" - """; + const string expression = "{{while x => int.Parse(x.counter) < 3}}{{counter}}{{counter:{{x => int.Parse(x.counter) + 1}}}}{{/while}}"; const string template = $"count: {expression}."; @@ -46,12 +40,7 @@ public void Should_honor_while_condition( ParseTemplateMethod parseMethod ) public void Should_honor_each_expression( ParseTemplateMethod parseMethod ) { // arrange - const string expression = - """ - {{each n:x => x.list.Split( \",\" )}} - World {{n}}, - {{/each}} - """; + const string expression = "{{each n:x => x.list.Split( \",\" )}}World {{n}},{{/each}}"; const string template = $"hello {expression}."; @@ -65,4 +54,27 @@ public void Should_honor_each_expression( ParseTemplateMethod parseMethod ) Assert.AreEqual( expected, result ); } + + [DataTestMethod] + [DataRow( ParseTemplateMethod.Buffered )] + [DataRow( ParseTemplateMethod.InMemory )] + public void Should_honor_each_expression_2( ParseTemplateMethod parseMethod ) + { + // arrange + const string expression = "{{each n:x => x.Select( t => t.Value ).Where( t => t.Value == RegEx.IsMatch( people ))}}{{/each}}"; + + + const string template = $"hello {expression}."; + + var parser = new TemplateParser { Variables = { ["people"] = "{John, Jane, Doe}" } }; + + + // act + var result = parser.Render( template, parseMethod ); + + // assert + var expected = "hello Doe."; + + Assert.AreEqual( expected, result ); + } } From 644367dd37d45f6b1754032eb2c07fcdce1fc2c8 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 16 Dec 2024 16:15:32 +0000 Subject: [PATCH 49/58] Updated code formatting to match rules in .editorconfig --- .../Compiler/RoslynTokenExpressionProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Hyperbee.Templating/Compiler/RoslynTokenExpressionProvider.cs b/src/Hyperbee.Templating/Compiler/RoslynTokenExpressionProvider.cs index b2b30e6..c11ce57 100644 --- a/src/Hyperbee.Templating/Compiler/RoslynTokenExpressionProvider.cs +++ b/src/Hyperbee.Templating/Compiler/RoslynTokenExpressionProvider.cs @@ -19,7 +19,7 @@ internal sealed class RoslynTokenExpressionProvider : ITokenExpressionProvider MetadataReference.CreateFromFile( typeof( RuntimeBinderException ).Assembly.Location ), MetadataReference.CreateFromFile( typeof( DynamicAttribute ).Assembly.Location ), MetadataReference.CreateFromFile( typeof( RoslynTokenExpressionProvider ).Assembly.Location ), - MetadataReference.CreateFromFile(typeof(System.Text.RegularExpressions.Regex).Assembly.Location) + MetadataReference.CreateFromFile( typeof( System.Text.RegularExpressions.Regex ).Assembly.Location ) ]; private sealed class RuntimeContext( ImmutableArray metadataReferences ) From f21d004103e3b6e54606ab766eb9073e44b7b04c Mon Sep 17 00:00:00 2001 From: "annette.findley" Date: Mon, 16 Dec 2024 11:32:09 -0500 Subject: [PATCH 50/58] Updated rosyyn provider --- .../Compiler/RoslynTokenExpressionProvider.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Hyperbee.Templating/Compiler/RoslynTokenExpressionProvider.cs b/src/Hyperbee.Templating/Compiler/RoslynTokenExpressionProvider.cs index c11ce57..67cd2dd 100644 --- a/src/Hyperbee.Templating/Compiler/RoslynTokenExpressionProvider.cs +++ b/src/Hyperbee.Templating/Compiler/RoslynTokenExpressionProvider.cs @@ -48,6 +48,8 @@ public static void Reset() private static TokenExpression Compile( string codeExpression ) { // Create a shim to compile the expression + //AF: I added the linq and regular Expression usings + //AF: the error is in the Regex as it doesn't know what the people are. var codeShim = $$$""" using Hyperbee.Templating.Text; From 0c277b36e0f89fecf2a7fdc1ad656ab8869b6226 Mon Sep 17 00:00:00 2001 From: Brenton Farmer Date: Wed, 18 Dec 2024 12:24:49 -0800 Subject: [PATCH 51/58] compiler enumerator handling --- .../Compiler/RoslynTokenExpressionProvider.cs | 60 ++++++++++++++----- .../Compiler/TokenExpression.cs | 2 +- .../Core/EnumeratorAdapter.cs | 8 ++- src/Hyperbee.Templating/Core/KeyHelper.cs | 7 +++ .../Text/TokenProcessor.cs | 22 ++++++- .../RoslynTokenExpressionProviderTests.cs | 14 ++--- .../Text/TemplateParser.LoopTests.cs | 22 ++++--- .../Text/TemplateParser.MethodTests.cs | 2 +- 8 files changed, 103 insertions(+), 34 deletions(-) diff --git a/src/Hyperbee.Templating/Compiler/RoslynTokenExpressionProvider.cs b/src/Hyperbee.Templating/Compiler/RoslynTokenExpressionProvider.cs index 67cd2dd..5112a52 100644 --- a/src/Hyperbee.Templating/Compiler/RoslynTokenExpressionProvider.cs +++ b/src/Hyperbee.Templating/Compiler/RoslynTokenExpressionProvider.cs @@ -1,7 +1,10 @@ -using System.Collections.Concurrent; +using System.Collections; +using System.Collections.Concurrent; using System.Collections.Immutable; using System.Reflection; using System.Runtime.CompilerServices; +using System.Text.Json.Serialization; +using System.Text.RegularExpressions; using Hyperbee.Templating.Text; using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp; @@ -15,11 +18,14 @@ internal sealed class RoslynTokenExpressionProvider : ITokenExpressionProvider private static readonly ImmutableArray MetadataReferences = [ MetadataReference.CreateFromFile( typeof( object ).Assembly.Location ), - MetadataReference.CreateFromFile( typeof( object ).Assembly.Location.Replace( "System.Private.CoreLib", "System.Runtime" ) ), MetadataReference.CreateFromFile( typeof( RuntimeBinderException ).Assembly.Location ), MetadataReference.CreateFromFile( typeof( DynamicAttribute ).Assembly.Location ), MetadataReference.CreateFromFile( typeof( RoslynTokenExpressionProvider ).Assembly.Location ), - MetadataReference.CreateFromFile( typeof( System.Text.RegularExpressions.Regex ).Assembly.Location ) + MetadataReference.CreateFromFile( typeof( Regex ).Assembly.Location ), + MetadataReference.CreateFromFile( typeof( Enumerable ).Assembly.Location ), + + MetadataReference.CreateFromFile( typeof(object).Assembly.Location.Replace( "System.Private.CoreLib", "System.Runtime" ) ), + MetadataReference.CreateFromFile( typeof( IList ).Assembly.Location.Replace( "System.Private.CoreLib", "System.Collections" ) ) ]; private sealed class RuntimeContext( ImmutableArray metadataReferences ) @@ -35,9 +41,9 @@ private sealed class RuntimeContext( ImmutableArray metadataR new( OutputKind.DynamicallyLinkedLibrary, optimizationLevel: OptimizationLevel.Release ); [MethodImpl( MethodImplOptions.AggressiveInlining )] - public TokenExpression GetTokenExpression( string codeExpression ) + public TokenExpression GetTokenExpression( string codeExpression, MemberDictionary members ) { - return __runtimeContext.TokenExpressions.GetOrAdd( codeExpression, Compile ); + return __runtimeContext.TokenExpressions.GetOrAdd( codeExpression, Compile( codeExpression, members ) ); } public static void Reset() @@ -45,19 +51,20 @@ public static void Reset() __runtimeContext = new RuntimeContext( MetadataReferences ); } - private static TokenExpression Compile( string codeExpression ) + private static TokenExpression Compile( string codeExpression, MemberDictionary members ) { // Create a shim to compile the expression //AF: I added the linq and regular Expression usings //AF: the error is in the Regex as it doesn't know what the people are. var codeShim = $$$""" - using Hyperbee.Templating.Text; - using Hyperbee.Templating.Compiler; using System; using System.Linq; using System.Text.RegularExpressions; - + using System.Collections; + using System.Collections.Generic; + using Hyperbee.Templating.Text; + using Hyperbee.Templating.Compiler; public static class TokenExpressionInvoker { @@ -86,7 +93,7 @@ public static object Invoke( {{{nameof( IReadOnlyMemberDictionary )}}} members ) }; // Rewrite the lambda expression to use the dictionary lookup - var rewriter = new TokenExpressionRewriter( parameterName ); + var rewriter = new TokenExpressionRewriter( parameterName, members ); var rewrittenSyntaxTree = rewriter.Visit( root ); var rewrittenCode = rewrittenSyntaxTree.ToFullString(); // Keep for debugging @@ -107,9 +114,9 @@ public static object Invoke( {{{nameof( IReadOnlyMemberDictionary )}}} members ) { var failures = result.Diagnostics.Where( diagnostic => diagnostic.IsWarningAsError || - diagnostic.Severity == DiagnosticSeverity.Error ); + diagnostic.Severity == DiagnosticSeverity.Error ).ToArray(); - throw new InvalidOperationException( "Compilation failed: " + string.Join( "\n", failures.Select( diagnostic => diagnostic.GetMessage() ) ) ); + throw new TokenExpressionProviderException( "Compilation failed: " + failures[0]?.GetMessage(), failures ); } peStream.Seek( 0, SeekOrigin.Begin ); @@ -124,6 +131,26 @@ public static object Invoke( {{{nameof( IReadOnlyMemberDictionary )}}} members ) } } +[Serializable] +internal class TokenExpressionProviderException : Exception +{ + public Diagnostic[] Diagnostic { get; } + public string Id => Diagnostic != null && Diagnostic.Length > 0 ? Diagnostic[0].Id : string.Empty; + + public TokenExpressionProviderException( string message, Diagnostic[] diagnostic ) + : base( message) + { + Diagnostic = diagnostic; + } + + public TokenExpressionProviderException( string message, Diagnostic[] diagnostic, Exception innerException ) + : base( message, innerException ) + { + Diagnostic = diagnostic; + } + +} + // This rewriter will transform the lambda expression to use dictionary lookup // for property access, method invocation, and 'generic' property casting. // @@ -133,7 +160,7 @@ public static object Invoke( {{{nameof( IReadOnlyMemberDictionary )}}} members ) // 2. x => x.someProp to x.GetValueAs("someProp") // 3. x => x.someMethod(..) to x.Invoke("someMethod", ..) -internal class TokenExpressionRewriter( string parameterName ) : CSharpSyntaxRewriter +internal class TokenExpressionRewriter( string parameterName, MemberDictionary members ) : CSharpSyntaxRewriter { private readonly HashSet _aliases = [parameterName]; @@ -157,7 +184,12 @@ memberAccess.Expression is IdentifierNameSyntax identifier && _aliases.Contains( identifier.Identifier.Text ) ) { // Handle method invocation rewrite - return RewriteMethodInvocation( memberAccess, node ); + + if ( members.Methods.ContainsKey( memberAccess.Name.Identifier.Text ) ) + return RewriteMethodInvocation( memberAccess, node ); + + return node.Update( node.Expression, (ArgumentListSyntax) VisitArgumentList( node.ArgumentList )! ); + //return RewriteMethodInvocation( memberAccess, node ); //BF this rewrite causes the error. we need to disambiguate calls to template lambdas methods } return base.VisitInvocationExpression( node ); diff --git a/src/Hyperbee.Templating/Compiler/TokenExpression.cs b/src/Hyperbee.Templating/Compiler/TokenExpression.cs index 67a1e2e..3bb0132 100644 --- a/src/Hyperbee.Templating/Compiler/TokenExpression.cs +++ b/src/Hyperbee.Templating/Compiler/TokenExpression.cs @@ -6,5 +6,5 @@ namespace Hyperbee.Templating.Compiler; public interface ITokenExpressionProvider { - public TokenExpression GetTokenExpression( string codeExpression ); + public TokenExpression GetTokenExpression( string codeExpression, MemberDictionary members ); } diff --git a/src/Hyperbee.Templating/Core/EnumeratorAdapter.cs b/src/Hyperbee.Templating/Core/EnumeratorAdapter.cs index 142cd45..8304153 100644 --- a/src/Hyperbee.Templating/Core/EnumeratorAdapter.cs +++ b/src/Hyperbee.Templating/Core/EnumeratorAdapter.cs @@ -8,8 +8,14 @@ internal sealed class EnumeratorAdapter : IEnumerator internal EnumeratorAdapter( IEnumerable enumerable ) { + if ( enumerable is not IEnumerable typedEnumerable ) + throw new ArgumentException( "The enumerable must be of type IEnumerable.", nameof( enumerable ) ); + + // take a snapshot of the enumerable to prevent changes during enumeration + var snapshot = new List( typedEnumerable ); + // ReSharper disable once GenericEnumeratorNotDisposed - _inner = enumerable?.GetEnumerator() ?? throw new ArgumentNullException( nameof( enumerable ) ); + _inner = snapshot.GetEnumerator(); } public string Current => (string) _inner.Current; diff --git a/src/Hyperbee.Templating/Core/KeyHelper.cs b/src/Hyperbee.Templating/Core/KeyHelper.cs index 8c76867..282779b 100644 --- a/src/Hyperbee.Templating/Core/KeyHelper.cs +++ b/src/Hyperbee.Templating/Core/KeyHelper.cs @@ -16,6 +16,13 @@ public static bool ValidateKey( string key ) return ValidateKey( key.AsSpan() ); } + // KeyScanner + // -Start + // Identifier + // -ArrayStart + // ArrayDigit + // -ArrayEnd + public static bool ValidateKey( ReadOnlySpan key ) { if ( key.IsEmpty || !char.IsLetter( key[0] ) ) diff --git a/src/Hyperbee.Templating/Text/TokenProcessor.cs b/src/Hyperbee.Templating/Text/TokenProcessor.cs index ca24655..008e98e 100644 --- a/src/Hyperbee.Templating/Text/TokenProcessor.cs +++ b/src/Hyperbee.Templating/Text/TokenProcessor.cs @@ -321,14 +321,32 @@ private bool TryInvokeTokenExpression( TokenDefinition token, out object result, { try { - var tokenExpression = _tokenExpressionProvider.GetTokenExpression( token.TokenExpression ); + var tokenExpression = _tokenExpressionProvider.GetTokenExpression( token.TokenExpression, _members ); result = tokenExpression( _members ); error = null; return true; } catch ( Exception ex ) { - error = ex.Message; + if ( ex is TokenExpressionProviderException providerException && providerException.Id == "CS1061" ) + { + string methodName = null; + + if ( providerException.Diagnostic.Length > 0 ) + { + var location = providerException.Diagnostic[0].Location; + var sourceTree = location.SourceTree; + var sourceSpan = location.SourceSpan; + methodName = sourceTree?.ToString().Substring( sourceSpan.Start, sourceSpan.Length ); + } + + error = $"Method '{methodName ?? ""}' not found."; + } + else + { + error = ex.Message; + } + result = null; return false; } diff --git a/test/Hyperbee.Templating.Tests/Compiler/RoslynTokenExpressionProviderTests.cs b/test/Hyperbee.Templating.Tests/Compiler/RoslynTokenExpressionProviderTests.cs index da32f1c..ab7e50c 100644 --- a/test/Hyperbee.Templating.Tests/Compiler/RoslynTokenExpressionProviderTests.cs +++ b/test/Hyperbee.Templating.Tests/Compiler/RoslynTokenExpressionProviderTests.cs @@ -22,8 +22,8 @@ public void Should_compile_expression() ["Value"] = "base" }; - var tokenExpression = compiler.GetTokenExpression( expression ); var variables = new MemberDictionary( tokens ); + var tokenExpression = compiler.GetTokenExpression( expression, variables ); // act @@ -48,8 +48,8 @@ public void Should_compile_cast_expression() ["Value"] = "1" }; - var tokenExpression = compiler.GetTokenExpression( expression ); var variables = new MemberDictionary( tokens ); + var tokenExpression = compiler.GetTokenExpression( expression, variables ); // act @@ -74,9 +74,9 @@ public void Should_compile_statement_expression() ["Value"] = "base" }; - var tokenExpression = compiler.GetTokenExpression( expression ); var variables = new MemberDictionary( tokens ); - + var tokenExpression = compiler.GetTokenExpression( expression, variables ); + // act var result = tokenExpression( variables ); @@ -98,11 +98,11 @@ public void Should_compile_multiple_expressions() var tokens = new Dictionary { ["Value"] = "base" }; - var tokenExpression1 = compiler.GetTokenExpression( expression1 ); - var tokenExpression2 = compiler.GetTokenExpression( expression2 ); - var variables = new MemberDictionary( tokens ); + var tokenExpression1 = compiler.GetTokenExpression( expression1, variables ); + var tokenExpression2 = compiler.GetTokenExpression( expression2, variables ); + // act var result1 = tokenExpression1( variables ); diff --git a/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs b/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs index 0fcc2a8..b5ba5ba 100644 --- a/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs +++ b/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs @@ -61,19 +61,25 @@ public void Should_honor_each_expression( ParseTemplateMethod parseMethod ) public void Should_honor_each_expression_2( ParseTemplateMethod parseMethod ) { // arrange - const string expression = "{{each n:x => x.Select( t => t.Value ).Where( t => t.Value == RegEx.IsMatch( people ))}}{{/each}}"; - - - const string template = $"hello {expression}."; - - var parser = new TemplateParser { Variables = { ["people"] = "{John, Jane, Doe}" } }; - + const string expression = "{{each n:x => x.Where( t => Regex.IsMatch( t.Key, \"people*\" ) ).Select( t => t.Value )}}hello {{n}}. {{/each}}"; + + const string template = $"{expression}"; + + var parser = new TemplateParser + { + Variables = //BF : we want the key validator to allow `[#]`, and `[#].` in the key + { + ["people0"] = "John", + ["people1"] = "Jane", + ["people2"] = "Doe" + } + }; // act var result = parser.Render( template, parseMethod ); // assert - var expected = "hello Doe."; + var expected = "hello John. hello Jane. hello Doe. "; Assert.AreEqual( expected, result ); } diff --git a/test/Hyperbee.Templating.Tests/Text/TemplateParser.MethodTests.cs b/test/Hyperbee.Templating.Tests/Text/TemplateParser.MethodTests.cs index 6cc8970..0b07aae 100644 --- a/test/Hyperbee.Templating.Tests/Text/TemplateParser.MethodTests.cs +++ b/test/Hyperbee.Templating.Tests/Text/TemplateParser.MethodTests.cs @@ -85,7 +85,7 @@ public void Should_not_replace_token_when_method_is_missing( ParseTemplateMethod // assert - var expected = template.Replace( expression, "{{Error (1):Failed to invoke method 'missing'.}}" ); + var expected = template.Replace( expression, "{{Error (1):Method 'missing' not found.}}" ); Assert.AreEqual( expected, result ); } From 8c01375259039c62dd153dcc8b6f7ac142f175de Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 18 Dec 2024 20:25:26 +0000 Subject: [PATCH 52/58] Updated code formatting to match rules in .editorconfig --- .../Compiler/RoslynTokenExpressionProvider.cs | 4 ++-- src/Hyperbee.Templating/Text/TokenProcessor.cs | 2 +- .../Compiler/RoslynTokenExpressionProviderTests.cs | 2 +- .../Text/TemplateParser.LoopTests.cs | 14 +++++++------- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Hyperbee.Templating/Compiler/RoslynTokenExpressionProvider.cs b/src/Hyperbee.Templating/Compiler/RoslynTokenExpressionProvider.cs index 5112a52..fb81598 100644 --- a/src/Hyperbee.Templating/Compiler/RoslynTokenExpressionProvider.cs +++ b/src/Hyperbee.Templating/Compiler/RoslynTokenExpressionProvider.cs @@ -24,7 +24,7 @@ internal sealed class RoslynTokenExpressionProvider : ITokenExpressionProvider MetadataReference.CreateFromFile( typeof( Regex ).Assembly.Location ), MetadataReference.CreateFromFile( typeof( Enumerable ).Assembly.Location ), - MetadataReference.CreateFromFile( typeof(object).Assembly.Location.Replace( "System.Private.CoreLib", "System.Runtime" ) ), + MetadataReference.CreateFromFile( typeof( object ).Assembly.Location.Replace( "System.Private.CoreLib", "System.Runtime" ) ), MetadataReference.CreateFromFile( typeof( IList ).Assembly.Location.Replace( "System.Private.CoreLib", "System.Collections" ) ) ]; @@ -138,7 +138,7 @@ internal class TokenExpressionProviderException : Exception public string Id => Diagnostic != null && Diagnostic.Length > 0 ? Diagnostic[0].Id : string.Empty; public TokenExpressionProviderException( string message, Diagnostic[] diagnostic ) - : base( message) + : base( message ) { Diagnostic = diagnostic; } diff --git a/src/Hyperbee.Templating/Text/TokenProcessor.cs b/src/Hyperbee.Templating/Text/TokenProcessor.cs index 008e98e..25d2ffb 100644 --- a/src/Hyperbee.Templating/Text/TokenProcessor.cs +++ b/src/Hyperbee.Templating/Text/TokenProcessor.cs @@ -339,7 +339,7 @@ private bool TryInvokeTokenExpression( TokenDefinition token, out object result, var sourceSpan = location.SourceSpan; methodName = sourceTree?.ToString().Substring( sourceSpan.Start, sourceSpan.Length ); } - + error = $"Method '{methodName ?? ""}' not found."; } else diff --git a/test/Hyperbee.Templating.Tests/Compiler/RoslynTokenExpressionProviderTests.cs b/test/Hyperbee.Templating.Tests/Compiler/RoslynTokenExpressionProviderTests.cs index ab7e50c..c64c953 100644 --- a/test/Hyperbee.Templating.Tests/Compiler/RoslynTokenExpressionProviderTests.cs +++ b/test/Hyperbee.Templating.Tests/Compiler/RoslynTokenExpressionProviderTests.cs @@ -76,7 +76,7 @@ public void Should_compile_statement_expression() var variables = new MemberDictionary( tokens ); var tokenExpression = compiler.GetTokenExpression( expression, variables ); - + // act var result = tokenExpression( variables ); diff --git a/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs b/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs index b5ba5ba..f0e1c07 100644 --- a/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs +++ b/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs @@ -65,14 +65,14 @@ public void Should_honor_each_expression_2( ParseTemplateMethod parseMethod ) const string template = $"{expression}"; - var parser = new TemplateParser - { + var parser = new TemplateParser + { Variables = //BF : we want the key validator to allow `[#]`, and `[#].` in the key - { - ["people0"] = "John", - ["people1"] = "Jane", - ["people2"] = "Doe" - } + { + ["people0"] = "John", + ["people1"] = "Jane", + ["people2"] = "Doe" + } }; // act From 888ada3b3b1e7d2a5e3706bc82bdf973d97eeb6c Mon Sep 17 00:00:00 2001 From: "annette.findley" Date: Thu, 19 Dec 2024 07:08:11 -0500 Subject: [PATCH 53/58] Added Bracket [] to Key Validator and test, updated docs --- docs/syntax/examples.md | 50 +++++++++++++++++++ docs/syntax/overview.md | 6 +++ docs/syntax/syntax.md | 17 ++++++- src/Hyperbee.Templating/Core/KeyHelper.cs | 27 +++++++++- .../Text/TemplateParser.LoopTests.cs | 33 +++++++++++- 5 files changed, 128 insertions(+), 5 deletions(-) diff --git a/docs/syntax/examples.md b/docs/syntax/examples.md index f794e3e..f154d2a 100644 --- a/docs/syntax/examples.md +++ b/docs/syntax/examples.md @@ -116,6 +116,56 @@ var result = parser.Render(template); Console.WriteLine(result); // Output: 012. ``` +### Each Statement + +```csharp +var template = "{{each n:x => x.list.Split( \",\" )}}World {{n}},{{/each}}"; + +var parser = new TemplateParser +{ + Variables = { ["list"] = "John,James,Sarah" } +}; + +var result = parser.Render(template); +Console.WriteLine(result); // hello World John,World James,World Sarah,. +``` + +```csharp + +var template = "{{each n:x => x.Where( t => Regex.IsMatch( t.Key, \"people*\" ) ).Select( t => t.Value )}}hello {{n}}. {{/each}}"; + +var parser = new TemplateParser +{ + Variables = + { + ["people0"] = "John", + ["people1"] = "Jane", + ["people2"] = "Doe" + } +}; + +var result = parser.Render(template); +Console.WriteLine(result); // hello John. hello Jane. hello Doe. +``` + +```csharp + +var template = " "{{each n:x => x.Where( t => Regex.IsMatch( t.Key, \"people*\" ) ).Select( t => t.Value )}}hello {{n}}. {{/each}}""; + +var parser = new TemplateParser +{ + Variables = + { + ["people[0]"] = "John", + ["people[1]"] = "Jane", + ["people[2]"] = "Doe" + } +}; + +var result = parser.Render(template); +Console.WriteLine(result); // hello John. hello Jane. hello Doe. +``` + ## Inline Definitions ```csharp diff --git a/docs/syntax/overview.md b/docs/syntax/overview.md index 8caa4dc..28060a5 100644 --- a/docs/syntax/overview.md +++ b/docs/syntax/overview.md @@ -64,6 +64,12 @@ The `while` statement repeats a template block while a condition is true. `{{ while condition }} ... {{ /while }}` +### Each Statement + +The `each` statement repeats a template block for each a condition. + +`{{ each condition }} ... {{ /each }}` + ## Inline Declarations You can declare variable tokens inline within the template. diff --git a/docs/syntax/syntax.md b/docs/syntax/syntax.md index 6dfb234..09b2a78 100644 --- a/docs/syntax/syntax.md +++ b/docs/syntax/syntax.md @@ -7,5 +7,18 @@ nav_order: 1 # Templating Syntax -Hyperbee Templating provides a variety of token syntaxes for different use cases. This section provides a -guide to to the available syntax forms. +Hyperbee Templating is a lightweight templating and variable substitution syntax engine. The library supports value replacements, +code expressions, token nesting, in-line definitions, conditional flow, and looping. It is designed to be lightweight and fast, +and does not rely on any external dependencies. + +## Features + +* Variable substitution syntax engine +* Value replacements +* Expression replacements +* Token nesting +* Conditional tokens +* Conditional flow +* Iterators +* User-defined methods + diff --git a/src/Hyperbee.Templating/Core/KeyHelper.cs b/src/Hyperbee.Templating/Core/KeyHelper.cs index 282779b..fc4dca1 100644 --- a/src/Hyperbee.Templating/Core/KeyHelper.cs +++ b/src/Hyperbee.Templating/Core/KeyHelper.cs @@ -28,10 +28,35 @@ public static bool ValidateKey( ReadOnlySpan key ) if ( key.IsEmpty || !char.IsLetter( key[0] ) ) return false; + for ( var i = 1; i < key.Length; i++ ) { - if ( !char.IsLetterOrDigit( key[i] ) && key[i] != '_' ) + if ( key[i] == '[' ) + { + i++; + if ( i >= key.Length || !char.IsDigit( key[i] ) ) + return false; + + int numberStart = i; + + while ( i < key.Length && char.IsDigit( key[i] ) ) + i++; + + if ( i >= key.Length || key[i] != ']' ) + return false; + + // Ensure that the bracket is at the end of the string + if ( i != key.Length - 1 ) + return false; + + // Ensure that the number inside the brackets is positive + if ( int.Parse( key.Slice( numberStart, i - numberStart ) ) <= -1 ) + return false; + } + else if ( !char.IsLetterOrDigit( key[i] ) && key[i] != '_' ) + { return false; + } } return true; diff --git a/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs b/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs index f0e1c07..f30442b 100644 --- a/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs +++ b/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs @@ -58,7 +58,7 @@ public void Should_honor_each_expression( ParseTemplateMethod parseMethod ) [DataTestMethod] [DataRow( ParseTemplateMethod.Buffered )] [DataRow( ParseTemplateMethod.InMemory )] - public void Should_honor_each_expression_2( ParseTemplateMethod parseMethod ) + public void Should_honor_each_expression_RegEx( ParseTemplateMethod parseMethod ) { // arrange const string expression = "{{each n:x => x.Where( t => Regex.IsMatch( t.Key, \"people*\" ) ).Select( t => t.Value )}}hello {{n}}. {{/each}}"; @@ -67,7 +67,7 @@ public void Should_honor_each_expression_2( ParseTemplateMethod parseMethod ) var parser = new TemplateParser { - Variables = //BF : we want the key validator to allow `[#]`, and `[#].` in the key + Variables = //TODO : we want the key validator to allow `[#].` in the key { ["people0"] = "John", ["people1"] = "Jane", @@ -83,4 +83,33 @@ public void Should_honor_each_expression_2( ParseTemplateMethod parseMethod ) Assert.AreEqual( expected, result ); } + + [DataTestMethod] + [DataRow( ParseTemplateMethod.Buffered )] + [DataRow( ParseTemplateMethod.InMemory )] + public void Should_honor_each_Key( ParseTemplateMethod parseMethod ) + { + // arrange + const string expression = "{{each n:x => x.Where( t => Regex.IsMatch( t.Key, \"people*\" ) ).Select( t => t.Value )}}hello {{n}}. {{/each}}"; + + const string template = $"{expression}"; + + var parser = new TemplateParser + { + Variables = + { + ["people[0]"] = "John", + ["people[1]"] = "Jane", + ["people[2]"] = "Doe" + } + }; + + // act + var result = parser.Render( template, parseMethod ); + + // assert + var expected = "hello John. hello Jane. hello Doe. "; + + Assert.AreEqual( expected, result ); + } } From d39e28d4b7868add8c756c2a9c0525b050c78219 Mon Sep 17 00:00:00 2001 From: "annette.findley" Date: Thu, 19 Dec 2024 08:41:49 -0500 Subject: [PATCH 54/58] Updated nugets --- src/Hyperbee.Templating/Hyperbee.Templating.csproj | 2 +- .../Hyperbee.Templating.Tests.csproj | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Hyperbee.Templating/Hyperbee.Templating.csproj b/src/Hyperbee.Templating/Hyperbee.Templating.csproj index 3cd9c17..2291a27 100644 --- a/src/Hyperbee.Templating/Hyperbee.Templating.csproj +++ b/src/Hyperbee.Templating/Hyperbee.Templating.csproj @@ -43,6 +43,6 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + \ No newline at end of file diff --git a/test/Hyperbee.Templating.Tests/Hyperbee.Templating.Tests.csproj b/test/Hyperbee.Templating.Tests/Hyperbee.Templating.Tests.csproj index 9c32c3a..a815591 100644 --- a/test/Hyperbee.Templating.Tests/Hyperbee.Templating.Tests.csproj +++ b/test/Hyperbee.Templating.Tests/Hyperbee.Templating.Tests.csproj @@ -13,9 +13,9 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive From 2adfe2f6da0bf529842a1ee69523e119c447ea86 Mon Sep 17 00:00:00 2001 From: "annette.findley" Date: Thu, 19 Dec 2024 08:53:50 -0500 Subject: [PATCH 55/58] Updated readme file --- README.md | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/README.md b/README.md index 557ca25..cfe2dd0 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,56 @@ var result = parser.Render(template); Console.WriteLine(result); // Output: 012. ``` +### Each Statement + +```csharp +var template = "{{each n:x => x.list.Split( \",\" )}}World {{n}},{{/each}}"; + +var parser = new TemplateParser +{ + Variables = { ["list"] = "John,James,Sarah" } +}; + +var result = parser.Render(template); +Console.WriteLine(result); // hello World John,World James,World Sarah,. +``` + +```csharp + +var template = "{{each n:x => x.Where( t => Regex.IsMatch( t.Key, \"people*\" ) ).Select( t => t.Value )}}hello {{n}}. {{/each}}"; + +var parser = new TemplateParser +{ + Variables = + { + ["people0"] = "John", + ["people1"] = "Jane", + ["people2"] = "Doe" + } +}; + +var result = parser.Render(template); +Console.WriteLine(result); // hello John. hello Jane. hello Doe. +``` + +```csharp + +var template = " "{{each n:x => x.Where( t => Regex.IsMatch( t.Key, \"people*\" ) ).Select( t => t.Value )}}hello {{n}}. {{/each}}""; + +var parser = new TemplateParser +{ + Variables = + { + ["people[0]"] = "John", + ["people[1]"] = "Jane", + ["people[2]"] = "Doe" + } +}; + +var result = parser.Render(template); +Console.WriteLine(result); // hello John. hello Jane. hello Doe. +``` + ### Methods You can invoke methods within token expressions. From 9035f5b5414e8e71e477e9a9039d33dc7c77204a Mon Sep 17 00:00:00 2001 From: Brenton Farmer Date: Thu, 19 Dec 2024 07:58:24 -0800 Subject: [PATCH 56/58] small clean up --- src/Hyperbee.Templating/Core/KeyHelper.cs | 33 +++++++------------ .../Text/TemplateParser.LoopTests.cs | 8 ++--- 2 files changed, 16 insertions(+), 25 deletions(-) diff --git a/src/Hyperbee.Templating/Core/KeyHelper.cs b/src/Hyperbee.Templating/Core/KeyHelper.cs index fc4dca1..d7bee63 100644 --- a/src/Hyperbee.Templating/Core/KeyHelper.cs +++ b/src/Hyperbee.Templating/Core/KeyHelper.cs @@ -16,44 +16,35 @@ public static bool ValidateKey( string key ) return ValidateKey( key.AsSpan() ); } - // KeyScanner - // -Start - // Identifier - // -ArrayStart - // ArrayDigit - // -ArrayEnd - public static bool ValidateKey( ReadOnlySpan key ) { if ( key.IsEmpty || !char.IsLetter( key[0] ) ) + { return false; + } + var length = key.Length; - for ( var i = 1; i < key.Length; i++ ) + for ( var i = 1; i < length; i++ ) { - if ( key[i] == '[' ) + var current = key[i]; + + if ( current == '[' ) { - i++; - if ( i >= key.Length || !char.IsDigit( key[i] ) ) + if ( ++i >= length || !char.IsDigit( key[i] ) ) return false; - int numberStart = i; - - while ( i < key.Length && char.IsDigit( key[i] ) ) + while ( i < length && char.IsDigit( key[i] ) ) i++; - if ( i >= key.Length || key[i] != ']' ) + if ( i >= length || key[i] != ']' ) return false; // Ensure that the bracket is at the end of the string - if ( i != key.Length - 1 ) - return false; - - // Ensure that the number inside the brackets is positive - if ( int.Parse( key.Slice( numberStart, i - numberStart ) ) <= -1 ) + if ( i != length - 1 ) return false; } - else if ( !char.IsLetterOrDigit( key[i] ) && key[i] != '_' ) + else if ( !char.IsLetterOrDigit( current ) && current != '_' ) { return false; } diff --git a/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs b/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs index f30442b..e6fdd68 100644 --- a/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs +++ b/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs @@ -67,11 +67,11 @@ public void Should_honor_each_expression_RegEx( ParseTemplateMethod parseMethod var parser = new TemplateParser { - Variables = //TODO : we want the key validator to allow `[#].` in the key + Variables = { - ["people0"] = "John", - ["people1"] = "Jane", - ["people2"] = "Doe" + ["people[0]"] = "John", + ["people[1]"] = "Jane", + ["people[2]"] = "Doe" } }; From 376f1ab36dbfd02108b5e59b0bdbe6809c829c20 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 19 Dec 2024 15:59:00 +0000 Subject: [PATCH 57/58] Updated code formatting to match rules in .editorconfig --- test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs b/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs index e6fdd68..abc9972 100644 --- a/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs +++ b/test/Hyperbee.Templating.Tests/Text/TemplateParser.LoopTests.cs @@ -67,7 +67,7 @@ public void Should_honor_each_expression_RegEx( ParseTemplateMethod parseMethod var parser = new TemplateParser { - Variables = + Variables = { ["people[0]"] = "John", ["people[1]"] = "Jane", From 0e59cbf50da938e0d116b44d9e17262828004f89 Mon Sep 17 00:00:00 2001 From: Brenton Farmer Date: Thu, 19 Dec 2024 08:01:51 -0800 Subject: [PATCH 58/58] fix doc duplicate example --- README.md | 18 ------------------ docs/syntax/examples.md | 18 ------------------ 2 files changed, 36 deletions(-) diff --git a/README.md b/README.md index cfe2dd0..e7d81b0 100644 --- a/README.md +++ b/README.md @@ -159,24 +159,6 @@ Console.WriteLine(result); // hello World John,World James,World Sarah,. var template = "{{each n:x => x.Where( t => Regex.IsMatch( t.Key, \"people*\" ) ).Select( t => t.Value )}}hello {{n}}. {{/each}}"; -var parser = new TemplateParser -{ - Variables = - { - ["people0"] = "John", - ["people1"] = "Jane", - ["people2"] = "Doe" - } -}; - -var result = parser.Render(template); -Console.WriteLine(result); // hello John. hello Jane. hello Doe. -``` - -```csharp - -var template = " "{{each n:x => x.Where( t => Regex.IsMatch( t.Key, \"people*\" ) ).Select( t => t.Value )}}hello {{n}}. {{/each}}""; - var parser = new TemplateParser { Variables = diff --git a/docs/syntax/examples.md b/docs/syntax/examples.md index f154d2a..0de4151 100644 --- a/docs/syntax/examples.md +++ b/docs/syntax/examples.md @@ -134,24 +134,6 @@ Console.WriteLine(result); // hello World John,World James,World Sarah,. var template = "{{each n:x => x.Where( t => Regex.IsMatch( t.Key, \"people*\" ) ).Select( t => t.Value )}}hello {{n}}. {{/each}}"; -var parser = new TemplateParser -{ - Variables = - { - ["people0"] = "John", - ["people1"] = "Jane", - ["people2"] = "Doe" - } -}; - -var result = parser.Render(template); -Console.WriteLine(result); // hello John. hello Jane. hello Doe. -``` - -```csharp - -var template = " "{{each n:x => x.Where( t => Regex.IsMatch( t.Key, \"people*\" ) ).Select( t => t.Value )}}hello {{n}}. {{/each}}""; - var parser = new TemplateParser { Variables =