Propagate regex comments to source-generated code

src/libraries/System.Text.RegularExpressions/gen/RegexGenerator.Emitter.cs

@@ -5804,6 +5804,16 @@ RegexNodeKind.BackreferenceConditional when node.Parent.Child(1) == node => "Not
 
                     string nodeDescription = DescribeNode(node, rm);
 
+                    // Write out any comments associated with this node.
+                    if (rm.Tree.NodeComments?.TryGetValue(node, out List<string>? comments) == true)
+                    {
+                        string indent = new string(' ', depth * 4);
+                        foreach (string comment in comments)
+                        {
+                            writer.WriteLine($"/// {indent}// {EscapeXmlComment(comment)}<br/>");
+                        }
+                    }
+
                     // Write out the line for the node.
                     const char BulletPoint = '\u25CB';
                     writer.WriteLine($"/// {new string(' ', depth * 4)}{BulletPoint} {tag}{EscapeXmlComment(nodeDescription)}<br/>");

src/libraries/System.Text.RegularExpressions/gen/RegexGenerator.cs

@@ -75,7 +75,7 @@ public void Initialize(IncrementalGeneratorInitializationContext context)
                     {
                         try
                         {
-                            RegexTree regexTree = RegexParser.Parse(method.Pattern, method.Options | RegexOptions.Compiled, method.Culture); // make sure Compiled is included to get all optimizations applied to it
+                            RegexTree regexTree = RegexParser.ParseForSourceGenerator(method.Pattern, method.Options | RegexOptions.Compiled, method.Culture); // make sure Compiled is included to get all optimizations applied to it
                             AnalysisResults analysis = RegexTreeAnalyzer.Analyze(regexTree);
                             return new RegexMethod(method.DeclaringType, method.IsProperty, method.DiagnosticLocation, method.MemberName, method.Modifiers, method.NullableRegex, method.Pattern, method.Options, method.MatchTimeout, regexTree, analysis, method.CompilationData);
                         }

src/libraries/System.Text.RegularExpressions/src/AssemblyInfo.cs

@@ -0,0 +1,6 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Runtime.CompilerServices;
+
+[assembly: InternalsVisibleTo("System.Text.RegularExpressions.Tests")]

src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexParser.cs

@@ -52,7 +52,11 @@ internal ref struct RegexParser
 
         private bool _ignoreNextParen; // flag to skip capturing a parentheses group
 
-        private RegexParser(string pattern, RegexOptions options, CultureInfo culture, Hashtable caps, int capsize, Hashtable? capnames, Span<int> optionSpan)
+        private bool _captureComments; // flag to enable comment capture for source generator
+        private Dictionary<RegexNode, List<string>>? _nodeComments; // side-channel for storing comments associated with nodes
+        private List<string>? _pendingComments; // comments waiting to be associated with the next node
+
+        private RegexParser(string pattern, RegexOptions options, CultureInfo culture, Hashtable caps, int capsize, Hashtable? capnames, Span<int> optionSpan, bool captureComments = false)
         {
             Debug.Assert(pattern != null, "Pattern must be set");
             Debug.Assert(culture != null, "Culture must be set");
@@ -79,6 +83,10 @@ private RegexParser(string pattern, RegexOptions options, CultureInfo culture, H
             _capnumlist = null;
             _capnamelist = null;
             _ignoreNextParen = false;
+
+            _captureComments = captureComments;
+            _nodeComments = captureComments ? new Dictionary<RegexNode, List<string>>() : null;
+            _pendingComments = captureComments ? new List<string>() : null;
         }
 
         /// <summary>Gets the culture to use based on the specified options.</summary>
@@ -102,7 +110,7 @@ public static RegexOptions ParseOptionsInPattern(string pattern, RegexOptions op
 
         public static RegexTree Parse(string pattern, RegexOptions options, CultureInfo culture)
         {
-            using var parser = new RegexParser(pattern, options, culture, new Hashtable(), 0, null, stackalloc int[OptionStackDefaultSize]);
+            using var parser = new RegexParser(pattern, options, culture, new Hashtable(), 0, null, stackalloc int[OptionStackDefaultSize], captureComments: false);
 
             parser.CountCaptures(out _);
             parser.Reset(options);
@@ -133,6 +141,40 @@ public static RegexTree Parse(string pattern, RegexOptions options, CultureInfo
             return new RegexTree(root, captureCount, parser._capnamelist?.ToArray(), parser._capnames!, sparseMapping, options, parser._hasIgnoreCaseBackreferenceNodes ? culture : null);
         }
 
+        /// <summary>Parses a regular expression and captures comments for source generation.</summary>
+        public static RegexTree ParseForSourceGenerator(string pattern, RegexOptions options, CultureInfo culture)
+        {
+            using var parser = new RegexParser(pattern, options, culture, new Hashtable(), 0, null, stackalloc int[OptionStackDefaultSize], captureComments: true);
+
+            parser.CountCaptures(out _);
+            parser.Reset(options);
+            RegexNode root = parser.ScanRegex();
+
+            int[]? captureNumberList = parser._capnumlist;
+            Hashtable? sparseMapping = parser._caps;
+            int captop = parser._captop;
+
+            int captureCount;
+            if (captureNumberList == null || captop == captureNumberList.Length)
+            {
+                // The capture list isn't sparse.  Null out the capture mapping as it's not necessary,
+                // and store the number of captures.
+                captureCount = captop;
+                sparseMapping = null;
+            }
+            else
+            {
+                // The capture list is sparse.  Store the number of captures, and populate the number-to-names-list.
+                captureCount = captureNumberList.Length;
+                for (int i = 0; i < captureNumberList.Length; i++)
+                {
+                    sparseMapping[captureNumberList[i]] = i;
+                }
+            }
+
+            return new RegexTree(root, captureCount, parser._capnamelist?.ToArray(), parser._capnames!, sparseMapping, options, parser._hasIgnoreCaseBackreferenceNodes ? culture : null, parser._nodeComments);
+        }
+
         /// <summary>This static call constructs a flat concatenation node given a replacement pattern.</summary>
         public static RegexReplacement ParseReplacement(string pattern, RegexOptions options, Hashtable caps, int capsize, Hashtable capnames)
         {
@@ -330,6 +372,7 @@ private RegexNode ScanRegex()
                     if (isQuantifier)
                     {
                         _unit = RegexNode.CreateOneWithCaseConversion(_pattern[endpos - 1], _options, _culture, ref _caseBehavior);
+                        AttachCommentsToNode(_unit);
                     }
                 }
 
@@ -345,13 +388,15 @@ private RegexNode ScanRegex()
                         {
                             string setString = ScanCharClass((_options & RegexOptions.IgnoreCase) != 0, scanOnly: false)!.ToStringClass();
                             _unit = new RegexNode(RegexNodeKind.Set, _options & ~RegexOptions.IgnoreCase, setString);
+                            AttachCommentsToNode(_unit);
                         }
                         break;
 
                     case '(':
                         _optionsStack.Append((int)_options);
                         if (ScanGroupOpen() is RegexNode grouper)
                         {
+                            AttachCommentsToNode(grouper);
                             PushGroup();
                             StartGroup(grouper);
                         }
@@ -388,20 +433,27 @@ private RegexNode ScanRegex()
                         }
 
                         _unit = ScanBackslash(scanOnly: false)!;
+                        if (_unit is not null)
+                        {
+                            AttachCommentsToNode(_unit);
+                        }
                         break;
 
                     case '^':
                         _unit = new RegexNode((_options & RegexOptions.Multiline) != 0 ? RegexNodeKind.Bol : RegexNodeKind.Beginning, _options);
+                        AttachCommentsToNode(_unit);
                         break;
 
                     case '$':
                         _unit = new RegexNode((_options & RegexOptions.Multiline) != 0 ? RegexNodeKind.Eol : RegexNodeKind.EndZ, _options);
+                        AttachCommentsToNode(_unit);
                         break;
 
                     case '.':
                         _unit = (_options & RegexOptions.Singleline) != 0 ?
                             new RegexNode(RegexNodeKind.Set, _options & ~RegexOptions.IgnoreCase, RegexCharClass.AnyClass) :
                             new RegexNode(RegexNodeKind.Notone, _options & ~RegexOptions.IgnoreCase, '\n');
+                        AttachCommentsToNode(_unit);
                         break;
 
                     case '{':
@@ -1048,21 +1100,41 @@ private void ScanBlank()
 
                 if ((_options & RegexOptions.IgnorePatternWhitespace) != 0 && _pos < _pattern.Length && _pattern[_pos] == '#')
                 {
+                    int commentStart = _pos + 1; // Skip the '#'
                     _pos = _pattern.IndexOf('\n', _pos);
                     if (_pos < 0)
                     {
                         _pos = _pattern.Length;
                     }
+
+                    if (_captureComments && commentStart < _pos)
+                    {
+                        string comment = _pattern.Substring(commentStart, _pos - commentStart).Trim();
+                        if (!string.IsNullOrEmpty(comment))
+                        {
+                            _pendingComments!.Add(comment);
+                        }
+                    }
                 }
                 else if (_pos + 2 < _pattern.Length && _pattern[_pos + 2] == '#' && _pattern[_pos + 1] == '?' && _pattern[_pos] == '(')
                 {
+                    int commentStart = _pos + 3; // Skip '(?#'
                     _pos = _pattern.IndexOf(')', _pos);
                     if (_pos < 0)
                     {
                         _pos = _pattern.Length;
                         throw MakeException(RegexParseError.UnterminatedComment, SR.UnterminatedComment);
                     }
 
+                    if (_captureComments && commentStart < _pos)
+                    {
+                        string comment = _pattern.Substring(commentStart, _pos - commentStart).Trim();
+                        if (!string.IsNullOrEmpty(comment))
+                        {
+                            _pendingComments!.Add(comment);
+                        }
+                    }
+
                     _pos++;
                 }
                 else
@@ -1072,6 +1144,22 @@ private void ScanBlank()
             }
         }
 
+        /// <summary>Attaches any pending comments to the specified node.</summary>
+        private void AttachCommentsToNode(RegexNode node)
+        {
+            if (_captureComments && _pendingComments!.Count > 0)
+            {
+                if (!_nodeComments!.TryGetValue(node, out List<string>? comments))
+                {
+                    comments = new List<string>();
+                    _nodeComments[node] = comments;
+                }
+
+                comments.AddRange(_pendingComments);
+                _pendingComments.Clear();
+            }
+        }
+
         /// <summary>Scans chars following a '\' (not counting the '\'), and returns a RegexNode for the type of atom scanned</summary>
         private RegexNode? ScanBackslash(bool scanOnly)
         {

src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexTree.cs

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System.Collections;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.Globalization;
 
@@ -39,8 +40,10 @@ internal sealed class RegexTree
         /// capture group number and the value is the index into <see cref="CaptureNames"/> for that capture group.
         /// </remarks>
         public readonly Hashtable? CaptureNumberSparseMapping;
+        /// <summary>A mapping of RegexNode to its associated comments from the pattern (for source generator use only).</summary>
+        internal readonly Dictionary<RegexNode, List<string>>? NodeComments;
 
-        internal RegexTree(RegexNode root, int captureCount, string[]? captureNames, Hashtable? captureNameToNumberMapping, Hashtable? captureNumberSparseMapping, RegexOptions options, CultureInfo? culture)
+        internal RegexTree(RegexNode root, int captureCount, string[]? captureNames, Hashtable? captureNameToNumberMapping, Hashtable? captureNumberSparseMapping, RegexOptions options, CultureInfo? culture, Dictionary<RegexNode, List<string>>? nodeComments = null)
         {
 #if DEBUG
             // Asserts to both demonstrate and validate the relationships between the various capture data structures.
@@ -77,6 +80,7 @@ internal RegexTree(RegexNode root, int captureCount, string[]? captureNames, Has
             CaptureNameToNumberMapping = captureNameToNumberMapping;
             CaptureNames = captureNames;
             Options = options;
+            NodeComments = nodeComments;
             FindOptimizations = RegexFindOptimizations.Create(root, options);
         }
     }

src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/RegexParserTests.netcoreapp.cs

@@ -312,5 +312,58 @@
                 throw new XunitException($"Expected RegexParseException or no exception -> Actual: ({e})");
             }
         }
+
+        [Fact]
+        public void ParseForSourceGenerator_CapturesComments()
+        {
+            // Test with (?x) inline option and # comments
+            string pattern = @"(?x)
+                ^           # Start of line
+                \w+         # Word characters
+            ";
+
+            RegexTree tree = RegexParser.ParseForSourceGenerator(pattern, RegexOptions.None, System.Globalization.CultureInfo.InvariantCulture);
+
+            Assert.NotNull(tree.NodeComments);
+            Assert.NotEmpty(tree.NodeComments);
+
+            // Verify some comments were captured
+            bool foundComment = false;
+            foreach (var kvp in tree.NodeComments)
+            {
+                if (kvp.Value.Count > 0)
+                {
+                    foundComment = true;
+                    break;
+                }
+            }
+            Assert.True(foundComment, "Expected to find at least one comment");
+        }
+
+        [Fact]
+        public void ParseForSourceGenerator_CapturesInlineComments()
+        {
+            // Test with (?# ) inline comments
+            string pattern = @"(?#This is a comment)abc(?#Another comment)";
+
+            RegexTree tree = RegexParser.ParseForSourceGenerator(pattern, RegexOptions.None, System.Globalization.CultureInfo.InvariantCulture);
+
+            Assert.NotNull(tree.NodeComments);
+            Assert.NotEmpty(tree.NodeComments);
+        }
+
+        [Fact]
+        public void Parse_DoesNotCaptureComments()
+        {
+            // Test that regular Parse doesn't capture comments
+            string pattern = @"(?x)
+                ^           # Start of line
+                \w+         # Word characters
+            ";
+
+            RegexTree tree = RegexParser.Parse(pattern, RegexOptions.None, System.Globalization.CultureInfo.InvariantCulture);
+
+            Assert.Null(tree.NodeComments);
+        }
     }
 }