Merge pull request #3519 from MDoerner/FixReportedLineNumbersOnParserErrors

retailcoder · web-flow · commit fdcb3c670bfb · 2017-10-28T10:08:30.000-04:00
Inverts the order of CodePane and attributes parser passes, massively reducing possible occurrences of parse errors in the attributes pass which would report line numbers that don't line up with what's in the code panes.
diff --git a/Rubberduck.Parsing/VBA/AttributeParser.cs b/Rubberduck.Parsing/VBA/AttributeParser.cs
@@ -26,18 +26,15 @@ public AttributeParser(IModuleExporter exporter, Func<IVBAPreprocessor> preproce
         /// Exports the specified component to a temporary file, loads, and then parses the exported file.
         /// </summary>
         /// <param name="component"></param>
-        /// <param name="token"></param>
-        /// <param name="stream"></param>
-        public IDictionary<Tuple<string, DeclarationType>, Attributes> Parse(IVBComponent component, CancellationToken token, out ITokenStream stream, out IParseTree tree)
+        /// <param name="cancellationToken"></param>
+        public (IParseTree tree, ITokenStream tokenStream, IDictionary<Tuple<string, DeclarationType>, Attributes> attributes) Parse(IVBComponent component, CancellationToken cancellationToken)
         {
-            token.ThrowIfCancellationRequested();
+            cancellationToken.ThrowIfCancellationRequested();
             var path = _exporter.Export(component);
             if (!File.Exists(path))
             {
                 // a document component without any code wouldn't be exported (file would be empty anyway).
-                stream = null;
-                tree = null;
-                return new Dictionary<Tuple<string, DeclarationType>, Attributes>();
+                return (null, null, new Dictionary<Tuple<string, DeclarationType>, Attributes>());
             }
             var code = File.ReadAllText(path);
             try
@@ -49,24 +46,24 @@ public IDictionary<Tuple<string, DeclarationType>, Attributes> Parse(IVBComponen
                 // Meh.
             }
            
-            token.ThrowIfCancellationRequested();
+            cancellationToken.ThrowIfCancellationRequested();
 
             var type = component.Type == ComponentType.StandardModule
                 ? DeclarationType.ProceduralModule
                 : DeclarationType.ClassModule;
             var tokenStreamProvider = new SimpleVBAModuleTokenStreamProvider();
             var tokens = tokenStreamProvider.Tokens(code);
             var preprocessor = _preprocessorFactory();
-            preprocessor.PreprocessTokenStream(component.Name, tokens, token);
+            preprocessor.PreprocessTokenStream(component.Name, tokens, cancellationToken);
             var listener = new AttributeListener(Tuple.Create(component.Name, type));
             // parse tree isn't usable for declarations because
             // line numbers are offset due to module header and attributes
             // (these don't show up in the VBE, that's why we're parsing an exported file)
 
-            tree = new VBAModuleParser().Parse(component.Name, tokens, new IParseTreeListener[] { listener }, new ExceptionErrorListener(), out stream);
+            var parseResults = new VBAModuleParser().Parse(component.Name, tokens, new IParseTreeListener[] { listener }, new ExceptionErrorListener());
 
-            token.ThrowIfCancellationRequested();
-            return listener.Attributes;
+            cancellationToken.ThrowIfCancellationRequested();
+            return (parseResults.tree, parseResults.tokenStream, listener.Attributes);
         }
     }
 }
diff --git a/Rubberduck.Parsing/VBA/ComponentParseTask.cs b/Rubberduck.Parsing/VBA/ComponentParseTask.cs
@@ -47,44 +47,40 @@ public ComponentParseTask(QualifiedModuleName module, IVBAPreprocessor preproces
             _parser = new VBAModuleParser();
         }
         
-        public void Start(CancellationToken token)
+        public void Start(CancellationToken cancellationToken)
         {
             try
             {
                 Logger.Trace($"Starting ParseTaskID {_taskId} on thread {Thread.CurrentThread.ManagedThreadId}.");
 
-                var tokenStream = RewriteAndPreprocess(token);
-                token.ThrowIfCancellationRequested();
-
-                IParseTree attributesTree;
-                IDictionary<Tuple<string, DeclarationType>, Attributes> attributes;
-                var attributesTokenStream = RunAttributesPass(token, out attributesTree, out attributes);
-
-                var rewriter = new MemberAttributesRewriter(_exporter, _component.CodeModule, new TokenStreamRewriter(attributesTokenStream ?? tokenStream));
+                var tokenStream = RewriteAndPreprocess(cancellationToken);
+                cancellationToken.ThrowIfCancellationRequested();  
 
                 // temporal coupling... comments must be acquired before we walk the parse tree for declarations
                 // otherwise none of the annotations get associated to their respective Declaration
                 var commentListener = new CommentListener();
                 var annotationListener = new AnnotationListener(new VBAParserAnnotationFactory(), _qualifiedName);
 
                 var stopwatch = Stopwatch.StartNew();
-                ITokenStream stream;
-                var tree = ParseInternal(_component.Name, tokenStream, new IParseTreeListener[]{ commentListener, annotationListener }, out stream);
+                var codePaneParseResults = ParseInternal(_component.Name, tokenStream, new IParseTreeListener[]{ commentListener, annotationListener });
                 stopwatch.Stop();
-                token.ThrowIfCancellationRequested();
+                cancellationToken.ThrowIfCancellationRequested();
 
                 var comments = QualifyAndUnionComments(_qualifiedName, commentListener.Comments, commentListener.RemComments);
-                token.ThrowIfCancellationRequested();
+                cancellationToken.ThrowIfCancellationRequested();
+
+                var attributesPassParseResults = RunAttributesPass(cancellationToken);
+                var rewriter = new MemberAttributesRewriter(_exporter, _component.CodeModule, new TokenStreamRewriter(attributesPassParseResults.tokenStream ?? tokenStream));
 
                 var completedHandler = ParseCompleted;
-                if (completedHandler != null && !token.IsCancellationRequested)
+                if (completedHandler != null && !cancellationToken.IsCancellationRequested)
                     completedHandler.Invoke(this, new ParseCompletionArgs
                     {
-                        ParseTree = tree,
-                        AttributesTree = attributesTree,
-                        Tokens = stream,
+                        ParseTree = codePaneParseResults.tree,
+                        AttributesTree = attributesPassParseResults.tree,
+                        Tokens = codePaneParseResults.tokenStream,
                         AttributesRewriter = rewriter,
-                        Attributes = attributes,
+                        Attributes = attributesPassParseResults.attributes,
                         Comments = comments,
                         Annotations = annotationListener.Annotations
                     });
@@ -128,14 +124,12 @@ public void Start(CancellationToken token)
             }
         }
 
-        private ITokenStream RunAttributesPass(CancellationToken token, out IParseTree attributesTree,
-            out IDictionary<Tuple<string, DeclarationType>, Attributes> attributes)
+        private (IParseTree tree, ITokenStream tokenStream, IDictionary<Tuple<string, DeclarationType>, Attributes> attributes) RunAttributesPass(CancellationToken cancellationToken)
         {
             Logger.Trace($"ParseTaskID {_taskId} begins attributes pass.");
-            ITokenStream attributesTokenStream;
-            attributes = _attributeParser.Parse(_component, token, out attributesTokenStream, out attributesTree);
+            var attributesParseResults = _attributeParser.Parse(_component, cancellationToken);
             Logger.Trace($"ParseTaskID {_taskId} finished attributes pass.");
-            return attributesTokenStream;
+            return attributesParseResults;
         }
 
         private static string GetCode(ICodeModule module)
@@ -152,20 +146,20 @@ private static string GetCode(ICodeModule module)
             return code;
         }
 
-        private CommonTokenStream RewriteAndPreprocess(CancellationToken token)
+        private CommonTokenStream RewriteAndPreprocess(CancellationToken cancellationToken)
         {
-            var code = _rewriter == null ? string.Join(Environment.NewLine, GetCode(_component.CodeModule)) : _rewriter.GetText();
+            var code = _rewriter?.GetText() ?? string.Join(Environment.NewLine, GetCode(_component.CodeModule));
             var tokenStreamProvider = new SimpleVBAModuleTokenStreamProvider();
             var tokens = tokenStreamProvider.Tokens(code);
-            _preprocessor.PreprocessTokenStream(_component.Name, tokens, token);
+            _preprocessor.PreprocessTokenStream(_component.Name, tokens, cancellationToken);
             return tokens;
         }
 
-        private IParseTree ParseInternal(string moduleName, CommonTokenStream tokenStream, IParseTreeListener[] listeners, out ITokenStream outStream)
+        private (IParseTree tree, ITokenStream tokenStream) ParseInternal(string moduleName, CommonTokenStream tokenStream, IParseTreeListener[] listeners)
         {
             //var errorNotifier = new SyntaxErrorNotificationListener();
             //errorNotifier.OnSyntaxError += ParserSyntaxError;
-            return _parser.Parse(moduleName, tokenStream, listeners, new ExceptionErrorListener(), out outStream);
+            return _parser.Parse(moduleName, tokenStream, listeners, new ExceptionErrorListener());
         }
 
         private IEnumerable<CommentNode> QualifyAndUnionComments(QualifiedModuleName qualifiedName, IEnumerable<VBAParser.CommentContext> comments, IEnumerable<VBAParser.RemCommentContext> remComments)
diff --git a/Rubberduck.Parsing/VBA/IAttributeParser.cs b/Rubberduck.Parsing/VBA/IAttributeParser.cs
@@ -10,6 +10,6 @@ namespace Rubberduck.Parsing.VBA
 {
     public interface IAttributeParser
     {
-        IDictionary<Tuple<string, DeclarationType>, Attributes> Parse(IVBComponent component, CancellationToken token, out ITokenStream stream, out IParseTree tree);
+        (IParseTree tree, ITokenStream tokenStream, IDictionary<Tuple<string, DeclarationType>, Attributes> attributes) Parse(IVBComponent component, CancellationToken cancellationToken);
     }
 }
diff --git a/Rubberduck.Parsing/VBA/VBAModuleParser.cs b/Rubberduck.Parsing/VBA/VBAModuleParser.cs
@@ -11,7 +11,7 @@ public sealed class VBAModuleParser
     {
         private static readonly Logger Logger = LogManager.GetCurrentClassLogger();
 
-        public IParseTree Parse(string moduleName, CommonTokenStream moduleTokens, IParseTreeListener[] listeners, BaseErrorListener errorListener, out ITokenStream outStream)
+        public (IParseTree tree, ITokenStream tokenStream) Parse(string moduleName, CommonTokenStream moduleTokens, IParseTreeListener[] listeners, BaseErrorListener errorListener)
         {
             moduleTokens.Reset();
             var parser = new VBAParser(moduleTokens);
@@ -34,8 +34,7 @@ public IParseTree Parse(string moduleName, CommonTokenStream moduleTokens, IPars
             {
                 ParseTreeWalker.Default.Walk(listener, tree);
             }
-            outStream = moduleTokens;
-            return tree;
+            return (tree, moduleTokens);
         }
     }
 }
diff --git a/RubberduckTests/Mocks/TestAttributeParser.cs b/RubberduckTests/Mocks/TestAttributeParser.cs
@@ -19,7 +19,7 @@ public TestAttributeParser(Func<IVBAPreprocessor> preprocessorFactory)
             _preprocessorFactory = preprocessorFactory;
         }
 
-        public IDictionary<Tuple<string, DeclarationType>, Attributes> Parse(IVBComponent component, CancellationToken token, out ITokenStream stream, out IParseTree tree)
+        public (IParseTree tree, ITokenStream tokenStream, IDictionary<Tuple<string, DeclarationType>, Attributes> attributes) Parse(IVBComponent component, CancellationToken cancellationToken)
         {
             var code = component.CodeModule.Content();
             var type = component.Type == ComponentType.StandardModule
@@ -28,16 +28,16 @@ public IDictionary<Tuple<string, DeclarationType>, Attributes> Parse(IVBComponen
             var tokenStreamProvider = new SimpleVBAModuleTokenStreamProvider();
             var tokens = tokenStreamProvider.Tokens(code);
             var preprocessor = _preprocessorFactory();
-            preprocessor.PreprocessTokenStream(component.Name, tokens, token);
+            preprocessor.PreprocessTokenStream(component.Name, tokens, cancellationToken);
             var listener = new AttributeListener(Tuple.Create(component.Name, type));
             // parse tree isn't usable for declarations because
             // line numbers are offset due to module header and attributes
             // (these don't show up in the VBE, that's why we're parsing an exported file)
 
-            tree = new VBAModuleParser().Parse(component.Name, tokens, new IParseTreeListener[] { listener }, new ExceptionErrorListener(), out stream);
+            var parseResults = new VBAModuleParser().Parse(component.Name, tokens, new IParseTreeListener[] { listener }, new ExceptionErrorListener());
 
-            token.ThrowIfCancellationRequested();
-            return listener.Attributes;
+            cancellationToken.ThrowIfCancellationRequested();
+            return (parseResults.tree, parseResults.tokenStream, listener.Attributes);
         }
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -10,6 +10,6 @@ namespace Rubberduck.Parsing.VBA`
`10`	`10`	`{`
`11`	`11`	`public interface IAttributeParser`
`12`	`12`	`{`
`13`		`- IDictionary<Tuple<string, DeclarationType>, Attributes> Parse(IVBComponent component, CancellationToken token, out ITokenStream stream, out IParseTree tree);`
	`13`	`+ (IParseTree tree, ITokenStream tokenStream, IDictionary<Tuple<string, DeclarationType>, Attributes> attributes) Parse(IVBComponent component, CancellationToken cancellationToken);`
`14`	`14`	`}`
`15`	`15`	`}`
Original file line number	Diff line number	Diff line change
`@@ -11,7 +11,7 @@ public sealed class VBAModuleParser`
`11`	`11`	`{`
`12`	`12`	`private static readonly Logger Logger = LogManager.GetCurrentClassLogger();`
`13`	`13`
`14`		`- public IParseTree Parse(string moduleName, CommonTokenStream moduleTokens, IParseTreeListener[] listeners, BaseErrorListener errorListener, out ITokenStream outStream)`
	`14`	`+ public (IParseTree tree, ITokenStream tokenStream) Parse(string moduleName, CommonTokenStream moduleTokens, IParseTreeListener[] listeners, BaseErrorListener errorListener)`
`15`	`15`	`{`
`16`	`16`	`moduleTokens.Reset();`
`17`	`17`	`var parser = new VBAParser(moduleTokens);`
`@@ -34,8 +34,7 @@ public IParseTree Parse(string moduleName, CommonTokenStream moduleTokens, IPars`
`34`	`34`	`{`
`35`	`35`	`ParseTreeWalker.Default.Walk(listener, tree);`
`36`	`36`	`}`
`37`		`- outStream = moduleTokens;`
`38`		`- return tree;`
	`37`	`+ return (tree, moduleTokens);`
`39`	`38`	`}`
`40`	`39`	`}`
`41`	`40`	`}`
Original file line number	Diff line number	Diff line change
`@@ -19,7 +19,7 @@ public TestAttributeParser(Func<IVBAPreprocessor> preprocessorFactory)`
`19`	`19`	`_preprocessorFactory = preprocessorFactory;`
`20`	`20`	`}`
`21`	`21`
`22`		`- public IDictionary<Tuple<string, DeclarationType>, Attributes> Parse(IVBComponent component, CancellationToken token, out ITokenStream stream, out IParseTree tree)`
	`22`	`+ public (IParseTree tree, ITokenStream tokenStream, IDictionary<Tuple<string, DeclarationType>, Attributes> attributes) Parse(IVBComponent component, CancellationToken cancellationToken)`
`23`	`23`	`{`
`24`	`24`	`var code = component.CodeModule.Content();`
`25`	`25`	`var type = component.Type == ComponentType.StandardModule`
`@@ -28,16 +28,16 @@ public IDictionary<Tuple<string, DeclarationType>, Attributes> Parse(IVBComponen`
`28`	`28`	`var tokenStreamProvider = new SimpleVBAModuleTokenStreamProvider();`
`29`	`29`	`var tokens = tokenStreamProvider.Tokens(code);`
`30`	`30`	`var preprocessor = _preprocessorFactory();`
`31`		`- preprocessor.PreprocessTokenStream(component.Name, tokens, token);`
	`31`	`+ preprocessor.PreprocessTokenStream(component.Name, tokens, cancellationToken);`
`32`	`32`	`var listener = new AttributeListener(Tuple.Create(component.Name, type));`
`33`	`33`	`// parse tree isn't usable for declarations because`
`34`	`34`	`// line numbers are offset due to module header and attributes`
`35`	`35`	`// (these don't show up in the VBE, that's why we're parsing an exported file)`
`36`	`36`
`37`		`- tree = new VBAModuleParser().Parse(component.Name, tokens, new IParseTreeListener[] { listener }, new ExceptionErrorListener(), out stream);`
	`37`	`+ var parseResults = new VBAModuleParser().Parse(component.Name, tokens, new IParseTreeListener[] { listener }, new ExceptionErrorListener());`
`38`	`38`
`39`		`- token.ThrowIfCancellationRequested();`
`40`		`- return listener.Attributes;`
	`39`	`+ cancellationToken.ThrowIfCancellationRequested();`
	`40`	`+ return (parseResults.tree, parseResults.tokenStream, listener.Attributes);`
`41`	`41`	`}`
`42`	`42`	`}`
`43`	`43`	`}`