Skip to content

Commit ec75bbc

Browse files
michaelnebelmichaelnebel
authored
Merge pull request #8203 from michaelnebel/csharp/extractor-option-buildless
C#: Refactoring - Move some of the standalone extractor code to the Standalone project.
2 parents 9e7b092 + 62dc23f commit ec75bbc

File tree

4 files changed

+171
-143
lines changed

4 files changed

+171
-143
lines changed

‎csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
using System;
2+
using System.Collections.Concurrent;
3+
using System.Collections.Generic;
4+
using System.Diagnostics;
5+
using System.Linq;
6+
using Microsoft.CodeAnalysis;
7+
using Microsoft.CodeAnalysis.CSharp;
8+
using Semmle.Util;
9+
using Semmle.Util.Logging;
10+
11+
namespace Semmle.Extraction.CSharp.Standalone
12+
{
13+
public static class Extractor
14+
{
15+
16+
private static IEnumerable<Action> GetResolvedReferencesStandalone(IEnumerable<string> referencePaths, BlockingCollection<MetadataReference> references)
17+
{
18+
return referencePaths.Select<string, Action>(path => () =>
19+
{
20+
var reference = MetadataReference.CreateFromFile(path);
21+
references.Add(reference);
22+
});
23+
}
24+
25+
private static void AnalyseStandalone(
26+
StandaloneAnalyser analyser,
27+
IEnumerable<string> sources,
28+
IEnumerable<string> referencePaths,
29+
CommonOptions options,
30+
IProgressMonitor progressMonitor,
31+
Stopwatch stopwatch)
32+
{
33+
CSharp.Extractor.Analyse(stopwatch, analyser, options,
34+
references => GetResolvedReferencesStandalone(referencePaths, references),
35+
(analyser, syntaxTrees) => CSharp.Extractor.ReadSyntaxTrees(sources, analyser, null, null, syntaxTrees),
36+
(syntaxTrees, references) => CSharpCompilation.Create("csharp.dll", syntaxTrees, references),
37+
(compilation, options) => analyser.Initialize(compilation, options),
38+
() => { },
39+
_ => { },
40+
() =>
41+
{
42+
foreach (var type in analyser.MissingNamespaces)
43+
{
44+
progressMonitor.MissingNamespace(type);
45+
}
46+
47+
foreach (var type in analyser.MissingTypes)
48+
{
49+
progressMonitor.MissingType(type);
50+
}
51+
52+
progressMonitor.MissingSummary(analyser.MissingTypes.Count(), analyser.MissingNamespaces.Count());
53+
});
54+
}
55+
56+
private static void ExtractStandalone(
57+
IEnumerable<string> sources,
58+
IEnumerable<string> referencePaths,
59+
IProgressMonitor pm,
60+
ILogger logger,
61+
CommonOptions options)
62+
{
63+
var stopwatch = new Stopwatch();
64+
stopwatch.Start();
65+
66+
var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
67+
var pathTransformer = new PathTransformer(canonicalPathCache);
68+
69+
using var analyser = new StandaloneAnalyser(pm, logger, false, pathTransformer);
70+
try
71+
{
72+
AnalyseStandalone(analyser, sources, referencePaths, options, pm, stopwatch);
73+
}
74+
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
75+
{
76+
analyser.Logger.Log(Severity.Error, " Unhandled exception: {0}", ex);
77+
}
78+
}
79+
80+
private class ExtractionProgress : IProgressMonitor
81+
{
82+
public ExtractionProgress(ILogger output)
83+
{
84+
logger = output;
85+
}
86+
87+
private readonly ILogger logger;
88+
89+
public void Analysed(int item, int total, string source, string output, TimeSpan time, AnalysisAction action)
90+
{
91+
logger.Log(Severity.Info, "[{0}/{1}] {2} ({3})", item, total, source,
92+
action == AnalysisAction.Extracted
93+
? time.ToString()
94+
: action == AnalysisAction.Excluded
95+
? "excluded"
96+
: "up to date");
97+
}
98+
99+
public void MissingType(string type)
100+
{
101+
logger.Log(Severity.Debug, "Missing type {0}", type);
102+
}
103+
104+
public void MissingNamespace(string @namespace)
105+
{
106+
logger.Log(Severity.Info, "Missing namespace {0}", @namespace);
107+
}
108+
109+
public void MissingSummary(int missingTypes, int missingNamespaces)
110+
{
111+
logger.Log(Severity.Info, "Failed to resolve {0} types in {1} namespaces", missingTypes, missingNamespaces);
112+
}
113+
}
114+
115+
public static ExitCode Run(Options options)
116+
{
117+
var stopwatch = new Stopwatch();
118+
stopwatch.Start();
119+
120+
using var logger = new ConsoleLogger(options.Verbosity);
121+
logger.Log(Severity.Info, "Running C# standalone extractor");
122+
using var a = new Analysis(logger, options);
123+
var sourceFileCount = a.Extraction.Sources.Count;
124+
125+
if (sourceFileCount == 0)
126+
{
127+
logger.Log(Severity.Error, "No source files found");
128+
return ExitCode.Errors;
129+
}
130+
131+
if (!options.SkipExtraction)
132+
{
133+
using var fileLogger = CSharp.Extractor.MakeLogger(options.Verbosity, false);
134+
135+
logger.Log(Severity.Info, "");
136+
logger.Log(Severity.Info, "Extracting...");
137+
ExtractStandalone(
138+
a.Extraction.Sources,
139+
a.References,
140+
new ExtractionProgress(logger),
141+
fileLogger,
142+
options);
143+
logger.Log(Severity.Info, $"Extraction completed in {stopwatch.Elapsed}");
144+
}
145+
146+
return ExitCode.Ok;
147+
}
148+
}
149+
}

‎csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs

Lines changed: 3 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
using System;
22
using System.Collections.Generic;
3+
using System.Diagnostics;
34
using Semmle.BuildAnalyser;
45
using Semmle.Util.Logging;
56

@@ -52,11 +53,10 @@ public class Program
5253
{
5354
public static int Main(string[] args)
5455
{
55-
Extractor.SetInvariantCulture();
56+
CSharp.Extractor.SetInvariantCulture();
5657

5758
var options = Options.Create(args);
5859
// options.CIL = true; // To do: Enable this
59-
using var output = new ConsoleLogger(options.Verbosity);
6060

6161
if (options.Help)
6262
{
@@ -67,69 +67,7 @@ public static int Main(string[] args)
6767
if (options.Errors)
6868
return 1;
6969

70-
var start = DateTime.Now;
71-
72-
output.Log(Severity.Info, "Running C# standalone extractor");
73-
using var a = new Analysis(output, options);
74-
var sourceFileCount = a.Extraction.Sources.Count;
75-
76-
if (sourceFileCount == 0)
77-
{
78-
output.Log(Severity.Error, "No source files found");
79-
return 1;
80-
}
81-
82-
if (!options.SkipExtraction)
83-
{
84-
using var fileLogger = new FileLogger(options.Verbosity, Extractor.GetCSharpLogPath());
85-
86-
output.Log(Severity.Info, "");
87-
output.Log(Severity.Info, "Extracting...");
88-
Extractor.ExtractStandalone(
89-
a.Extraction.Sources,
90-
a.References,
91-
new ExtractionProgress(output),
92-
fileLogger,
93-
options);
94-
output.Log(Severity.Info, $"Extraction completed in {DateTime.Now - start}");
95-
}
96-
97-
return 0;
98-
}
99-
100-
private class ExtractionProgress : IProgressMonitor
101-
{
102-
public ExtractionProgress(ILogger output)
103-
{
104-
logger = output;
105-
}
106-
107-
private readonly ILogger logger;
108-
109-
public void Analysed(int item, int total, string source, string output, TimeSpan time, AnalysisAction action)
110-
{
111-
logger.Log(Severity.Info, "[{0}/{1}] {2} ({3})", item, total, source,
112-
action == AnalysisAction.Extracted
113-
? time.ToString()
114-
: action == AnalysisAction.Excluded
115-
? "excluded"
116-
: "up to date");
117-
}
118-
119-
public void MissingType(string type)
120-
{
121-
logger.Log(Severity.Debug, "Missing type {0}", type);
122-
}
123-
124-
public void MissingNamespace(string @namespace)
125-
{
126-
logger.Log(Severity.Info, "Missing namespace {0}", @namespace);
127-
}
128-
129-
public void MissingSummary(int missingTypes, int missingNamespaces)
130-
{
131-
logger.Log(Severity.Info, "Failed to resolve {0} types in {1} namespaces", missingTypes, missingNamespaces);
132-
}
70+
return (int)Extractor.Run(options);
13371
}
13472
}
13573
}

‎csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs renamed to ‎csharp/extractor/Semmle.Extraction.CSharp.Standalone/StandaloneAnalyser.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ public StandaloneAnalyser(IProgressMonitor pm, ILogger logger, bool addAssemblyT
1111
{
1212
}
1313

14-
public void InitializeStandalone(CSharpCompilation compilationIn, CommonOptions options)
14+
public void Initialize(CSharpCompilation compilationIn, CommonOptions options)
1515
{
1616
compilation = compilationIn;
1717
layout = new Layout();

‎csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs

Lines changed: 18 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,15 @@
1616

1717
namespace Semmle.Extraction.CSharp
1818
{
19-
public static class Extractor
19+
public enum ExitCode
2020
{
21-
public enum ExitCode
22-
{
23-
Ok, // Everything worked perfectly
24-
Errors, // Trap was generated but there were processing errors
25-
Failed // Trap could not be generated
26-
}
21+
Ok, // Everything worked perfectly
22+
Errors, // Trap was generated but there were processing errors
23+
Failed // Trap could not be generated
24+
}
2725

26+
public static class Extractor
27+
{
2828
private class LogProgressMonitor : IProgressMonitor
2929
{
3030
private readonly ILogger logger;
@@ -69,6 +69,14 @@ public static void SetInvariantCulture()
6969
Thread.CurrentThread.CurrentUICulture = culture;
7070
}
7171

72+
public static ILogger MakeLogger(Verbosity verbosity, bool includeConsole)
73+
{
74+
var fileLogger = new FileLogger(verbosity, GetCSharpLogPath());
75+
return includeConsole
76+
? new CombinedLogger(new ConsoleLogger(verbosity), fileLogger)
77+
: (ILogger)fileLogger;
78+
}
79+
7280
/// <summary>
7381
/// Command-line driver for the extractor.
7482
/// </summary>
@@ -89,10 +97,7 @@ public static ExitCode Run(string[] args)
8997
var options = Options.CreateWithEnvironment(args);
9098
Entities.Compilation.Settings = (Directory.GetCurrentDirectory(), options.CompilerArguments.ToArray());
9199

92-
var fileLogger = new FileLogger(options.Verbosity, GetCSharpLogPath());
93-
using var logger = options.Console
94-
? new CombinedLogger(new ConsoleLogger(options.Verbosity), fileLogger)
95-
: (ILogger)fileLogger;
100+
using var logger = MakeLogger(options.Verbosity, options.Console);
96101

97102
if (Environment.GetEnvironmentVariable("SEMMLE_CLRTRACER") == "1" && !options.ClrTracer)
98103
{
@@ -276,7 +281,7 @@ private static IEnumerable<Action> ResolveReferences(Microsoft.CodeAnalysis.Comm
276281
/// The constructed syntax trees will be added (thread-safely) to the supplied
277282
/// list <paramref name="ret"/>.
278283
/// </summary>
279-
private static IEnumerable<Action> ReadSyntaxTrees(IEnumerable<string> sources, Analyser analyser, CSharpParseOptions? parseOptions, Encoding? encoding, IList<SyntaxTree> ret)
284+
public static IEnumerable<Action> ReadSyntaxTrees(IEnumerable<string> sources, Analyser analyser, CSharpParseOptions? parseOptions, Encoding? encoding, IList<SyntaxTree> ret)
280285
{
281286
return sources.Select<string, Action>(path => () =>
282287
{
@@ -298,31 +303,7 @@ private static IEnumerable<Action> ReadSyntaxTrees(IEnumerable<string> sources,
298303
});
299304
}
300305

301-
public static void ExtractStandalone(
302-
IEnumerable<string> sources,
303-
IEnumerable<string> referencePaths,
304-
IProgressMonitor pm,
305-
ILogger logger,
306-
CommonOptions options)
307-
{
308-
var stopwatch = new Stopwatch();
309-
stopwatch.Start();
310-
311-
var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
312-
var pathTransformer = new PathTransformer(canonicalPathCache);
313-
314-
using var analyser = new StandaloneAnalyser(pm, logger, false, pathTransformer);
315-
try
316-
{
317-
AnalyseStandalone(analyser, sources, referencePaths, options, pm, stopwatch);
318-
}
319-
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
320-
{
321-
analyser.Logger.Log(Severity.Error, " Unhandled exception: {0}", ex);
322-
}
323-
}
324-
325-
private static ExitCode Analyse(Stopwatch stopwatch, Analyser analyser, CommonOptions options,
306+
public static ExitCode Analyse(Stopwatch stopwatch, Analyser analyser, CommonOptions options,
326307
Func<BlockingCollection<MetadataReference>, IEnumerable<Action>> getResolvedReferenceTasks,
327308
Func<Analyser, List<SyntaxTree>, IEnumerable<Action>> getSyntaxTreeTasks,
328309
Func<IEnumerable<SyntaxTree>, IEnumerable<MetadataReference>, CSharpCompilation> getCompilation,
@@ -395,37 +376,6 @@ private static ExitCode Analyse(Stopwatch stopwatch, Analyser analyser, CommonOp
395376
return analyser.TotalErrors == 0 ? ExitCode.Ok : ExitCode.Errors;
396377
}
397378

398-
private static void AnalyseStandalone(
399-
StandaloneAnalyser analyser,
400-
IEnumerable<string> sources,
401-
IEnumerable<string> referencePaths,
402-
CommonOptions options,
403-
IProgressMonitor progressMonitor,
404-
Stopwatch stopwatch)
405-
{
406-
Analyse(stopwatch, analyser, options,
407-
references => GetResolvedReferencesStandalone(referencePaths, references),
408-
(analyser, syntaxTrees) => ReadSyntaxTrees(sources, analyser, null, null, syntaxTrees),
409-
(syntaxTrees, references) => CSharpCompilation.Create("csharp.dll", syntaxTrees, references),
410-
(compilation, options) => analyser.InitializeStandalone(compilation, options),
411-
() => { },
412-
_ => { },
413-
() =>
414-
{
415-
foreach (var type in analyser.MissingNamespaces)
416-
{
417-
progressMonitor.MissingNamespace(type);
418-
}
419-
420-
foreach (var type in analyser.MissingTypes)
421-
{
422-
progressMonitor.MissingType(type);
423-
}
424-
425-
progressMonitor.MissingSummary(analyser.MissingTypes.Count(), analyser.MissingNamespaces.Count());
426-
});
427-
}
428-
429379
private static ExitCode AnalyseTracing(
430380
TracingAnalyser analyser,
431381
CSharpCommandLineArguments compilerArguments,
@@ -468,15 +418,6 @@ private static ExitCode AnalyseTracing(
468418
() => { });
469419
}
470420

471-
private static IEnumerable<Action> GetResolvedReferencesStandalone(IEnumerable<string> referencePaths, BlockingCollection<MetadataReference> references)
472-
{
473-
return referencePaths.Select<string, Action>(path => () =>
474-
{
475-
var reference = MetadataReference.CreateFromFile(path);
476-
references.Add(reference);
477-
});
478-
}
479-
480421
/// <summary>
481422
/// Gets the path to the `csharp.log` file written to by the C# extractor.
482423
/// </summary>

0 commit comments

Comments
 (0)