Skip to content

Commit 24e52b4

Browse files
jakebaileyjakebailey
committed
Add configurable concurency options
1 parent 8e7cc78 commit 24e52b4

File tree

9 files changed

+119
-38
lines changed

9 files changed

+119
-38
lines changed

‎internal/compiler/concurrency.go

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
package compiler
2+
3+
import (
4+
"runtime"
5+
"strconv"
6+
"strings"
7+
8+
"github.com/microsoft/typescript-go/internal/core"
9+
)
10+
11+
type concurrency struct {
12+
checkerCount int
13+
}
14+
15+
func parseConcurrency(options *core.CompilerOptions, numFiles int) concurrency {
16+
if options.SingleThreaded.IsTrue() {
17+
return concurrency{
18+
checkerCount: 1,
19+
}
20+
}
21+
22+
checkerCount := 4
23+
24+
switch strings.ToLower(options.Concurrency) {
25+
case "default", "auto":
26+
break
27+
case "single", "none":
28+
checkerCount = 1
29+
case "max":
30+
checkerCount = runtime.GOMAXPROCS(0)
31+
case "half":
32+
checkerCount = max(1, runtime.GOMAXPROCS(0)/2)
33+
case "checker-per-file":
34+
checkerCount = -1
35+
default:
36+
if v, err := strconv.Atoi(options.Concurrency); err == nil && v > 0 {
37+
checkerCount = v
38+
}
39+
}
40+
41+
return concurrency{
42+
checkerCount: checkerCount,
43+
}
44+
}
45+
46+
func (c concurrency) isSingleThreaded() bool {
47+
return c.checkerCount == 1
48+
}
49+
50+
func (c concurrency) getCheckerCount(numFiles int) int {
51+
if c.checkerCount == -1 {
52+
return max(1, numFiles)
53+
}
54+
return max(1, c.checkerCount)
55+
}

‎internal/compiler/program.go

Lines changed: 37 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ type ProgramOptions struct {
2626
Host CompilerHost
2727
Config *tsoptions.ParsedCommandLine
2828
UseSourceOfProjectReference bool
29-
SingleThreaded core.Tristate
3029
CreateCheckerPool func(*Program) CheckerPool
3130
TypingsLocation string
3231
ProjectName string
@@ -39,7 +38,11 @@ func (p *ProgramOptions) canUseProjectReferenceSource() bool {
3938
type Program struct {
4039
opts ProgramOptions
4140
nodeModules map[string]*ast.SourceFile
42-
checkerPool CheckerPool
41+
42+
concurrency concurrency
43+
concurrencyOnce sync.Once
44+
checkerPool CheckerPool
45+
checkerPoolOnce sync.Once
4346

4447
comparePathsOptions tspath.ComparePathsOptions
4548

@@ -191,9 +194,6 @@ func NewProgram(opts ProgramOptions) *Program {
191194
if p.opts.Host == nil {
192195
panic("host required")
193196
}
194-
p.initCheckerPool()
195-
196-
// p.maxNodeModuleJsDepth = p.options.MaxNodeModuleJsDepth
197197

198198
// TODO(ercornel): !!! tracing?
199199
// tracing?.push(tracing.Phase.Program, "createProgram", { configFilePath: options.configFilePath, rootDir: options.rootDir }, /*separateBeginAndEnd*/ true);
@@ -239,7 +239,6 @@ func (p *Program) UpdateProgram(changedFilePath tspath.Path) (*Program, bool) {
239239
currentNodeModulesDepth: p.currentNodeModulesDepth,
240240
usesUriStyleNodeCoreModules: p.usesUriStyleNodeCoreModules,
241241
}
242-
result.initCheckerPool()
243242
index := core.FindIndex(result.files, func(file *ast.SourceFile) bool { return file.Path() == newFile.Path() })
244243
result.files = slices.Clone(result.files)
245244
result.files[index] = newFile
@@ -248,14 +247,6 @@ func (p *Program) UpdateProgram(changedFilePath tspath.Path) (*Program, bool) {
248247
return result, true
249248
}
250249

251-
func (p *Program) initCheckerPool() {
252-
if p.opts.CreateCheckerPool != nil {
253-
p.checkerPool = p.opts.CreateCheckerPool(p)
254-
} else {
255-
p.checkerPool = newCheckerPool(core.IfElse(p.singleThreaded(), 1, 4), p)
256-
}
257-
}
258-
259250
func canReplaceFileInProgram(file1 *ast.SourceFile, file2 *ast.SourceFile) bool {
260251
// TODO(jakebailey): metadata??
261252
return file2 != nil &&
@@ -299,8 +290,26 @@ func (p *Program) GetConfigFileParsingDiagnostics() []*ast.Diagnostic {
299290
return slices.Clip(p.opts.Config.GetConfigFileParsingDiagnostics())
300291
}
301292

293+
func (p *Program) getConcurrency() concurrency {
294+
p.concurrencyOnce.Do(func() {
295+
p.concurrency = parseConcurrency(p.Options(), len(p.files))
296+
})
297+
return p.concurrency
298+
}
299+
302300
func (p *Program) singleThreaded() bool {
303-
return p.opts.SingleThreaded.DefaultIfUnknown(p.Options().SingleThreaded).IsTrue()
301+
return p.getConcurrency().isSingleThreaded()
302+
}
303+
304+
func (p *Program) getCheckerPool() CheckerPool {
305+
p.checkerPoolOnce.Do(func() {
306+
if p.opts.CreateCheckerPool != nil {
307+
p.checkerPool = p.opts.CreateCheckerPool(p)
308+
} else {
309+
p.checkerPool = newCheckerPool(p.getConcurrency().getCheckerCount(len(p.files)), p)
310+
}
311+
})
312+
return p.checkerPool
304313
}
305314

306315
func (p *Program) BindSourceFiles() {
@@ -317,11 +326,11 @@ func (p *Program) BindSourceFiles() {
317326

318327
func (p *Program) CheckSourceFiles(ctx context.Context) {
319328
wg := core.NewWorkGroup(p.singleThreaded())
320-
checkers, done := p.checkerPool.GetAllCheckers(ctx)
329+
checkers, done := p.getCheckerPool().GetAllCheckers(ctx)
321330
defer done()
322331
for _, checker := range checkers {
323332
wg.Queue(func() {
324-
for file := range p.checkerPool.Files(checker) {
333+
for file := range p.getCheckerPool().Files(checker) {
325334
checker.CheckSourceFile(ctx, file)
326335
}
327336
})
@@ -331,19 +340,19 @@ func (p *Program) CheckSourceFiles(ctx context.Context) {
331340

332341
// Return the type checker associated with the program.
333342
func (p *Program) GetTypeChecker(ctx context.Context) (*checker.Checker, func()) {
334-
return p.checkerPool.GetChecker(ctx)
343+
return p.getCheckerPool().GetChecker(ctx)
335344
}
336345

337346
func (p *Program) GetTypeCheckers(ctx context.Context) ([]*checker.Checker, func()) {
338-
return p.checkerPool.GetAllCheckers(ctx)
347+
return p.getCheckerPool().GetAllCheckers(ctx)
339348
}
340349

341350
// Return a checker for the given file. We may have multiple checkers in concurrent scenarios and this
342351
// method returns the checker that was tasked with checking the file. Note that it isn't possible to mix
343352
// types obtained from different checkers, so only non-type data (such as diagnostics or string
344353
// representations of types) should be obtained from checkers returned by this method.
345354
func (p *Program) GetTypeCheckerForFile(ctx context.Context, file *ast.SourceFile) (*checker.Checker, func()) {
346-
return p.checkerPool.GetCheckerForFile(ctx, file)
355+
return p.getCheckerPool().GetCheckerForFile(ctx, file)
347356
}
348357

349358
func (p *Program) GetResolvedModule(file ast.HasFileName, moduleReference string, mode core.ResolutionMode) *module.ResolvedModule {
@@ -385,7 +394,7 @@ func (p *Program) GetSuggestionDiagnostics(ctx context.Context, sourceFile *ast.
385394

386395
func (p *Program) GetGlobalDiagnostics(ctx context.Context) []*ast.Diagnostic {
387396
var globalDiagnostics []*ast.Diagnostic
388-
checkers, done := p.checkerPool.GetAllCheckers(ctx)
397+
checkers, done := p.getCheckerPool().GetAllCheckers(ctx)
389398
defer done()
390399
for _, checker := range checkers {
391400
globalDiagnostics = append(globalDiagnostics, checker.GetGlobalDiagnostics()...)
@@ -432,11 +441,11 @@ func (p *Program) getSemanticDiagnosticsForFile(ctx context.Context, sourceFile
432441
var fileChecker *checker.Checker
433442
var done func()
434443
if sourceFile != nil {
435-
fileChecker, done = p.checkerPool.GetCheckerForFile(ctx, sourceFile)
444+
fileChecker, done = p.getCheckerPool().GetCheckerForFile(ctx, sourceFile)
436445
defer done()
437446
}
438447
diags := slices.Clip(sourceFile.BindDiagnostics())
439-
checkers, closeCheckers := p.checkerPool.GetAllCheckers(ctx)
448+
checkers, closeCheckers := p.getCheckerPool().GetAllCheckers(ctx)
440449
defer closeCheckers()
441450

442451
// Ask for diags from all checkers; checking one file may add diagnostics to other files.
@@ -523,13 +532,13 @@ func (p *Program) getSuggestionDiagnosticsForFile(ctx context.Context, sourceFil
523532
var fileChecker *checker.Checker
524533
var done func()
525534
if sourceFile != nil {
526-
fileChecker, done = p.checkerPool.GetCheckerForFile(ctx, sourceFile)
535+
fileChecker, done = p.getCheckerPool().GetCheckerForFile(ctx, sourceFile)
527536
defer done()
528537
}
529538

530539
diags := slices.Clip(sourceFile.BindSuggestionDiagnostics)
531540

532-
checkers, closeCheckers := p.checkerPool.GetAllCheckers(ctx)
541+
checkers, closeCheckers := p.getCheckerPool().GetAllCheckers(ctx)
533542
defer closeCheckers()
534543

535544
// Ask for diags from all checkers; checking one file may add diagnostics to other files.
@@ -640,7 +649,7 @@ func (p *Program) SymbolCount() int {
640649
for _, file := range p.files {
641650
count += file.SymbolCount
642651
}
643-
checkers, done := p.checkerPool.GetAllCheckers(context.Background())
652+
checkers, done := p.getCheckerPool().GetAllCheckers(context.Background())
644653
defer done()
645654
for _, checker := range checkers {
646655
count += int(checker.SymbolCount)
@@ -650,7 +659,7 @@ func (p *Program) SymbolCount() int {
650659

651660
func (p *Program) TypeCount() int {
652661
var count int
653-
checkers, done := p.checkerPool.GetAllCheckers(context.Background())
662+
checkers, done := p.getCheckerPool().GetAllCheckers(context.Background())
654663
defer done()
655664
for _, checker := range checkers {
656665
count += int(checker.TypeCount)
@@ -660,7 +669,7 @@ func (p *Program) TypeCount() int {
660669

661670
func (p *Program) InstantiationCount() int {
662671
var count int
663-
checkers, done := p.checkerPool.GetAllCheckers(context.Background())
672+
checkers, done := p.getCheckerPool().GetAllCheckers(context.Background())
664673
defer done()
665674
for _, checker := range checkers {
666675
count += int(checker.TotalInstantiationCount)

‎internal/core/compileroptions.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@ type CompilerOptions struct {
146146

147147
PprofDir string `json:"pprofDir,omitzero"`
148148
SingleThreaded Tristate `json:"singleThreaded,omitzero"`
149+
Concurrency string `json:"concurrency,omitzero"`
149150
Quiet Tristate `json:"quiet,omitzero"`
150151

151152
sourceFileAffectingCompilerOptionsOnce sync.Once

‎internal/core/tristate.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@ const (
1313
TSTrue
1414
)
1515

16+
func (t Tristate) IsUnknown() bool {
17+
return t == TSUnknown
18+
}
19+
1620
func (t Tristate) IsTrue() bool {
1721
return t == TSTrue
1822
}

‎internal/diagnostics/diagnostics_generated.go

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎internal/diagnostics/extraDiagnosticMessages.json

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,9 @@
1010
"Generate pprof CPU/memory profiles to the given directory.": {
1111
"category": "Message",
1212
"code": 100002
13+
},
14+
"Specify how much concurrency will be used.": {
15+
"category": "Message",
16+
"code": 100003
1317
}
1418
}

‎internal/testutil/harnessutil/harnessutil.go

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,11 @@ func CompileFilesEx(
182182
compilerOptions.TypeRoots[i] = tspath.GetNormalizedAbsolutePath(typeRoot, currentDirectory)
183183
}
184184

185+
if compilerOptions.Concurrency == "" && compilerOptions.SingleThreaded.IsUnknown() && testutil.TestProgramIsSingleThreaded() {
186+
// TODO(jakebailey): replace TestProgramIsSingleThreaded with passed in value
187+
compilerOptions.Concurrency = "1"
188+
}
189+
185190
// Create fake FS for testing
186191
testfs := map[string]any{}
187192
for _, file := range inputFiles {
@@ -841,18 +846,11 @@ func (c *CompilationResult) GetSourceMapRecord() string {
841846
}
842847

843848
func createProgram(host compiler.CompilerHost, config *tsoptions.ParsedCommandLine) *compiler.Program {
844-
var singleThreaded core.Tristate
845-
if testutil.TestProgramIsSingleThreaded() {
846-
singleThreaded = core.TSTrue
847-
}
848-
849849
programOptions := compiler.ProgramOptions{
850-
Config: config,
851-
Host: host,
852-
SingleThreaded: singleThreaded,
850+
Config: config,
851+
Host: host,
853852
}
854-
program := compiler.NewProgram(programOptions)
855-
return program
853+
return compiler.NewProgram(programOptions)
856854
}
857855

858856
func EnumerateFiles(folder string, testRegex *regexp.Regexp, recursive bool) ([]string, error) {

‎internal/tsoptions/declscompiler.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,12 @@ var optionsForCompiler = []*CommandLineOption{
225225
Category: diagnostics.Command_line_Options,
226226
Description: diagnostics.Run_in_single_threaded_mode,
227227
},
228+
{
229+
Name: "concurrency",
230+
Kind: CommandLineOptionTypeString,
231+
Category: diagnostics.Command_line_Options,
232+
Description: diagnostics.Specify_how_much_concurrency_will_be_used,
233+
},
228234
{
229235
Name: "pprofDir",
230236
Kind: CommandLineOptionTypeString,

‎internal/tsoptions/parsinghelpers.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -430,6 +430,8 @@ func parseCompilerOptions(key string, value any, allOptions *core.CompilerOption
430430
allOptions.PprofDir = parseString(value)
431431
case "singleThreaded":
432432
allOptions.SingleThreaded = parseTristate(value)
433+
case "concurrency":
434+
allOptions.Concurrency = parseString(value)
433435
case "quiet":
434436
allOptions.Quiet = parseTristate(value)
435437
default:

0 commit comments

Comments
 (0)