Simplify parser options passing, share ASTs even more (#1158)

Author: jakebailey

internal/api/encoder/encoder_test.go

@@ -13,18 +13,21 @@ import (
 	"github.com/microsoft/typescript-go/internal/core"
 	"github.com/microsoft/typescript-go/internal/parser"
 	"github.com/microsoft/typescript-go/internal/repo"
-	"github.com/microsoft/typescript-go/internal/scanner"
 	"github.com/microsoft/typescript-go/internal/testutil/baseline"
 	"gotest.tools/v3/assert"
 )
 
-var parseCompilerOptions = &core.SourceFileAffectingCompilerOptions{
+var parseCompilerOptions = core.SourceFileAffectingCompilerOptions{
 	EmitScriptTarget: core.ScriptTargetLatest,
 }
 
 func TestEncodeSourceFile(t *testing.T) {
 	t.Parallel()
-	sourceFile := parser.ParseSourceFile("/test.ts", "/test.ts", "import { bar } from \"bar\";\nexport function foo<T, U>(a: string, b: string): any {}\nfoo();", parseCompilerOptions, nil, scanner.JSDocParsingModeParseAll)
+	sourceFile := parser.ParseSourceFile(ast.SourceFileParseOptions{
+		FileName:        "/test.ts",
+		Path:            "/test.ts",
+		CompilerOptions: parseCompilerOptions,
+	}, "import { bar } from \"bar\";\nexport function foo<T, U>(a: string, b: string): any {}\nfoo();", core.ScriptKindTS)
 	t.Run("baseline", func(t *testing.T) {
 		t.Parallel()
 		buf, err := encoder.EncodeSourceFile(sourceFile, "")
@@ -42,14 +45,11 @@ func BenchmarkEncodeSourceFile(b *testing.B) {
 	filePath := filepath.Join(repo.TypeScriptSubmodulePath, "src/compiler/checker.ts")
 	fileContent, err := os.ReadFile(filePath)
 	assert.NilError(b, err)
-	sourceFile := parser.ParseSourceFile(
-		"/checker.ts",
-		"/checker.ts",
-		string(fileContent),
-		parseCompilerOptions,
-		nil,
-		scanner.JSDocParsingModeParseAll,
-	)
+	sourceFile := parser.ParseSourceFile(ast.SourceFileParseOptions{
+		FileName:        "/checker.ts",
+		Path:            "/checker.ts",
+		CompilerOptions: parseCompilerOptions,
+	}, string(fileContent), core.ScriptKindTS)
 
 	for b.Loop() {
 		_, err := encoder.EncodeSourceFile(sourceFile, "")

internal/ast/ast.go

@@ -9985,17 +9985,14 @@ type SourceFile struct {
 	compositeNodeBase
 
 	// Fields set by NewSourceFile
-
-	text       string
-	fileName   string
-	path       tspath.Path
-	Statements *NodeList // NodeList[*Statement]
+	fileName     string // For debugging convenience
+	parseOptions SourceFileParseOptions
+	text         string
+	Statements   *NodeList // NodeList[*Statement]
 
 	// Fields set by parser
-
 	diagnostics                 []*Diagnostic
 	jsdocDiagnostics            []*Diagnostic
-	LanguageVersion             core.ScriptTarget
 	LanguageVariant             core.LanguageVariant
 	ScriptKind                  core.ScriptKind
 	IsDeclarationFile           bool
@@ -10040,30 +10037,36 @@ type SourceFile struct {
 	tokenCache   map[core.TextRange]*Node
 }
 
-func (f *NodeFactory) NewSourceFile(text string, fileName string, path tspath.Path, statements *NodeList) *Node {
-	if (tspath.GetEncodedRootLength(fileName) == 0 && !strings.HasPrefix(fileName, "^/")) || fileName != tspath.NormalizePath(fileName) {
-		panic(fmt.Sprintf("fileName should be normalized and absolute: %q", fileName))
+func (f *NodeFactory) NewSourceFile(opts SourceFileParseOptions, text string, statements *NodeList) *Node {
+	if (tspath.GetEncodedRootLength(opts.FileName) == 0 && !strings.HasPrefix(opts.FileName, "^/")) || opts.FileName != tspath.NormalizePath(opts.FileName) {
+		panic(fmt.Sprintf("fileName should be normalized and absolute: %q", opts.FileName))
 	}
-
 	data := &SourceFile{}
+	data.fileName = opts.FileName
+	data.parseOptions = opts
 	data.text = text
-	data.fileName = fileName
-	data.path = path
 	data.Statements = statements
-	data.LanguageVersion = core.ScriptTargetLatest
 	return f.newNode(KindSourceFile, data)
 }
 
+func (node *SourceFile) ParseOptions() SourceFileParseOptions {
+	return node.parseOptions
+}
+
+func (node *SourceFile) LanguageVersion() core.ScriptTarget {
+	return node.parseOptions.CompilerOptions.EmitScriptTarget
+}
+
 func (node *SourceFile) Text() string {
 	return node.text
 }
 
 func (node *SourceFile) FileName() string {
-	return node.fileName
+	return node.parseOptions.FileName
 }
 
 func (node *SourceFile) Path() tspath.Path {
-	return node.path
+	return node.parseOptions.Path
 }
 
 func (node *SourceFile) OriginalFileName() string {
@@ -10120,7 +10123,6 @@ func (node *SourceFile) IsJS() bool {
 
 func (node *SourceFile) copyFrom(other *SourceFile) {
 	// Do not copy fields set by NewSourceFile (Text, FileName, Path, or Statements)
-	node.LanguageVersion = other.LanguageVersion
 	node.LanguageVariant = other.LanguageVariant
 	node.ScriptKind = other.ScriptKind
 	node.IsDeclarationFile = other.IsDeclarationFile
@@ -10141,7 +10143,7 @@ func (node *SourceFile) copyFrom(other *SourceFile) {
 }
 
 func (node *SourceFile) Clone(f NodeFactoryCoercible) *Node {
-	updated := f.AsNodeFactory().NewSourceFile(node.Text(), node.FileName(), node.Path(), node.Statements)
+	updated := f.AsNodeFactory().NewSourceFile(node.parseOptions, node.text, node.Statements)
 	newFile := updated.AsSourceFile()
 	newFile.copyFrom(node)
 	return cloneNode(updated, node.AsNode(), f.AsNodeFactory().hooks)
@@ -10153,7 +10155,7 @@ func (node *SourceFile) computeSubtreeFacts() SubtreeFacts {
 
 func (f *NodeFactory) UpdateSourceFile(node *SourceFile, statements *StatementList) *Node {
 	if statements != node.Statements {
-		updated := f.NewSourceFile(node.Text(), node.fileName, node.path, statements).AsSourceFile()
+		updated := f.NewSourceFile(node.parseOptions, node.text, statements).AsSourceFile()
 		updated.copyFrom(node)
 		return updateNode(updated.AsNode(), node.AsNode(), f.hooks)
 	}

internal/ast/parseoptions.go

@@ -0,0 +1,162 @@
+package ast
+
+import (
+	"github.com/microsoft/typescript-go/internal/core"
+	"github.com/microsoft/typescript-go/internal/tspath"
+)
+
+type JSDocParsingMode int
+
+const (
+	JSDocParsingModeParseAll JSDocParsingMode = iota
+	JSDocParsingModeParseNone
+	JSDocParsingModeParseForTypeErrors
+	JSDocParsingModeParseForTypeInfo
+)
+
+type SourceFileParseOptions struct {
+	FileName                       string
+	Path                           tspath.Path
+	CompilerOptions                core.SourceFileAffectingCompilerOptions
+	ExternalModuleIndicatorOptions ExternalModuleIndicatorOptions
+	JSDocParsingMode               JSDocParsingMode
+}
+
+func GetSourceFileAffectingCompilerOptions(fileName string, options *core.CompilerOptions) core.SourceFileAffectingCompilerOptions {
+	// Declaration files are not parsed/bound differently depending on compiler options.
+	if tspath.IsDeclarationFileName(fileName) {
+		return core.SourceFileAffectingCompilerOptions{}
+	}
+	return options.SourceFileAffecting()
+}
+
+type ExternalModuleIndicatorOptions struct {
+	jsx   bool
+	force bool
+}
+
+func GetExternalModuleIndicatorOptions(fileName string, options *core.CompilerOptions, metadata SourceFileMetaData) ExternalModuleIndicatorOptions {
+	if tspath.IsDeclarationFileName(fileName) {
+		return ExternalModuleIndicatorOptions{}
+	}
+
+	switch options.GetEmitModuleDetectionKind() {
+	case core.ModuleDetectionKindForce:
+		// All non-declaration files are modules, declaration files still do the usual isFileProbablyExternalModule
+		return ExternalModuleIndicatorOptions{force: true}
+	case core.ModuleDetectionKindLegacy:
+		// Files are modules if they have imports, exports, or import.meta
+		return ExternalModuleIndicatorOptions{}
+	case core.ModuleDetectionKindAuto:
+		// If module is nodenext or node16, all esm format files are modules
+		// If jsx is react-jsx or react-jsxdev then jsx tags force module-ness
+		// otherwise, the presence of import or export statments (or import.meta) implies module-ness
+		return ExternalModuleIndicatorOptions{
+			jsx:   options.Jsx == core.JsxEmitReactJSX || options.Jsx == core.JsxEmitReactJSXDev,
+			force: isFileForcedToBeModuleByFormat(fileName, options, metadata),
+		}
+	default:
+		return ExternalModuleIndicatorOptions{}
+	}
+}
+
+var isFileForcedToBeModuleByFormatExtensions = []string{tspath.ExtensionCjs, tspath.ExtensionCts, tspath.ExtensionMjs, tspath.ExtensionMts}
+
+func isFileForcedToBeModuleByFormat(fileName string, options *core.CompilerOptions, metadata SourceFileMetaData) bool {
+	// Excludes declaration files - they still require an explicit `export {}` or the like
+	// for back compat purposes. The only non-declaration files _not_ forced to be a module are `.js` files
+	// that aren't esm-mode (meaning not in a `type: module` scope).
+	if GetImpliedNodeFormatForEmitWorker(fileName, options.GetEmitModuleKind(), metadata) == core.ModuleKindESNext || tspath.FileExtensionIsOneOf(fileName, isFileForcedToBeModuleByFormatExtensions) {
+		return true
+	}
+	return false
+}
+
+func SetExternalModuleIndicator(file *SourceFile, opts ExternalModuleIndicatorOptions) {
+	file.ExternalModuleIndicator = getExternalModuleIndicator(file, opts)
+}
+
+func getExternalModuleIndicator(file *SourceFile, opts ExternalModuleIndicatorOptions) *Node {
+	if file.ScriptKind == core.ScriptKindJSON {
+		return nil
+	}
+
+	if node := isFileProbablyExternalModule(file); node != nil {
+		return node
+	}
+
+	if file.IsDeclarationFile {
+		return nil
+	}
+
+	if opts.jsx {
+		if node := isFileModuleFromUsingJSXTag(file); node != nil {
+			return node
+		}
+	}
+
+	if opts.force {
+		return file.AsNode()
+	}
+
+	return nil
+}
+
+func isFileProbablyExternalModule(sourceFile *SourceFile) *Node {
+	for _, statement := range sourceFile.Statements.Nodes {
+		if IsExternalModuleIndicator(statement) {
+			return statement
+		}
+	}
+	return getImportMetaIfNecessary(sourceFile)
+}
+
+func getImportMetaIfNecessary(sourceFile *SourceFile) *Node {
+	if sourceFile.AsNode().Flags&NodeFlagsPossiblyContainsImportMeta != 0 {
+		return findChildNode(sourceFile.AsNode(), IsImportMeta)
+	}
+	return nil
+}
+
+func findChildNode(root *Node, check func(*Node) bool) *Node {
+	var result *Node
+	var visit func(*Node) bool
+	visit = func(node *Node) bool {
+		if check(node) {
+			result = node
+			return true
+		}
+		return node.ForEachChild(visit)
+	}
+	visit(root)
+	return result
+}
+
+func isFileModuleFromUsingJSXTag(file *SourceFile) *Node {
+	return walkTreeForJSXTags(file.AsNode())
+}
+
+// This is a somewhat unavoidable full tree walk to locate a JSX tag - `import.meta` requires the same,
+// but we avoid that walk (or parts of it) if at all possible using the `PossiblyContainsImportMeta` node flag.
+// Unfortunately, there's no `NodeFlag` space to do the same for JSX.
+func walkTreeForJSXTags(node *Node) *Node {
+	var found *Node
+
+	var visitor func(node *Node) bool
+	visitor = func(node *Node) bool {
+		if found != nil {
+			return true
+		}
+		if node.SubtreeFacts()&SubtreeContainsJsx == 0 {
+			return false
+		}
+		if IsJsxOpeningElement(node) || IsJsxFragment(node) {
+			found = node
+			return true
+		}
+		return node.ForEachChild(visitor)
+	}
+	visitor(node)
+
+	return found
+}

internal/ast/utilities.go

@@ -2549,27 +2549,24 @@ func GetImpliedNodeFormatForFile(path string, packageJsonType string) core.Modul
 	return impliedNodeFormat
 }
 
-func GetEmitModuleFormatOfFileWorker(fileName string, options *core.CompilerOptions, sourceFileMetaData *SourceFileMetaData) core.ModuleKind {
+func GetEmitModuleFormatOfFileWorker(fileName string, options *core.CompilerOptions, sourceFileMetaData SourceFileMetaData) core.ModuleKind {
 	result := GetImpliedNodeFormatForEmitWorker(fileName, options.GetEmitModuleKind(), sourceFileMetaData)
 	if result != core.ModuleKindNone {
 		return result
 	}
 	return options.GetEmitModuleKind()
 }
 
-func GetImpliedNodeFormatForEmitWorker(fileName string, emitModuleKind core.ModuleKind, sourceFileMetaData *SourceFileMetaData) core.ResolutionMode {
+func GetImpliedNodeFormatForEmitWorker(fileName string, emitModuleKind core.ModuleKind, sourceFileMetaData SourceFileMetaData) core.ResolutionMode {
 	if core.ModuleKindNode16 <= emitModuleKind && emitModuleKind <= core.ModuleKindNodeNext {
-		if sourceFileMetaData == nil {
-			return core.ModuleKindNone
-		}
 		return sourceFileMetaData.ImpliedNodeFormat
 	}
-	if sourceFileMetaData != nil && sourceFileMetaData.ImpliedNodeFormat == core.ModuleKindCommonJS &&
+	if sourceFileMetaData.ImpliedNodeFormat == core.ModuleKindCommonJS &&
 		(sourceFileMetaData.PackageJsonType == "commonjs" ||
 			tspath.FileExtensionIsOneOf(fileName, []string{tspath.ExtensionCjs, tspath.ExtensionCts})) {
 		return core.ModuleKindCommonJS
 	}
-	if sourceFileMetaData != nil && sourceFileMetaData.ImpliedNodeFormat == core.ModuleKindESNext &&
+	if sourceFileMetaData.ImpliedNodeFormat == core.ModuleKindESNext &&
 		(sourceFileMetaData.PackageJsonType == "module" ||
 			tspath.FileExtensionIsOneOf(fileName, []string{tspath.ExtensionMjs, tspath.ExtensionMts})) {
 		return core.ModuleKindESNext
@@ -2899,15 +2896,6 @@ func GetClassLikeDeclarationOfSymbol(symbol *Symbol) *Node {
 	return core.Find(symbol.Declarations, IsClassLike)
 }
 
-func GetLanguageVariant(scriptKind core.ScriptKind) core.LanguageVariant {
-	switch scriptKind {
-	case core.ScriptKindTSX, core.ScriptKindJSX, core.ScriptKindJS, core.ScriptKindJSON:
-		// .tsx and .jsx files are treated as jsx language variant.
-		return core.LanguageVariantJSX
-	}
-	return core.LanguageVariantStandard
-}
-
 func IsCallLikeExpression(node *Node) bool {
 	switch node.Kind {
 	case KindJsxOpeningElement, KindJsxSelfClosingElement, KindJsxOpeningFragment, KindCallExpression, KindNewExpression,