Add Full TypeScript Chapter (#1376)

* Add ts-morph

* Add Full TS chapter

* Add tests and comments

* Add support for builtins and preludes

* Add modules support

Author: zhaojj2209

package.json

@@ -33,6 +33,7 @@
   "dependencies": {
     "@babel/parser": "^7.19.4",
     "@joeychenofficial/alt-ergo-modified": "^2.4.0",
+    "@ts-morph/bootstrap": "^0.18.0",
     "@types/estree": "0.0.52",
     "acorn": "^8.0.3",
     "acorn-class-fields": "^1.0.0",

src/createContext.ts

@@ -430,7 +430,7 @@ const createContext = <T>(
   externalContext?: T,
   externalBuiltIns: CustomBuiltIns = defaultBuiltIns
 ): Context => {
-  if (chapter === Chapter.FULL_JS) {
+  if (chapter === Chapter.FULL_JS || chapter === Chapter.FULL_TS) {
     // fullJS will include all builtins and preludes of source 4
     return {
       ...createContext(
@@ -440,7 +440,7 @@ const createContext = <T>(
         externalContext,
         externalBuiltIns
       ),
-      chapter: Chapter.FULL_JS
+      chapter
     } as Context
   }
 

src/index.ts

@@ -329,7 +329,7 @@ export async function runFilesInContext(
     return resolvedErrorPromise
   }
 
-  if (context.chapter === Chapter.FULL_JS) {
+  if (context.chapter === Chapter.FULL_JS || context.chapter === Chapter.FULL_TS) {
     const program = parse(code, context)
     if (program === null) {
       return resolvedErrorPromise

src/parser/__tests__/__snapshots__/fullTS.ts.snap

@@ -0,0 +1,151 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`fullTS parser returns ESTree compliant program 1`] = `
+Node {
+  "body": Array [
+    Node {
+      "declarations": Array [
+        Node {
+          "end": 72,
+          "id": Node {
+            "end": 68,
+            "loc": SourceLocation {
+              "end": Position {
+                "column": 29,
+                "line": 2,
+              },
+              "identifierName": "x",
+              "source": undefined,
+              "start": Position {
+                "column": 12,
+                "line": 2,
+              },
+            },
+            "name": "x",
+            "start": 51,
+            "type": "Identifier",
+            "typeAnnotation": Node {
+              "end": 68,
+              "loc": SourceLocation {
+                "end": Position {
+                  "column": 29,
+                  "line": 2,
+                },
+                "identifierName": undefined,
+                "source": undefined,
+                "start": Position {
+                  "column": 13,
+                  "line": 2,
+                },
+              },
+              "start": 52,
+              "type": "TSTypeAnnotation",
+              "typeAnnotation": Node {
+                "end": 68,
+                "loc": SourceLocation {
+                  "end": Position {
+                    "column": 29,
+                    "line": 2,
+                  },
+                  "identifierName": undefined,
+                  "source": undefined,
+                  "start": Position {
+                    "column": 15,
+                    "line": 2,
+                  },
+                },
+                "start": 54,
+                "type": "TSTypeReference",
+                "typeName": Node {
+                  "end": 68,
+                  "loc": SourceLocation {
+                    "end": Position {
+                      "column": 29,
+                      "line": 2,
+                    },
+                    "identifierName": "StringOrNumber",
+                    "source": undefined,
+                    "start": Position {
+                      "column": 15,
+                      "line": 2,
+                    },
+                  },
+                  "name": "StringOrNumber",
+                  "start": 54,
+                  "type": "Identifier",
+                },
+              },
+            },
+          },
+          "init": Node {
+            "end": 72,
+            "loc": SourceLocation {
+              "end": Position {
+                "column": 33,
+                "line": 2,
+              },
+              "identifierName": undefined,
+              "source": undefined,
+              "start": Position {
+                "column": 32,
+                "line": 2,
+              },
+            },
+            "raw": "1",
+            "start": 71,
+            "type": "Literal",
+            "value": 1,
+          },
+          "loc": SourceLocation {
+            "end": Position {
+              "column": 33,
+              "line": 2,
+            },
+            "identifierName": undefined,
+            "source": undefined,
+            "start": Position {
+              "column": 12,
+              "line": 2,
+            },
+          },
+          "start": 51,
+          "type": "VariableDeclarator",
+        },
+      ],
+      "end": 73,
+      "kind": "const",
+      "loc": SourceLocation {
+        "end": Position {
+          "column": 34,
+          "line": 2,
+        },
+        "identifierName": undefined,
+        "source": undefined,
+        "start": Position {
+          "column": 6,
+          "line": 2,
+        },
+      },
+      "start": 45,
+      "type": "VariableDeclaration",
+    },
+  ],
+  "end": 78,
+  "interpreter": null,
+  "loc": SourceLocation {
+    "end": Position {
+      "column": 4,
+      "line": 3,
+    },
+    "identifierName": undefined,
+    "source": undefined,
+    "start": Position {
+      "column": 0,
+      "line": 1,
+    },
+  },
+  "sourceType": "module",
+  "start": 0,
+  "type": "Program",
+}
+`;

src/parser/__tests__/fullTS.ts

@@ -0,0 +1,53 @@
+import { parseError } from '../..'
+import { mockContext } from '../../mocks/context'
+import { Chapter } from '../../types'
+import { FullTSParser } from '../fullTS'
+
+const parser = new FullTSParser()
+let context = mockContext(Chapter.FULL_TS)
+
+beforeEach(() => {
+  context = mockContext(Chapter.FULL_TS)
+})
+
+describe('fullTS parser', () => {
+  it('formats errors correctly', () => {
+    const code = `type StringOrNumber = string | number;
+      const x: StringOrNumber = true;
+    `
+
+    parser.parse(code, context)
+    expect(parseError(context.errors)).toMatchInlineSnapshot(
+      `"Line 2: Type \'boolean\' is not assignable to type \'StringOrNumber\'."`
+    )
+  })
+
+  it('allows usage of builtins/preludes', () => {
+    const code = `const xs = list(1);
+      const ys = list(1);
+      equal(xs, ys);
+    `
+
+    parser.parse(code, context)
+    expect(parseError(context.errors)).toMatchInlineSnapshot(`""`)
+  })
+
+  it('allows usage of imports/modules', () => {
+    const code = `import { show, heart } from "rune";
+      show(heart);
+    `
+
+    parser.parse(code, context)
+    expect(parseError(context.errors)).toMatchInlineSnapshot(`""`)
+  })
+
+  it('returns ESTree compliant program', () => {
+    const code = `type StringOrNumber = string | number;
+      const x: StringOrNumber = 1;
+    `
+
+    // Resulting program should not have node for type alias declaration
+    const parsedProgram = parser.parse(code, context)
+    expect(parsedProgram).toMatchSnapshot()
+  })
+})

src/parser/fullTS/index.ts

@@ -0,0 +1,122 @@
+import { parse as babelParse } from '@babel/parser'
+import { createProjectSync, ts } from '@ts-morph/bootstrap'
+import { Program } from 'estree'
+
+import { Context } from '../..'
+import * as TypedES from '../../typeChecker/tsESTree'
+import { removeTSNodes } from '../../typeChecker/typeErrorChecker'
+import { FatalSyntaxError } from '../errors'
+import { transformBabelASTToESTreeCompliantAST } from '../source/typed/utils'
+import { AcornOptions, Parser } from '../types'
+import { defaultBabelOptions, positionToSourceLocation } from '../utils'
+
+const IMPORT_TOP_LEVEL_ERROR =
+  'An import declaration can only be used at the top level of a namespace or module.'
+const START_OF_MODULE_ERROR = 'Cannot find module '
+
+export class FullTSParser implements Parser<AcornOptions> {
+  parse(
+    programStr: string,
+    context: Context,
+    options?: Partial<AcornOptions>,
+    throwOnError?: boolean
+  ): Program | null {
+    let code = ''
+    // Add builtins to code
+    // Each declaration is replaced with a single constant declaration with type `any`
+    // to reduce evaluation time
+    for (const builtin of context.nativeStorage.builtins) {
+      code += `const ${builtin[0]}: any = 1\n`
+    }
+    // Add prelude functions to code
+    // Each declaration is replaced with a single constant declaration with type `any`
+    // to reduce evaluation time
+    if (context.prelude) {
+      const preludeFns = context.prelude.split('\nfunction ').slice(1)
+      preludeFns.forEach(fnString => {
+        const fnName = fnString.split('(')[0]
+        // Functions in prelude that start with $ are not added
+        if (fnName.startsWith('$')) {
+          return
+        }
+        code += `const ${fnName}: any = 1\n`
+      })
+    }
+    // Get line offset
+    const lineOffset = code.split('\n').length - 1
+
+    // Add program string to code string,
+    // wrapping it in a block to allow redeclaration of variables
+    code = code + '{' + programStr + '}'
+    // Initialize file to analyze
+    const project = createProjectSync({ useInMemoryFileSystem: true })
+    const filename = 'program.ts'
+    project.createSourceFile(filename, code)
+
+    // Get TS diagnostics from file, formatted as TS error string
+    const diagnostics = ts.getPreEmitDiagnostics(project.createProgram())
+    const formattedString = project.formatDiagnosticsWithColorAndContext(diagnostics)
+
+    // Reformat TS error string to Source error by getting line number using regex
+    // This is because logic to retrieve line number is only present in
+    // formatDiagnosticsWithColorAndContext and cannot be called directly
+    const lineNumRegex = /(?<=\[7m)\d+/
+    diagnostics.forEach(diagnostic => {
+      const message = diagnostic.messageText.toString()
+      // Ignore errors regarding imports
+      // as TS does not have information about Source modules
+      if (message === IMPORT_TOP_LEVEL_ERROR || message.startsWith(START_OF_MODULE_ERROR)) {
+        return
+      }
+      const lineNumRegExpArr = lineNumRegex.exec(formattedString.split(message)[1])
+      const lineNum = (lineNumRegExpArr === null ? 0 : parseInt(lineNumRegExpArr[0])) - lineOffset
+      // Ignore any errors that occur in builtins/prelude (line number <= 0)
+      if (lineNum <= 0) {
+        return
+      }
+      const position = { line: lineNum, column: 0, offset: 0 }
+      context.errors.push(new FatalSyntaxError(positionToSourceLocation(position), message))
+    })
+
+    if (context.errors.length > 0) {
+      return null
+    }
+
+    // Parse code into Babel AST, which supports type syntax
+    const ast = babelParse(programStr, {
+      ...defaultBabelOptions,
+      sourceFilename: options?.sourceFile,
+      errorRecovery: throwOnError ?? true
+    })
+
+    if (ast.errors.length) {
+      ast.errors
+        .filter(error => error instanceof SyntaxError)
+        .forEach(error => {
+          context.errors.push(
+            new FatalSyntaxError(
+              positionToSourceLocation((error as any).loc, options?.sourceFile),
+              error.toString()
+            )
+          )
+        })
+
+      return null
+    }
+
+    // Transform Babel AST into ESTree AST
+    const typedProgram: TypedES.Program = ast.program as TypedES.Program
+    const transpiledProgram: Program = removeTSNodes(typedProgram)
+    transformBabelASTToESTreeCompliantAST(transpiledProgram)
+
+    return transpiledProgram
+  }
+
+  validate(_ast: Program, _context: Context, _throwOnError: boolean): boolean {
+    return true
+  }
+
+  toString(): string {
+    return 'FullTSParser'
+  }
+}

src/parser/parser.ts

@@ -3,6 +3,7 @@ import { Program } from 'estree'
 import { Context } from '..'
 import { Chapter, Variant } from '../types'
 import { FullJSParser } from './fullJS'
+import { FullTSParser } from './fullTS'
 import { SourceParser } from './source'
 import { SourceTypedParser } from './source/typed'
 import { AcornOptions, Parser } from './types'
@@ -18,6 +19,9 @@ export function parse<TOptions extends AcornOptions>(
     case Chapter.FULL_JS:
       parser = new FullJSParser()
       break
+    case Chapter.FULL_TS:
+      parser = new FullTSParser()
+      break
     default:
       switch (context.variant) {
         case Variant.TYPED: