Make stdlib available to modules (#1388)

* Add import support to ec evaluator

* Add support for import checking

* Run format

* Fix module loader test

* Add tests

* Run format

* Add require provider

* Improve error handling

* Make stdlib available

* Update module loading and tests

* Fix require provider not properly providing js-slang

---------

Co-authored-by: Martin Henz <henz@comp.nus.edu.sg>

Author: leeyi45

src/__tests__/environment.ts

@@ -1,6 +1,6 @@
 import { Program } from 'estree'
 
-import { evaluate } from '../interpreter/interpreter'
+import { evaluateProgram as evaluate } from '../interpreter/interpreter'
 import { mockContext } from '../mocks/context'
 import { parse } from '../parser/parser'
 import { Chapter } from '../types'
@@ -18,7 +18,7 @@ test('Function params and body identifiers are in different environment', () =>
   const context = mockContext(Chapter.SOURCE_4)
   context.prelude = null // hide the unneeded prelude
   const parsed = parse(code, context)
-  const it = evaluate(parsed as any as Program, context)
+  const it = evaluate(parsed as any as Program, context, false, false)
   const stepsToComment = 13 // manually counted magic number
   for (let i = 0; i < stepsToComment; i += 1) {
     it.next()

src/constants.ts

@@ -6,6 +6,7 @@ import { Chapter, Language, Variant } from './types'
 export const DEFAULT_ECMA_VERSION = 6
 export const ACORN_PARSE_OPTIONS: AcornOptions = { ecmaVersion: DEFAULT_ECMA_VERSION }
 
+export const REQUIRE_PROVIDER_ID = 'requireProvider'
 export const CUT = 'cut' // cut operator for Source 4.3
 export const TRY_AGAIN = 'retry' // command for Source 4.3
 export const GLOBAL = typeof window === 'undefined' ? global : window

src/ec-evaluator/__tests__/__snapshots__/ec-evaluator-errors.ts.snap

@@ -792,6 +792,19 @@ Object {
 }
 `;
 
+exports[`Importing unknown variables throws UndefinedImport error: expectParsedError 1`] = `
+Object {
+  "alertResult": Array [],
+  "code": "import { foo1 } from 'one_module';",
+  "displayResult": Array [],
+  "numErrors": 1,
+  "parsedErrors": "'one_module' does not contain a definition for 'foo1'",
+  "result": undefined,
+  "resultStatus": "error",
+  "visualiseListResult": Array [],
+}
+`;
+
 exports[`In a block, every going-to-be-defined variable in the block cannot be accessed until it has been defined in the block.: expectParsedError 1`] = `
 Object {
   "alertResult": Array [],

src/ec-evaluator/__tests__/__snapshots__/ec-evaluator.ts.snap

@@ -1,5 +1,19 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`Imports are properly handled: expectResult 1`] = `
+Object {
+  "alertResult": Array [],
+  "code": "import { foo } from 'one_module';
+foo();",
+  "displayResult": Array [],
+  "numErrors": 0,
+  "parsedErrors": "",
+  "result": "foo",
+  "resultStatus": "finished",
+  "visualiseListResult": Array [],
+}
+`;
+
 exports[`Simple tail call returns work: expectResult 1`] = `
 Object {
   "alertResult": Array [],

src/ec-evaluator/__tests__/ec-evaluator-errors.ts

@@ -1,4 +1,6 @@
 /* tslint:disable:max-line-length */
+import * as _ from 'lodash'
+
 import { Chapter, Variant } from '../../types'
 import { stripIndent } from '../../utils/formatters'
 import {
@@ -8,6 +10,20 @@ import {
   expectResult
 } from '../../utils/testing'
 
+jest.spyOn(_, 'memoize').mockImplementation(func => func as any)
+
+const mockXMLHttpRequest = (xhr: Partial<XMLHttpRequest> = {}) => {
+  const xhrMock: Partial<XMLHttpRequest> = {
+    open: jest.fn(() => {}),
+    send: jest.fn(() => {}),
+    status: 200,
+    responseText: 'Hello World!',
+    ...xhr
+  }
+  jest.spyOn(window, 'XMLHttpRequest').mockImplementationOnce(() => xhrMock as XMLHttpRequest)
+  return xhrMock
+}
+
 const undefinedVariable = stripIndent`
 im_undefined;
 `
@@ -1000,3 +1016,35 @@ test('Shadowed variables may not be assigned to until declared in the current sc
     optionEC3
   ).toMatchInlineSnapshot(`"Line 3: Name variable not declared."`)
 })
+
+test('Importing unknown variables throws UndefinedImport error', () => {
+  // for getModuleFile
+  mockXMLHttpRequest({
+    responseText: `{
+    "one_module": {
+      "tabs": []
+    },
+    "another_module": {
+      "tabs": []
+    }
+  }`
+  })
+
+  // for bundle body
+  mockXMLHttpRequest({
+    responseText: `
+      require => {
+        return {
+          foo: () => 'foo',
+        }
+      }
+    `
+  })
+
+  return expectParsedError(
+    stripIndent`
+    import { foo1 } from 'one_module';
+  `,
+    optionEC
+  ).toMatchInlineSnapshot("\"'one_module' does not contain a definition for 'foo1'\"")
+})

src/ec-evaluator/__tests__/ec-evaluator.ts

@@ -2,6 +2,23 @@ import { Chapter, Variant } from '../../types'
 import { stripIndent } from '../../utils/formatters'
 import { expectResult } from '../../utils/testing'
 
+// jest.mock('lodash', () => ({
+//   ...jest.requireActual('lodash'),
+//   memoize: jest.fn(func => func)
+// }))
+
+const mockXMLHttpRequest = (xhr: Partial<XMLHttpRequest> = {}) => {
+  const xhrMock: Partial<XMLHttpRequest> = {
+    open: jest.fn(() => {}),
+    send: jest.fn(() => {}),
+    status: 200,
+    responseText: 'Hello World!',
+    ...xhr
+  }
+  jest.spyOn(window, 'XMLHttpRequest').mockImplementationOnce(() => xhrMock as XMLHttpRequest)
+  return xhrMock
+}
+
 const optionEC = { variant: Variant.EXPLICIT_CONTROL }
 const optionEC3 = { chapter: Chapter.SOURCE_3, variant: Variant.EXPLICIT_CONTROL }
 const optionEC4 = { chapter: Chapter.SOURCE_4, variant: Variant.EXPLICIT_CONTROL }
@@ -297,3 +314,36 @@ test('streams can be created and functions with no return statements are still e
     optionEC4
   ).toMatchInlineSnapshot(`false`)
 })
+
+test('Imports are properly handled', () => {
+  // for getModuleFile
+  mockXMLHttpRequest({
+    responseText: `{
+    "one_module": {
+      "tabs": []
+    },
+    "another_module": {
+      "tabs": []
+    }
+  }`
+  })
+
+  // for bundle body
+  mockXMLHttpRequest({
+    responseText: `
+      require => {
+        return {
+          foo: () => 'foo',
+        }
+      }
+    `
+  })
+
+  return expectResult(
+    stripIndent`
+    import { foo } from 'one_module';
+    foo();
+  `,
+    optionEC
+  ).toEqual('foo')
+})

src/ec-evaluator/interpreter.ts

@@ -7,12 +7,15 @@
 
 /* tslint:disable:max-classes-per-file */
 import * as es from 'estree'
-import { uniqueId } from 'lodash'
+import { partition, uniqueId } from 'lodash'
 
 import { UNKNOWN_LOCATION } from '../constants'
 import * as errors from '../errors/errors'
 import { RuntimeSourceError } from '../errors/runtimeSourceError'
 import Closure from '../interpreter/closure'
+import { UndefinedImportError } from '../modules/errors'
+import { loadModuleBundle, loadModuleTabs } from '../modules/moduleLoader'
+import { ModuleFunctions } from '../modules/moduleTypes'
 import { checkEditorBreakpoints } from '../stdlib/inspector'
 import { Context, ContiguousArrayElements, Result, Value } from '../types'
 import * as ast from '../utils/astCreator'
@@ -43,6 +46,7 @@ import {
   createEnvironment,
   currentEnvironment,
   declareFunctionsAndVariables,
+  declareIdentifier,
   defineVariable,
   getVariable,
   handleRuntimeError,
@@ -92,11 +96,18 @@ export class Stash extends Stack<Value> {
 export function evaluate(program: es.Program, context: Context): Value {
   try {
     context.runtime.isRunning = true
-    context.runtime.agenda = new Agenda(program)
+
+    const nonImportNodes = evaluateImports(program, context, true, true)
+
+    context.runtime.agenda = new Agenda({
+      ...program,
+      body: nonImportNodes
+    })
     context.runtime.stash = new Stash()
     return runECEMachine(context, context.runtime.agenda, context.runtime.stash)
   } catch (error) {
-    return new ECError()
+    // console.error('ecerror:', error)
+    return new ECError(error)
   } finally {
     context.runtime.isRunning = false
   }
@@ -115,12 +126,63 @@ export function resumeEvaluate(context: Context) {
     context.runtime.isRunning = true
     return runECEMachine(context, context.runtime.agenda!, context.runtime.stash!)
   } catch (error) {
-    return new ECError()
+    return new ECError(error)
   } finally {
     context.runtime.isRunning = false
   }
 }
 
+function evaluateImports(
+  program: es.Program,
+  context: Context,
+  loadTabs: boolean,
+  checkImports: boolean
+) {
+  const [importNodes, otherNodes] = partition(
+    program.body,
+    ({ type }) => type === 'ImportDeclaration'
+  ) as [es.ImportDeclaration[], es.Statement[]]
+
+  const moduleFunctions: Record<string, ModuleFunctions> = {}
+
+  try {
+    for (const node of importNodes) {
+      const moduleName = node.source.value
+      if (typeof moduleName !== 'string') {
+        throw new Error(`ImportDeclarations should have string sources, got ${moduleName}`)
+      }
+
+      if (!(moduleName in moduleFunctions)) {
+        context.moduleContexts[moduleName] = {
+          state: null,
+          tabs: loadTabs ? loadModuleTabs(moduleName, node) : null
+        }
+        moduleFunctions[moduleName] = loadModuleBundle(moduleName, context, node)
+      }
+
+      const functions = moduleFunctions[moduleName]
+      const environment = currentEnvironment(context)
+      for (const spec of node.specifiers) {
+        if (spec.type !== 'ImportSpecifier') {
+          throw new Error(`Only ImportSpecifiers are supported, got: ${spec.type}`)
+        }
+
+        if (checkImports && !(spec.imported.name in functions)) {
+          throw new UndefinedImportError(spec.imported.name, moduleName, node)
+        }
+
+        declareIdentifier(context, spec.local.name, node, environment)
+        defineVariable(context, spec.local.name, functions[spec.imported.name], true, node)
+      }
+    }
+  } catch (error) {
+    // console.log(error)
+    handleRuntimeError(context, error)
+  }
+
+  return otherNodes
+}
+
 /**
  * Function that returns the appropriate Promise<Result> given the output of ec evaluating, depending
  * on whether the program is finished evaluating, ran into a breakpoint or ran into an error.
@@ -339,6 +401,9 @@ const cmdEvaluators: { [type: string]: CmdEvaluator } = {
   ) {
     agenda.push(instr.breakInstr())
   },
+  ImportDeclaration: function () {
+    throw new Error('Import Declarations should already have been removed.')
+  },
 
   /**
    * Expressions

src/ec-evaluator/types.ts

@@ -105,4 +105,6 @@ export class ECEBreak {}
 
 // Special value that cannot be found on the stash so is safe to be used
 // as an indicator of an error from running the ECE machine
-export class ECError {}
+export class ECError {
+  constructor(public readonly error: any) {}
+}

src/ec-evaluator/utils.ts

@@ -205,7 +205,7 @@ export const createBlockEnvironment = (
 
 const DECLARED_BUT_NOT_YET_ASSIGNED = Symbol('Used to implement hoisting')
 
-function declareIdentifier(
+export function declareIdentifier(
   context: Context,
   name: string,
   node: es.Node,
@@ -259,7 +259,7 @@ export function defineVariable(
   name: string,
   value: Value,
   constant = false,
-  node: es.VariableDeclaration
+  node: es.VariableDeclaration | es.ImportDeclaration
 ) {
   const environment = currentEnvironment(context)
 

src/errors/moduleErrors.ts

@@ -36,7 +36,7 @@ export class ModuleNotFoundError extends RuntimeSourceError {
 }
 
 export class ModuleInternalError extends RuntimeSourceError {
-  constructor(public moduleName: string, node?: es.Node) {
+  constructor(public moduleName: string, public error?: any, node?: es.Node) {
     super(node)
   }