try to parse JS files without using the supported extensions

Author: erik-krogh

javascript/extractor/src/com/semmle/jcorn/Options.java

@@ -40,7 +40,7 @@ public void call(
   }
 
   private boolean allowHashBang, allowReturnOutsideFunction, allowImportExportEverywhere, allowGeneratedCodeExprs;
-  private boolean preserveParens, mozExtensions, jscript, esnext, v8Extensions, e4x;
+  private boolean preserveParens, mozExtensions, jscript, esnext, v8Extensions, e4x, allowFlowTypes;
   private int ecmaVersion;
   private AllowReserved allowReserved;
   private String sourceType;
@@ -70,6 +70,7 @@ public Options() {
     this.v8Extensions = false;
     this.e4x = false;
     this.onRecoverableError = null;
+    this.allowFlowTypes = false;
   }
 
   public Options(Options that) {
@@ -92,6 +93,7 @@ public Options(Options that) {
     this.onComment = that.onComment;
     this.program = that.program;
     this.onRecoverableError = that.onRecoverableError;
+    this.allowFlowTypes = that.allowFlowTypes;
   }
 
   public boolean allowHashBang() {
@@ -130,6 +132,10 @@ public boolean v8Extensions() {
     return v8Extensions;
   }
 
+  public boolean allowFlowTypes() {
+    return allowFlowTypes;
+  }
+
   public boolean e4x() {
     return e4x;
   }
@@ -202,6 +208,10 @@ public void v8Extensions(boolean v8Extensions) {
     this.v8Extensions = v8Extensions;
   }
 
+  public void allowFlowTypes(boolean allowFlowTypes) {
+    this.allowFlowTypes = allowFlowTypes;
+  }
+
   public void e4x(boolean e4x) {
     this.e4x = e4x;
   }

javascript/extractor/src/com/semmle/jcorn/flow/FlowParser.java

@@ -830,7 +830,7 @@ private void flowParseTypeAnnotatableIdentifier(
 
   /** Should Flow syntax be allowed? */
   private boolean flow() {
-    return options.esnext();
+    return options.allowFlowTypes();
   }
 
   @Override

javascript/extractor/src/com/semmle/js/extractor/Main.java

@@ -41,7 +41,7 @@ public class Main {
    * A version identifier that should be updated every time the extractor changes in such a way that
    * it may produce different tuples for the same file under the same {@link ExtractorConfig}.
    */
-  public static final String EXTRACTOR_VERSION = "2022-08-25";
+  public static final String EXTRACTOR_VERSION = "2022-09-19";
 
   public static final Pattern NEWLINE = Pattern.compile("\n");
 

javascript/extractor/src/com/semmle/js/parser/JcornWrapper.java

@@ -28,20 +28,29 @@ public static JSParser.Result parse(
             .onToken(tokens)
             .preserveParens(true)
             .allowReturnOutsideFunction(true);
-    if (config.isMozExtensions()) options.mozExtensions(true);
-    if (config.isJscript()) options.jscript(true);
-    if (config.isJsx()) options = new JSXOptions(options);
     if (config.isEsnext()) options.esnext(true);
-    if (config.isV8Extensions()) options.v8Extensions(true);
-    if (config.isE4X()) options.e4x(true);
 
     Program program = null;
     List<ParseError> errors = new ArrayList<>();
+    
     try {
-      if (config.isTolerateParseErrors())
-        options.onRecoverableError((err) -> errors.add(mkParseError(err)));
-
-      program = sourceType.createParser(options, source, 0).parse();
+      try {
+        // First try to parse as a regular JavaScript program.
+        program = sourceType.createParser(options, source, 0).parse();
+      } catch (SyntaxError e) {
+        // If that fails, try to enable all the extensions that we support.
+        if (config.isTolerateParseErrors())
+          options.onRecoverableError((err) -> errors.add(mkParseError(err)));
+        comments.clear();
+        tokens.clear();
+        if (config.isMozExtensions()) options.mozExtensions(true);
+        if (config.isJscript()) options.jscript(true);
+        if (config.isJsx()) options = new JSXOptions(options);
+        if (config.isV8Extensions()) options.v8Extensions(true);
+        if (config.isE4X()) options.e4x(true);
+        if (config.isEsnext()) options.allowFlowTypes(true); // allow the flow-parser to parse types.
+        program = sourceType.createParser(options, source, 0).parse();
+      }
     } catch (SyntaxError e) {
       errors.add(mkParseError(e));
     }

javascript/extractor/tests/mozilla/output/trap/odasa-2593.js.trap

@@ -133,76 +133,71 @@ locations_default(#20044,#10000,1,17,4,1)
 hasLocation(#20043,#20044)
 stmt_containers(#20043,#20039)
 #20045=*
-scopes(#20045,4)
-scopenodes(#20043,#20045)
-scopenesting(#20045,#20041)
-#20046=@"var;{arguments};{#20045}"
-variables(#20046,"arguments",#20045)
+stmts(#20045,2,#20043,0,"arguments;")
+#20046=@"loc,{#10000},2,5,2,14"
+locations_default(#20046,#10000,2,5,2,14)
+hasLocation(#20045,#20046)
+stmt_containers(#20045,#20039)
 #20047=*
-stmts(#20047,2,#20043,0,"arguments;")
-#20048=@"loc,{#10000},2,5,2,14"
-locations_default(#20048,#10000,2,5,2,14)
-hasLocation(#20047,#20048)
-stmt_containers(#20047,#20039)
-#20049=*
-exprs(#20049,79,#20047,0,"arguments")
-hasLocation(#20049,#20021)
-enclosing_stmt(#20049,#20047)
-expr_containers(#20049,#20039)
-literals("arguments","arguments",#20049)
-bind(#20049,#20046)
+exprs(#20047,79,#20045,0,"arguments")
+hasLocation(#20047,#20021)
+enclosing_stmt(#20047,#20045)
+expr_containers(#20047,#20039)
+literals("arguments","arguments",#20047)
+bind(#20047,#20042)
+#20048=*
+stmts(#20048,2,#20043,1,"let (arguments);")
+#20049=@"loc,{#10000},3,5,3,20"
+locations_default(#20049,#10000,3,5,3,20)
+hasLocation(#20048,#20049)
+stmt_containers(#20048,#20039)
 #20050=*
-stmts(#20050,24,#20043,1,"let (arguments);")
-#20051=@"loc,{#10000},3,5,3,20"
-locations_default(#20051,#10000,3,5,3,20)
+exprs(#20050,13,#20048,0,"let (arguments)")
+#20051=@"loc,{#10000},3,5,3,19"
+locations_default(#20051,#10000,3,5,3,19)
 hasLocation(#20050,#20051)
-stmt_containers(#20050,#20039)
+enclosing_stmt(#20050,#20048)
+expr_containers(#20050,#20039)
 #20052=*
-scopes(#20052,4)
-scopenodes(#20050,#20052)
-scopenesting(#20052,#20045)
-#20053=@"var;{arguments};{#20052}"
-variables(#20053,"arguments",#20052)
+exprs(#20052,79,#20050,-1,"let")
+hasLocation(#20052,#20025)
+enclosing_stmt(#20052,#20048)
+expr_containers(#20052,#20039)
+literals("let","let",#20052)
+#20053=@"var;{let};{#20000}"
+variables(#20053,"let",#20000)
+bind(#20052,#20053)
 #20054=*
-exprs(#20054,64,#20050,0,"arguments")
+exprs(#20054,79,#20050,0,"arguments")
 hasLocation(#20054,#20029)
-enclosing_stmt(#20054,#20050)
+enclosing_stmt(#20054,#20048)
 expr_containers(#20054,#20039)
+literals("arguments","arguments",#20054)
+bind(#20054,#20042)
 #20055=*
-exprs(#20055,78,#20054,0,"arguments")
-hasLocation(#20055,#20029)
-enclosing_stmt(#20055,#20050)
-expr_containers(#20055,#20039)
-literals("arguments","arguments",#20055)
-decl(#20055,#20053)
-#20056=*
-stmts(#20056,0,#20050,-1,";")
-hasLocation(#20056,#20033)
-stmt_containers(#20056,#20039)
+entry_cfg_node(#20055,#20001)
+#20056=@"loc,{#10000},1,1,1,0"
+locations_default(#20056,#10000,1,1,1,0)
+hasLocation(#20055,#20056)
 #20057=*
-entry_cfg_node(#20057,#20001)
-#20058=@"loc,{#10000},1,1,1,0"
-locations_default(#20058,#10000,1,1,1,0)
-hasLocation(#20057,#20058)
+exit_cfg_node(#20057,#20001)
+hasLocation(#20057,#20036)
+successor(#20039,#20057)
+#20058=*
+entry_cfg_node(#20058,#20039)
+hasLocation(#20058,#20056)
 #20059=*
-exit_cfg_node(#20059,#20001)
+exit_cfg_node(#20059,#20039)
 hasLocation(#20059,#20036)
-successor(#20039,#20059)
-#20060=*
-entry_cfg_node(#20060,#20039)
-hasLocation(#20060,#20058)
-#20061=*
-exit_cfg_node(#20061,#20039)
-hasLocation(#20061,#20036)
-successor(#20043,#20047)
-successor(#20050,#20055)
-successor(#20056,#20061)
-successor(#20055,#20054)
-successor(#20054,#20056)
-successor(#20047,#20049)
-successor(#20049,#20050)
-successor(#20060,#20043)
+successor(#20043,#20045)
+successor(#20048,#20052)
+successor(#20054,#20050)
+successor(#20052,#20054)
+successor(#20050,#20059)
+successor(#20045,#20047)
+successor(#20047,#20048)
+successor(#20058,#20043)
 successor(#20040,#20039)
-successor(#20057,#20040)
+successor(#20055,#20040)
 numlines(#10000,4,4,0)
 filetype(#10000,"javascript")

javascript/ql/test/query-tests/LanguageFeatures/SyntaxError/SyntaxError.expected

@@ -1,4 +1,3 @@
 | arrows.js:1:5:1:5 | Error: Argument name clash | Error: Argument name clash |
 | destructingPrivate.js:4:6:4:6 | Error: Unexpected token | Error: Unexpected token |
-| notFlow.js:1:35:1:35 | Error: Unexpected token | Error: Unexpected token |
 | tst.js:2:12:2:12 | Error: Unterminated string constant | Error: Unterminated string constant |