Merge pull request #10470 from erik-krogh/flowParse

JS: Try to parse files without using our parser extensions before enabling the extensions

Author: erik-krogh

javascript/extractor/src/com/semmle/jcorn/Options.java

@@ -40,7 +40,7 @@ public void call(
   }
 
   private boolean allowHashBang, allowReturnOutsideFunction, allowImportExportEverywhere, allowGeneratedCodeExprs;
-  private boolean preserveParens, mozExtensions, jscript, esnext, v8Extensions, e4x;
+  private boolean preserveParens, mozExtensions, jscript, esnext, v8Extensions, e4x, allowFlowTypes;
   private int ecmaVersion;
   private AllowReserved allowReserved;
   private String sourceType;
@@ -70,6 +70,7 @@ public Options() {
     this.v8Extensions = false;
     this.e4x = false;
     this.onRecoverableError = null;
+    this.allowFlowTypes = false;
   }
 
   public Options(Options that) {
@@ -92,6 +93,7 @@ public Options(Options that) {
     this.onComment = that.onComment;
     this.program = that.program;
     this.onRecoverableError = that.onRecoverableError;
+    this.allowFlowTypes = that.allowFlowTypes;
   }
 
   public boolean allowHashBang() {
@@ -130,6 +132,10 @@ public boolean v8Extensions() {
     return v8Extensions;
   }
 
+  public boolean allowFlowTypes() {
+    return allowFlowTypes;
+  }
+
   public boolean e4x() {
     return e4x;
   }
@@ -202,6 +208,10 @@ public void v8Extensions(boolean v8Extensions) {
     this.v8Extensions = v8Extensions;
   }
 
+  public void allowFlowTypes(boolean allowFlowTypes) {
+    this.allowFlowTypes = allowFlowTypes;
+  }
+
   public void e4x(boolean e4x) {
     this.e4x = e4x;
   }

javascript/extractor/src/com/semmle/jcorn/flow/FlowParser.java

@@ -830,7 +830,7 @@ private void flowParseTypeAnnotatableIdentifier(
 
   /** Should Flow syntax be allowed? */
   private boolean flow() {
-    return options.esnext();
+    return options.allowFlowTypes();
   }
 
   @Override

javascript/extractor/src/com/semmle/js/extractor/Main.java

@@ -41,7 +41,7 @@ public class Main {
    * A version identifier that should be updated every time the extractor changes in such a way that
    * it may produce different tuples for the same file under the same {@link ExtractorConfig}.
    */
-  public static final String EXTRACTOR_VERSION = "2022-08-25";
+  public static final String EXTRACTOR_VERSION = "2022-09-19";
 
   public static final Pattern NEWLINE = Pattern.compile("\n");
 

javascript/extractor/src/com/semmle/js/parser/JcornWrapper.java

@@ -28,20 +28,29 @@ public static JSParser.Result parse(
             .onToken(tokens)
             .preserveParens(true)
             .allowReturnOutsideFunction(true);
-    if (config.isMozExtensions()) options.mozExtensions(true);
-    if (config.isJscript()) options.jscript(true);
-    if (config.isJsx()) options = new JSXOptions(options);
     if (config.isEsnext()) options.esnext(true);
-    if (config.isV8Extensions()) options.v8Extensions(true);
-    if (config.isE4X()) options.e4x(true);
 
     Program program = null;
     List<ParseError> errors = new ArrayList<>();
+    
     try {
-      if (config.isTolerateParseErrors())
-        options.onRecoverableError((err) -> errors.add(mkParseError(err)));
-
-      program = sourceType.createParser(options, source, 0).parse();
+      try {
+        // First try to parse as a regular JavaScript program.
+        program = sourceType.createParser(options, source, 0).parse();
+      } catch (SyntaxError e) {
+        // If that fails, try to enable all the extensions that we support.
+        if (config.isTolerateParseErrors())
+          options.onRecoverableError((err) -> errors.add(mkParseError(err)));
+        comments.clear();
+        tokens.clear();
+        if (config.isMozExtensions()) options.mozExtensions(true);
+        if (config.isJscript()) options.jscript(true);
+        if (config.isJsx()) options = new JSXOptions(options);
+        if (config.isV8Extensions()) options.v8Extensions(true);
+        if (config.isE4X()) options.e4x(true);
+        if (config.isEsnext()) options.allowFlowTypes(true); // allow the flow-parser to parse types.
+        program = sourceType.createParser(options, source, 0).parse();
+      }
     } catch (SyntaxError e) {
       errors.add(mkParseError(e));
     }

javascript/extractor/tests/mozilla/output/trap/odasa-2593.js.trap

@@ -133,76 +133,71 @@ locations_default(#20044,#10000,1,17,4,1)
 hasLocation(#20043,#20044)
 stmt_containers(#20043,#20039)
 #20045=*
-scopes(#20045,4)
-scopenodes(#20043,#20045)
-scopenesting(#20045,#20041)
-#20046=@"var;{arguments};{#20045}"
-variables(#20046,"arguments",#20045)
+stmts(#20045,2,#20043,0,"arguments;")
+#20046=@"loc,{#10000},2,5,2,14"
+locations_default(#20046,#10000,2,5,2,14)
+hasLocation(#20045,#20046)
+stmt_containers(#20045,#20039)
 #20047=*
-stmts(#20047,2,#20043,0,"arguments;")
-#20048=@"loc,{#10000},2,5,2,14"
-locations_default(#20048,#10000,2,5,2,14)
-hasLocation(#20047,#20048)
-stmt_containers(#20047,#20039)
-#20049=*
-exprs(#20049,79,#20047,0,"arguments")
-hasLocation(#20049,#20021)
-enclosing_stmt(#20049,#20047)
-expr_containers(#20049,#20039)
-literals("arguments","arguments",#20049)
-bind(#20049,#20046)
+exprs(#20047,79,#20045,0,"arguments")
+hasLocation(#20047,#20021)
+enclosing_stmt(#20047,#20045)
+expr_containers(#20047,#20039)
+literals("arguments","arguments",#20047)
+bind(#20047,#20042)
+#20048=*
+stmts(#20048,2,#20043,1,"let (arguments);")
+#20049=@"loc,{#10000},3,5,3,20"
+locations_default(#20049,#10000,3,5,3,20)
+hasLocation(#20048,#20049)
+stmt_containers(#20048,#20039)
 #20050=*
-stmts(#20050,24,#20043,1,"let (arguments);")
-#20051=@"loc,{#10000},3,5,3,20"
-locations_default(#20051,#10000,3,5,3,20)
+exprs(#20050,13,#20048,0,"let (arguments)")
+#20051=@"loc,{#10000},3,5,3,19"
+locations_default(#20051,#10000,3,5,3,19)
 hasLocation(#20050,#20051)
-stmt_containers(#20050,#20039)
+enclosing_stmt(#20050,#20048)
+expr_containers(#20050,#20039)
 #20052=*
-scopes(#20052,4)
-scopenodes(#20050,#20052)
-scopenesting(#20052,#20045)
-#20053=@"var;{arguments};{#20052}"
-variables(#20053,"arguments",#20052)
+exprs(#20052,79,#20050,-1,"let")
+hasLocation(#20052,#20025)
+enclosing_stmt(#20052,#20048)
+expr_containers(#20052,#20039)
+literals("let","let",#20052)
+#20053=@"var;{let};{#20000}"
+variables(#20053,"let",#20000)
+bind(#20052,#20053)
 #20054=*
-exprs(#20054,64,#20050,0,"arguments")
+exprs(#20054,79,#20050,0,"arguments")
 hasLocation(#20054,#20029)
-enclosing_stmt(#20054,#20050)
+enclosing_stmt(#20054,#20048)
 expr_containers(#20054,#20039)
+literals("arguments","arguments",#20054)
+bind(#20054,#20042)
 #20055=*
-exprs(#20055,78,#20054,0,"arguments")
-hasLocation(#20055,#20029)
-enclosing_stmt(#20055,#20050)
-expr_containers(#20055,#20039)
-literals("arguments","arguments",#20055)
-decl(#20055,#20053)
-#20056=*
-stmts(#20056,0,#20050,-1,";")
-hasLocation(#20056,#20033)
-stmt_containers(#20056,#20039)
+entry_cfg_node(#20055,#20001)
+#20056=@"loc,{#10000},1,1,1,0"
+locations_default(#20056,#10000,1,1,1,0)
+hasLocation(#20055,#20056)
 #20057=*
-entry_cfg_node(#20057,#20001)
-#20058=@"loc,{#10000},1,1,1,0"
-locations_default(#20058,#10000,1,1,1,0)
-hasLocation(#20057,#20058)
+exit_cfg_node(#20057,#20001)
+hasLocation(#20057,#20036)
+successor(#20039,#20057)
+#20058=*
+entry_cfg_node(#20058,#20039)
+hasLocation(#20058,#20056)
 #20059=*
-exit_cfg_node(#20059,#20001)
+exit_cfg_node(#20059,#20039)
 hasLocation(#20059,#20036)
-successor(#20039,#20059)
-#20060=*
-entry_cfg_node(#20060,#20039)
-hasLocation(#20060,#20058)
-#20061=*
-exit_cfg_node(#20061,#20039)
-hasLocation(#20061,#20036)
-successor(#20043,#20047)
-successor(#20050,#20055)
-successor(#20056,#20061)
-successor(#20055,#20054)
-successor(#20054,#20056)
-successor(#20047,#20049)
-successor(#20049,#20050)
-successor(#20060,#20043)
+successor(#20043,#20045)
+successor(#20048,#20052)
+successor(#20054,#20050)
+successor(#20052,#20054)
+successor(#20050,#20059)
+successor(#20045,#20047)
+successor(#20047,#20048)
+successor(#20058,#20043)
 successor(#20040,#20039)
-successor(#20057,#20040)
+successor(#20055,#20040)
 numlines(#10000,4,4,0)
 filetype(#10000,"javascript")

javascript/ql/src/change-notes/2022-09-22-plain-parsing.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Improved how the JavaScript parser handles ambiguities between plain JavaScript and dialects such as Flow and E4X that use the same file extension. The parser now prefers plain JavaScript if possible, falling back to dialects only if the source code can not be parsed as plain JavaScript. Previously, there were rare cases where parsing would fail because the parser would erroneously attempt to parse dialect-specific syntax in a regular JavaScript file.

javascript/ql/test/query-tests/LanguageFeatures/SyntaxError/notFlow.js

@@ -0,0 +1 @@
+foo ? () => ({}) : (name) => "foo";