Merge pull request #9593 from erik-krogh/param2

QL: followup fixes to parameterized modules

Author: erik-krogh

.github/workflows/ql-for-ql-build.yml

@@ -50,9 +50,6 @@ jobs:
           path: ${{ runner.temp }}/query-pack.zip
 
   extractors:
-    strategy:
-      fail-fast: false
-
     runs-on: ubuntu-latest
 
     steps:
@@ -195,9 +192,36 @@ jobs:
           category: "ql-for-ql-${{ matrix.folder }}"
       - name: Copy sarif file to CWD
         run: cp ../results/ql.sarif ./${{ matrix.folder }}.sarif
+      - name: Fixup the $scema in sarif  # Until https://github.com/microsoft/sarif-vscode-extension/pull/436/ is part in a stable release
+        run: |
+          sed -i 's/\$schema.*/\$schema": "https:\/\/raw.githubusercontent.com\/oasis-tcs\/sarif-spec\/master\/Schemata\/sarif-schema-2.1.0",/' ${{ matrix.folder }}.sarif 
       - name: Sarif as artifact
         uses: actions/upload-artifact@v3
         with:
           name: ${{ matrix.folder }}.sarif
           path: ${{ matrix.folder }}.sarif
 
+  combine:
+    runs-on: ubuntu-latest
+    needs:
+      - analyze
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Make a folder for artifacts. 
+        run: mkdir -p results
+      - name: Download all sarif files
+        uses: actions/download-artifact@v3
+        with: 
+            path: results
+      - uses: actions/setup-node@v3
+        with:
+            node-version: 16
+      - name: Combine all sarif files
+        run: | 
+          node ./ql/scripts/merge-sarif.js results/**/*.sarif combined.sarif
+      - name: Upload combined sarif file
+        uses: actions/upload-artifact@v3
+        with:
+            name: combined.sarif
+            path: combined.sarif

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -36,7 +36,7 @@ jobs:
             ql/target
           key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
       - name: Build Extractor
-        run: cd ql; env "PATH=$PATH:`dirname ${CODEQL}`" ./create-extractor-pack.sh
+        run: cd ql; env "PATH=$PATH:`dirname ${CODEQL}`" ./scripts/create-extractor-pack.sh
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - name: Checkout ${{ matrix.repo }}

.github/workflows/ql-for-ql-tests.yml

@@ -36,7 +36,7 @@ jobs:
         run: |
           cd ql;
           codeqlpath=$(dirname ${{ steps.find-codeql.outputs.codeql-path }});
-          env "PATH=$PATH:$codeqlpath" ./create-extractor-pack.sh
+          env "PATH=$PATH:$codeqlpath" ./scripts/create-extractor-pack.sh
       - name: Run QL tests
         run: |
           "${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}/ql/extractor-pack" --consistency-queries ql/ql/consistency-queries ql/ql/test

javascript/ql/src/Expressions/TypoDatabase.qll

@@ -5793,6 +5793,8 @@ predicate typos(string wrong, string right) {
   or
   wrong = "paramters" and right = "parameters"
   or
+  wrong = "parametarized" and right = "parameterized"
+  or
   wrong = "paranthesis" and right = "parenthesis"
   or
   wrong = "paraphenalia" and right = "paraphernalia"

ql/README.md

@@ -21,14 +21,14 @@ cargo build --release
 The generated `ql/src/ql.dbscheme` and `ql/src/codeql_ql/ast/internal/TreeSitter.qll` files are included in the repository, but they can be re-generated as follows:
 
 ```bash
-./create-extractor-pack.sh
+./scripts/create-extractor-pack.sh
 ```
 
 ## Building a CodeQL database for a QL program
 
 First, get an extractor pack:
 
-Run `./create-extractor-pack.sh` (Linux/Mac) or `.\create-extractor-pack.ps1` (Windows PowerShell) and the pack will be created in the `extractor-pack` directory.
+Run `./scripts/create-extractor-pack.sh` (Linux/Mac) or `.\scripts\create-extractor-pack.ps1` (Windows PowerShell) and the pack will be created in the `extractor-pack` directory.
 
 Then run
 

ql/ql/src/codeql_ql/ast/Ast.qll

@@ -688,7 +688,7 @@ class TypeExpr extends TType, TypeRef {
     // resolve type
     resolveTypeExpr(this, result)
     or
-    // if it resolves to a module
+    // if it resolves to a module,
     exists(FileOrModule mod | resolveModuleRef(this, mod) | result = mod.toType())
   }
 
@@ -1181,7 +1181,7 @@ class Import extends TImport, ModuleMember, TypeRef {
    */
   string getImportString() {
     exists(string selec |
-      not exists(getSelectionName(_)) and selec = ""
+      not exists(this.getSelectionName(_)) and selec = ""
       or
       selec =
         "::" + strictconcat(int i, string q | q = this.getSelectionName(i) | q, "::" order by i)
@@ -2231,9 +2231,7 @@ class ModuleExpr extends TModuleExpr, TypeRef {
     or
     not exists(me.getName()) and result = me.getChild().(QL::SimpleId).getValue()
     or
-    exists(QL::ModuleInstantiation instantiation | instantiation.getParent() = me |
-      result = instantiation.getName().getChild().getValue()
-    )
+    result = me.getAFieldOrChild().(QL::ModuleInstantiation).getName().getChild().getValue()
   }
 
   /**
@@ -2268,9 +2266,7 @@ class ModuleExpr extends TModuleExpr, TypeRef {
    * The result is either a `PredicateExpr` or a `TypeExpr`.
    */
   SignatureExpr getArgument(int i) {
-    exists(QL::ModuleInstantiation instantiation | instantiation.getParent() = me |
-      result.toQL() = instantiation.getChild(i)
-    )
+    result.toQL() = me.getAFieldOrChild().(QL::ModuleInstantiation).getChild(i)
   }
 }
 

ql/ql/src/codeql_ql/ast/internal/Module.qll

@@ -218,6 +218,20 @@ private module Cached {
     not exists(me.(ModuleExpr).getQualifier()) and
     exists(ContainerOrModule enclosing, string name | resolveModuleRefHelper(me, enclosing, name) |
       definesModule(enclosing, name, m, _)
+    ) and
+    (
+      not me instanceof TypeExpr
+      or
+      // remove some spurious results that can happen with `TypeExpr`
+      me instanceof TypeExpr and
+      m instanceof Module_ and // TypeExpr can only resolve to a Module, and only in some scenarios
+      (
+        // only possible if this is inside a moduleInstantiation.
+        me = any(ModuleExpr mod).getArgument(_).asType()
+        or
+        // or if it's a parameter to a parameterized module
+        me = any(SignatureExpr sig, Module mod | mod.hasParameter(_, _, sig) | sig).asType()
+      )
     )
     or
     exists(FileOrModule mid |
@@ -229,7 +243,8 @@ private module Cached {
   pragma[noinline]
   private predicate resolveModuleRefHelper(TypeRef me, ContainerOrModule enclosing, string name) {
     enclosing = getEnclosingModule(me).getEnclosing*() and
-    name = [me.(ModuleExpr).getName(), me.(TypeExpr).getClassName()]
+    name = [me.(ModuleExpr).getName(), me.(TypeExpr).getClassName()] and
+    (not me instanceof ModuleExpr or not enclosing instanceof Folder_) // module expressions are not imports, so they can't resolve to a file (which is contained in a folder).
   }
 }
 
@@ -332,7 +347,19 @@ module ModConsistency {
    *  }
    */
 
-  query predicate noName(Module mod) { not exists(mod.getName()) }
+  query predicate noName(AstNode mod) {
+    mod instanceof Module and
+    not exists(mod.(Module).getName())
+    or
+    mod instanceof ModuleExpr and
+    not exists(mod.(ModuleExpr).getName())
+  }
 
-  query predicate nonUniqueName(Module mod) { count(mod.getName()) >= 2 }
+  query predicate nonUniqueName(AstNode mod) {
+    mod instanceof Module and
+    count(mod.(Module).getName()) >= 2
+    or
+    mod instanceof ModuleExpr and
+    count(mod.(ModuleExpr).getName()) >= 2
+  }
 }

ql/ql/src/codeql_ql/ast/internal/Predicate.qll

@@ -31,7 +31,8 @@ private predicate definesPredicate(
       name = alias.getName() and
       resolvePredicateExpr(alias.getAlias(), p) and
       public = getPublicBool(alias) and
-      arity = alias.getArity()
+      arity = alias.getArity() and
+      arity = p.getArity()
     )
   )
 }
@@ -48,15 +49,18 @@ private module Cached {
       m = pe.getQualifier().getResolvedModule() and
       public = true
     |
-      definesPredicate(m, pe.getName(), p.getArity(), p, public)
+      definesPredicate(m, pe.getName(), p.getArity(), p, public) and
+      p.getArity() = pe.getArity()
     )
   }
 
   private predicate resolvePredicateCall(PredicateCall pc, PredicateOrBuiltin p) {
+    // calls to class methods
     exists(Class c, ClassType t |
       c = pc.getParent*() and
       t = c.getType() and
-      p = t.getClassPredicate(pc.getPredicateName(), pc.getNumberOfArguments())
+      p = t.getClassPredicate(pc.getPredicateName(), pc.getNumberOfArguments()) and
+      not exists(pc.getQualifier()) // no module qualifier, because then it's not a call to a class method.
     )
     or
     exists(FileOrModule m, boolean public |
@@ -85,22 +89,47 @@ private module Cached {
     )
   }
 
+  /**
+   *  Holds if `mc` is a `this.method()` call to a predicate defined in the same class.
+   * helps avoid spuriously resolving to predicates in super-classes.
+   */
+  private predicate resolveSelfClassCalls(MemberCall mc, PredicateOrBuiltin p) {
+    exists(Class c |
+      mc.getBase() instanceof ThisAccess and
+      c = mc.getEnclosingPredicate().getParent() and
+      p = c.getClassPredicate(mc.getMemberName()) and
+      p.getArity() = mc.getNumberOfArguments()
+    )
+  }
+
   private predicate resolveMemberCall(MemberCall mc, PredicateOrBuiltin p) {
+    resolveSelfClassCalls(mc, p)
+    or
+    not resolveSelfClassCalls(mc, _) and
     exists(Type t |
       t = mc.getBase().getType() and
       p = t.getClassPredicate(mc.getMemberName(), mc.getNumberOfArguments())
     )
     or
     // super calls - and `this.method()` calls in charpreds. (Basically: in charpreds there is no difference between super and this.)
-    exists(AstNode sup, ClassType type, Type supertype |
+    exists(AstNode sup, Type supertype |
       sup instanceof Super
       or
       sup.(ThisAccess).getEnclosingPredicate() instanceof CharPred
     |
       mc.getBase() = sup and
-      sup.getEnclosingPredicate().getParent().(Class).getType() = type and
-      supertype in [type.getASuperType(), type.getAnInstanceofType()] and
-      p = supertype.getClassPredicate(mc.getMemberName(), mc.getNumberOfArguments())
+      p = supertype.getClassPredicate(mc.getMemberName(), mc.getNumberOfArguments()) and
+      (
+        // super.method()
+        not exists(mc.getSuperType()) and
+        exists(ClassType type |
+          sup.getEnclosingPredicate().getParent().(Class).getType() = type and
+          supertype in [type.getASuperType(), type.getAnInstanceofType()]
+        )
+        or
+        // Class.super.method()
+        supertype = mc.getSuperType().getResolvedType()
+      )
     )
   }
 
@@ -172,24 +201,29 @@ module PredConsistency {
   }
 
   query predicate multipleResolvePredicateExpr(PredicateExpr pe, int c, ClasslessPredicate p) {
-    c = strictcount(ClasslessPredicate p0 | resolvePredicateExpr(pe, p0)) and
+    c =
+      strictcount(ClasslessPredicate p0 |
+        resolvePredicateExpr(pe, p0) and
+        // aliases are expected to resolve to multiple.
+        not exists(p0.getAlias())
+      ) and
     c > 1 and
     resolvePredicateExpr(pe, p)
   }
-  // This can happen with parametarized modules
-  /*
-   * query predicate multipleResolveCall(Call call, int c, PredicateOrBuiltin p) {
-   *    c =
-   *      strictcount(PredicateOrBuiltin p0 |
-   *        resolveCall(call, p0) and
-   *        // aliases are expected to resolve to multiple.
-   *        not exists(p0.(ClasslessPredicate).getAlias()) and
-   *        // overridden predicates may have multiple targets
-   *        not p0.(ClassPredicate).isOverride()
-   *      ) and
-   *    c > 1 and
-   *    resolveCall(call, p)
-   *  }
-   */
 
+  query predicate multipleResolveCall(Call call, int c, PredicateOrBuiltin p) {
+    c =
+      strictcount(PredicateOrBuiltin p0 |
+        resolveCall(call, p0) and
+        // aliases are expected to resolve to multiple.
+        not exists(p0.(ClasslessPredicate).getAlias()) and
+        // overridden predicates may have multiple targets
+        not p0.(ClassPredicate).isOverride() and
+        not p0 instanceof Relation // <- DB relations resolve to both a relation and a predicate.
+      ) and
+    c > 1 and
+    resolveCall(call, p) and
+    // parameterized modules are expected to resolve to multiple.
+    not exists(Predicate sig | not exists(sig.getBody()) and resolveCall(call, sig))
   }
+}

ql/ql/src/codeql_ql/dataflow/DataFlow.qll

@@ -204,7 +204,7 @@ class SuperNode extends LocalFlow::TSuperNode {
   Node getANode() { LocalFlow::getRepr(result) = repr }
 
   /** Gets an AST node from any of the nodes in this super node. */
-  AstNode asAstNode() { result = getANode().asAstNode() }
+  AstNode asAstNode() { result = this.getANode().asAstNode() }
 
   /**
    * Gets a single node from this super node.
@@ -214,23 +214,25 @@ class SuperNode extends LocalFlow::TSuperNode {
    * - An `AstNodeNode` is preferred over other nodes.
    * - A node occurring earlier is preferred over one occurring later.
    */
-  Node getArbitraryRepr() { result = min(Node n | n = getANode() | n order by getInternalId(n)) }
+  Node getArbitraryRepr() {
+    result = min(Node n | n = this.getANode() | n order by getInternalId(n))
+  }
 
   /**
    * Gets the predicate containing all nodes that are part of this super node.
    */
-  Predicate getEnclosingPredicate() { result = getANode().getEnclosingPredicate() }
+  Predicate getEnclosingPredicate() { result = this.getANode().getEnclosingPredicate() }
 
   /** Gets a string representation of this super node. */
   string toString() {
     exists(int c |
-      c = strictcount(getANode()) and
-      result = "Super node of " + c + " nodes in " + getEnclosingPredicate().getName()
+      c = strictcount(this.getANode()) and
+      result = "Super node of " + c + " nodes in " + this.getEnclosingPredicate().getName()
     )
   }
 
   /** Gets the location of an arbitrary node in this super node. */
-  Location getLocation() { result = getArbitraryRepr().getLocation() }
+  Location getLocation() { result = this.getArbitraryRepr().getLocation() }
 
   /** Gets any member call whose receiver is in the same super node. */
   MemberCall getALocalMemberCall() { superNode(result.getBase()) = this }
@@ -287,7 +289,7 @@ class SuperNode extends LocalFlow::TSuperNode {
   cached
   private string getAStringValue(Tracker t) {
     t.start() and
-    result = asAstNode().(String).getValue()
+    result = this.asAstNode().(String).getValue()
     or
     exists(SuperNode pred, Tracker t2 |
       this = pred.track(t2, t) and

ql/ql/src/codeql_ql/style/TypoDatabase.qll

@@ -5793,6 +5793,8 @@ predicate typos(string wrong, string right) {
   or
   wrong = "paramters" and right = "parameters"
   or
+  wrong = "parametarized" and right = "parameterized"
+  or
   wrong = "paranthesis" and right = "parenthesis"
   or
   wrong = "paraphenalia" and right = "paraphernalia"