Merge branch 'main' into closure-expr-as-cfg-callable

Author: MathiasVP

ql/Cargo.lock

@@ -10,7 +10,7 @@ edition = "2018"
 flate2 = "1.0"
 node-types = { path = "../node-types" }
 tree-sitter = "0.19"
-tree-sitter-ql = { git = "https://github.com/tausbn/tree-sitter-ql.git", rev = "725395405e65814f10095a451404b0ced5dc6289" }
+tree-sitter-ql = { git = "https://github.com/tausbn/tree-sitter-ql.git", rev = "99f3bc30fa772b07ad19a23799fe7433dc15f765" }
 clap = "2.33"
 tracing = "0.1"
 tracing-subscriber = { version = "0.3.3", features = ["env-filter"] }

ql/extractor/Cargo.toml

@@ -11,4 +11,4 @@ clap = "2.33"
 node-types = { path = "../node-types" }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3.3", features = ["env-filter"] }
-tree-sitter-ql = { git = "https://github.com/tausbn/tree-sitter-ql.git", rev = "725395405e65814f10095a451404b0ced5dc6289" }
+tree-sitter-ql = { git = "https://github.com/tausbn/tree-sitter-ql.git", rev = "99f3bc30fa772b07ad19a23799fe7433dc15f765" }

ql/generator/Cargo.toml

@@ -673,7 +673,7 @@ class TypeExpr extends TType, AstNode {
    * Gets the module of the type, if it exists.
    * E.g. `DataFlow` in `DataFlow::Node`.
    */
-  ModuleExpr getModule() { toQL(result) = type.getChild() }
+  ModuleExpr getModule() { toQL(result) = type.getQualifier() }
 
   /**
    * Gets the type that this type reference refers to.

ql/ql/src/codeql_ql/ast/Ast.qll

@@ -137,13 +137,13 @@ module QL {
     /** Gets the node corresponding to the field `name`. */
     final LiteralId getName() { ql_arityless_predicate_expr_def(this, result) }
 
-    /** Gets the child of this node. */
-    final ModuleExpr getChild() { ql_arityless_predicate_expr_child(this, result) }
+    /** Gets the node corresponding to the field `qualifier`. */
+    final ModuleExpr getQualifier() { ql_arityless_predicate_expr_qualifier(this, result) }
 
     /** Gets a field or child node of this node. */
     final override AstNode getAFieldOrChild() {
       ql_arityless_predicate_expr_def(this, result) or
-      ql_arityless_predicate_expr_child(this, result)
+      ql_arityless_predicate_expr_qualifier(this, result)
     }
   }
 
@@ -944,15 +944,24 @@ module QL {
     /** Gets the name of the primary QL class for this element. */
     final override string getAPrimaryQlClass() { result = "Module" }
 
+    /** Gets the node corresponding to the field `implements`. */
+    final SignatureExpr getImplements(int i) { ql_module_implements(this, i, result) }
+
     /** Gets the node corresponding to the field `name`. */
     final ModuleName getName() { ql_module_def(this, result) }
 
+    /** Gets the node corresponding to the field `parameter`. */
+    final ModuleParam getParameter(int i) { ql_module_parameter(this, i, result) }
+
     /** Gets the `i`th child of this node. */
     final AstNode getChild(int i) { ql_module_child(this, i, result) }
 
     /** Gets a field or child node of this node. */
     final override AstNode getAFieldOrChild() {
-      ql_module_def(this, result) or ql_module_child(this, _, result)
+      ql_module_implements(this, _, result) or
+      ql_module_def(this, result) or
+      ql_module_parameter(this, _, result) or
+      ql_module_child(this, _, result)
     }
   }
 
@@ -985,6 +994,18 @@ module QL {
     }
   }
 
+  /** A class representing `moduleInstantiation` nodes. */
+  class ModuleInstantiation extends @ql_module_instantiation, AstNode {
+    /** Gets the name of the primary QL class for this element. */
+    final override string getAPrimaryQlClass() { result = "ModuleInstantiation" }
+
+    /** Gets the `i`th child of this node. */
+    final SignatureExpr getChild(int i) { ql_module_instantiation_child(this, i, result) }
+
+    /** Gets a field or child node of this node. */
+    final override AstNode getAFieldOrChild() { ql_module_instantiation_child(this, _, result) }
+  }
+
   /** A class representing `moduleMember` nodes. */
   class ModuleMember extends @ql_module_member, AstNode {
     /** Gets the name of the primary QL class for this element. */
@@ -1009,6 +1030,23 @@ module QL {
     final override AstNode getAFieldOrChild() { ql_module_name_def(this, result) }
   }
 
+  /** A class representing `moduleParam` nodes. */
+  class ModuleParam extends @ql_module_param, AstNode {
+    /** Gets the name of the primary QL class for this element. */
+    final override string getAPrimaryQlClass() { result = "ModuleParam" }
+
+    /** Gets the node corresponding to the field `parameter`. */
+    final SimpleId getParameter() { ql_module_param_def(this, result, _) }
+
+    /** Gets the node corresponding to the field `signature`. */
+    final SignatureExpr getSignature() { ql_module_param_def(this, _, result) }
+
+    /** Gets a field or child node of this node. */
+    final override AstNode getAFieldOrChild() {
+      ql_module_param_def(this, result, _) or ql_module_param_def(this, _, result)
+    }
+  }
+
   /** A class representing `mul_expr` nodes. */
   class MulExpr extends @ql_mul_expr, AstNode {
     /** Gets the name of the primary QL class for this element. */
@@ -1277,6 +1315,23 @@ module QL {
     final override AstNode getAFieldOrChild() { ql_set_literal_child(this, _, result) }
   }
 
+  /** A class representing `signatureExpr` nodes. */
+  class SignatureExpr extends @ql_signature_expr, AstNode {
+    /** Gets the name of the primary QL class for this element. */
+    final override string getAPrimaryQlClass() { result = "SignatureExpr" }
+
+    /** Gets the node corresponding to the field `predicate`. */
+    final PredicateExpr getPredicate() { ql_signature_expr_predicate(this, result) }
+
+    /** Gets the node corresponding to the field `type_expr`. */
+    final TypeExpr getTypeExpr() { ql_signature_expr_type_expr(this, result) }
+
+    /** Gets a field or child node of this node. */
+    final override AstNode getAFieldOrChild() {
+      ql_signature_expr_predicate(this, result) or ql_signature_expr_type_expr(this, result)
+    }
+  }
+
   /** A class representing `simpleId` tokens. */
   class SimpleId extends @ql_token_simple_id, Token {
     /** Gets the name of the primary QL class for this element. */
@@ -1357,12 +1412,17 @@ module QL {
     /** Gets the node corresponding to the field `name`. */
     final ClassName getName() { ql_type_expr_name(this, result) }
 
+    /** Gets the node corresponding to the field `qualifier`. */
+    final ModuleExpr getQualifier() { ql_type_expr_qualifier(this, result) }
+
     /** Gets the child of this node. */
     final AstNode getChild() { ql_type_expr_child(this, result) }
 
     /** Gets a field or child node of this node. */
     final override AstNode getAFieldOrChild() {
-      ql_type_expr_name(this, result) or ql_type_expr_child(this, result)
+      ql_type_expr_name(this, result) or
+      ql_type_expr_qualifier(this, result) or
+      ql_type_expr_child(this, result)
     }
   }
 

ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll

@@ -95,9 +95,9 @@ ql_annotation_def(
   int name: @ql_token_annot_name ref
 );
 
-ql_arityless_predicate_expr_child(
+ql_arityless_predicate_expr_qualifier(
   unique int ql_arityless_predicate_expr: @ql_arityless_predicate_expr ref,
-  unique int child: @ql_module_expr ref
+  unique int qualifier: @ql_module_expr ref
 );
 
 ql_arityless_predicate_expr_def(
@@ -601,6 +601,20 @@ ql_member_predicate_def(
   int return_type: @ql_memberPredicate_returnType_type ref
 );
 
+#keyset[ql_module, index]
+ql_module_implements(
+  int ql_module: @ql_module ref,
+  int index: int ref,
+  unique int implements: @ql_signature_expr ref
+);
+
+#keyset[ql_module, index]
+ql_module_parameter(
+  int ql_module: @ql_module ref,
+  int index: int ref,
+  unique int parameter: @ql_module_param ref
+);
+
 @ql_module_child_type = @ql_module_alias_body | @ql_module_member
 
 #keyset[ql_module, index]
@@ -625,13 +639,24 @@ ql_module_expr_name(
   unique int name: @ql_token_simple_id ref
 );
 
-@ql_moduleExpr_child_type = @ql_module_expr | @ql_token_simple_id
+@ql_moduleExpr_child_type = @ql_module_expr | @ql_module_instantiation | @ql_token_simple_id
 
 ql_module_expr_def(
   unique int id: @ql_module_expr,
   int child: @ql_moduleExpr_child_type ref
 );
 
+#keyset[ql_module_instantiation, index]
+ql_module_instantiation_child(
+  int ql_module_instantiation: @ql_module_instantiation ref,
+  int index: int ref,
+  unique int child: @ql_signature_expr ref
+);
+
+ql_module_instantiation_def(
+  unique int id: @ql_module_instantiation
+);
+
 @ql_moduleMember_child_type = @ql_annotation | @ql_classless_predicate | @ql_dataclass | @ql_datatype | @ql_import_directive | @ql_module | @ql_select | @ql_token_qldoc
 
 #keyset[ql_module_member, index]
@@ -650,6 +675,12 @@ ql_module_name_def(
   int child: @ql_token_simple_id ref
 );
 
+ql_module_param_def(
+  unique int id: @ql_module_param,
+  int parameter: @ql_token_simple_id ref,
+  int signature: @ql_signature_expr ref
+);
+
 @ql_mul_expr_left_type = @ql_add_expr | @ql_aggregate | @ql_call_or_unqual_agg_expr | @ql_comp_term | @ql_conjunction | @ql_disjunction | @ql_expr_annotation | @ql_if_term | @ql_implication | @ql_in_expr | @ql_instance_of | @ql_literal | @ql_mul_expr | @ql_negation | @ql_par_expr | @ql_prefix_cast | @ql_qualified_expr | @ql_quantified | @ql_range | @ql_set_literal | @ql_special_call | @ql_super_ref | @ql_unary_expr | @ql_variable
 
 @ql_mul_expr_right_type = @ql_add_expr | @ql_aggregate | @ql_call_or_unqual_agg_expr | @ql_comp_term | @ql_conjunction | @ql_disjunction | @ql_expr_annotation | @ql_if_term | @ql_implication | @ql_in_expr | @ql_instance_of | @ql_literal | @ql_mul_expr | @ql_negation | @ql_par_expr | @ql_prefix_cast | @ql_qualified_expr | @ql_quantified | @ql_range | @ql_set_literal | @ql_special_call | @ql_super_ref | @ql_unary_expr | @ql_variable
@@ -855,6 +886,20 @@ ql_set_literal_def(
   unique int id: @ql_set_literal
 );
 
+ql_signature_expr_predicate(
+  unique int ql_signature_expr: @ql_signature_expr ref,
+  unique int predicate: @ql_predicate_expr ref
+);
+
+ql_signature_expr_type_expr(
+  unique int ql_signature_expr: @ql_signature_expr ref,
+  unique int type_expr: @ql_type_expr ref
+);
+
+ql_signature_expr_def(
+  unique int id: @ql_signature_expr
+);
+
 ql_special_call_def(
   unique int id: @ql_special_call,
   int child: @ql_token_special_id ref
@@ -883,7 +928,12 @@ ql_type_expr_name(
   unique int name: @ql_token_class_name ref
 );
 
-@ql_typeExpr_child_type = @ql_module_expr | @ql_token_dbtype | @ql_token_primitive_type
+ql_type_expr_qualifier(
+  unique int ql_type_expr: @ql_type_expr ref,
+  unique int qualifier: @ql_module_expr ref
+);
+
+@ql_typeExpr_child_type = @ql_token_dbtype | @ql_token_primitive_type
 
 ql_type_expr_child(
   unique int ql_type_expr: @ql_type_expr ref,
@@ -1057,7 +1107,7 @@ case @ql_token.kind of
 ;
 
 
-@ql_ast_node = @ql_add_expr | @ql_aggregate | @ql_annot_arg | @ql_annotation | @ql_arityless_predicate_expr | @ql_as_expr | @ql_as_exprs | @ql_body | @ql_bool | @ql_call_body | @ql_call_or_unqual_agg_expr | @ql_charpred | @ql_class_member | @ql_classless_predicate | @ql_comp_term | @ql_conjunction | @ql_dataclass | @ql_datatype | @ql_datatype_branch | @ql_datatype_branches | @ql_db_annotation | @ql_db_args_annotation | @ql_db_branch | @ql_db_case_decl | @ql_db_col_type | @ql_db_column | @ql_db_entry | @ql_db_repr_type | @ql_db_table | @ql_db_table_name | @ql_db_union_decl | @ql_disjunction | @ql_expr_aggregate_body | @ql_expr_annotation | @ql_field | @ql_full_aggregate_body | @ql_higher_order_term | @ql_if_term | @ql_implication | @ql_import_directive | @ql_import_module_expr | @ql_in_expr | @ql_instance_of | @ql_literal | @ql_member_predicate | @ql_module | @ql_module_alias_body | @ql_module_expr | @ql_module_member | @ql_module_name | @ql_mul_expr | @ql_negation | @ql_order_by | @ql_order_bys | @ql_par_expr | @ql_predicate_alias_body | @ql_predicate_expr | @ql_prefix_cast | @ql_ql | @ql_qual_module_expr | @ql_qualified_expr | @ql_qualified_rhs | @ql_quantified | @ql_range | @ql_select | @ql_set_literal | @ql_special_call | @ql_super_ref | @ql_token | @ql_type_alias_body | @ql_type_expr | @ql_type_union_body | @ql_unary_expr | @ql_unqual_agg_body | @ql_var_decl | @ql_var_name | @ql_variable | @ql_yaml_comment | @ql_yaml_entry | @ql_yaml_key | @ql_yaml_keyvaluepair | @ql_yaml_listitem
+@ql_ast_node = @ql_add_expr | @ql_aggregate | @ql_annot_arg | @ql_annotation | @ql_arityless_predicate_expr | @ql_as_expr | @ql_as_exprs | @ql_body | @ql_bool | @ql_call_body | @ql_call_or_unqual_agg_expr | @ql_charpred | @ql_class_member | @ql_classless_predicate | @ql_comp_term | @ql_conjunction | @ql_dataclass | @ql_datatype | @ql_datatype_branch | @ql_datatype_branches | @ql_db_annotation | @ql_db_args_annotation | @ql_db_branch | @ql_db_case_decl | @ql_db_col_type | @ql_db_column | @ql_db_entry | @ql_db_repr_type | @ql_db_table | @ql_db_table_name | @ql_db_union_decl | @ql_disjunction | @ql_expr_aggregate_body | @ql_expr_annotation | @ql_field | @ql_full_aggregate_body | @ql_higher_order_term | @ql_if_term | @ql_implication | @ql_import_directive | @ql_import_module_expr | @ql_in_expr | @ql_instance_of | @ql_literal | @ql_member_predicate | @ql_module | @ql_module_alias_body | @ql_module_expr | @ql_module_instantiation | @ql_module_member | @ql_module_name | @ql_module_param | @ql_mul_expr | @ql_negation | @ql_order_by | @ql_order_bys | @ql_par_expr | @ql_predicate_alias_body | @ql_predicate_expr | @ql_prefix_cast | @ql_ql | @ql_qual_module_expr | @ql_qualified_expr | @ql_qualified_rhs | @ql_quantified | @ql_range | @ql_select | @ql_set_literal | @ql_signature_expr | @ql_special_call | @ql_super_ref | @ql_token | @ql_type_alias_body | @ql_type_expr | @ql_type_union_body | @ql_unary_expr | @ql_unqual_agg_body | @ql_var_decl | @ql_var_name | @ql_variable | @ql_yaml_comment | @ql_yaml_entry | @ql_yaml_key | @ql_yaml_keyvaluepair | @ql_yaml_listitem
 
 @ql_ast_node_parent = @file | @ql_ast_node
 

ql/ql/src/ql.dbscheme

@@ -49,14 +49,15 @@ static void archiveFile(const SwiftExtractorConfiguration& config, swift::Source
   }
 }
 
-static void extractFile(const SwiftExtractorConfiguration& config,
-                        swift::CompilerInstance& compiler,
-                        swift::SourceFile& file) {
+static void extractDeclarations(const SwiftExtractorConfiguration& config,
+                                swift::CompilerInstance& compiler,
+                                llvm::StringRef filename,
+                                llvm::ArrayRef<swift::Decl*> topLevelDecls) {
   // The extractor can be called several times from different processes with
   // the same input file(s)
   // We are using PID to avoid concurrent access
   // TODO: find a more robust approach to avoid collisions?
-  std::string tempTrapName = file.getFilename().str() + '.' + std::to_string(getpid()) + ".trap";
+  std::string tempTrapName = filename.str() + '.' + std::to_string(getpid()) + ".trap";
   llvm::SmallString<PATH_MAX> tempTrapPath(config.trapDir);
   llvm::sys::path::append(tempTrapPath, tempTrapName);
 
@@ -84,7 +85,7 @@ static void extractFile(const SwiftExtractorConfiguration& config,
   TrapOutput trap{trapStream};
   TrapArena arena{};
 
-  // TODO move default location emission elsewhere, possibly in a separate global trap file
+  // TODO: move default location emission elsewhere, possibly in a separate global trap file
   auto unknownFileLabel = arena.allocateLabel<FileTag>();
   // the following cannot conflict with actual files as those have an absolute path starting with /
   trap.assignKey(unknownFileLabel, "unknown");
@@ -93,22 +94,23 @@ static void extractFile(const SwiftExtractorConfiguration& config,
   trap.assignKey(unknownLocationLabel, "unknown");
   trap.emit(LocationsTrap{unknownLocationLabel, unknownFileLabel});
 
-  // In the case of emtpy files, the dispatcher is not called, but we still want to 'record' the
-  // fact that the file was extracted
-  // TODO: to be moved elsewhere
-  llvm::SmallString<PATH_MAX> srcFilePath(file.getFilename());
-  llvm::sys::fs::make_absolute(srcFilePath);
-  auto fileLabel = arena.allocateLabel<FileTag>();
-  trap.assignKey(fileLabel, srcFilePath.str().str());
-  trap.emit(FilesTrap{fileLabel, srcFilePath.str().str()});
-
   SwiftVisitor visitor(compiler.getSourceMgr(), arena, trap);
-  for (swift::Decl* decl : file.getTopLevelDecls()) {
+  for (swift::Decl* decl : topLevelDecls) {
     visitor.extract(decl);
   }
+  if (topLevelDecls.empty()) {
+    // In the case of emtpy files, the dispatcher is not called, but we still want to 'record' the
+    // fact that the file was extracted
+    // TODO: to be moved elsewhere
+    llvm::SmallString<PATH_MAX> srcFilePath(filename);
+    llvm::sys::fs::make_absolute(srcFilePath);
+    auto fileLabel = arena.allocateLabel<FileTag>();
+    trap.assignKey(fileLabel, srcFilePath.str().str());
+    trap.emit(FilesTrap{fileLabel, srcFilePath.str().str()});
+  }
 
   // TODO: Pick a better name to avoid collisions
-  std::string trapName = file.getFilename().str() + ".trap";
+  std::string trapName = filename.str() + ".trap";
   llvm::SmallString<PATH_MAX> trapPath(config.trapDir);
   llvm::sys::path::append(trapPath, trapName);
 
@@ -121,10 +123,24 @@ static void extractFile(const SwiftExtractorConfiguration& config,
 
 void codeql::extractSwiftFiles(const SwiftExtractorConfiguration& config,
                                swift::CompilerInstance& compiler) {
-  // The extraction will only work if one (or more) `-primary-file` CLI option is provided, which
-  // is what always happen in case of `swift build` and `xcodebuild`
-  for (auto s : compiler.getPrimarySourceFiles()) {
-    archiveFile(config, *s);
-    extractFile(config, compiler, *s);
+  for (auto& [_, module] : compiler.getASTContext().getLoadedModules()) {
+    // We only extract system and builtin modules here as the other "user" modules can be built
+    // during the build process and then re-used at a later stage. In this case, we extract the
+    // user code twice: once during the module build in a form of a source file, and then as
+    // a pre-built module during building of the dependent source files.
+    if (module->isSystemModule() || module->isBuiltinModule()) {
+      llvm::SmallVector<swift::Decl*> decls;
+      module->getTopLevelDecls(decls);
+      // TODO: pass ModuleDecl directly when we have module extraction in place?
+      extractDeclarations(config, compiler, module->getModuleFilename(), decls);
+    } else {
+      // The extraction will only work if one (or more) `-primary-file` CLI option is provided,
+      // which is what always happens in case of `swift build` and `xcodebuild`
+      for (auto primaryFile : module->getPrimarySourceFiles()) {
+        archiveFile(config, *primaryFile);
+        extractDeclarations(config, compiler, primaryFile->getFilename(),
+                            primaryFile->getTopLevelDecls());
+      }
+    }
   }
 }

swift/extractor/SwiftExtractor.cpp

@@ -6,5 +6,5 @@ import codeql.swift.controlflow.ControlFlowGraph
 import codeql.swift.controlflow.internal.ControlFlowGraphImpl::TestOutput
 
 class MyRelevantNode extends RelevantNode {
-  MyRelevantNode() { exists(this) }
+  MyRelevantNode() { getScope().getLocation().getFile().getName().matches("%swift/ql/test%") }
 }