Merge pull request #9521 from github/redsun82/swift-qltestgen

Swift: generated extractor tests

Author: redsun82

.github/workflows/swift-codegen.yml

@@ -22,7 +22,7 @@ jobs:
         run: |
           bazel run //swift/codegen
           git add swift
-          git diff --exit-code --stat HEAD
+          git diff --exit-code HEAD
       - name: Generate C++ files
         run: |
           bazel run //swift/codegen:codegen -- --generate=trap,cpp --cpp-output=$PWD/swift-generated-headers

swift/codegen/codegen.py

@@ -31,6 +31,8 @@ def _parse_args() -> argparse.Namespace:
     p.add_argument("--ql-stub-output", type=_abspath, default=paths.swift_dir / "ql/lib/codeql/swift/elements",
                    help="output directory for QL stub/customization files (default %(default)s). Defines also the "
                         "generated qll file importing every class file")
+    p.add_argument("--ql-test-output", type=_abspath, default=paths.swift_dir / "ql/test/extractor-tests/generated",
+                   help="output directory for QL generated extractor test files (default %(default)s)")
     p.add_argument("--ql-format", action="store_true", default=True,
                    help="use codeql to autoformat QL files (which is the default)")
     p.add_argument("--no-ql-format", action="store_false", dest="ql_format", help="do not format QL files")

swift/codegen/generators/qlgen.py

@@ -3,6 +3,7 @@
 import logging
 import pathlib
 import subprocess
+import typing
 
 import inflection
 
@@ -11,41 +12,48 @@
 log = logging.getLogger(__name__)
 
 
+class FormatError(Exception):
+    pass
+
+
 def get_ql_property(cls: schema.Class, prop: schema.Property):
+    common_args = dict(
+        type=prop.type if not prop.is_predicate else "predicate",
+        skip_qltest="no_qltest" in prop.tags,
+        is_child=prop.is_child,
+        is_optional=prop.is_optional,
+        is_predicate=prop.is_predicate,
+    )
     if prop.is_single:
         return ql.Property(
+            **common_args,
             singular=inflection.camelize(prop.name),
-            type=prop.type,
             tablename=inflection.tableize(cls.name),
-            tableparams=["this"] + ["result" if p is prop else "_" for p in cls.properties if p.is_single],
-            is_child=prop.is_child,
+            tableparams=[
+                "this"] + ["result" if p is prop else "_" for p in cls.properties if p.is_single],
         )
     elif prop.is_repeated:
         return ql.Property(
+            **common_args,
             singular=inflection.singularize(inflection.camelize(prop.name)),
             plural=inflection.pluralize(inflection.camelize(prop.name)),
-            type=prop.type,
             tablename=inflection.tableize(f"{cls.name}_{prop.name}"),
             tableparams=["this", "index", "result"],
-            is_optional=prop.is_optional,
-            is_child=prop.is_child,
         )
     elif prop.is_optional:
         return ql.Property(
+            **common_args,
             singular=inflection.camelize(prop.name),
-            type=prop.type,
             tablename=inflection.tableize(f"{cls.name}_{prop.name}"),
             tableparams=["this", "result"],
-            is_optional=True,
-            is_child=prop.is_child,
         )
     elif prop.is_predicate:
         return ql.Property(
-            singular=inflection.camelize(prop.name, uppercase_first_letter=False),
-            type="predicate",
+            **common_args,
+            singular=inflection.camelize(
+                prop.name, uppercase_first_letter=False),
             tablename=inflection.underscore(f"{cls.name}_{prop.name}"),
             tableparams=["this"],
-            is_predicate=True,
         )
 
 
@@ -56,6 +64,7 @@ def get_ql_class(cls: schema.Class):
         final=not cls.derived,
         properties=[get_ql_property(cls, p) for p in cls.properties],
         dir=cls.dir,
+        skip_qltest="no_qltest" in cls.tags,
     )
 
 
@@ -77,48 +86,106 @@ def get_classes_used_by(cls: ql.Class):
 
 def is_generated(file):
     with open(file) as contents:
-        return next(contents).startswith("// generated")
+        for line in contents:
+            return line.startswith("// generated")
+        return False
 
 
 def format(codeql, files):
     format_cmd = [codeql, "query", "format", "--in-place", "--"]
-    format_cmd.extend(str(f) for f in files)
-    res = subprocess.run(format_cmd, check=True, stderr=subprocess.PIPE, text=True)
+    format_cmd.extend(str(f) for f in files if f.suffix in (".qll", ".ql"))
+    res = subprocess.run(format_cmd, stderr=subprocess.PIPE, text=True)
+    if res.returncode:
+        for line in res.stderr.splitlines():
+            log.error(line.strip())
+        raise FormatError("QL format failed")
     for line in res.stderr.splitlines():
         log.debug(line.strip())
 
 
+def _get_all_properties(cls: ql.Class, lookup: typing.Dict[str, ql.Class]) -> typing.Iterable[
+        typing.Tuple[ql.Class, ql.Property]]:
+    for b in cls.bases:
+        base = lookup[b]
+        for item in _get_all_properties(base, lookup):
+            yield item
+    for p in cls.properties:
+        yield cls, p
+
+
+def _get_all_properties_to_be_tested(cls: ql.Class, lookup: typing.Dict[str, ql.Class]) -> typing.Iterable[
+        ql.PropertyForTest]:
+    # deduplicate using id
+    already_seen = set()
+    for c, p in _get_all_properties(cls, lookup):
+        if not (c.skip_qltest or p.skip_qltest or id(p) in already_seen):
+            already_seen.add(id(p))
+            yield ql.PropertyForTest(p.getter, p.type, p.is_single, p.is_predicate, p.is_repeated)
+
+
+def _partition(l, pred):
+    """ partitions a list according to boolean predicate """
+    res = ([], [])
+    for x in l:
+        res[not pred(x)].append(x)
+    return res
+
+
 def generate(opts, renderer):
     input = opts.schema
     out = opts.ql_output
     stub_out = opts.ql_stub_output
+    test_out = opts.ql_test_output
+    missing_test_source_filename = "MISSING_SOURCE.txt"
     existing = {q for q in out.rglob("*.qll")}
     existing |= {q for q in stub_out.rglob("*.qll") if is_generated(q)}
+    existing |= {q for q in test_out.rglob("*.ql")}
+    existing |= {q for q in test_out.rglob(missing_test_source_filename)}
 
     data = schema.load(input)
 
     classes = [get_ql_class(cls) for cls in data.classes]
-    classes.sort(key=lambda cls: cls.name)
+    lookup = {cls.name: cls for cls in classes}
+    classes.sort(key=lambda cls: (cls.dir, cls.name))
     imports = {}
 
     for c in classes:
         imports[c.name] = get_import(stub_out / c.path, opts.swift_dir)
 
     for c in classes:
-        qll = (out / c.path).with_suffix(".qll")
+        qll = out / c.path.with_suffix(".qll")
         c.imports = [imports[t] for t in get_classes_used_by(c)]
         renderer.render(c, qll)
-        stub_file = (stub_out / c.path).with_suffix(".qll")
+        stub_file = stub_out / c.path.with_suffix(".qll")
         if not stub_file.is_file() or is_generated(stub_file):
-            stub = ql.Stub(name=c.name, base_import=get_import(qll, opts.swift_dir))
+            stub = ql.Stub(
+                name=c.name, base_import=get_import(qll, opts.swift_dir))
             renderer.render(stub, stub_file)
 
     # for example path/to/elements -> path/to/elements.qll
     include_file = stub_out.with_suffix(".qll")
-    all_imports = ql.ImportList(list(sorted(imports.values())))
-    renderer.render(all_imports, include_file)
+    renderer.render(ql.ImportList(list(imports.values())), include_file)
+
+    renderer.render(ql.GetParentImplementation(
+        classes), out / 'GetImmediateParent.qll')
 
-    renderer.render(ql.GetParentImplementation(classes), out / 'GetImmediateParent.qll')
+    for c in classes:
+        if not c.final or c.skip_qltest:
+            continue
+        test_dir = test_out / c.path
+        test_dir.mkdir(parents=True, exist_ok=True)
+        if not any(test_dir.glob("*.swift")):
+            log.warning(f"no test source in {c.path}")
+            renderer.render(ql.MissingTestInstructions(),
+                            test_dir / missing_test_source_filename)
+            continue
+        total_props, partial_props = _partition(_get_all_properties_to_be_tested(c, lookup),
+                                                lambda p: p.is_single or p.is_predicate)
+        renderer.render(ql.ClassTester(class_name=c.name,
+                                       properties=total_props), test_dir / f"{c.name}.ql")
+        for p in partial_props:
+            renderer.render(ql.PropertyTester(class_name=c.name,
+                                              property=p), test_dir / f"{c.name}_{p.getter}.ql")
 
     renderer.cleanup(existing)
     if opts.ql_format:

swift/codegen/lib/ql.py

@@ -37,6 +37,7 @@ class Property:
     is_optional: bool = False
     is_predicate: bool = False
     is_child: bool = False
+    skip_qltest: bool = False
 
     def __post_init__(self):
         if self.tableparams:
@@ -63,6 +64,10 @@ def type_is_class(self):
     def is_repeated(self):
         return bool(self.plural)
 
+    @property
+    def is_single(self):
+        return not (self.is_optional or self.is_repeated or self.is_predicate)
+
 
 @dataclass
 class Class:
@@ -74,6 +79,7 @@ class Class:
     properties: List[Property] = field(default_factory=list)
     dir: pathlib.Path = pathlib.Path()
     imports: List[str] = field(default_factory=list)
+    skip_qltest: bool = False
 
     def __post_init__(self):
         self.bases = sorted(self.bases)
@@ -113,3 +119,33 @@ class GetParentImplementation:
     template: ClassVar = 'ql_parent'
 
     classes: List[Class] = field(default_factory=list)
+
+
+@dataclass
+class PropertyForTest:
+    getter: str
+    type: str = None
+    is_single: bool = False
+    is_predicate: bool = False
+    is_repeated: bool = False
+
+
+@dataclass
+class ClassTester:
+    template: ClassVar = 'ql_test_class'
+
+    class_name: str
+    properties: List[PropertyForTest] = field(default_factory=list)
+
+
+@dataclass
+class PropertyTester:
+    template: ClassVar = 'ql_test_property'
+
+    class_name: str
+    property: PropertyForTest
+
+
+@dataclass
+class MissingTestInstructions:
+    template: ClassVar = 'ql_test_missing'

swift/codegen/lib/schema.py

@@ -3,10 +3,17 @@
 import pathlib
 import re
 from dataclasses import dataclass, field
-from typing import List, Set, Dict, ClassVar
+from typing import List, Set, Union, Dict, ClassVar
 
 import yaml
 
+
+class Error(Exception):
+
+    def __str__(self):
+        return f"schema.Error{args}"
+
+
 root_class_name = "Element"
 
 
@@ -20,6 +27,7 @@ class Property:
     name: str
     type: str = None
     is_child: bool = False
+    tags: List[str] = field(default_factory=list)
 
 
 @dataclass
@@ -55,6 +63,7 @@ class Class:
     derived: Set[str] = field(default_factory=set)
     properties: List[Property] = field(default_factory=list)
     dir: pathlib.Path = pathlib.Path()
+    tags: List[str] = field(default_factory=list)
 
 
 @dataclass
@@ -63,18 +72,24 @@ class Schema:
     includes: Set[str] = field(default_factory=set)
 
 
-def _parse_property(name: str, type: str, is_child: bool = False):
-    assert not (is_child and type[0].islower()), f"children must have class type, got {type} for {name}"
+def _parse_property(name: str, type: Union[str, Dict[str, str]], is_child: bool = False):
+    if isinstance(type, dict):
+        tags = type.get("_tags", [])
+        type = type["type"]
+    else:
+        tags = []
+    if is_child and type[0].islower():
+        raise Error(f"children must have class type, got {type} for {name}")
     if type.endswith("?*"):
-        return RepeatedOptionalProperty(name, type[:-2], is_child=is_child)
+        return RepeatedOptionalProperty(name, type[:-2], is_child=is_child, tags=tags)
     elif type.endswith("*"):
-        return RepeatedProperty(name, type[:-1], is_child=is_child)
+        return RepeatedProperty(name, type[:-1], is_child=is_child, tags=tags)
     elif type.endswith("?"):
-        return OptionalProperty(name, type[:-1], is_child=is_child)
+        return OptionalProperty(name, type[:-1], is_child=is_child, tags=tags)
     elif type == "predicate":
-        return PredicateProperty(name)
+        return PredicateProperty(name, tags=tags)
     else:
-        return SingleProperty(name, type, is_child=is_child)
+        return SingleProperty(name, type, is_child=is_child, tags=tags)
 
 
 class _DirSelector:
@@ -98,7 +113,8 @@ def load(path):
     for name, info in data.items():
         if name.startswith("_"):
             continue
-        assert name[0].isupper()
+        if not name[0].isupper():
+            raise Error(f"keys in the schema file must be capitalized class names or metadata, got {name}")
         cls = classes[name]
         for k, v in info.items():
             if not k.startswith("_"):
@@ -113,6 +129,10 @@ def load(path):
                 cls.dir = pathlib.Path(v)
             elif k == "_children":
                 cls.properties.extend(_parse_property(kk, vv, is_child=True) for kk, vv in v.items())
+            elif k == "_tags":
+                cls.tags = v
+            else:
+                raise Error(f"unknown metadata {k} for class {name}")
         if not cls.bases and cls.name != root_class_name:
             cls.bases.add(root_class_name)
             classes[root_class_name].derived.add(name)

swift/codegen/schema.yml

@@ -13,15 +13,20 @@ _directories:
   stmt: Stmt$
 
 Element:
-  is_unknown: predicate
+  is_unknown:
+    type: predicate
+    _tags: [ no_qltest ]
 
 File:
   name: string
 
 Locatable:
-  location: Location
+  location:
+    type: Location
+    _tags: [no_qltest]
 
 Location:
+  _tags: [ no_qltest ]
   file: File
   start_line: int
   start_column: int