Swift: implement python schema

The information that was contained in `schema.yml` is now in
`swift/schema.py`, which allows a more integrated IDE experience
for writing and navigating it.

Another minor change is that `schema.Class` now has a `str` `group`
field instead of a `pathlib.Path` `dir` one.

Author: redsun82

swift/.pep8

@@ -0,0 +1,3 @@
+[pep8]
+ignore = E302
+max_line_length = 120

swift/BUILD.bazel

@@ -3,6 +3,18 @@ load("@rules_pkg//:install.bzl", "pkg_install")
 load("//:defs.bzl", "codeql_platform")
 load("//misc/bazel:pkg_runfiles.bzl", "pkg_runfiles")
 
+filegroup(
+    name = "schema",
+    srcs = ["schema.py"],
+    visibility = ["//swift:__subpackages__"],
+)
+
+filegroup(
+    name = "schema_includes",
+    srcs = glob(["*.dbscheme"]),
+    visibility = ["//swift:__subpackages__"],
+)
+
 pkg_files(
     name = "dbscheme_files",
     srcs = [

swift/codegen/.pep8

@@ -1,3 +1,2 @@
 [pep8]
 max_line_length = 120
-ignore = E302

swift/codegen/BUILD.bazel

@@ -1,26 +1,16 @@
 load("@swift_codegen_deps//:requirements.bzl", "requirement")
 
-filegroup(
-    name = "schema",
-    srcs = ["schema.yml"],
-    visibility = ["//swift:__subpackages__"],
-)
-
-filegroup(
-    name = "schema_includes",
-    srcs = glob(["*.dbscheme"]),
-    visibility = ["//swift:__subpackages__"],
-)
-
 py_binary(
     name = "codegen",
     srcs = ["codegen.py"],
     data = [
-        ":schema",
-        ":schema_includes",
+        "//swift:schema",
+        "//swift:schema_includes",
         "//swift/codegen/templates:cpp",
         "//swift/codegen/templates:trap",
     ],
     visibility = ["//swift:__subpackages__"],
-    deps = ["//swift/codegen/generators"],
+    deps = [
+        "//swift/codegen/generators",
+    ],
 )

swift/codegen/codegen.py

@@ -23,7 +23,7 @@ def _parse_args() -> argparse.Namespace:
     p.add_argument("--swift-dir", type=_abspath, default=paths.swift_dir,
                    help="the directory that should be regarded as the root of the swift codebase. Used to compute QL "
                         "imports and in some comments (default %(default)s)")
-    p.add_argument("--schema", type=_abspath, default=paths.swift_dir / "codegen/schema.yml",
+    p.add_argument("--schema", type=_abspath, default=paths.swift_dir / "schema.py",
                    help="input schema file (default %(default)s)")
     p.add_argument("--dbscheme", type=_abspath, default=paths.swift_dir / "ql/lib/swift.dbscheme",
                    help="output file for dbscheme generation, input file for trap generation (default %(default)s)")

swift/codegen/generators/cppgen.py

@@ -70,16 +70,16 @@ def _get_class(self, name: str) -> cpp.Class:
         )
 
     def get_classes(self):
-        ret = {pathlib.Path(): []}
+        ret = {'': []}
         for k, cls in self._classmap.items():
-            ret.setdefault(cls.dir, []).append(self._get_class(cls.name))
+            ret.setdefault(cls.group, []).append(self._get_class(cls.name))
         return ret
 
 
 def generate(opts, renderer):
     assert opts.cpp_output
-    processor = Processor(schema.load(opts.schema).classes)
+    processor = Processor(schema.load_file(opts.schema).classes)
     out = opts.cpp_output
     for dir, classes in processor.get_classes().items():
-        include_parent = (dir != pathlib.Path())
-        renderer.render(cpp.ClassList(classes, opts.schema, include_parent), out / dir / "TrapClasses")
+        renderer.render(cpp.ClassList(classes, opts.schema,
+                        include_parent=bool(dir)), out / dir / "TrapClasses")

swift/codegen/generators/dbschemegen.py

@@ -14,12 +14,11 @@
 The type hierarchy will be translated to corresponding `union` declarations.
 """
 
-import pathlib
-
 import inflection
 
 from swift.codegen.lib import schema
 from swift.codegen.lib.dbscheme import *
+from typing import Set, List
 
 log = logging.getLogger(__name__)
 
@@ -35,7 +34,7 @@ def cls_to_dbscheme(cls: schema.Class):
     """ Yield all dbscheme entities needed to model class `cls` """
     if cls.derived:
         yield Union(dbtype(cls.name), (dbtype(c) for c in cls.derived))
-    dir = cls.dir if cls.dir != pathlib.Path() else None
+    dir = pathlib.Path(cls.group) if cls.group else None
     # output a table specific to a class only if it is a leaf class or it has 1-to-1 properties
     # Leaf classes need a table to bind the `@` ids
     # 1-to-1 properties are added to a class specific table
@@ -104,7 +103,7 @@ def generate(opts, renderer):
     input = opts.schema
     out = opts.dbscheme
 
-    data = schema.load(input)
+    data = schema.load_file(input)
 
     dbscheme = Scheme(src=input.relative_to(opts.swift_dir),
                       includes=get_includes(data, include_dir=input.parent, swift_dir=opts.swift_dir),

swift/codegen/generators/qlgen.py

@@ -51,6 +51,10 @@ class RootElementHasChildren(Error):
     pass
 
 
+class NoClasses(Error):
+    pass
+
+
 def get_ql_property(cls: schema.Class, prop: schema.Property, prev_child: str = "") -> ql.Property:
     args = dict(
         type=prop.type if not prop.is_predicate else "predicate",
@@ -103,7 +107,7 @@ def get_ql_class(cls: schema.Class, lookup: typing.Dict[str, schema.Class]):
         bases=cls.bases,
         final=not cls.derived,
         properties=properties,
-        dir=cls.dir,
+        dir=pathlib.Path(cls.group or ""),
         ipa=bool(cls.ipa),
         **pragmas,
     )
@@ -127,7 +131,7 @@ def get_ql_ipa_class_db(name: str) -> ql.Synth.FinalClassDb:
 def get_ql_ipa_class(cls: schema.Class):
     if cls.derived:
         return ql.Synth.NonFinalClass(name=cls.name, derived=sorted(cls.derived),
-                                      root=(cls.name == schema.root_class_name))
+                                      root=not cls.bases)
     if cls.ipa and cls.ipa.from_class is not None:
         source = cls.ipa.from_class
         get_ql_ipa_class_db(source).subtract_type(cls.name)
@@ -253,12 +257,14 @@ def generate(opts, renderer):
     existing |= {q for q in test_out.rglob("*.ql")}
     existing |= {q for q in test_out.rglob(missing_test_source_filename)}
 
-    data = schema.load(input)
+    data = schema.load_file(input)
 
     classes = {name: get_ql_class(cls, data.classes) for name, cls in data.classes.items()}
-    # element root is absent in tests
-    if schema.root_class_name in classes and classes[schema.root_class_name].has_children:
-        raise RootElementHasChildren
+    if not classes:
+        raise NoClasses
+    root = next(iter(classes.values()))
+    if root.has_children:
+        raise RootElementHasChildren(root)
 
     imports = {}
 
@@ -288,10 +294,10 @@ def generate(opts, renderer):
     for c in data.classes.values():
         if _should_skip_qltest(c, data.classes):
             continue
-        test_dir = test_out / c.dir / c.name
+        test_dir = test_out / c.group / c.name
         test_dir.mkdir(parents=True, exist_ok=True)
         if not any(test_dir.glob("*.swift")):
-            log.warning(f"no test source in {c.dir / c.name}")
+            log.warning(f"no test source in {test_dir.relative_to(test_out)}")
             renderer.render(ql.MissingTestInstructions(),
                             test_dir / missing_test_source_filename)
             continue
@@ -308,12 +314,12 @@ def generate(opts, renderer):
     constructor_imports = []
     ipa_constructor_imports = []
     stubs = {}
-    for cls in sorted(data.classes.values(), key=lambda cls: (cls.dir, cls.name)):
+    for cls in sorted(data.classes.values(), key=lambda cls: (cls.group, cls.name)):
         ipa_type = get_ql_ipa_class(cls)
         if ipa_type.is_final:
             final_ipa_types.append(ipa_type)
             if ipa_type.has_params:
-                stub_file = stub_out / cls.dir / f"{cls.name}Constructor.qll"
+                stub_file = stub_out / cls.group / f"{cls.name}Constructor.qll"
                 if not stub_file.is_file() or _is_generated_stub(stub_file):
                     # stub rendering must be postponed as we might not have yet all subtracted ipa types in `ipa_type`
                     stubs[stub_file] = ql.Synth.ConstructorStub(ipa_type)
@@ -326,7 +332,7 @@ def generate(opts, renderer):
 
     for stub_file, data in stubs.items():
         renderer.render(data, stub_file)
-    renderer.render(ql.Synth.Types(schema.root_class_name, final_ipa_types, non_final_ipa_types), out / "Synth.qll")
+    renderer.render(ql.Synth.Types(root.name, final_ipa_types, non_final_ipa_types), out / "Synth.qll")
     renderer.render(ql.ImportList(constructor_imports), out / "SynthConstructors.qll")
     renderer.render(ql.ImportList(ipa_constructor_imports), out / "PureSynthConstructors.qll")
 

swift/codegen/lib/schema/defs.py

@@ -1 +1,121 @@
-optional = list = int = string = boolean = predicate = include = group = child = synth = qltest = cpp = object()
+from typing import Callable as _Callable, Union as _Union
+from functools import singledispatch as _singledispatch
+from swift.codegen.lib import schema as _schema
+import inspect as _inspect
+
+
+class _ChildModifier(_schema.PropertyModifier):
+    def modify(self, prop: _schema.Property):
+        if prop.type is None or prop.type[0].islower():
+            raise _schema.Error("Non-class properties cannot be children")
+        prop.is_child = True
+
+
+def include(source: str):
+    # add to `includes` variable in calling context
+    _inspect.currentframe().f_back.f_locals.setdefault(
+        "__includes", []).append(source)
+
+
+class _Pragma(_schema.PropertyModifier):
+    """ A class or property pragma.
+    For properties, it functions similarly to a `_PropertyModifier` with `|`, adding the pragma.
+    For schema classes it acts as a python decorator with `@`.
+    """
+
+    def __init__(self, pragma):
+        self.pragma = pragma
+
+    def modify(self, prop: _schema.Property):
+        prop.pragmas.append(self.pragma)
+
+    def __call__(self, cls: type) -> type:
+        """ use this pragma as a decorator on classes """
+        if "pragmas" in cls.__dict__:  # not using hasattr as we don't want to land on inherited pragmas
+            cls.pragmas.append(self.pragma)
+        else:
+            cls.pragmas = [self.pragma]
+        return cls
+
+
+class _Optionalizer(_schema.PropertyModifier):
+    def modify(self, prop: _schema.Property):
+        K = _schema.Property.Kind
+        if prop.kind != K.SINGLE:
+            raise _schema.Error(
+                "Optional should only be applied to simple property types")
+        prop.kind = K.OPTIONAL
+
+
+class _Listifier(_schema.PropertyModifier):
+    def modify(self, prop: _schema.Property):
+        K = _schema.Property.Kind
+        if prop.kind == K.SINGLE:
+            prop.kind = K.REPEATED
+        elif prop.kind == K.OPTIONAL:
+            prop.kind = K.REPEATED_OPTIONAL
+        else:
+            raise _schema.Error(
+                "Repeated should only be applied to simple or optional property types")
+
+
+class _TypeModifier:
+    """ Modifies types using get item notation """
+
+    def __init__(self, modifier: _schema.PropertyModifier):
+        self.modifier = modifier
+
+    def __getitem__(self, item):
+        return item | self.modifier
+
+
+class _Namespace:
+    """ simple namespacing mechanism """
+
+    def __init__(self, **kwargs):
+        self.__dict__.update(kwargs)
+
+
+_ClassDecorator = _Callable[[type], type]
+
+
+def _annotate(**kwargs) -> _ClassDecorator:
+    def f(cls: type) -> type:
+        for k, v in kwargs.items():
+            setattr(cls, k, v)
+        return cls
+
+    return f
+
+
+boolean = "boolean"
+int = "int"
+string = "string"
+
+predicate = _schema.predicate_marker
+optional = _TypeModifier(_Optionalizer())
+list = _TypeModifier(_Listifier())
+
+child = _ChildModifier()
+
+qltest = _Namespace(
+    skip=_Pragma("qltest_skip"),
+    collapse_hierarchy=_Pragma("qltest_collapse_hierarchy"),
+    uncollapse_hierarchy=_Pragma("qltest_uncollapse_hierarchy"),
+)
+
+cpp = _Namespace(
+    skip=_Pragma("cpp_skip"),
+)
+
+
+def group(name: str = "") -> _ClassDecorator:
+    return _annotate(group=name)
+
+
+synth = _Namespace(
+    from_class=lambda ref: _annotate(ipa=_schema.IpaInfo(
+        from_class=_schema.get_type_name(ref))),
+    on_arguments=lambda **kwargs: _annotate(
+        ipa=_schema.IpaInfo(on_arguments={k: _schema.get_type_name(t) for k, t in kwargs.items()}))
+)