chore: file generation cleanup (#1736)

This PR does too many things :( 

1. Major refactoring of the dev/scripts and dev/code-generation folders. 

Primarily this was removing duplicate code and cleaning up some poorly written code snippets as well as making them more idempotent so then can be re-run over and over again but still maintain the same results. This is working on my machine, but I've been having problems in CI and comparing diffs so running generators in CI will have to wait. 

2. Re-Implement using the generated api routes for testing

This was a _huge_ refactor that touched damn near every test file but now we have auto-generated typed routes with inline hints and it's used for nearly every test excluding a few that use classes for better parameterization. This should greatly reduce errors when writing new tests. 

3. Minor Perf improvements for the All Recipes endpoint

  A. Removed redundant loops
  B. Uses orjson to do the encoding directly and returns a byte response instead of relying on the default 
       jsonable_encoder.

4. Fix some TS type errors that cropped up for seemingly no reason half way through the PR.

See this issue phillipdupuis/pydantic-to-typescript#28

Basically, the generated TS type is not-correct since Pydantic will automatically fill in null fields. The resulting TS type is generated with a ? to indicate it can be null even though we _know_ that i can't be.

Author: hay-kot

.gitignore

@@ -158,3 +158,4 @@ dev/code-generation/generated/openapi.json
 dev/code-generation/generated/test_routes.py
 mealie/services/parser_services/crfpp/model.crfmodel
 lcov.info
+dev/code-generation/openapi.json

dev/code-generation/_static.py

@@ -1,8 +1,8 @@
 from dataclasses import dataclass
 from pathlib import Path
 
-from _gen_utils import log, render_python_template
 from slugify import slugify
+from utils import render_python_template
 
 CWD = Path(__file__).parent
 
@@ -25,9 +25,7 @@ def from_path(cls, path: Path):
 
         # Remove any file extension
         var = var.split(".")[0]
-
         var = var.replace("'", "")
-
         var = slugify(var, separator="_")
 
         return cls(var, rel_path)
@@ -97,8 +95,6 @@ def recursive_rename(p: Path):
 
 
 def main():
-    log.info("Starting Template Generation")
-
     rename_non_compliant_paths()
 
     GENERATED.mkdir(exist_ok=True)
@@ -115,8 +111,6 @@ def main():
         {"children": all_children},
     )
 
-    log.info("Finished Template Generation")
-
 
 if __name__ == "__main__":
     main()

dev/scripts/api_docs_gen.py renamed to dev/code-generation/gen_docs_api.py

@@ -0,0 +1,84 @@
+import json
+from pathlib import Path
+
+from fastapi import FastAPI
+from jinja2 import Template
+from pydantic import BaseModel
+from utils import PROJECT_DIR, CodeTemplates, HTTPRequest, RouteObject
+
+CWD = Path(__file__).parent
+
+OUTFILE = PROJECT_DIR / "tests" / "utils" / "api_routes" / "__init__.py"
+
+
+class PathObject(BaseModel):
+    route_object: RouteObject
+    http_verbs: list[HTTPRequest]
+
+    class Config:
+        arbitrary_types_allowed = True
+
+
+def get_path_objects(app: FastAPI):
+    paths = []
+
+    for key, value in app.openapi().items():
+        if key == "paths":
+            for key, value in value.items():
+
+                paths.append(
+                    PathObject(
+                        route_object=RouteObject(key),
+                        http_verbs=[HTTPRequest(request_type=k, **v) for k, v in value.items()],
+                    )
+                )
+
+    return paths
+
+
+def dump_open_api(app: FastAPI):
+    """Writes the Open API as JSON to a json file"""
+    OPEN_API_FILE = CWD / "openapi.json"
+
+    with open(OPEN_API_FILE, "w") as f:
+        f.write(json.dumps(app.openapi()))
+
+
+def read_template(file: Path):
+    with open(file) as f:
+        return f.read()
+
+
+def generate_python_templates(static_paths: list[PathObject], function_paths: list[PathObject]):
+
+    template = Template(read_template(CodeTemplates.pytest_routes))
+    content = template.render(
+        paths={
+            "prefix": "/api",
+            "static_paths": static_paths,
+            "function_paths": function_paths,
+        }
+    )
+    with open(OUTFILE, "w") as f:
+        f.write(content)
+
+    return
+
+
+def main():
+    from mealie.app import app
+
+    dump_open_api(app)
+    paths = get_path_objects(app)
+
+    static_paths = [x.route_object for x in paths if not x.route_object.is_function]
+    function_paths = [x.route_object for x in paths if x.route_object.is_function]
+
+    static_paths.sort(key=lambda x: x.router_slug)
+    function_paths.sort(key=lambda x: x.router_slug)
+
+    generate_python_templates(static_paths, function_paths)
+
+
+if __name__ == "__main__":
+    main()

dev/code-generation/gen_test_data_paths.py renamed to dev/code-generation/gen_py_pytest_data_paths.py

@@ -0,0 +1,102 @@
+import pathlib
+import re
+from dataclasses import dataclass, field
+
+from utils import PROJECT_DIR, log, render_python_template
+
+template = """# This file is auto-generated by gen_schema_exports.py
+{% for file in data.module.files %}{{ file.import_str() }}
+{% endfor %}
+
+__all__ = [
+    {% for file in data.module.files %}
+    {%- for class in file.classes -%}
+    "{{ class }}",
+    {%- endfor -%}
+    {%- endfor %}
+]
+
+"""
+
+SCHEMA_PATH = PROJECT_DIR / "mealie" / "schema"
+
+SKIP = {"static", "__pycache__"}
+
+
+class PyFile:
+    import_path: str
+    """The import path of the file"""
+
+    classes: list[str]
+    """A list of classes in the file"""
+
+    def __init__(self, path: pathlib.Path):
+        self.import_path = path.stem
+        self.classes = []
+
+        self.classes = PyFile.extract_classes(path)
+        self.classes.sort()
+
+    def import_str(self) -> str:
+        """Returns a string that can be used to import the file"""
+        return f"from .{self.import_path} import {', '.join(self.classes)}"
+
+    @staticmethod
+    def extract_classes(file_path: pathlib.Path) -> list[str]:
+        name = file_path.stem
+
+        if name == "__init__" or name.startswith("_"):
+            return []
+
+        classes = re.findall(r"(?m)^class\s(\w+)", file_path.read_text())
+        return classes
+
+
+@dataclass
+class Modules:
+    directory: pathlib.Path
+    """The directory to search for modules"""
+
+    files: list[PyFile] = field(default_factory=list)
+    """A list of files in the directory"""
+
+    def __post_init__(self):
+        for file in self.directory.glob("*.py"):
+            if file.name.startswith("_"):
+                continue
+
+            pfile = PyFile(file)
+
+            if len(pfile.classes) > 0:
+                self.files.append(pfile)
+
+            else:
+                log.debug(f"Skipping {file.name} as it has no classes")
+
+
+def find_modules(root: pathlib.Path) -> list[Modules]:
+    """Finds all the top level modules in the provided folder"""
+    modules: list[Modules] = []
+    for file in root.iterdir():
+        if file.is_dir() and file.name not in SKIP:
+
+            modules.append(Modules(directory=file))
+
+    return modules
+
+
+def main():
+
+    modules = find_modules(SCHEMA_PATH)
+
+    for module in modules:
+        log.debug(f"Module: {module.directory.name}")
+        for file in module.files:
+            log.debug(f"  File: {file.import_path}")
+            log.debug(f"    Classes: [{', '.join(file.classes)}]")
+
+        render_python_template(template, module.directory / "__init__.py", {"module": module})
+
+
+if __name__ == "__main__":
+    main()