added and passed test_postgres_build

Author: wangpatrick57

benchmark/tests/integtest_benchmark.py

@@ -20,11 +20,13 @@
 )
 
 
-class TestBenchmark(unittest.TestCase):
+class BenchmarkTests(unittest.TestCase):
     DBGYM_CONFIG_PATH = Path("benchmark/tests/benchmark_integtest_dbgym_config.yaml")
 
     def setUp(self) -> None:
-        workspace_path = get_workspace_path_from_config(TestBenchmark.DBGYM_CONFIG_PATH)
+        workspace_path = get_workspace_path_from_config(
+            BenchmarkTests.DBGYM_CONFIG_PATH
+        )
         # Get a clean start each time.
         if workspace_path.exists():
             shutil.rmtree(workspace_path)

dbms/postgres/build_repo.sh renamed to dbms/postgres/_build_repo.sh

@@ -3,14 +3,14 @@
 """
 
 import logging
-import os
 import shutil
 import subprocess
 from pathlib import Path
 from typing import Optional
 
 import click
 import sqlalchemy
+from gymlib.symlinks_paths import get_pgbin_symlink_path, get_repo_symlink_path
 
 from benchmark.constants import DEFAULT_SCALE_FACTOR
 from benchmark.job.load_info import JobLoadInfo
@@ -35,7 +35,6 @@
     fully_resolve_path,
     get_dbdata_tgz_filename,
     get_default_dbdata_parent_dpath,
-    get_default_pgbin_path,
     is_fully_resolved,
     is_ssd,
     link_result,
@@ -61,7 +60,38 @@ def postgres_group(dbgym_workspace: DBGymWorkspace) -> None:
     help="Include this flag to rebuild Postgres even if it already exists.",
 )
 def postgres_build(dbgym_workspace: DBGymWorkspace, rebuild: bool) -> None:
-    _build_repo(dbgym_workspace, rebuild)
+    _postgres_build(dbgym_workspace, rebuild)
+
+
+def _postgres_build(dbgym_workspace: DBGymWorkspace, rebuild: bool) -> None:
+    """
+    This function exists as a hook for integration tests.
+    """
+    expected_repo_symlink_dpath = get_repo_symlink_path(
+        dbgym_workspace.dbgym_workspace_path
+    )
+    if not rebuild and expected_repo_symlink_dpath.exists():
+        logging.getLogger(DBGYM_LOGGER_NAME).info(
+            f"Skipping _postgres_build: {expected_repo_symlink_dpath}"
+        )
+        return
+
+    logging.getLogger(DBGYM_LOGGER_NAME).info(
+        f"Setting up repo in {expected_repo_symlink_dpath}"
+    )
+    repo_real_dpath = dbgym_workspace.dbgym_this_run_path / "repo"
+    repo_real_dpath.mkdir(parents=False, exist_ok=False)
+    subprocess_run(
+        f"./_build_repo.sh {repo_real_dpath}",
+        cwd=dbgym_workspace.base_dbgym_repo_dpath / "dbms" / "postgres",
+    )
+
+    # only link at the end so that the link only ever points to a complete repo
+    repo_symlink_dpath = dbgym_workspace.link_result(repo_real_dpath)
+    assert expected_repo_symlink_dpath.samefile(repo_symlink_dpath)
+    logging.getLogger(DBGYM_LOGGER_NAME).info(
+        f"Set up repo in {expected_repo_symlink_dpath}"
+    )
 
 
 @postgres_group.command(
@@ -75,7 +105,7 @@ def postgres_build(dbgym_workspace: DBGymWorkspace, rebuild: bool) -> None:
     "--pgbin-path",
     type=Path,
     default=None,
-    help=f"The path to the bin containing Postgres executables. The default is {get_default_pgbin_path(WORKSPACE_PATH_PLACEHOLDER)}.",
+    help=f"The path to the bin containing Postgres executables. The default is {get_pgbin_symlink_path(WORKSPACE_PATH_PLACEHOLDER)}.",
 )
 @click.option(
     "--intended-dbdata-hardware",
@@ -99,7 +129,7 @@ def postgres_dbdata(
 ) -> None:
     # Set args to defaults programmatically (do this before doing anything else in the function)
     if pgbin_path is None:
-        pgbin_path = get_default_pgbin_path(dbgym_workspace.dbgym_workspace_path)
+        pgbin_path = get_pgbin_symlink_path(dbgym_workspace.dbgym_workspace_path)
     if dbdata_parent_dpath is None:
         dbdata_parent_dpath = get_default_dbdata_parent_dpath(
             dbgym_workspace.dbgym_workspace_path
@@ -127,40 +157,6 @@ def postgres_dbdata(
     )
 
 
-def _get_pgbin_symlink_path(dbgym_workspace: DBGymWorkspace) -> Path:
-    return dbgym_workspace.cur_symlinks_build_path(
-        "repo.link", "boot", "build", "postgres", "bin"
-    )
-
-
-def _get_repo_symlink_path(dbgym_workspace: DBGymWorkspace) -> Path:
-    return dbgym_workspace.cur_symlinks_build_path("repo.link")
-
-
-def _build_repo(dbgym_workspace: DBGymWorkspace, rebuild: bool) -> None:
-    expected_repo_symlink_dpath = _get_repo_symlink_path(dbgym_workspace)
-    if not rebuild and expected_repo_symlink_dpath.exists():
-        logging.getLogger(DBGYM_LOGGER_NAME).info(
-            f"Skipping _build_repo: {expected_repo_symlink_dpath}"
-        )
-        return
-
-    logging.getLogger(DBGYM_LOGGER_NAME).info(
-        f"Setting up repo in {expected_repo_symlink_dpath}"
-    )
-    repo_real_dpath = dbgym_workspace.cur_task_runs_build_path("repo", mkdir=True)
-    subprocess_run(
-        f"./build_repo.sh {repo_real_dpath}", cwd=dbgym_workspace.cur_source_path()
-    )
-
-    # only link at the end so that the link only ever points to a complete repo
-    repo_symlink_dpath = link_result(dbgym_workspace, repo_real_dpath)
-    assert expected_repo_symlink_dpath.samefile(repo_symlink_dpath)
-    logging.getLogger(DBGYM_LOGGER_NAME).info(
-        f"Set up repo in {expected_repo_symlink_dpath}"
-    )
-
-
 def _create_dbdata(
     dbgym_workspace: DBGymWorkspace,
     benchmark_name: str,
@@ -216,7 +212,9 @@ def _create_dbdata(
 
 def _generic_dbdata_setup(dbgym_workspace: DBGymWorkspace) -> None:
     # get necessary vars
-    pgbin_real_dpath = _get_pgbin_symlink_path(dbgym_workspace).resolve()
+    pgbin_real_dpath = get_pgbin_symlink_path(
+        dbgym_workspace.dbgym_workspace_path
+    ).resolve()
     assert pgbin_real_dpath.exists()
     dbgym_pguser = DBGYM_POSTGRES_USER
     dbgym_pgpass = DBGYM_POSTGRES_PASS

dbms/postgres/cli.py

@@ -0,0 +1 @@
+dbgym_workspace_path: ../dbgym_dbms_integtest_workspace/

dbms/tests/__init__.py

@@ -0,0 +1,42 @@
+import shutil
+import unittest
+from pathlib import Path
+
+from gymlib.symlinks_paths import get_repo_symlink_path
+
+from dbms.postgres.cli import _postgres_build
+from util.workspace import (
+    DBGymWorkspace,
+    fully_resolve_path,
+    get_workspace_path_from_config,
+)
+
+
+class DBMSTests(unittest.TestCase):
+    DBGYM_CONFIG_PATH = Path("dbms/tests/dbms_integtest_dbgym_config.yaml")
+
+    def setUp(self) -> None:
+        workspace_path = get_workspace_path_from_config(DBMSTests.DBGYM_CONFIG_PATH)
+        # Get a clean start each time.
+        if workspace_path.exists():
+            shutil.rmtree(workspace_path)
+
+        # Reset this to avoid the error of it being created twice.
+        # In real usage, the second run would be a different Python process so DBGymWorkspace.num_times_created_this_run would be 0.
+        DBGymWorkspace.num_times_created_this_run = 0
+        self.workspace = DBGymWorkspace(workspace_path)
+
+    def tearDown(self) -> None:
+        if self.workspace.dbgym_workspace_path.exists():
+            shutil.rmtree(self.workspace.dbgym_workspace_path)
+
+    def test_postgres_build(self) -> None:
+        repo_path = get_repo_symlink_path(self.workspace.dbgym_workspace_path)
+        self.assertFalse(repo_path.exists())
+        _postgres_build(self.workspace, False)
+        self.assertTrue(repo_path.exists())
+        self.assertTrue(fully_resolve_path(repo_path).exists())
+
+
+if __name__ == "__main__":
+    unittest.main()

dbms/tests/dbms_integtest_dbgym_config.yaml

@@ -4,15 +4,18 @@
 from typing import Optional
 
 # TODO: remove symlinks_paths from the import
-from gymlib.symlinks_paths import get_workload_suffix, get_workload_symlink_path
+from gymlib.symlinks_paths import (
+    get_pgbin_symlink_path,
+    get_workload_suffix,
+    get_workload_symlink_path,
+)
 
 from benchmark.tpch.constants import DEFAULT_TPCH_SEED
 from env.tuning_artifacts import TuningMetadata
 from util.workspace import (
     DBGymWorkspace,
     fully_resolve_path,
     get_default_dbdata_parent_dpath,
-    get_default_pgbin_path,
     get_default_pristine_dbdata_snapshot_path,
     get_workspace_path_from_config,
 )
@@ -91,7 +94,7 @@ def get_default_metadata() -> TuningMetadata:
                     dbgym_workspace.dbgym_workspace_path,
                     GymlibIntegtestManager.BENCHMARK,
                     GymlibIntegtestManager.SCALE_FACTOR,
-                    suffix
+                    suffix,
                 ),
             ),
             pristine_dbdata_snapshot_path=fully_resolve_path(
@@ -105,6 +108,6 @@ def get_default_metadata() -> TuningMetadata:
                 get_default_dbdata_parent_dpath(dbgym_workspace.dbgym_workspace_path),
             ),
             pgbin_path=fully_resolve_path(
-                get_default_pgbin_path(dbgym_workspace.dbgym_workspace_path),
+                get_pgbin_symlink_path(dbgym_workspace.dbgym_workspace_path),
             ),
         )

dbms/tests/integtest_dbms.py

@@ -59,3 +59,11 @@ def get_workload_symlink_path(
         / DBGYM_APP_NAME
         / (get_workload_dirname(benchmark, scale_factor, suffix) + ".link")
     )
+
+
+def get_repo_symlink_path(workspace_path: Path) -> Path:
+    return workspace_path / SYMLINKS_DNAME / DBGYM_APP_NAME / "repo.link"
+
+
+def get_pgbin_symlink_path(workspace_path: Path) -> Path:
+    return get_repo_symlink_path(workspace_path) / "boot" / "build" / "postgres" / "bin"

env/tests/gymlib_integtest_util.py

@@ -45,9 +45,6 @@ def get_latest_run_path_from_workspace_path(workspace_path: Path) -> Path:
     return get_runs_path_from_workspace_path(workspace_path) / "latest_run.link"
 
 
-# Other parameters
-
-
 # Paths of config files in the codebase. These are always relative paths.
 # The reason these can be relative paths instead of functions taking in codebase_path as input is because relative paths are relative to the codebase root
 DEFAULT_BOOT_CONFIG_FPATH = POSTGRES_PATH / "default_boot_config.yaml"
@@ -109,19 +106,6 @@ def get_default_dbdata_parent_dpath(workspace_path: Path) -> Path:
     return get_tmp_path_from_workspace_path(workspace_path)
 
 
-def get_default_repo_path(workspace_path: Path) -> Path:
-    return (
-        get_symlinks_path_from_workspace_path(workspace_path)
-        / "dbgym_dbms_postgres"
-        / "build"
-        / "repo.link"
-    )
-
-
-def get_default_pgbin_path(workspace_path: Path) -> Path:
-    return get_default_repo_path(workspace_path) / "boot" / "build" / "postgres" / "bin"
-
-
 class DBGymWorkspace:
     """
     Global configurations that apply to all parts of DB-Gym