Integration test CI

.github/workflows/unittest_ci.yml → .github/workflows/tests_ci.yml

@@ -1,4 +1,4 @@
-name: Unit Tests
+name: Unit and Integration Tests
 
 on:
   push: {}
@@ -26,10 +26,22 @@ jobs:
     # Note that the GHA runners are stateful. Dependencies installed from previous runs will still be on the runner.
     # This means this step will usually be pretty fast as most dependencies will already be cached. However, it also
     # means that past runs might interfere with the current run, so you sometimes may need to restart the GHA runners.
+
+    # We need to do `. "$HOME/.cargo/env"` in each step for it to work.
     - name: Install dependencies
       run: |
         ./dependencies/install_dependencies.sh
-        . "$HOME/.cargo/env"
 
     - name: Run unit tests
-      run: python scripts/run_unittests.py
+      run: |
+        . "$HOME/.cargo/env"
+        python scripts/run_unit_tests.py
+
+    - name: Run integration test
+      # Delete the workspace. Run once with a clean workspace. Run again from the existing workspace.
+      # Need to run with a non-root user in order to start Postgres.
+      run: |
+        . "$HOME/.cargo/env"
+        rm -rf ../dbgym_integtest_workspace
+        ./scripts/integration_test.sh ssd
+        ./scripts/integration_test.sh ssd

benchmark/tpch/cli.py

@@ -67,7 +67,7 @@ def _clone(dbgym_cfg: DBGymConfig):
         f"./tpch_setup.sh {real_build_path}", cwd=dbgym_cfg.cur_source_path()
     )
     symlink_dpath = link_result(dbgym_cfg, real_build_path / "tpch-kit")
-    assert os.path.samefile(expected_symlink_dpath, symlink_dpath)
+    assert expected_symlink_dpath.samefile(symlink_dpath)
     benchmark_tpch_logger.info(f"Cloned: {expected_symlink_dpath}")
 
 
@@ -97,7 +97,7 @@ def _generate_queries(dbgym_cfg: DBGymConfig, seed_start: int, seed_end: int, sc
                 verbose=False,
             )
         queries_symlink_dpath = link_result(dbgym_cfg, real_dir)
-        assert os.path.samefile(queries_symlink_dpath, expected_queries_symlink_dpath)
+        assert queries_symlink_dpath.samefile(expected_queries_symlink_dpath)
     benchmark_tpch_logger.info(
         f"Generated queries: {data_path} [{seed_start}, {seed_end}]"
     )
@@ -119,7 +119,7 @@ def _generate_data(dbgym_cfg: DBGymConfig, scale_factor: float):
     subprocess_run(f"mv ./*.tbl {real_dir}", cwd=tpch_kit_dpath / "dbgen")
 
     tables_symlink_dpath = link_result(dbgym_cfg, real_dir)
-    assert os.path.samefile(tables_symlink_dpath, expected_tables_symlink_dpath)
+    assert tables_symlink_dpath.samefile(expected_tables_symlink_dpath)
     benchmark_tpch_logger.info(f"Generated: {expected_tables_symlink_dpath}")
 
 
@@ -130,12 +130,12 @@ def _generate_workload(
     query_subset: str,
     scale_factor: float,
 ):
-    symlink_data_dir = dbgym_cfg.cur_symlinks_data_path(mkdir=True)
+    symlink_data_dpath = dbgym_cfg.cur_symlinks_data_path(mkdir=True)
     workload_name = workload_name_fn(scale_factor, seed_start, seed_end, query_subset)
-    expected_workload_symlink_dpath = symlink_data_dir / (workload_name + ".link")
+    expected_workload_symlink_dpath = symlink_data_dpath / (workload_name + ".link")
 
     benchmark_tpch_logger.info(f"Generating: {expected_workload_symlink_dpath}")
-    real_dir = dbgym_cfg.cur_task_runs_data_path(
+    real_dpath = dbgym_cfg.cur_task_runs_data_path(
         workload_name, mkdir=True
     )
 
@@ -147,15 +147,15 @@ def _generate_workload(
     elif query_subset == "odd":
         queries = [f"{i}" for i in range(1, 22 + 1) if i % 2 == 1]
 
-    with open(real_dir / "order.txt", "w") as f:
+    with open(real_dpath / "order.txt", "w") as f:
         for seed in range(seed_start, seed_end + 1):
             for qnum in queries:
-                sql_fpath = (symlink_data_dir / (_get_queries_dname(seed, scale_factor) + ".link")).resolve() / f"{qnum}.sql"
+                sql_fpath = (symlink_data_dpath / (_get_queries_dname(seed, scale_factor) + ".link")).resolve() / f"{qnum}.sql"
                 assert sql_fpath.exists() and not sql_fpath.is_symlink() and sql_fpath.is_absolute(), "We should only write existent real absolute paths to a file"
                 output = ",".join([f"S{seed}-Q{qnum}", str(sql_fpath)])
                 print(output, file=f)
                 # TODO(WAN): add option to deep-copy the workload.
 
-    workload_symlink_dpath = link_result(dbgym_cfg, real_dir)
+    workload_symlink_dpath = link_result(dbgym_cfg, real_dpath)
     assert workload_symlink_dpath == expected_workload_symlink_dpath
     benchmark_tpch_logger.info(f"Generated: {expected_workload_symlink_dpath}")

benchmark/tpch/tpch_setup.sh

@@ -7,7 +7,7 @@ TPCH_REPO_ROOT="$1"
 if [ ! -d "${TPCH_REPO_ROOT}/tpch-kit" ]; then
   mkdir -p "${TPCH_REPO_ROOT}"
   cd "${TPCH_REPO_ROOT}"
-  git clone git@github.com:lmwnshn/tpch-kit.git --single-branch --branch master --depth 1
+  git clone https://github.com/lmwnshn/tpch-kit.git --single-branch --branch master --depth 1
   cd ./tpch-kit/dbgen
   make MACHINE=LINUX DATABASE=POSTGRESQL
 fi

dbms/postgres/build_repo.sh

@@ -7,7 +7,7 @@ REPO_REAL_PARENT_DPATH="$1"
 # Download and make postgres from the boot repository.
 mkdir -p "${REPO_REAL_PARENT_DPATH}"
 cd "${REPO_REAL_PARENT_DPATH}"
-git clone git@github.com:lmwnshn/boot.git --single-branch --branch vldb_2024 --depth 1
+git clone https://github.com/lmwnshn/boot.git --single-branch --branch vldb_2024 --depth 1
 cd ./boot
 ./cmudb/build/configure.sh release "${REPO_REAL_PARENT_DPATH}/boot/build/postgres"
 make clean
@@ -25,7 +25,7 @@ make install -j
 cd "${REPO_REAL_PARENT_DPATH}/boot"
 
 # Download and make hypopg.
-git clone git@github.com:HypoPG/hypopg.git
+git clone https://github.com/HypoPG/hypopg.git
 cd ./hypopg
 PG_CONFIG="${REPO_REAL_PARENT_DPATH}/boot/build/postgres/bin/pg_config" make install
 cd "${REPO_REAL_PARENT_DPATH}/boot"

dbms/postgres/cli.py

@@ -104,7 +104,7 @@ def _build_repo(dbgym_cfg: DBGymConfig, rebuild):
 
     # only link at the end so that the link only ever points to a complete repo
     repo_symlink_dpath = link_result(dbgym_cfg, repo_real_dpath)
-    assert os.path.samefile(expected_repo_symlink_dpath, repo_symlink_dpath)
+    assert expected_repo_symlink_dpath.samefile(repo_symlink_dpath)
     dbms_postgres_logger.info(f"Set up repo in {expected_repo_symlink_dpath}")
 
 

manage/cli.py

@@ -169,11 +169,11 @@ def clean_workspace(dbgym_cfg: DBGymConfig, mode: str="safe", verbose=False) ->
             if not is_child_path(real_fordpath, dbgym_cfg.dbgym_runs_path):
                 continue
 
-            assert not os.path.samefile(real_fordpath, dbgym_cfg.dbgym_runs_path)
+            assert not real_fordpath.samefile(dbgym_cfg.dbgym_runs_path)
 
             # Figure out the task_run_child_fordpath to put into task_run_child_fordpaths_to_keep
             task_run_child_fordpath = None
-            if os.path.samefile(parent_dpath_of_path(real_fordpath), dbgym_cfg.dbgym_runs_path):
+            if parent_dpath_of_path(real_fordpath).samefile(dbgym_cfg.dbgym_runs_path):
                 # While it's true that it shouldn't be possible to symlink to a directory directly in task_runs/,
                 #   we'll just not delete it if the user happens to have one like this. Even if the user messed up
                 #   the structure somehow, it's just a good idea not to delete it.
@@ -183,10 +183,10 @@ def clean_workspace(dbgym_cfg: DBGymConfig, mode: str="safe", verbose=False) ->
                 #   However, as with above, we won't just nuke files if the workspace doesn't follow this rule for
                 #   some reason.
                 task_run_child_fordpath = real_fordpath
-                while not os.path.samefile(parent_dpath_of_path(task_run_child_fordpath), dbgym_cfg.dbgym_runs_path):
+                while not parent_dpath_of_path(task_run_child_fordpath).samefile(dbgym_cfg.dbgym_runs_path):
                     task_run_child_fordpath = parent_dpath_of_path(task_run_child_fordpath)
             assert task_run_child_fordpath != None
-            assert os.path.samefile(parent_dpath_of_path(task_run_child_fordpath), dbgym_cfg.dbgym_runs_path), f"task_run_child_fordpath ({task_run_child_fordpath}) is not a direct child of dbgym_cfg.dbgym_runs_path"
+            assert parent_dpath_of_path(task_run_child_fordpath).samefile(dbgym_cfg.dbgym_runs_path), f"task_run_child_fordpath ({task_run_child_fordpath}) is not a direct child of dbgym_cfg.dbgym_runs_path"
             task_run_child_fordpaths_to_keep.add(task_run_child_fordpath)
 
             # If on safe mode, add symlinks inside the task_run_child_fordpath to be processed