made all defaults above workload path fns

Author: wangpatrick57

scripts/run_protox_e2e_test.py

@@ -11,14 +11,14 @@
 from benchmark.tpch.constants import DEFAULT_TPCH_SEED
 from util.pg import get_is_postgres_running
 from util.workspace import (
-    default_embedder_path,
-    default_hpoed_agent_params_path,
     default_pristine_dbdata_snapshot_path,
     default_replay_data_fpath,
     default_repo_path,
-    default_tables_path,
-    default_traindata_path,
     default_tuning_steps_dpath,
+    get_default_embedder_path,
+    get_default_hpoed_agent_params_path,
+    get_default_tables_path,
+    get_default_traindata_path,
     get_default_workload_path,
     get_workload_name,
 )
@@ -96,7 +96,9 @@ def run_e2e_for_benchmark(benchmark_name: str, intended_dbdata_hardware: str) ->
 
     # Run the full Proto-X training pipeline, asserting things along the way
     # Setup (workload and database)
-    tables_dpath = default_tables_path(workspace_dpath, benchmark_name, scale_factor)
+    tables_dpath = get_default_tables_path(
+        workspace_dpath, benchmark_name, scale_factor
+    )
     if Stage.Tables in STAGES_TO_RUN:
         assert not tables_dpath.exists()
         subprocess.run(
@@ -135,7 +137,7 @@ def run_e2e_for_benchmark(benchmark_name: str, intended_dbdata_hardware: str) ->
         assert pristine_dbdata_snapshot_fpath.exists()
 
     # Tuning (embedding, HPO, and actual tuning)
-    traindata_dpath = default_traindata_path(
+    traindata_dpath = get_default_traindata_path(
         workspace_dpath, benchmark_name, workload_name
     )
     if Stage.EmbeddingData in STAGES_TO_RUN:
@@ -146,7 +148,7 @@ def run_e2e_for_benchmark(benchmark_name: str, intended_dbdata_hardware: str) ->
         )
         assert traindata_dpath.exists()
 
-    embedder_dpath = default_embedder_path(
+    embedder_dpath = get_default_embedder_path(
         workspace_dpath, benchmark_name, workload_name
     )
     if Stage.EmbeddingModel in STAGES_TO_RUN:
@@ -157,7 +159,7 @@ def run_e2e_for_benchmark(benchmark_name: str, intended_dbdata_hardware: str) ->
         )
         assert embedder_dpath.exists()
 
-    hpoed_agent_params_fpath = default_hpoed_agent_params_path(
+    hpoed_agent_params_fpath = get_default_hpoed_agent_params_path(
         workspace_dpath, benchmark_name, workload_name
     )
     if Stage.TuneHPO in STAGES_TO_RUN:

tune/protox/agent/hpo.py

@@ -36,12 +36,12 @@
     DBGymConfig,
     TuningMode,
     default_dbdata_parent_dpath,
-    default_embedder_path,
     default_pgbin_path,
     default_pristine_dbdata_snapshot_path,
     fully_resolve_path,
     get_default_benchbase_config_path,
     get_default_benchmark_config_path,
+    get_default_embedder_path,
     get_default_hpoed_agent_params_fname,
     get_default_workload_name_suffix,
     get_default_workload_path,
@@ -120,7 +120,7 @@ def __init__(
     "--embedder-path",
     type=Path,
     default=None,
-    help=f"The path to the directory that contains an `embedder.pth` file with a trained encoder and decoder as well as a `config` file. The default is {default_embedder_path(WORKSPACE_PATH_PLACEHOLDER, BENCHMARK_NAME_PLACEHOLDER, WORKLOAD_NAME_PLACEHOLDER)}",
+    help=f"The path to the directory that contains an `embedder.pth` file with a trained encoder and decoder as well as a `config` file. The default is {get_default_embedder_path(WORKSPACE_PATH_PLACEHOLDER, BENCHMARK_NAME_PLACEHOLDER, WORKLOAD_NAME_PLACEHOLDER)}",
 )
 @click.option(
     "--benchmark-config-path",
@@ -270,7 +270,7 @@ def hpo(
         workload_name_suffix = get_default_workload_name_suffix(benchmark_name)
     workload_name = get_workload_name(scale_factor, workload_name_suffix)
     if embedder_path is None:
-        embedder_path = default_embedder_path(
+        embedder_path = get_default_embedder_path(
             dbgym_cfg.dbgym_workspace_path, benchmark_name, workload_name
         )
     if benchmark_config_path is None:

tune/protox/agent/tune.py

@@ -18,8 +18,8 @@
     WORKSPACE_PATH_PLACEHOLDER,
     DBGymConfig,
     TuningMode,
-    default_hpoed_agent_params_path,
     fully_resolve_path,
+    get_default_hpoed_agent_params_path,
     get_default_tuning_steps_dname,
     get_default_workload_name_suffix,
     get_workload_name,
@@ -48,7 +48,7 @@
     "--hpoed-agent-params-path",
     default=None,
     type=Path,
-    help=f"The path to best params found by the agent HPO process. The default is {default_hpoed_agent_params_path(WORKSPACE_PATH_PLACEHOLDER, BENCHMARK_NAME_PLACEHOLDER, WORKLOAD_NAME_PLACEHOLDER)}",
+    help=f"The path to best params found by the agent HPO process. The default is {get_default_hpoed_agent_params_path(WORKSPACE_PATH_PLACEHOLDER, BENCHMARK_NAME_PLACEHOLDER, WORKLOAD_NAME_PLACEHOLDER)}",
 )
 @click.option(
     "--enable-boot-during-tune",
@@ -83,7 +83,7 @@ def tune(
         workload_name_suffix = get_default_workload_name_suffix(benchmark_name)
     workload_name = get_workload_name(scale_factor, workload_name_suffix)
     if hpoed_agent_params_path is None:
-        hpoed_agent_params_path = default_hpoed_agent_params_path(
+        hpoed_agent_params_path = get_default_hpoed_agent_params_path(
             dbgym_cfg.dbgym_workspace_path, benchmark_name, workload_name
         )
 

tune/protox/embedding/train.py

@@ -29,9 +29,9 @@
     WORKLOAD_NAME_PLACEHOLDER,
     WORKSPACE_PATH_PLACEHOLDER,
     DBGymConfig,
-    default_traindata_path,
     fully_resolve_path,
     get_default_benchmark_config_path,
+    get_default_traindata_path,
     get_default_workload_name_suffix,
     get_default_workload_path,
     get_workload_name,
@@ -66,7 +66,7 @@
     "--traindata-path",
     type=Path,
     default=None,
-    help=f"The path to the .parquet file containing the training data to use to train the embedding models. The default is {default_traindata_path(WORKSPACE_PATH_PLACEHOLDER, BENCHMARK_NAME_PLACEHOLDER, WORKLOAD_NAME_PLACEHOLDER)}.",
+    help=f"The path to the .parquet file containing the training data to use to train the embedding models. The default is {get_default_traindata_path(WORKSPACE_PATH_PLACEHOLDER, BENCHMARK_NAME_PLACEHOLDER, WORKLOAD_NAME_PLACEHOLDER)}.",
 )
 @click.option(
     "--seed",
@@ -201,7 +201,7 @@ def train(
         workload_name_suffix = get_default_workload_name_suffix(benchmark_name)
     workload_name = get_workload_name(scale_factor, workload_name_suffix)
     if traindata_path is None:
-        traindata_path = default_traindata_path(
+        traindata_path = get_default_traindata_path(
             dbgym_cfg.dbgym_workspace_path, benchmark_name, workload_name
         )
     # TODO(phw2): figure out whether different scale factors use the same config

util/workspace.py

@@ -149,38 +149,51 @@ def get_default_replay_data_fname(
 #    folder called run_*/dbgym_agent_protox_tune/tuning_steps. However, replay itself generates an replay_info.log file, which goes in
 #    run_*/dbgym_agent_protox_tune/tuning_steps/. The bug was that my replay function was overwriting the replay_info.log file of the
 #    tuning run. By naming all symlinks "*.link", we avoid the possibility of subtle bugs like this happening.
-default_traindata_path: Callable[[Path, str, str], Path] = (
-    lambda workspace_path, benchmark_name, workload_name: get_symlinks_path_from_workspace_path(
-        workspace_path
+def get_default_traindata_path(
+    workspace_path: Path, benchmark_name: str, workload_name: str
+) -> Path:
+    return (
+        get_symlinks_path_from_workspace_path(workspace_path)
+        / "dbgym_tune_protox_embedding"
+        / "data"
+        / (get_default_traindata_fname(benchmark_name, workload_name) + ".link")
     )
-    / "dbgym_tune_protox_embedding"
-    / "data"
-    / (get_default_traindata_fname(benchmark_name, workload_name) + ".link")
-)
-default_embedder_path: Callable[[Path, str, str], Path] = (
-    lambda workspace_path, benchmark_name, workload_name: get_symlinks_path_from_workspace_path(
-        workspace_path
+
+
+def get_default_embedder_path(
+    workspace_path: Path, benchmark_name: str, workload_name: str
+) -> Path:
+    return (
+        get_symlinks_path_from_workspace_path(workspace_path)
+        / "dbgym_tune_protox_embedding"
+        / "data"
+        / (get_default_embedder_dname(benchmark_name, workload_name) + ".link")
     )
-    / "dbgym_tune_protox_embedding"
-    / "data"
-    / (get_default_embedder_dname(benchmark_name, workload_name) + ".link")
-)
-default_hpoed_agent_params_path: Callable[[Path, str, str], Path] = (
-    lambda workspace_path, benchmark_name, workload_name: get_symlinks_path_from_workspace_path(
-        workspace_path
+
+
+def get_default_hpoed_agent_params_path(
+    workspace_path: Path, benchmark_name: str, workload_name: str
+) -> Path:
+    return (
+        get_symlinks_path_from_workspace_path(workspace_path)
+        / "dbgym_tune_protox_agent"
+        / "data"
+        / (
+            get_default_hpoed_agent_params_fname(benchmark_name, workload_name)
+            + ".link"
+        )
     )
-    / "dbgym_tune_protox_agent"
-    / "data"
-    / (get_default_hpoed_agent_params_fname(benchmark_name, workload_name) + ".link")
-)
-default_tables_path: Callable[[Path, str, float | str], Path] = (
-    lambda workspace_path, benchmark_name, scale_factor: get_symlinks_path_from_workspace_path(
-        workspace_path
+
+
+def get_default_tables_path(
+    workspace_path: Path, benchmark_name: str, scale_factor: float | str
+) -> Path:
+    return (
+        get_symlinks_path_from_workspace_path(workspace_path)
+        / f"dbgym_benchmark_{benchmark_name}"
+        / "data"
+        / (get_default_tables_dname(scale_factor) + ".link")
     )
-    / f"dbgym_benchmark_{benchmark_name}"
-    / "data"
-    / (get_default_tables_dname(scale_factor) + ".link")
-)
 
 
 def get_default_workload_path(