moved workspace and its tests to gymlib

Author: wangpatrick57

benchmark/cli.py

@@ -1,8 +1,8 @@
 import click
+from gymlib.workspace import DBGymWorkspace
 
 from benchmark.job.cli import job_group
 from benchmark.tpch.cli import tpch_group
-from util.workspace import DBGymWorkspace
 
 
 @click.group(name="benchmark")

benchmark/job/cli.py

@@ -2,16 +2,16 @@
 from typing import Optional
 
 import click
-from gymlib import (
+from gymlib.shell import subprocess_run
+from gymlib.symlinks_paths import (
     get_tables_dirname,
     get_workload_dirname,
     get_workload_suffix,
     name_to_linkname,
 )
-from gymlib.shell import subprocess_run
+from gymlib.workspace import DBGymWorkspace, fully_resolve_path
 
 from benchmark.constants import DEFAULT_SCALE_FACTOR
-from util.workspace import DBGymWorkspace, fully_resolve_path
 
 JOB_TABLES_URL = "https://event.cwi.nl/da/job/imdb.tgz"
 JOB_QUERIES_URL = "https://event.cwi.nl/da/job/job.tgz"

benchmark/job/load_info.py

@@ -1,11 +1,11 @@
 from pathlib import Path
 from typing import Optional
 
-from gymlib import get_tables_symlink_path
+from gymlib.symlinks_paths import get_tables_symlink_path
+from gymlib.workspace import DBGymWorkspace, fully_resolve_path
 
 from benchmark.constants import DEFAULT_SCALE_FACTOR
 from dbms.load_info_base_class import LoadInfoBaseClass
-from util.workspace import DBGymWorkspace, fully_resolve_path
 
 JOB_SCHEMA_FNAME = "job_schema.sql"
 

benchmark/tests/integtest_benchmark.py

@@ -2,22 +2,22 @@
 import unittest
 from pathlib import Path
 
-from gymlib import (
+from gymlib.symlinks_paths import (
     get_tables_symlink_path,
     get_workload_suffix,
     get_workload_symlink_path,
 )
+from gymlib.workspace import (
+    DBGymWorkspace,
+    fully_resolve_path,
+    get_workspace_path_from_config,
+)
 
 # It's ok to import private functions from the benchmark module because this is an integration test.
 from benchmark.constants import DEFAULT_SCALE_FACTOR
 from benchmark.job.cli import _job_tables, _job_workload
 from benchmark.tpch.cli import _tpch_tables, _tpch_workload
 from benchmark.tpch.constants import DEFAULT_TPCH_SEED
-from util.workspace import (
-    DBGymWorkspace,
-    fully_resolve_path,
-    get_workspace_path_from_config,
-)
 
 
 class BenchmarkTests(unittest.TestCase):

benchmark/tpch/cli.py

@@ -1,7 +1,8 @@
 import logging
 
 import click
-from gymlib import (
+from gymlib.shell import subprocess_run
+from gymlib.symlinks_paths import (
     get_scale_factor_string,
     get_tables_dirname,
     get_tables_symlink_path,
@@ -10,11 +11,10 @@
     linkname_to_name,
     name_to_linkname,
 )
-from gymlib.shell import subprocess_run
+from gymlib.workspace import DBGymWorkspace, fully_resolve_path, is_fully_resolved
 
 from benchmark.constants import DEFAULT_SCALE_FACTOR
 from benchmark.tpch.constants import DEFAULT_TPCH_SEED, NUM_TPCH_QUERIES
-from util.workspace import DBGymWorkspace, fully_resolve_path, is_fully_resolved
 
 TPCH_KIT_DIRNAME = "tpch-kit"
 

benchmark/tpch/load_info.py

@@ -1,10 +1,10 @@
 from pathlib import Path
 from typing import Optional
 
-from gymlib import get_tables_symlink_path
+from gymlib.symlinks_paths import get_tables_symlink_path
+from gymlib.workspace import DBGymWorkspace, fully_resolve_path
 
 from dbms.load_info_base_class import LoadInfoBaseClass
-from util.workspace import DBGymWorkspace, fully_resolve_path
 
 TPCH_SCHEMA_FNAME = "tpch_schema.sql"
 TPCH_CONSTRAINTS_FNAME = "tpch_constraints.sql"

dbms/cli.py

@@ -1,7 +1,7 @@
 import click
+from gymlib.workspace import DBGymWorkspace
 
 from dbms.postgres.cli import postgres_group
-from util.workspace import DBGymWorkspace
 
 
 @click.group(name="dbms")

dbms/postgres/cli.py

@@ -13,27 +13,27 @@
 import psutil
 import psycopg
 import sqlalchemy
-from gymlib import (
+from gymlib.shell import subprocess_run
+from gymlib.symlinks_paths import (
     get_dbdata_tgz_symlink_path,
     get_pgbin_symlink_path,
     get_repo_symlink_path,
     linkname_to_name,
 )
-from gymlib.shell import subprocess_run
-from sqlalchemy import create_engine, text
-
-from benchmark.constants import DEFAULT_SCALE_FACTOR
-from benchmark.job.load_info import JobLoadInfo
-from benchmark.tpch.load_info import TpchLoadInfo
-from dbms.load_info_base_class import LoadInfoBaseClass
-from util.workspace import (
+from gymlib.workspace import (
     WORKSPACE_PATH_PLACEHOLDER,
     DBGymWorkspace,
     fully_resolve_path,
     get_tmp_path_from_workspace_path,
     is_fully_resolved,
     is_ssd,
 )
+from sqlalchemy import create_engine, text
+
+from benchmark.constants import DEFAULT_SCALE_FACTOR
+from benchmark.job.load_info import JobLoadInfo
+from benchmark.tpch.load_info import TpchLoadInfo
+from dbms.load_info_base_class import LoadInfoBaseClass
 
 DBGYM_POSTGRES_USER = "dbgym_user"
 DBGYM_POSTGRES_PASS = "dbgym_pass"

dbms/tests/integtest_dbms.py

@@ -3,16 +3,16 @@
 import unittest
 from pathlib import Path
 
-from gymlib import get_dbdata_tgz_symlink_path, get_repo_symlink_path
-
-from benchmark.tpch.cli import _tpch_tables
-from dbms.postgres.cli import _postgres_build, _postgres_dbdata
-from util.workspace import (
+from gymlib.symlinks_paths import get_dbdata_tgz_symlink_path, get_repo_symlink_path
+from gymlib.workspace import (
     DBGymWorkspace,
     fully_resolve_path,
     get_workspace_path_from_config,
 )
 
+from benchmark.tpch.cli import _tpch_tables
+from dbms.postgres.cli import _postgres_build, _postgres_dbdata
+
 
 class DBMSTests(unittest.TestCase):
     DBGYM_CONFIG_PATH = Path("dbms/tests/dbms_integtest_dbgym_config.yaml")

env/pg_conn.py

@@ -19,6 +19,7 @@
 import psutil
 import psycopg
 import yaml
+from gymlib.workspace import DBGymWorkspace, parent_path_of_path
 from plumbum import local
 from psycopg.errors import ProgramLimitExceeded, QueryCanceled
 
@@ -27,7 +28,6 @@
     SHARED_PRELOAD_LIBRARIES,
     get_kv_connstr,
 )
-from util.workspace import DBGymWorkspace, parent_path_of_path
 
 CONNECT_TIMEOUT = 300
 

env/replay.py

@@ -1,11 +1,12 @@
 from collections import defaultdict
 from pathlib import Path
 
+from gymlib.workspace import DBGymWorkspace
+
 from env.pg_conn import PostgresConn
 from env.tuning_artifacts import TuningArtifactsReader
 from env.workload import Workload
 from util.pg import DEFAULT_POSTGRES_PORT
-from util.workspace import DBGymWorkspace
 
 
 def replay(

env/tests/gymlib_integtest_util.py

@@ -4,22 +4,22 @@
 from typing import Optional
 
 # TODO: remove symlinks_paths from the import
-from gymlib import (
+from gymlib.symlinks_paths import (
     get_dbdata_tgz_symlink_path,
     get_pgbin_symlink_path,
     get_workload_suffix,
     get_workload_symlink_path,
 )
-
-from benchmark.tpch.constants import DEFAULT_TPCH_SEED
-from env.tuning_artifacts import TuningMetadata
-from util.workspace import (
+from gymlib.workspace import (
     DBGymWorkspace,
     fully_resolve_path,
     get_tmp_path_from_workspace_path,
     get_workspace_path_from_config,
 )
 
+from benchmark.tpch.constants import DEFAULT_TPCH_SEED
+from env.tuning_artifacts import TuningMetadata
+
 
 class GymlibIntegtestManager:
     """

env/tests/integtest_pg_conn.py

@@ -2,6 +2,7 @@
 import unittest
 
 import psycopg
+from gymlib.workspace import DBGymWorkspace
 
 from env.pg_conn import PostgresConn
 from env.tests.gymlib_integtest_util import GymlibIntegtestManager
@@ -10,7 +11,6 @@
     get_is_postgres_running,
     get_running_postgres_ports,
 )
-from util.workspace import DBGymWorkspace
 
 
 class PostgresConnTests(unittest.TestCase):

env/tests/integtest_replay.py

@@ -1,5 +1,7 @@
 import unittest
 
+from gymlib.workspace import DBGymWorkspace
+
 from benchmark.tpch.constants import DEFAULT_TPCH_SEED
 from env.replay import replay
 from env.tests.gymlib_integtest_util import GymlibIntegtestManager
@@ -10,7 +12,6 @@
     SysKnobsDelta,
     TuningArtifactsWriter,
 )
-from util.workspace import DBGymWorkspace
 
 
 class ReplayTests(unittest.TestCase):

env/tests/integtest_tuning_artifacts.py

@@ -1,5 +1,7 @@
 import unittest
 
+from gymlib.workspace import DBGymWorkspace
+
 from env.tests.gymlib_integtest_util import GymlibIntegtestManager
 from env.tuning_artifacts import (
     DBMSConfigDelta,
@@ -9,7 +11,6 @@
     TuningArtifactsReader,
     TuningArtifactsWriter,
 )
-from util.workspace import DBGymWorkspace
 
 
 class PostgresConnTests(unittest.TestCase):

env/tests/integtest_workload.py

@@ -1,9 +1,10 @@
 import unittest
 
+from gymlib.workspace import DBGymWorkspace
+
 from benchmark.tpch.constants import DEFAULT_TPCH_SEED, NUM_TPCH_QUERIES
 from env.tests.gymlib_integtest_util import GymlibIntegtestManager
 from env.workload import Workload
-from util.workspace import DBGymWorkspace
 
 
 class WorkloadTests(unittest.TestCase):

env/tuning_artifacts.py

@@ -3,7 +3,7 @@
 from pathlib import Path
 from typing import Any, NewType
 
-from util.workspace import DBGymWorkspace, is_fully_resolved
+from gymlib.workspace import DBGymWorkspace, is_fully_resolved
 
 # PostgresConn doesn't use these types because PostgresConn is used internally by tuning agents
 # while these types are only used in the interface between the orchestrator and the tuning agents.

env/workload.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-from util.workspace import DBGymWorkspace, is_fully_resolved
+from gymlib.workspace import DBGymWorkspace, is_fully_resolved
 
 
 class Workload:

gymlib_package/__init__.py

@@ -1,5 +1 @@
-# Everything at the "core" of gymlib will be imported directly (e.g. `from .[module] import *`).
-# Everything not as important will be only be imported as a module (e.g. `from . import [module]`).
-from . import shell
-from .magic import *
-from .symlinks_paths import *
+from . import magic, shell, symlinks_paths, workspace

gymlib_package/gymlib/__init__.py

@@ -3,7 +3,7 @@
 import unittest
 from pathlib import Path
 
-from util.tests.filesystem_unittest_util import (
+from gymlib.tests.filesystem_unittest_util import (
     FilesystemStructure,
     create_structure,
     verify_structure,

gymlib_package/gymlib/tests/__init__.py

@@ -6,13 +6,13 @@
 from pathlib import Path
 from typing import Optional
 
-from util.tests.filesystem_unittest_util import (
+from gymlib.tests.filesystem_unittest_util import (
     FilesystemStructure,
     create_structure,
     make_workspace_structure,
     verify_structure,
 )
-from util.workspace import DBGymWorkspace
+from gymlib.workspace import DBGymWorkspace
 
 
 class WorkspaceTests(unittest.TestCase):

util/tests/filesystem_unittest_util.py renamed to gymlib_package/gymlib/tests/filesystem_unittest_util.py

@@ -12,7 +12,7 @@
 from typing import IO, Any, Optional
 
 import yaml
-from gymlib import is_linkname, name_to_linkname
+from gymlib.symlinks_paths import is_linkname, name_to_linkname
 
 WORKSPACE_PATH_PLACEHOLDER = Path("[workspace]")
 

util/tests/unittest_filesystem_unittest_util.py renamed to gymlib_package/gymlib/tests/unittest_filesystem_unittest_util.py

@@ -5,8 +5,7 @@
 from pathlib import Path
 
 import click
-
-from util.workspace import (
+from gymlib.workspace import (
     DBGymWorkspace,
     get_runs_path_from_workspace_path,
     get_symlinks_path_from_workspace_path,

util/tests/unittest_workspace.py renamed to gymlib_package/gymlib/tests/unittest_workspace.py

@@ -3,14 +3,15 @@
 import unittest
 from pathlib import Path
 
-from manage.cli import MockDBGymWorkspace, clean_workspace
-from util.tests.filesystem_unittest_util import (
+from gymlib.tests.filesystem_unittest_util import (
     FilesystemStructure,
     create_structure,
     make_workspace_structure,
     verify_structure,
 )
 
+from manage.cli import MockDBGymWorkspace, clean_workspace
+
 # This is here instead of on `if __name__ == "__main__"` because we often run individual tests, which
 #   does not go through the `if __name__ == "__main__"` codepath.
 # Make it DEBUG to see logs from verify_structure(). Make it CRITICAL to not see any logs.

util/workspace.py renamed to gymlib_package/gymlib/workspace.py

@@ -1,9 +1,9 @@
 import click
+from gymlib.workspace import make_standard_dbgym_workspace
 
 from benchmark.cli import benchmark_group
 from dbms.cli import dbms_group
 from manage.cli import manage_group
-from util.workspace import make_standard_dbgym_workspace
 
 # TODO(phw2): Save commit, git diff, and run command.
 # TODO(phw2): Remove write permissions on old run_*/ dirs to enforce that they are immutable.