11"""
2- At a high level, this file's goal is to (1) install+ build postgres and (2) create pgdata.
2+ At a high level, this file's goal is to (1) build postgres and (2) create dbdata (aka pgdata) .
33On the other hand, the goal of tune.protox.env.util.postgres is to provide helpers to manage
44 a Postgres instance during agent tuning.
55util.pg provides helpers used by *both* of the above files (as well as other files).
1010import subprocess
1111from pathlib import Path
1212import click
13- import ssd_checker
1413
1514from benchmark .tpch .load_info import TpchLoadInfo
1615from dbms .load_info_base_class import LoadInfoBaseClass
17- from misc .utils import DBGymConfig , conv_inputpath_to_realabspath , link_result , open_and_save , save_file , get_pgdata_tgz_name , default_pgbin_path , WORKSPACE_PATH_PLACEHOLDER , default_pgdata_parent_dpath
16+ from misc .utils import DBGymConfig , conv_inputpath_to_realabspath , link_result , open_and_save , save_file , get_dbdata_tgz_name , default_pgbin_path , WORKSPACE_PATH_PLACEHOLDER , default_dbdata_parent_dpath , is_ssd
1817from util .shell import subprocess_run
1918from sqlalchemy import Connection
2019from util .pg import SHARED_PRELOAD_LIBRARIES , conn_execute , sql_file_execute , DBGYM_POSTGRES_DBNAME , create_conn , DEFAULT_POSTGRES_PORT , DBGYM_POSTGRES_USER , DBGYM_POSTGRES_PASS , DEFAULT_POSTGRES_DBNAME
@@ -32,7 +31,7 @@ def postgres_group(dbgym_cfg: DBGymConfig):
3231
3332@postgres_group .command (
3433 name = "build" ,
35- help = "Download and build the Postgres repository and all necessary extensions/shared libraries. Does not create pgdata ." ,
34+ help = "Download and build the Postgres repository and all necessary extensions/shared libraries. Does not create dbdata ." ,
3635)
3736@click .pass_obj
3837@click .option ("--rebuild" , is_flag = True , help = "Include this flag to rebuild Postgres even if it already exists." )
@@ -41,46 +40,46 @@ def postgres_build(dbgym_cfg: DBGymConfig, rebuild: bool):
4140
4241
4342@postgres_group .command (
44- name = "pgdata " ,
45- help = "Build a .tgz file of pgdata with various specifications for its contents." ,
43+ name = "dbdata " ,
44+ help = "Build a .tgz file of dbdata with various specifications for its contents." ,
4645)
4746@click .pass_obj
4847@click .argument ("benchmark_name" , type = str )
4948@click .option ("--scale-factor" , type = float , default = 1 )
5049@click .option ("--pgbin-path" , type = Path , default = None , help = f"The path to the bin containing Postgres executables. The default is { default_pgbin_path (WORKSPACE_PATH_PLACEHOLDER )} )
5150@click .option (
52- "--intended-pgdata -hardware" ,
51+ "--intended-dbdata -hardware" ,
5352 type = click .Choice (["hdd" , "ssd" ]),
5453 default = "hdd" ,
55- help = f"The intended hardware pgdata should be on. Used as a sanity check for --pgdata -parent-dpath." ,
54+ help = f"The intended hardware dbdata should be on. Used as a sanity check for --dbdata -parent-dpath." ,
5655)
5756@click .option (
58- "--pgdata -parent-dpath" ,
57+ "--dbdata -parent-dpath" ,
5958 default = None ,
6059 type = Path ,
61- help = f"The path to the parent directory of the pgdata which will be actively tuned. The default is { default_pgdata_parent_dpath (WORKSPACE_PATH_PLACEHOLDER )} ,
60+ help = f"The path to the parent directory of the dbdata which will be actively tuned. The default is { default_dbdata_parent_dpath (WORKSPACE_PATH_PLACEHOLDER )} ,
6261)
63- def postgres_pgdata (dbgym_cfg : DBGymConfig , benchmark_name : str , scale_factor : float , pgbin_path : Path , intended_pgdata_hardware : str , pgdata_parent_dpath : Path ):
62+ def postgres_dbdata (dbgym_cfg : DBGymConfig , benchmark_name : str , scale_factor : float , pgbin_path : Path , intended_dbdata_hardware : str , dbdata_parent_dpath : Path ):
6463 # Set args to defaults programmatically (do this before doing anything else in the function)
6564 if pgbin_path == None :
6665 pgbin_path = default_pgbin_path (dbgym_cfg .dbgym_workspace_path )
67- if pgdata_parent_dpath == None :
68- pgdata_parent_dpath = default_pgdata_parent_dpath (dbgym_cfg .dbgym_workspace_path )
66+ if dbdata_parent_dpath == None :
67+ dbdata_parent_dpath = default_dbdata_parent_dpath (dbgym_cfg .dbgym_workspace_path )
6968
7069 # Convert all input paths to absolute paths
7170 pgbin_path = conv_inputpath_to_realabspath (dbgym_cfg , pgbin_path )
72- pgdata_parent_dpath = conv_inputpath_to_realabspath (dbgym_cfg , pgdata_parent_dpath )
71+ dbdata_parent_dpath = conv_inputpath_to_realabspath (dbgym_cfg , dbdata_parent_dpath )
7372
7473 # Check assertions on args
75- if intended_pgdata_hardware == "hdd" :
76- assert not ssd_checker . is_ssd (pgdata_parent_dpath ), f"Intended hardware is HDD but pgdata_parent_dpath ({ pgdata_parent_dpath }
77- elif intended_pgdata_hardware == "ssd" :
78- assert ssd_checker . is_ssd (pgdata_parent_dpath ), f"Intended hardware is SSD but pgdata_parent_dpath ({ pgdata_parent_dpath }
74+ if intended_dbdata_hardware == "hdd" :
75+ assert not is_ssd (dbdata_parent_dpath ), f"Intended hardware is HDD but dbdata_parent_dpath ({ dbdata_parent_dpath }
76+ elif intended_dbdata_hardware == "ssd" :
77+ assert is_ssd (dbdata_parent_dpath ), f"Intended hardware is SSD but dbdata_parent_dpath ({ dbdata_parent_dpath }
7978 else :
8079 assert False
8180
82- # Create pgdata
83- _create_pgdata (dbgym_cfg , benchmark_name , scale_factor , pgbin_path , pgdata_parent_dpath )
81+ # Create dbdata
82+ _create_dbdata (dbgym_cfg , benchmark_name , scale_factor , pgbin_path , dbdata_parent_dpath )
8483
8584
8685def _get_pgbin_symlink_path (dbgym_cfg : DBGymConfig ) -> Path :
@@ -109,52 +108,52 @@ def _build_repo(dbgym_cfg: DBGymConfig, rebuild):
109108 dbms_postgres_logger .info (f"Set up repo in { expected_repo_symlink_dpath } )
110109
111110
112- def _create_pgdata (dbgym_cfg : DBGymConfig , benchmark_name : str , scale_factor : float , pgbin_path : Path , pgdata_parent_dpath : Path ) -> None :
111+ def _create_dbdata (dbgym_cfg : DBGymConfig , benchmark_name : str , scale_factor : float , pgbin_path : Path , dbdata_parent_dpath : Path ) -> None :
113112 """
114- I chose *not* for this function to skip by default if pgdata_tgz_symlink_path already exists. This
113+ I chose *not* for this function to skip by default if dbdata_tgz_symlink_path already exists. This
115114 is because, while the generated data is deterministic given benchmark_name and scale_factor, any
116- change in the _create_pgdata () function would result in a different pgdata . Since _create_pgdata ()
115+ change in the _create_dbdata () function would result in a different dbdata . Since _create_dbdata ()
117116 may change somewhat frequently, I decided to get rid of the footgun of having changes to
118- _create_pgdata () not propagate to [pgdata ].tgz by default.
117+ _create_dbdata () not propagate to [dbdata ].tgz by default.
119118 """
120119
121- # It's ok for the pgdata / directory to be temporary. It just matters that the .tgz is saved in a safe place.
122- pgdata_dpath = pgdata_parent_dpath / "pgdata_being_created "
123- # We might be reusing the same pgdata_parent_dpath , so delete pgdata_dpath if it already exists
124- if pgdata_dpath .exists ():
125- shutil .rmtree (pgdata_dpath )
120+ # It's ok for the dbdata / directory to be temporary. It just matters that the .tgz is saved in a safe place.
121+ dbdata_dpath = dbdata_parent_dpath / "dbdata_being_created "
122+ # We might be reusing the same dbdata_parent_dpath , so delete dbdata_dpath if it already exists
123+ if dbdata_dpath .exists ():
124+ shutil .rmtree (dbdata_dpath )
126125
127126 # Call initdb.
128127 # Save any script we call from pgbin_symlink_dpath because they are dependencies generated from another task run.
129128 save_file (dbgym_cfg , pgbin_path / "initdb" )
130- subprocess_run (f'./initdb -D "{ pgdata_dpath } , cwd = pgbin_path )
129+ subprocess_run (f'./initdb -D "{ dbdata_dpath } , cwd = pgbin_path )
131130
132- # Start Postgres (all other pgdata setup requires postgres to be started).
131+ # Start Postgres (all other dbdata setup requires postgres to be started).
133132 # Note that subprocess_run() never returns when running "pg_ctl start", so I'm using subprocess.run() instead.
134- start_postgres (dbgym_cfg , pgbin_path , pgdata_dpath )
133+ start_postgres (dbgym_cfg , pgbin_path , dbdata_dpath )
135134
136135 # Set up Postgres.
137- _generic_pgdata_setup (dbgym_cfg )
138- _load_benchmark_into_pgdata (dbgym_cfg , benchmark_name , scale_factor )
136+ _generic_dbdata_setup (dbgym_cfg )
137+ _load_benchmark_into_dbdata (dbgym_cfg , benchmark_name , scale_factor )
139138
140139 # Stop Postgres so that we don't "leak" processes.
141- stop_postgres (dbgym_cfg , pgbin_path , pgdata_dpath )
140+ stop_postgres (dbgym_cfg , pgbin_path , dbdata_dpath )
142141
143142 # Create .tgz file.
144- # Note that you can't pass "[pgdata ].tgz" as an arg to cur_task_runs_data_path() because that would create "[pgdata ].tgz" as a dir.
145- pgdata_tgz_real_fpath = dbgym_cfg .cur_task_runs_data_path (
143+ # Note that you can't pass "[dbdata ].tgz" as an arg to cur_task_runs_data_path() because that would create "[dbdata ].tgz" as a dir.
144+ dbdata_tgz_real_fpath = dbgym_cfg .cur_task_runs_data_path (
146145 mkdir = True
147- ) / get_pgdata_tgz_name (benchmark_name , scale_factor )
148- # We need to cd into pgdata_dpath so that the tar file does not contain folders for the whole path of pgdata_dpath .
149- subprocess_run (f"tar -czf { pgdata_tgz_real_fpath } , cwd = pgdata_dpath )
146+ ) / get_dbdata_tgz_name (benchmark_name , scale_factor )
147+ # We need to cd into dbdata_dpath so that the tar file does not contain folders for the whole path of dbdata_dpath .
148+ subprocess_run (f"tar -czf { dbdata_tgz_real_fpath } , cwd = dbdata_dpath )
150149
151150 # Create symlink.
152- # Only link at the end so that the link only ever points to a complete pgdata .
153- pgdata_tgz_symlink_path = link_result (dbgym_cfg , pgdata_tgz_real_fpath )
154- dbms_postgres_logger .info (f"Created pgdata in { pgdata_tgz_symlink_path } )
151+ # Only link at the end so that the link only ever points to a complete dbdata .
152+ dbdata_tgz_symlink_path = link_result (dbgym_cfg , dbdata_tgz_real_fpath )
153+ dbms_postgres_logger .info (f"Created dbdata in { dbdata_tgz_symlink_path } )
155154
156155
157- def _generic_pgdata_setup (dbgym_cfg : DBGymConfig ):
156+ def _generic_dbdata_setup (dbgym_cfg : DBGymConfig ):
158157 # get necessary vars
159158 pgbin_real_dpath = _get_pgbin_symlink_path (dbgym_cfg ).resolve ()
160159 assert pgbin_real_dpath .exists ()
@@ -182,29 +181,29 @@ def _generic_pgdata_setup(dbgym_cfg: DBGymConfig):
182181 cwd = pgbin_real_dpath ,
183182 )
184183
185- # Create the dbgym database. since one pgdata dir maps to one benchmark, all benchmarks will use the same database
186- # as opposed to using databases named after the benchmark
184+ # Create the dbgym database. Since one dbdata dir maps to one benchmark, all benchmarks will use the same database
185+ # as opposed to using databases named after the benchmark.
187186 subprocess_run (
188187 f"./psql -c \" create database { DBGYM_POSTGRES_DBNAME } { dbgym_pguser } \" { DEFAULT_POSTGRES_DBNAME } { pgport } ,
189188 cwd = pgbin_real_dpath ,
190189 )
191190
192191
193- def _load_benchmark_into_pgdata (
192+ def _load_benchmark_into_dbdata (
194193 dbgym_cfg : DBGymConfig , benchmark_name : str , scale_factor : float
195194):
196195 with create_conn (use_psycopg = False ) as conn :
197196 if benchmark_name == "tpch" :
198197 load_info = TpchLoadInfo (dbgym_cfg , scale_factor )
199198 else :
200199 raise AssertionError (
201- f"_load_benchmark_into_pgdata (): the benchmark of name { benchmark_name }
200+ f"_load_benchmark_into_dbdata (): the benchmark of name { benchmark_name }
202201 )
203202
204- _load_into_pgdata (dbgym_cfg , conn , load_info )
203+ _load_into_dbdata (dbgym_cfg , conn , load_info )
205204
206205
207- def _load_into_pgdata (dbgym_cfg : DBGymConfig , conn : Connection , load_info : LoadInfoBaseClass ):
206+ def _load_into_dbdata (dbgym_cfg : DBGymConfig , conn : Connection , load_info : LoadInfoBaseClass ):
208207 sql_file_execute (dbgym_cfg , conn , load_info .get_schema_fpath ())
209208
210209 # truncate all tables first before even loading a single one
@@ -223,29 +222,29 @@ def _load_into_pgdata(dbgym_cfg: DBGymConfig, conn: Connection, load_info: LoadI
223222 sql_file_execute (dbgym_cfg , conn , constraints_fpath )
224223
225224
226- def start_postgres (dbgym_cfg : DBGymConfig , pgbin_path : Path , pgdata_dpath : Path ) -> None :
227- _start_or_stop_postgres (dbgym_cfg , pgbin_path , pgdata_dpath , True )
225+ def start_postgres (dbgym_cfg : DBGymConfig , pgbin_path : Path , dbdata_dpath : Path ) -> None :
226+ _start_or_stop_postgres (dbgym_cfg , pgbin_path , dbdata_dpath , True )
228227
229228
230- def stop_postgres (dbgym_cfg : DBGymConfig , pgbin_path : Path , pgdata_dpath : Path ) -> None :
231- _start_or_stop_postgres (dbgym_cfg , pgbin_path , pgdata_dpath , False )
229+ def stop_postgres (dbgym_cfg : DBGymConfig , pgbin_path : Path , dbdata_dpath : Path ) -> None :
230+ _start_or_stop_postgres (dbgym_cfg , pgbin_path , dbdata_dpath , False )
232231
233232
234- def _start_or_stop_postgres (dbgym_cfg : DBGymConfig , pgbin_path : Path , pgdata_dpath : Path , is_start : bool ) -> None :
233+ def _start_or_stop_postgres (dbgym_cfg : DBGymConfig , pgbin_path : Path , dbdata_dpath : Path , is_start : bool ) -> None :
235234 # They should be absolute paths and should exist
236235 assert pgbin_path .is_absolute () and pgbin_path .exists ()
237- assert pgdata_dpath .is_absolute () and pgdata_dpath .exists ()
236+ assert dbdata_dpath .is_absolute () and dbdata_dpath .exists ()
238237 # The inputs may be symlinks so we need to resolve them first
239238 pgbin_real_dpath = pgbin_path .resolve ()
240- pgdata_dpath = pgdata_dpath .resolve ()
239+ dbdata_dpath = dbdata_dpath .resolve ()
241240 pgport = DEFAULT_POSTGRES_PORT
242241 save_file (dbgym_cfg , pgbin_real_dpath / "pg_ctl" )
243242
244243 if is_start :
245244 # We use subprocess.run() because subprocess_run() never returns when running "pg_ctl start".
246245 # The reason subprocess_run() never returns is because pg_ctl spawns a postgres process so .poll() always returns None.
247246 # On the other hand, subprocess.run() does return normally, like calling `./pg_ctl` on the command line would do.
248- result = subprocess .run (f"./pg_ctl -D \" { pgdata_dpath } \" -o '-p { pgport } , cwd = pgbin_real_dpath , shell = True )
247+ result = subprocess .run (f"./pg_ctl -D \" { dbdata_dpath } \" -o '-p { pgport } , cwd = pgbin_real_dpath , shell = True )
249248 result .check_returncode ()
250249 else :
251- subprocess_run (f"./pg_ctl -D \" { pgdata_dpath } \" -o '-p { pgport } , cwd = pgbin_real_dpath )
250+ subprocess_run (f"./pg_ctl -D \" { dbdata_dpath } \" -o '-p { pgport } , cwd = pgbin_real_dpath )
0 commit comments