Skip to content

Commit 2e87631

Browse files
kachuliskachulis
authored
Automatically deliver QC'd results file to delivery bucket (#108)
1 parent 49406be commit 2e87631

File tree

1 file changed

+128
-26
lines changed

1 file changed

+128
-26
lines changed

‎ImputationPipeline/notebooks/CreateManuallyQCdPRSResultsFile.ipynb

Lines changed: 128 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,16 @@
88
"\n",
99
"Running this notebook will start a GUI to make edits to the PRS results file for a selected lab batch. If no edits need to be made for a particular batch, you shoudl still run the notebook, select the batch, load initial results, and save QC'd results. This will save an appropriately named QC'd results file, with extra required columns, and update the sample_set table in the workspace to reflect the QC'd file path. \n",
1010
"\n",
11-
"QC'd files are saved in the bucket associated with to the workspace this notebook is run in, in a \"directory\" `manually_qcd_prs_results`, and named `{lab_batch}_{date_and_time_saved}_manually_qcd_prs_results.csv`"
11+
"QC'd files are saved in the bucket associated with to the workspace this notebook is run in, in a \"directory\" `manually_qcd_prs_results`, and named `{lab_batch}_{date_and_time_saved}_manually_qcd_prs_results.csv`. If the workspace variable `delivery-bucket` is set, this file will also be delivered to the specified bucket, after dialog prompt and confirmation. "
1212
]
1313
},
1414
{
1515
"cell_type": "code",
1616
"execution_count": null,
17-
"metadata": {},
17+
"metadata": {
18+
"hide_input": true,
19+
"scrolled": false
20+
},
1821
"outputs": [],
1922
"source": [
2023
"import ipywidgets as widgets\n",
@@ -26,6 +29,23 @@
2629
"from datetime import datetime, timezone\n",
2730
"import pytz\n",
2831
"import os\n",
32+
"from abc import ABC, abstractmethod\n",
33+
"\n",
34+
"def get_bucket_and_blob(uri):\n",
35+
" return uri.replace(\"gs://\",\"\").split(\"/\",1)\n",
36+
" \n",
37+
"\n",
38+
"class color:\n",
39+
" PURPLE = '\\033[95m'\n",
40+
" CYAN = '\\033[96m'\n",
41+
" DARKCYAN = '\\033[36m'\n",
42+
" BLUE = '\\033[94m'\n",
43+
" GREEN = '\\033[92m'\n",
44+
" YELLOW = '\\033[93m'\n",
45+
" RED = '\\033[91m'\n",
46+
" BOLD = '\\033[1m'\n",
47+
" UNDERLINE = '\\033[4m'\n",
48+
" END = '\\033[0m'\n",
2949
"\n",
3050
"class TableSelectionGUI:\n",
3151
" def __init__(self, workspace_namespace, workspace_name, workspace_bucket_name, available_tables):\n",
@@ -53,18 +73,37 @@
5373
" self.app = ResultsModificationGUI(self.workspace_namespace, self.workspace_name, self.workspace_bucket_name, change['new'])\n",
5474
" self.app.run()\n",
5575
"\n",
56-
"class ResultsModificationGUI:\n",
76+
"class WidgetGUI(ABC): \n",
77+
" def __init__(self):\n",
78+
" self.registered_widgets = []\n",
79+
" \n",
80+
" @abstractmethod\n",
81+
" def run(self):\n",
82+
" pass\n",
83+
" \n",
84+
" def close_widgets(self):\n",
85+
" for widget in self.registered_widgets:\n",
86+
" widget.close()\n",
87+
" \n",
88+
" def register_widget(self, widget):\n",
89+
" #test\n",
90+
" self.registered_widgets.append(widget)\n",
91+
" display(widget)\n",
92+
" \n",
93+
" \n",
94+
"class ResultsModificationGUI(WidgetGUI):\n",
5795
" eastern_tz = pytz.timezone('US/Eastern')\n",
5896
" \n",
5997
" def __init__(self, workspace_namespace, workspace_name, workspace_bucket_name, table_name):\n",
98+
" super().__init__()\n",
6099
" self.workspace_namespace = workspace_namespace\n",
61100
" self.workspace_name = workspace_name\n",
62101
" self.workspace_bucket_name = workspace_bucket_name\n",
102+
" self.delivery_bucket = None\n",
63103
" self.table_name = table_name\n",
64104
" self.lab_batch_output_box = widgets.Output(layout={'border': '1px solid black'})\n",
65105
" self.status_output_box = widgets.Output(layout={'border': '1px solid black'})\n",
66106
" self.selected_batch_gui = None\n",
67-
" self.registered_widgets = []\n",
68107
" self.register_widget(self.status_output_box)\n",
69108
" self.register_widget(self.lab_batch_output_box)\n",
70109
" self.initialize_workspace_info_and_gcloud()\n",
@@ -74,15 +113,11 @@
74113
" self.build_lab_batch_selection_section()\n",
75114
" \n",
76115
" def close_widgets(self):\n",
77-
" for widget in self.registered_widgets:\n",
78-
" widget.close()\n",
116+
" super().close_widgets()\n",
117+
" \n",
79118
" if self.selected_batch_gui != None:\n",
80119
" self.selected_batch_gui.close_widgets()\n",
81120
" \n",
82-
" def register_widget(self, widget):\n",
83-
" self.registered_widgets.append(widget)\n",
84-
" display(widget)\n",
85-
" \n",
86121
" def initialize_workspace_info_and_gcloud(self):\n",
87122
" with self.status_output_box:\n",
88123
" print(f\"Getting {self.table_name} information for workspace \" + self.workspace_namespace + \"/\" + self.workspace_name + \"...\")\n",
@@ -105,9 +140,27 @@
105140
"\n",
106141
" with self.status_output_box:\n",
107142
" print(\"Done\")\n",
143+
" \n",
144+
" self.lookup_delivery_bucket()\n",
108145
"\n",
109146
" self.storage_client = storage.Client()\n",
110147
" \n",
148+
" def lookup_delivery_bucket(self):\n",
149+
" with self.status_output_box:\n",
150+
" print(\"Looking up delivery-bucket\")\n",
151+
" workspace_response = fapi.get_workspace(self.workspace_namespace, self.workspace_name)\n",
152+
" if not workspace_response.ok:\n",
153+
" raise RuntimeError(f'ERROR: {workspace_response.text}')\n",
154+
" \n",
155+
" workspace_attributes = workspace_response.json()['workspace']['attributes']\n",
156+
" if 'delivery-bucket' in workspace_attributes:\n",
157+
" self.delivery_bucket = workspace_attributes['delivery-bucket']\n",
158+
" with self.status_output_box:\n",
159+
" print(f\"Delivery bucket set to {color.BOLD} {color.BLUE} {self.delivery_bucket} {color.END}\")\n",
160+
" else:\n",
161+
" with self.status_output_box:\n",
162+
" print(\"No delivery bucket found\")\n",
163+
" \n",
111164
" def build_lab_batch_selection_section(self):\n",
112165
" self.lab_batch_selection_dropdown = widgets.Combobox(options=list(self.lab_batch_map.keys()),\n",
113166
" description = 'Select Lab Batch',\n",
@@ -153,14 +206,11 @@
153206
" self.lab_batch_download_button.disabled = False\n",
154207
" \n",
155208
" def get_time_created(self, uri):\n",
156-
" bucket_name, path = self.get_bucket_and_path(uri)\n",
209+
" bucket_name, path = get_bucket_and_blob(uri)\n",
157210
" bucket = self.storage_client.get_bucket(bucket_name)\n",
158211
" blob = bucket.get_blob(path)\n",
159212
" return blob.time_created\n",
160213
" \n",
161-
" def get_bucket_and_path(self, uri):\n",
162-
" return uri.replace(\"gs://\",\"\").split(\"/\",1)\n",
163-
" \n",
164214
" def print_lab_batch_result_info(self, lab_batch_name):\n",
165215
" with self.lab_batch_output_box:\n",
166216
" clear_output()\n",
@@ -213,7 +263,9 @@
213263
" self.lab_batch_selection_dropdown.value = ''\n",
214264
" self.lab_batch = None\n",
215265
" self.selected_batch_gui = None\n",
216-
" \n",
266+
" if self.delivery_bucket:\n",
267+
" self.delivery_confirmation_box = DeliveryConfirmationBox(path, self.delivery_bucket, self.lab_batch_selection_dropdown, self.status_output_box)\n",
268+
" self.delivery_confirmation_box.run()\n",
217269
" def update_sample_sets_table_with_modified_results(self, path):\n",
218270
" with self.status_output_box:\n",
219271
" print(f\"Updating {self.table_name} data table for \" + self.lab_batch + \"...\")\n",
@@ -243,7 +295,66 @@
243295
" modified_batch_results.to_csv(path)\n",
244296
" return path\n",
245297
"\n",
246-
"class SelectedBatchModificationGui():\n",
298+
"\n",
299+
"class DeliveryConfirmationBox(WidgetGUI):\n",
300+
" \n",
301+
" def __init__(self, source_path, delivery_bucket, drop_down_button, higher_status_output_box):\n",
302+
" super().__init__()\n",
303+
" self.source_path = source_path\n",
304+
" self.delivery_bucket = delivery_bucket\n",
305+
" self.higher_status_output_box = higher_status_output_box\n",
306+
" self.drop_down_button = drop_down_button\n",
307+
" self.drop_down_button.disabled = True\n",
308+
" self.delivery_output_box = widgets.Output(layout={'border': '1px solid black'})\n",
309+
" self.register_widget(self.delivery_output_box)\n",
310+
" \n",
311+
" def run(self):\n",
312+
" self.build_confirmation_box()\n",
313+
" \n",
314+
" def build_confirmation_box(self):\n",
315+
" with self.delivery_output_box:\n",
316+
" print(f'{color.BOLD} Are you sure you want to deliver {self.source_path} to bucket {color.BLUE} {self.delivery_bucket}? {color.END}')\n",
317+
" self.deliver_button = widgets.Button(description = \"Yes, deliver\",\n",
318+
" layout = widgets.Layout(width='auto'),\n",
319+
" button_style = 'success'\n",
320+
" )\n",
321+
" \n",
322+
" self.dont_deliver_button = widgets.Button(description = \"No, do not deliver\",\n",
323+
" layout = widgets.Layout(width='auto'),\n",
324+
" button_style = 'danger'\n",
325+
" )\n",
326+
" \n",
327+
" self.button_hbox = widgets.HBox([self.deliver_button,\n",
328+
" self.dont_deliver_button\n",
329+
" ])\n",
330+
" self.deliver_button.on_click(self.deliver_button_clicked)\n",
331+
" self.dont_deliver_button.on_click(self.dont_deliver_button_clicked)\n",
332+
" \n",
333+
" self.register_widget(self.button_hbox)\n",
334+
" \n",
335+
" def deliver_button_clicked(self, button):\n",
336+
" self.deliver_modified_batch_results()\n",
337+
" self.close_widgets()\n",
338+
" self.drop_down_button.disabled = False\n",
339+
"\n",
340+
" def dont_deliver_button_clicked(self, button):\n",
341+
" self.close_widgets()\n",
342+
" self.drop_down_button.disabled = False \n",
343+
" \n",
344+
" def deliver_modified_batch_results(self):\n",
345+
" source_bucket_name, source_blob_name = get_bucket_and_blob(self.source_path)\n",
346+
" storage_client = storage.Client()\n",
347+
" source_bucket = storage_client.bucket(source_bucket_name)\n",
348+
" source_blob = source_bucket.blob(source_blob_name)\n",
349+
"\n",
350+
" destination_bucket = storage_client.bucket(self.delivery_bucket.replace(\"gs://\", \"\"))\n",
351+
" destination_blob_name = source_blob_name.split(\"/\")[-1]\n",
352+
" source_bucket.copy_blob(source_blob, destination_bucket, destination_blob_name)\n",
353+
" with self.higher_status_output_box:\n",
354+
" print(f'{color.BOLD} {color.BLUE} Delivering qcd results to bucket {destination_bucket.name} {color.END}') \n",
355+
" \n",
356+
" \n",
357+
"class SelectedBatchModificationGui(WidgetGUI):\n",
247358
" result_to_status_dict = {\"HIGH\" : \"PASS\",\n",
248359
" \"NOT_HIGH\" : \"PASS\",\n",
249360
" \"NOT_RESULTED\" : \"FAIL\",\n",
@@ -254,13 +365,13 @@
254365
" \"INFO\" : \"info\"}\n",
255366
" \n",
256367
" def __init__(self, results, lab_batch):\n",
368+
" super().__init__()\n",
257369
" self.manual_fails = {}\n",
258370
" self.drop_down_label_dict = {}\n",
259371
" self.drop_down_custom_dict = {}\n",
260372
" self.custom_failure_label_dict = {}\n",
261373
" self.failed_conditions = {}\n",
262374
" self.manual_failure_hbox_dict = {}\n",
263-
" self.registered_widgets = []\n",
264375
" self.results = results\n",
265376
" self.lab_batch = lab_batch\n",
266377
" self.modified_results = results.copy()\n",
@@ -280,14 +391,6 @@
280391
" def run(self):\n",
281392
" self.build_failed_imputation_section()\n",
282393
" \n",
283-
" def close_widgets(self):\n",
284-
" for widget in self.registered_widgets:\n",
285-
" widget.close()\n",
286-
" \n",
287-
" def register_widget(self, widget):\n",
288-
" self.registered_widgets.append(widget)\n",
289-
" display(widget)\n",
290-
" \n",
291394
" def build_failed_imputation_section(self): \n",
292395
" self.out_failed_imputation = widgets.Output(layout={'border': '1px solid black'})\n",
293396
" \n",
@@ -794,7 +897,6 @@
794897
}
795898
],
796899
"metadata": {
797-
"hide_input": false,
798900
"kernelspec": {
799901
"display_name": "Python 3",
800902
"language": "python",

0 commit comments

Comments
 (0)