bugfix: enforce gcloud vars

Author: bpblanken

v03_pipeline/lib/tasks/dataproc/create_dataproc_cluster.py

@@ -167,6 +167,9 @@ def complete(self) -> bool:
         return cluster.status.state == RUNNING_STATE
 
     def run(self):
+        if not Env.GCLOUD_PROJECT or not Env.GCLOUD_REGION or not Env.GCLOUD_ZONE:
+            msg = 'Environment Variables GCLOUD_PROJECT, GCLOUD_REGION, GCLOUD_ZONE are required for running the pipeline on dataproc.'
+            raise RuntimeError(msg)
         operation = self.client.create_cluster(
             request={
                 'project_id': Env.GCLOUD_PROJECT,

v03_pipeline/lib/tasks/dataproc/create_dataproc_cluster_test.py

@@ -12,6 +12,12 @@
 )
 
 
+@patch.multiple(
+    'v03_pipeline.lib.tasks.dataproc.create_dataproc_cluster.Env',
+    GCLOUD_PROJECT='proj',
+    GCLOUD_REGION='us-a',
+    GCLOUD_ZONE='us-a-1',
+)
 @patch(
     'v03_pipeline.lib.tasks.dataproc.create_dataproc_cluster.get_service_account_credentials',
     return_value=SimpleNamespace(
@@ -26,7 +32,7 @@ class CreateDataprocClusterTaskTest(unittest.TestCase):
     def test_dataset_type_unsupported(
         self,
         mock_cluster_controller: Mock,
-        _: Mock,
+        *_: Mock,
     ) -> None:
         task = CreateDataprocClusterTask(
             reference_genome=ReferenceGenome.GRCh38,
@@ -38,7 +44,7 @@ def test_dataset_type_unsupported(
     def test_spinup_cluster_already_exists_failed(
         self,
         mock_cluster_controller: Mock,
-        _: Mock,
+        *_: Mock,
     ) -> None:
         mock_client = mock_cluster_controller.return_value
         mock_client.get_cluster.return_value = SimpleNamespace(
@@ -61,7 +67,7 @@ def test_spinup_cluster_already_exists_failed(
     def test_spinup_cluster_already_exists_success(
         self,
         mock_cluster_controller: Mock,
-        _: Mock,
+        *_: Mock,
     ) -> None:
         mock_client = mock_cluster_controller.return_value
         mock_client.get_cluster.return_value = SimpleNamespace(
@@ -85,7 +91,7 @@ def test_spinup_cluster_doesnt_exist_failed(
         self,
         mock_logger: Mock,
         mock_cluster_controller: Mock,
-        _: Mock,
+        *_: Mock,
     ) -> None:
         mock_client = mock_cluster_controller.return_value
         mock_client.get_cluster.side_effect = google.api_core.exceptions.NotFound(
@@ -111,7 +117,7 @@ def test_spinup_cluster_doesnt_exist_success(
         self,
         mock_logger: Mock,
         mock_cluster_controller: Mock,
-        _: Mock,
+        *_: Mock,
     ) -> None:
         mock_client = mock_cluster_controller.return_value
         mock_client.get_cluster.side_effect = google.api_core.exceptions.NotFound(