bugfix: handle missing metrics in tdr (#1060)

* handle missing metrics in tdr

* ruff

Author: bpblanken

v03_pipeline/lib/misc/terra_data_repository.py

@@ -10,10 +10,10 @@
 from v03_pipeline.lib.misc.requests import requests_retry_session
 
 BIGQUERY_METRICS = [
-    'collaborator_sample_id',
     'predicted_sex',
     'contamination_rate',
     'percent_bases_at_20x',
+    'collaborator_sample_id',
     'mean_coverage',
 ]
 BIGQUERY_RESOURCE = 'bigquery'
@@ -63,9 +63,18 @@ def bq_metrics_query(bq_table_name: str) -> google.cloud.bigquery.table.RowItera
         msg = f'{bq_table_name} does not match expected pattern'
         raise ValueError(msg)
     client = bigquery.Client()
+
+    # not all columns are guaranteed to be present, coalesce if missing
+    table_ddl = next(
+        client.query_and_wait(
+            f"""
+        SELECT ddl FROM `{bq_table_name}`.INFORMATION_SCHEMA.TABLES where table_name='sample';
+        """,  # noqa: S608
+        ),
+    )[0]
+    metrics = [(m if m in table_ddl else f'NULL AS {m}') for m in BIGQUERY_METRICS]
     return client.query_and_wait(
         f"""
-        SELECT {','.join(BIGQUERY_METRICS)}
-        FROM `{bq_table_name}.sample`
-    """,  # noqa: S608
+        SELECT {','.join(metrics)} FROM `{bq_table_name}.sample`;
+        """,  # noqa: S608
     )

v03_pipeline/lib/misc/terra_data_repository_test.py

@@ -2,13 +2,14 @@
 import os
 import unittest
 from types import SimpleNamespace
-from unittest.mock import Mock, patch
+from unittest.mock import Mock, call, patch
 
 import responses
 
 from v03_pipeline.lib.misc.terra_data_repository import (
     TDR_ROOT_URL,
     _get_dataset_ids,
+    bq_metrics_query,
     gen_bq_table_names,
 )
 
@@ -301,3 +302,20 @@ def test_gen_bq_table_names(self, _: Mock) -> None:
                 'datarepo-aada2e3b.datarepo_RP_3059',
             ],
         )
+
+    @patch('v03_pipeline.lib.misc.terra_data_repository.bigquery.Client')
+    def test_bq_metrics_query_missing_metrics(
+        self,
+        mock_bq_client: Mock,
+        _: Mock,
+    ) -> None:
+        mock_bq_client.return_value.query_and_wait.return_value = iter(
+            [['predicted_sex,contamination_rate,percent_bases_at_20x']],
+        )
+        bq_metrics_query('datarepo-7242affb.datarepo_RP_3053')
+        self.assertEqual(
+            mock_bq_client.return_value.query_and_wait.mock_calls[1],
+            call(
+                '\n        SELECT predicted_sex,contamination_rate,percent_bases_at_20x,NULL AS collaborator_sample_id,NULL AS mean_coverage FROM `datarepo-7242affb.datarepo_RP_3053.sample`;\n        ',
+            ),
+        )