only hard fail on samples in pedigree

Author: jklugherz

v03_pipeline/lib/misc/validation.py

@@ -2,6 +2,7 @@
 
 import hail as hl
 
+from v03_pipeline.lib.misc.pedigree import Family
 from v03_pipeline.lib.model import (
     DatasetType,
     ReferenceGenome,
@@ -132,6 +133,7 @@ def validate_imputed_sex_ploidy(
     mt: hl.MatrixTable,
     # NB: sex_check_ht will be undefined if sex checking is disabled for the run
     sex_check_ht: hl.Table | None = None,
+    pedigree_families: set[Family] | None = None,
     **_: Any,
 ) -> None:
     if not sex_check_ht:
@@ -161,8 +163,15 @@ def validate_imputed_sex_ploidy(
     discrepant_samples = mt.aggregate_cols(
         hl.agg.filter(mt.discrepant, hl.agg.collect_as_set(mt.s)),
     )
-    if discrepant_samples:
-        sorted_discrepant_samples = sorted(discrepant_samples)
+    loading_samples = (
+        {sample_id for family in pedigree_families for sample_id in family.samples}
+        if pedigree_families
+        else set()
+    )
+    discrepant_loading_samples = discrepant_samples & loading_samples
+
+    if discrepant_loading_samples:
+        sorted_discrepant_samples = sorted(discrepant_loading_samples)
         msg = f'Found samples with misaligned ploidy with their provided imputed sex (first 10, if applicable) : {sorted_discrepant_samples[:10]}'
         raise SeqrValidationError(
             msg,

v03_pipeline/lib/misc/validation_test.py

@@ -2,6 +2,7 @@
 
 import hail as hl
 
+from v03_pipeline.lib.misc.pedigree import Family, Sample
 from v03_pipeline.lib.misc.validation import (
     SeqrValidationError,
     validate_allele_type,
@@ -11,7 +12,7 @@
     validate_no_duplicate_variants,
     validate_sample_type,
 )
-from v03_pipeline.lib.model import DatasetType, ReferenceGenome, SampleType
+from v03_pipeline.lib.model import DatasetType, ReferenceGenome, SampleType, Sex
 
 TEST_SEX_CHECK_1 = 'v03_pipeline/var/test/sex_check/test_sex_check_1.ht'
 TEST_MITO_MT = 'v03_pipeline/var/test/callsets/mito_1.mt'
@@ -171,7 +172,20 @@ def test_validate_imputed_sex_ploidy(self) -> None:
             .key_rows_by('locus')
             .key_cols_by('s')
         )
-        validate_imputed_sex_ploidy(mt, sex_check_ht)
+        pedigree_families = {
+            Family(
+                family_guid='',
+                samples={
+                    female_sample: Sample(female_sample, Sex.FEMALE),
+                    male_sample_1: Sample(male_sample_1, Sex.MALE),
+                    x0_sample: Sample(x0_sample, Sex.X0),
+                    xxy_sample: Sample(xxy_sample, Sex.XXY),
+                    xyy_sample: Sample(xyy_sample, Sex.XYY),
+                    xxx_sample: Sample(xxx_sample, Sex.XXX),
+                },
+            ),
+        }
+        validate_imputed_sex_ploidy(mt, sex_check_ht, pedigree_families)
 
         # All calls on Y chromosome are valid
         mt = (
@@ -211,7 +225,7 @@ def test_validate_imputed_sex_ploidy(self) -> None:
             .key_rows_by('locus')
             .key_cols_by('s')
         )
-        validate_imputed_sex_ploidy(mt, sex_check_ht)
+        validate_imputed_sex_ploidy(mt, sex_check_ht, pedigree_families)
 
         # Invalid X chromosome case
         mt = (
@@ -259,8 +273,12 @@ def test_validate_imputed_sex_ploidy(self) -> None:
             validate_imputed_sex_ploidy,
             mt,
             sex_check_ht,
+            pedigree_families,
         )
 
+        # Invalid X chromosome case, but invalid samples are missing from pedigree
+        validate_imputed_sex_ploidy(mt, sex_check_ht, pedigree_families=set())
+
     def test_validate_imported_field_types(self) -> None:
         mt = hl.read_matrix_table(TEST_MITO_MT)
         validate_imported_field_types(mt, DatasetType.MITO, {})

v03_pipeline/lib/tasks/validate_callset.py

@@ -2,6 +2,8 @@
 import luigi
 import luigi.util
 
+from v03_pipeline.lib.misc.io import import_pedigree
+from v03_pipeline.lib.misc.pedigree import parse_pedigree_ht_to_families
 from v03_pipeline.lib.misc.validation import (
     SeqrValidationError,
     validate_allele_type,
@@ -19,7 +21,7 @@
 from v03_pipeline.lib.reference_datasets.reference_dataset import ReferenceDataset
 from v03_pipeline.lib.tasks.base.base_loading_run_params import BaseLoadingRunParams
 from v03_pipeline.lib.tasks.base.base_update import BaseUpdateTask
-from v03_pipeline.lib.tasks.files import CallsetTask, GCSorLocalTarget
+from v03_pipeline.lib.tasks.files import CallsetTask, GCSorLocalTarget, RawFileTask
 from v03_pipeline.lib.tasks.reference_data.updated_reference_dataset import (
     UpdatedReferenceDatasetTask,
 )
@@ -52,6 +54,10 @@ def get_validation_dependencies(self) -> dict[str, hl.Table]:
                     self.callset_path,
                 ),
             )
+            deps['pedigree_families'] = parse_pedigree_ht_to_families(
+                import_pedigree(self.input()[1].path),
+            )
+
         return deps
 
     def complete(self) -> luigi.Target:
@@ -74,6 +80,7 @@ def output(self) -> luigi.Target:
     def requires(self) -> list[luigi.Task]:
         requirements = [
             self.clone(WriteImportedCallsetTask),
+            RawFileTask(self.project_pedigree_paths[self.project_i]),
         ]
         if not self.skip_validation and self.dataset_type.can_run_validation:
             requirements = [