From 0973dc95184be97386ebc34997b06f7ed6eb95d3 Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Sun, 19 May 2024 13:14:04 -0400 Subject: [PATCH 01/14] Finish validity check test --- .../lib/reference_data/compare_globals.py | 38 +++++-- .../reference_data/compare_globals_test.py | 100 +++++++++++++++--- 2 files changed, 116 insertions(+), 22 deletions(-) diff --git a/v03_pipeline/lib/reference_data/compare_globals.py b/v03_pipeline/lib/reference_data/compare_globals.py index 0b5c2993d..e881759de 100644 --- a/v03_pipeline/lib/reference_data/compare_globals.py +++ b/v03_pipeline/lib/reference_data/compare_globals.py @@ -21,7 +21,7 @@ class Globals: paths: dict[str, str] versions: dict[str, str] enums: dict[str, dict[str, list[str]]] - selects: dict[str, set[str]] + selects: dict[str, dict[str, hl.dtype]] def __getitem__(self, name: str): return getattr(self, name) @@ -50,7 +50,11 @@ def from_dataset_configs( dataset_ht = dataset_ht.transmute( **get_enum_select_fields(dataset_ht, dataset_config), ) - selects[dataset] = set(dataset_ht.row) - set(dataset_ht.key) + selects[dataset] = { + k: v.dtype + for k, v in dict(dataset_ht.row).items() + if k not in set(dataset_ht.key) + } return cls(paths, versions, enums, selects) @classmethod @@ -69,9 +73,9 @@ def from_ht( if dataset in ht.row: # NB: handle an edge case (mito high constraint) where we annotate a bool from the reference dataset collection selects[dataset] = ( - set(ht[dataset]) + {k: v.dtype for k, v in dict(ht[dataset]).items()} if isinstance(ht[dataset], hl.StructExpression) - else set() + else {} ) return cls(paths, versions, enums, selects) @@ -82,11 +86,9 @@ def get_datasets_to_update( validate_selects: bool = True, ) -> list[str]: datasets_to_update = set() - for field in dataclasses.fields(Globals): - if field.name == 'selects' and not validate_selects: + if field.name == 'selects': continue - datasets_to_update.update( ht1_globals[field.name].keys() ^ ht2_globals[field.name].keys(), ) @@ -97,4 +99,26 @@ def get_datasets_to_update( logger.info(f'{field.name} mismatch for {dataset}') datasets_to_update.add(dataset) + # Selects are a special case and are handled separately + if validate_selects: + ht1_selects, ht2_selects = ht1_globals['selects'], ht2_globals['selects'] + datasets_to_update.update(ht1_selects.keys() ^ ht2_globals[field.name].keys()) + for dataset in ht1_selects.keys() & ht2_selects.keys(): + # Special integrity check to ensure that fields do not change + # without an explicit configuration change. Because we check + # that the dataset isn't already "to_update", this must run + # after all other fields have been validated. + if ( + dataset not in datasets_to_update and + (ht1_selects.get(dataset).keys() == ht2_selects.get(dataset).keys()) and + (ht1_selects.get(dataset) != ht2_selects.get(dataset)) + ): + differing_items = ht2_selects.get(dataset).items() - ht1_selects.get(dataset).items() + msg = f'Unexpected field types detected in {dataset}: {differing_items}' + raise ValueError(msg) + if ht1_globals[field.name].get(dataset) != ht2_globals[field.name].get( + dataset, + ): + logger.info(f'{field.name} mismatch for {dataset}') + datasets_to_update.add(dataset) return sorted(datasets_to_update) diff --git a/v03_pipeline/lib/reference_data/compare_globals_test.py b/v03_pipeline/lib/reference_data/compare_globals_test.py index 0d290489b..4c498609e 100644 --- a/v03_pipeline/lib/reference_data/compare_globals_test.py +++ b/v03_pipeline/lib/reference_data/compare_globals_test.py @@ -103,8 +103,15 @@ def test_create_globals_from_dataset_configs( self.assertTrue( dataset_config_globals.selects == { - 'a': {'test_select', 'test_enum_id'}, - 'b': {'test_select', 'field2', 'test_enum_id'}, + 'a': { + 'test_select': hl.tint32, + 'test_enum_id': hl.tint32, + }, + 'b': { + 'test_select': hl.tint32, + 'field2': hl.tint32, + 'test_enum_id': hl.tint32, + }, }, ) @@ -134,7 +141,11 @@ def test_create_globals_from_dataset_configs_single_dataset(self, mock_read_tabl self.assertTrue( dataset_config_globals.selects == { - 'b': {'test_select', 'field2', 'test_enum_id'}, + 'b': { + 'test_select': hl.tint32, + 'field2': hl.tint32, + 'test_enum_id': hl.tint32, + }, }, ) @@ -183,11 +194,12 @@ def test_from_rdc_or_annotations_ht(self): self.assertTrue( rdc_globals.enums == {'screen': {'region_type': ['C', 'D']}}, ) + print(rdc_globals.selects) self.assertTrue( rdc_globals.selects == { - 'gnomad_non_coding_constraint': {'z_score'}, - 'screen': {'region_type_ids'}, + 'gnomad_non_coding_constraint': {'z_score': hl.tfloat32}, + 'screen': {'region_type_ids': hl.tarray(hl.tint32)}, }, ) @@ -198,13 +210,13 @@ def test_get_datasets_to_update_version_different(self): # 'a' has a different version, 'c' is missing version in ht2_globals versions={'a': 'v2', 'b': 'v2', 'c': 'v1'}, enums={'a': {}, 'b': {}, 'c': {}}, - selects={'a': set(), 'b': set()}, + selects={'a': {}, 'b': {}}, ), ht2_globals=Globals( paths={'a': 'a_path', 'b': 'b_path'}, versions={'a': 'v1', 'b': 'v2'}, enums={'a': {}, 'b': {}}, - selects={'a': set(), 'b': set()}, + selects={'a': {}, 'b': {}}, ), ) self.assertTrue(result == ['a', 'c']) @@ -216,13 +228,13 @@ def test_get_datasets_to_update_path_different(self): paths={'a': 'a_path', 'b': 'old_b_path', 'c': 'extra_c_path'}, versions={'a': 'v1', 'b': 'v2'}, enums={'a': {}, 'b': {}}, - selects={'a': set(), 'b': set()}, + selects={'a': {}, 'b': {}}, ), ht2_globals=Globals( paths={'a': 'a_path', 'b': 'b_path'}, versions={'a': 'v1', 'b': 'v2'}, enums={'a': {}, 'b': {}}, - selects={'a': set(), 'b': set()}, + selects={'a': {}, 'b': {}}, ), ) self.assertTrue(result == ['b', 'c']) @@ -238,13 +250,13 @@ def test_get_datasets_to_update_enum_different(self): 'b': {'enum_key_1': []}, 'c': {}, }, - selects={'a': set(), 'b': set()}, + selects={'a': {}, 'b': {}}, ), ht2_globals=Globals( paths={'a': 'a_path', 'b': 'b_path'}, versions={'a': 'v1', 'b': 'v2'}, enums={'a': {'test_enum': ['C', 'D']}, 'b': {'enum_key_2': []}}, - selects={'a': set(), 'b': set()}, + selects={'a': {}, 'b': {}}, ), ) self.assertTrue(result == ['a', 'b', 'c']) @@ -257,16 +269,74 @@ def test_get_datasets_to_update_select_different(self): enums={'a': {}, 'b': {}}, # 'a' has extra select, 'b' has different select, 'c' is missing select in ht2_globals selects={ - 'a': {'field1', 'field2'}, - 'b': {'test_select'}, - 'c': set('test_select'), + 'a': {'field1': hl.tint32, 'field2': hl.tint32}, + 'b': {'test_select': hl.tint32}, + 'c': {'test_select': hl.tint32}, }, ), ht2_globals=Globals( paths={'a': 'a_path', 'b': 'b_path'}, versions={'a': 'v1', 'b': 'v2'}, enums={'a': {}, 'b': {}}, - selects={'a': {'field1'}, 'b': {'test_select_2'}}, + selects={'a': {'field1': hl.tint32}, 'b': {'test_select_2': hl.tint32}}, ), ) self.assertTrue(result == ['a', 'b', 'c']) + + def test_get_datasets_to_update_select_type_validation(self): + self.assertRaisesRegex( + ValueError, + "Unexpected field types detected in a: {\\('field1', dtype\\('int32'\\)\\)}", + get_datasets_to_update, + ht1_globals=Globals( + paths={'a': 'a_path'}, + versions={'a': 'v1'}, + enums={'a': {}}, + selects={ + # field1 is an array in ht1 but an int in ht2. + 'a': {'field1': hl.tarray(hl.tint32)}, + }, + ), + ht2_globals=Globals( + paths={'a': 'a_path'}, + versions={'a': 'v1'}, + enums={'a': {}}, + selects={'a': {'field1': hl.tint32}}, + ), + ) + result = get_datasets_to_update( + ht1_globals=Globals( + paths={'a': 'a_path'}, + versions={'a': 'v1'}, + enums={'a': {}}, + selects={ + 'a': {'field1': hl.tarray(hl.tint32)}, + }, + ), + ht2_globals=Globals( + paths={'a': 'a_path'}, + versions={'a': 'v1'}, + enums={'a': {}}, + # additional field + selects={'a': {'field1': hl.tint32, 'field2': hl.tint32}}, + ), + ) + self.assertTrue(result == ['a']) + result = get_datasets_to_update( + ht1_globals=Globals( + paths={'a': 'a_path'}, + versions={'a': 'v1'}, + enums={'a': {}}, + selects={ + 'a': {'field1': hl.tarray(hl.tint32)}, + }, + ), + ht2_globals=Globals( + paths={'a': 'a_path'}, + versions={'a': 'v2'}, # version bump + enums={'a': {}}, + selects={'a': {'field1': hl.tint32}}, + ), + ) + self.assertTrue(result == ['a']) + From 27ac8d49d4974e42b4d4082b07286d2a8044c4e8 Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Mon, 20 May 2024 12:11:53 -0400 Subject: [PATCH 02/14] ruff --- v03_pipeline/lib/reference_data/compare_globals.py | 10 ++++++---- .../lib/reference_data/compare_globals_test.py | 3 +-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/v03_pipeline/lib/reference_data/compare_globals.py b/v03_pipeline/lib/reference_data/compare_globals.py index e881759de..ca4578867 100644 --- a/v03_pipeline/lib/reference_data/compare_globals.py +++ b/v03_pipeline/lib/reference_data/compare_globals.py @@ -109,11 +109,13 @@ def get_datasets_to_update( # that the dataset isn't already "to_update", this must run # after all other fields have been validated. if ( - dataset not in datasets_to_update and - (ht1_selects.get(dataset).keys() == ht2_selects.get(dataset).keys()) and - (ht1_selects.get(dataset) != ht2_selects.get(dataset)) + dataset not in datasets_to_update + and (ht1_selects.get(dataset).keys() == ht2_selects.get(dataset).keys()) + and (ht1_selects.get(dataset) != ht2_selects.get(dataset)) ): - differing_items = ht2_selects.get(dataset).items() - ht1_selects.get(dataset).items() + differing_items = ( + ht2_selects.get(dataset).items() - ht1_selects.get(dataset).items() + ) msg = f'Unexpected field types detected in {dataset}: {differing_items}' raise ValueError(msg) if ht1_globals[field.name].get(dataset) != ht2_globals[field.name].get( diff --git a/v03_pipeline/lib/reference_data/compare_globals_test.py b/v03_pipeline/lib/reference_data/compare_globals_test.py index 4c498609e..39fcdaf8e 100644 --- a/v03_pipeline/lib/reference_data/compare_globals_test.py +++ b/v03_pipeline/lib/reference_data/compare_globals_test.py @@ -333,10 +333,9 @@ def test_get_datasets_to_update_select_type_validation(self): ), ht2_globals=Globals( paths={'a': 'a_path'}, - versions={'a': 'v2'}, # version bump + versions={'a': 'v2'}, # version bump enums={'a': {}}, selects={'a': {'field1': hl.tint32}}, ), ) self.assertTrue(result == ['a']) - From f227a05f68036e12f5882b69fe3d9b86bbe67ca0 Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Mon, 20 May 2024 12:57:46 -0400 Subject: [PATCH 03/14] update dbnsfp field --- .../test_combined_1.ht/.README.txt.crc | Bin 12 -> 12 bytes .../test_combined_1.ht/.metadata.json.gz.crc | Bin 16 -> 16 bytes .../test_combined_1.ht/README.txt | 4 ++-- .../.index.crc | Bin .../.metadata.json.gz.crc | Bin .../index | Bin .../metadata.json.gz | Bin .../test_combined_1.ht/metadata.json.gz | Bin 725 -> 724 bytes .../rows/.metadata.json.gz.crc | Bin 20 -> 20 bytes .../test_combined_1.ht/rows/metadata.json.gz | Bin 1064 -> 1062 bytes ...0-963ef6be-cd5f-443a-970c-8b5cf2bcd090.crc | Bin 12 -> 0 bytes ...0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.crc | Bin 0 -> 12 bytes ...art-0-963ef6be-cd5f-443a-970c-8b5cf2bcd090 | Bin 107 -> 0 bytes ...art-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d | Bin 0 -> 103 bytes 14 files changed, 2 insertions(+), 2 deletions(-) rename v03_pipeline/var/test/reference_data/test_combined_1.ht/index/{part-0-963ef6be-cd5f-443a-970c-8b5cf2bcd090.idx => part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.idx}/.index.crc (100%) rename v03_pipeline/var/test/reference_data/test_combined_1.ht/index/{part-0-963ef6be-cd5f-443a-970c-8b5cf2bcd090.idx => part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.idx}/.metadata.json.gz.crc (100%) rename v03_pipeline/var/test/reference_data/test_combined_1.ht/index/{part-0-963ef6be-cd5f-443a-970c-8b5cf2bcd090.idx => part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.idx}/index (100%) rename v03_pipeline/var/test/reference_data/test_combined_1.ht/index/{part-0-963ef6be-cd5f-443a-970c-8b5cf2bcd090.idx => part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.idx}/metadata.json.gz (100%) delete mode 100644 v03_pipeline/var/test/reference_data/test_combined_1.ht/rows/parts/.part-0-963ef6be-cd5f-443a-970c-8b5cf2bcd090.crc create mode 100644 v03_pipeline/var/test/reference_data/test_combined_1.ht/rows/parts/.part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.crc delete mode 100644 v03_pipeline/var/test/reference_data/test_combined_1.ht/rows/parts/part-0-963ef6be-cd5f-443a-970c-8b5cf2bcd090 create mode 100644 v03_pipeline/var/test/reference_data/test_combined_1.ht/rows/parts/part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d diff --git a/v03_pipeline/var/test/reference_data/test_combined_1.ht/.README.txt.crc b/v03_pipeline/var/test/reference_data/test_combined_1.ht/.README.txt.crc index a98fcdeadc0fcaf79fa2103c4c51f29df3150945..62a72838065f36b5d5539bfb643254f95da0b339 100644 GIT binary patch literal 12 TcmYc;N@ieSU}A8;sbl~E5t{<7 literal 12 TcmYc;N@ieSU}D(#Q0D~z6ek1W diff --git a/v03_pipeline/var/test/reference_data/test_combined_1.ht/.metadata.json.gz.crc b/v03_pipeline/var/test/reference_data/test_combined_1.ht/.metadata.json.gz.crc index b2ace029a7d4822ca27d6492cfef89141512034c..11f6d03d5ee0f27eddfe86535d46c9d5282ac93b 100644 GIT binary patch literal 16 XcmYc;N@ieSU}E_9{a$TCspWhCEV2eT literal 16 XcmYc;N@ieSU}6x_xbB~3c>F&AA8`cw diff --git a/v03_pipeline/var/test/reference_data/test_combined_1.ht/README.txt b/v03_pipeline/var/test/reference_data/test_combined_1.ht/README.txt index 856600d4e..b427c10e6 100644 --- a/v03_pipeline/var/test/reference_data/test_combined_1.ht/README.txt +++ b/v03_pipeline/var/test/reference_data/test_combined_1.ht/README.txt @@ -1,3 +1,3 @@ This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.128-eead8100a1c1 - Created at 2024/05/09 20:02:21 \ No newline at end of file + Written with version 0.2.130-bea04d9c79b5 + Created at 2024/05/20 12:56:42 \ No newline at end of file diff --git a/v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-963ef6be-cd5f-443a-970c-8b5cf2bcd090.idx/.index.crc b/v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.idx/.index.crc similarity index 100% rename from v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-963ef6be-cd5f-443a-970c-8b5cf2bcd090.idx/.index.crc rename to v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.idx/.index.crc diff --git a/v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-963ef6be-cd5f-443a-970c-8b5cf2bcd090.idx/.metadata.json.gz.crc b/v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.idx/.metadata.json.gz.crc similarity index 100% rename from v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-963ef6be-cd5f-443a-970c-8b5cf2bcd090.idx/.metadata.json.gz.crc rename to v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.idx/.metadata.json.gz.crc diff --git a/v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-963ef6be-cd5f-443a-970c-8b5cf2bcd090.idx/index b/v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.idx/index similarity index 100% rename from v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-963ef6be-cd5f-443a-970c-8b5cf2bcd090.idx/index rename to v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.idx/index diff --git a/v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-963ef6be-cd5f-443a-970c-8b5cf2bcd090.idx/metadata.json.gz b/v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.idx/metadata.json.gz similarity index 100% rename from v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-963ef6be-cd5f-443a-970c-8b5cf2bcd090.idx/metadata.json.gz rename to v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.idx/metadata.json.gz diff --git a/v03_pipeline/var/test/reference_data/test_combined_1.ht/metadata.json.gz b/v03_pipeline/var/test/reference_data/test_combined_1.ht/metadata.json.gz index 2ad9c32f8c2ad53e857f2b08f4041690baa24bd4..e3a900771bb206aeead61dd8347234d908b3d317 100644 GIT binary patch delta 713 zcmV;)0yh2C1=IzQ7k|}4x3+o9D700fjR2uZ2w5gMz?)-dwrPtH|NHDDPMoBb_Oge) zAm4ZQ-LKr)9K8*pfL=gI!kHI*yXyC@eQ$>etwHbntaoVIfR(h4z75_2YO>&KL^ z7c8upteM-L6h~ zoFzo%TheNJW6*|1K-tdpdPF(SJK)Yh^NX)d2syN+|ctB-Ho5WhvSMj(n`lo9si@P`L{8hjhsY-y4Bs~lra}j z8+0}`0e`A6YFPQ*?X8BVy|`6L@6}tES$)a&l1_(<;kXrk_Ar{apor5$wgc8f_lt1a zQogh3^{_97v-xG05E-K;y3yN`rfBjw)|DrExSM)?Tj)11=4r+mFeSa<=oQajeq=Z` z&*F4(JFq6R4C4G{t)5eBf6^#_{d-<+tVtQxdqSIUZoc@ggUG4|xu)&R+Al9w#*g5c vxdNc!p#n=z(<&iS-%_!nhetR-;syExg1JI1KmSLNQ%`>YiyACQiUXMY2dR)}$um@{E}KcZ!%mp+K zy-iJkDu0YxUVeXj>*47xZe7xQ^)_YJU$VQTi^+O2bJO2k`^$&vs>3Ma^pqWd4blBN zTy%`@EP6BS>&bF;-3LU*sjc1VZAnu!f1K&c6FuB*yS^>-8yNF6;|!RRUU2rx=dU<2 zoSJ8GvA!K!mt_WV@v_#>srApbmcRWyu5i|%3`6U^EjG7Ud^bR3P2kC`U8TwLM^`dM^HfXKbzsZ#fk_30PQ+sKL7v# diff --git a/v03_pipeline/var/test/reference_data/test_combined_1.ht/rows/.metadata.json.gz.crc b/v03_pipeline/var/test/reference_data/test_combined_1.ht/rows/.metadata.json.gz.crc index e8f7e30d106255c979a53f01228ac53ff9f2b2d7..17dc1bdefc98dfd1700af97cb1aa711d86ee2a28 100644 GIT binary patch literal 20 bcmYc;N@ieSU}9*Uzv)D6R`cn@26touMXd+= literal 20 ccmYc;N@ieSU}AWB_x6Efo+}lfwO3vT08XI^E&u=k diff --git a/v03_pipeline/var/test/reference_data/test_combined_1.ht/rows/metadata.json.gz b/v03_pipeline/var/test/reference_data/test_combined_1.ht/rows/metadata.json.gz index 2ad21d8dcd715d9518bc9718f1db801ff2c5d73a..974e67dc32458e9bf43439084b52ffda5279f380 100644 GIT binary patch literal 1062 zcmV+>1lju^iwFP!000000Ns~cZ`w)_fd7l1&QVQIxD-;}fI@=~Nf1O))d*ROcMRLw zYoGN}g+~7GXV%8{uI(gE)r(XfkUcv)H_v=tKUkauU=6J~r*IE+b^C4|f!ncc%lQmP zYj|h*p?iq&p7=ia5L+8uazhHPQnZE-xrLi!1awj?pcW_*uDqI#NepW#4z7Gi%?%Ba z8*<5ZO2#-5M>`5*CRxZ)GUu|}N$itv6pVx*$Gcp0W4nY>)7tW4HGNlZb$!Oa5MkUq98)(}#g>WTzXNc*E;=x=Wf%pjEA%Q%$|RfWp;e*mfW)R-9($2FaI5L< zba4vkRbgs9pWRfWT!#KJ+5_)67q_ETb@lQfiE8&765~KP5jL_);xuyh?R0(9YuK|? zdKIyUvls-<<-ZGsGo1X2LXs$keK|4Apw~kE|Mh+~*>}(XF5r~SA1b1)6POGnRe)4Q z(X%8f>^CAvCs3U~kKm_?KAD?3dn)Ob;4v0*>NEhy&uG(7+A86i zeMN=-n>XElecQ)a(sb(!_hEH`1l661`e71G9sVjg%yU_K78Kws9HIR6>wi*pCG~-L z>51=~j)D67z+)no_)zXMboYBUo;Hp1U8##J!=HGBOzhC}Vvt27g!3?KAk#;ZY?@vP zWhz=C2CG#9FMWR$QPpCqMDvFVufOSwtDA1+IqVs{ni<6L#Nwo4+b5Yex`p~5x#>lo(ZYi z|6s*<7?dip?4uqpvK~gS?q0MpoVAy|N4}T0`gc=hIAGrmb{t!EiyZ(W~ gItvUlhe9@0E|eruxezIZ@{}b10}PQzf3y$)0O0xcrvLx| literal 1064 zcmV+@1lRi?iwFP!000000Ns~gZ`w!@z`u*1daCIqkkEuTpwOU_1VI#?u0qzbJBDrT zwQs#tp^?A)&Dz-BwVmctU0bPni0s+f`L{Da){l-LAvgnPCKx;bTi(B0#X!4`>v*5x z;0!)GK}7cnzLOw;01{`7OEh9YwW2e4EG*m|VqoKP0kuGjaOL%6L=spjxu@zeGdHkA zG!lw$wTufQ4{jVKT=7VtWF}Ovo4QBeC>Tmf4j&8EwMz+Owzb98a`LX)N&_yw5NYe= z_GWC5Yq=PKA8@L;*e;0LMOzS@VhrfA@>p$;4V{C>4KsV8O4n<58wG5oh)fEFr#6x# z0jX0)S?MxgTf51A6LO_Mn!yz5M}lRwYObX&aC9{=1D|M#7w;sX>uC^?tVHlYsHHZX zdW+k|_3*D!$sJCO=dm|rpNiP=?Fy9n7KPSY$jjFoV`J)WMHCX|!9x`4LR$eC-fk*u z#gplxh}FAYdhh4Aqv5>TJH=JgmJv+p9xL)|{D-k!_3OphIFwlOkSOp7x0DhZNifus z82h%N#UN0`vvmtqY?%c9*@M96vIA3BMRBOR!mPr$O0umUMir_KNL{uOi67eox18Kh z=Es0u=cf8+?rJu@t4Fzrfh7#% z&qC|f#6Heq7<$(q<{D=>`89U4;yFNAeAw)LA7t$+G@p!4yBDKs$#v+C2h30zAxS=?O-pH` zg)9CQ75eAry?%4s--%+`))(%>>H-PtJ5&9KNpOAntK~2&WZ7L%LZESk@;7h(%+$3s z0ODsizW;O$G=C3#E)#`^@{pq&4;gQp*8Q&1#g)Miyh5gKb+*>^}Op@zr0i1Qh6eg-?4eIXy18B%xX|o$F`C>+IUQ) znsWW!ji2KBrE}7`?DxR$Z=gfjx!*ZE>yggo+Y{Qk*ql?pyP@pl@+A9A|Fk4x3nRP} zGQI!NN$@hLbYl5OJwD_;j9%$Zo?RWPt+VBiIs2m)YJ85kKD0N!#O AY5)KL literal 0 HcmV?d00001 From fa6db6d61397740b400e21743d35c3b14a2d65d2 Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Mon, 20 May 2024 13:30:33 -0400 Subject: [PATCH 04/14] More types --- ...table_with_updated_reference_dataset_test.py | 2 +- .../test_interval_1.ht/.README.txt.crc | Bin 12 -> 12 bytes .../test_interval_1.ht/.metadata.json.gz.crc | Bin 12 -> 12 bytes .../test_interval_1.ht/README.txt | 4 ++-- .../globals/.metadata.json.gz.crc | Bin 12 -> 12 bytes .../test_interval_1.ht/globals/metadata.json.gz | Bin 359 -> 357 bytes .../globals/parts/.part-0.crc | Bin 12 -> 12 bytes .../test_interval_1.ht/globals/parts/part-0 | Bin 221 -> 217 bytes .../.index.crc | Bin 0 -> 12 bytes .../.metadata.json.gz.crc | Bin 0 -> 12 bytes .../index | Bin 0 -> 79 bytes .../metadata.json.gz | Bin 0 -> 176 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 65 -> 0 bytes .../metadata.json.gz | Bin 176 -> 0 bytes .../test_interval_1.ht/metadata.json.gz | Bin 372 -> 372 bytes .../rows/.metadata.json.gz.crc | Bin 16 -> 16 bytes .../test_interval_1.ht/rows/metadata.json.gz | Bin 647 -> 643 bytes ...t-0-1224c3b3-ab5b-49d7-8d6d-6084ccbbc683.crc | Bin 0 -> 12 bytes ...t-0-2d30884d-a682-4d9e-9214-4bf4b5156c98.crc | Bin 12 -> 0 bytes .../part-0-1224c3b3-ab5b-49d7-8d6d-6084ccbbc683 | Bin 0 -> 60 bytes .../part-0-2d30884d-a682-4d9e-9214-4bf4b5156c98 | Bin 47 -> 0 bytes 23 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 v03_pipeline/var/test/reference_data/test_interval_1.ht/index/part-0-1224c3b3-ab5b-49d7-8d6d-6084ccbbc683.idx/.index.crc create mode 100644 v03_pipeline/var/test/reference_data/test_interval_1.ht/index/part-0-1224c3b3-ab5b-49d7-8d6d-6084ccbbc683.idx/.metadata.json.gz.crc create mode 100644 v03_pipeline/var/test/reference_data/test_interval_1.ht/index/part-0-1224c3b3-ab5b-49d7-8d6d-6084ccbbc683.idx/index create mode 100644 v03_pipeline/var/test/reference_data/test_interval_1.ht/index/part-0-1224c3b3-ab5b-49d7-8d6d-6084ccbbc683.idx/metadata.json.gz delete mode 100644 v03_pipeline/var/test/reference_data/test_interval_1.ht/index/part-0-2d30884d-a682-4d9e-9214-4bf4b5156c98.idx/.index.crc delete mode 100644 v03_pipeline/var/test/reference_data/test_interval_1.ht/index/part-0-2d30884d-a682-4d9e-9214-4bf4b5156c98.idx/.metadata.json.gz.crc delete mode 100644 v03_pipeline/var/test/reference_data/test_interval_1.ht/index/part-0-2d30884d-a682-4d9e-9214-4bf4b5156c98.idx/index delete mode 100644 v03_pipeline/var/test/reference_data/test_interval_1.ht/index/part-0-2d30884d-a682-4d9e-9214-4bf4b5156c98.idx/metadata.json.gz create mode 100644 v03_pipeline/var/test/reference_data/test_interval_1.ht/rows/parts/.part-0-1224c3b3-ab5b-49d7-8d6d-6084ccbbc683.crc delete mode 100644 v03_pipeline/var/test/reference_data/test_interval_1.ht/rows/parts/.part-0-2d30884d-a682-4d9e-9214-4bf4b5156c98.crc create mode 100644 v03_pipeline/var/test/reference_data/test_interval_1.ht/rows/parts/part-0-1224c3b3-ab5b-49d7-8d6d-6084ccbbc683 delete mode 100644 v03_pipeline/var/test/reference_data/test_interval_1.ht/rows/parts/part-0-2d30884d-a682-4d9e-9214-4bf4b5156c98 diff --git a/v03_pipeline/lib/tasks/reference_data/update_variant_annotations_table_with_updated_reference_dataset_test.py b/v03_pipeline/lib/tasks/reference_data/update_variant_annotations_table_with_updated_reference_dataset_test.py index 842686c5c..f3a62969d 100644 --- a/v03_pipeline/lib/tasks/reference_data/update_variant_annotations_table_with_updated_reference_dataset_test.py +++ b/v03_pipeline/lib/tasks/reference_data/update_variant_annotations_table_with_updated_reference_dataset_test.py @@ -45,7 +45,7 @@ hl.tstruct( locus=hl.tlocus('GRCh38'), alleles=hl.tarray(hl.tstr), - PHRED=hl.tint32, + PHRED=hl.tfloat32, ), key=['locus', 'alleles'], globals=hl.Struct( diff --git a/v03_pipeline/var/test/reference_data/test_interval_1.ht/.README.txt.crc b/v03_pipeline/var/test/reference_data/test_interval_1.ht/.README.txt.crc index 2985d33a727d77c8feffc21c93a074438d31d566..7132fdfe3cc37620d0e1e1ae3d399aaf4189568d 100644 GIT binary patch literal 12 TcmYc;N@ieSU}BiO_6rLD6Tbs< literal 12 TcmYc;N@ieSU}AWYyT}p%6XXMH diff --git a/v03_pipeline/var/test/reference_data/test_interval_1.ht/.metadata.json.gz.crc b/v03_pipeline/var/test/reference_data/test_interval_1.ht/.metadata.json.gz.crc index 8eabeebfec2ca55da91ecac106845a179b9f8060..2cacfc95c330dc4a8ec4d55314704d9cbaf0a608 100644 GIT binary patch literal 12 TcmYc;N@ieSU}C8K_{k6e6kG$$ literal 12 TcmYc;N@ieSU}Dhv-roZN61M|) diff --git a/v03_pipeline/var/test/reference_data/test_interval_1.ht/README.txt b/v03_pipeline/var/test/reference_data/test_interval_1.ht/README.txt index 2c105134d..3d9a5ac98 100644 --- a/v03_pipeline/var/test/reference_data/test_interval_1.ht/README.txt +++ b/v03_pipeline/var/test/reference_data/test_interval_1.ht/README.txt @@ -1,3 +1,3 @@ This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.114-cc8d36408b36 - Created at 2023/07/13 19:51:12 \ No newline at end of file + Written with version 0.2.130-bea04d9c79b5 + Created at 2024/05/20 13:22:32 \ No newline at end of file diff --git a/v03_pipeline/var/test/reference_data/test_interval_1.ht/globals/.metadata.json.gz.crc b/v03_pipeline/var/test/reference_data/test_interval_1.ht/globals/.metadata.json.gz.crc index d23992107450550c243e584968da657db5d685d6..d68bbbd080a8b2d387bbb0849ac99b19979e41c7 100644 GIT binary patch literal 12 TcmYc;N@ieSU}89MpW_e!6Ym40 literal 12 TcmYc;N@ieSU}6y8?EeM;5q|?f diff --git a/v03_pipeline/var/test/reference_data/test_interval_1.ht/globals/metadata.json.gz b/v03_pipeline/var/test/reference_data/test_interval_1.ht/globals/metadata.json.gz index 380be3d033640e983bae3dc117df74e24092ebcf..c16ad768c557bd7860fd640812bfb5b00bc126e3 100644 GIT binary patch literal 357 zcmV-r0h<0FiwFP!000000G(68Zh}A*{Fl9!CasO8;Z|CEYSKiT9%_tPcb~Ww9%Oeh zp@e_$d!iuLgO&@s^Ja$G30n!2DTq+KTMZWUdQ4#RbA_jUD( zx+D#82sxx^c1QL6-=eAILVd4_GbJ0Qa)?+;h%OPu7ZQo&pQmn~`8ZPCuS!IfwxZ*m4Uc3+D@5iA;hZoQ2g< z@LuL1N6i3PKoCx@#u2wLbEXik%DLOvumi<9OSxR?Op~$FF}}2QhAZuesWdlbV1)tD z&E5w)FjgV)0<)f1V+FcMFAUW({YkVIH4tOCrNatv@MH`x=f~^|?*qvlw1=U(c89t; zMV*rdIfxun)Vq^<{%=v&a;Cmh#UCZq71Y*E0f#Tq@mIxq4W{=~O~%pfU>LQdfQgMt z2x{eE*kNn^Vf49{&pOKM=XHzUJcm zmdWK=k_D``B4TR-^4#Cw6tpe(Gl@MN_)1ZMlflS!Dwm@D^U|HInb%#zxefn098F%~ RtHBHO0|z`*byh`*{c#-FXD#78dCirKY78rRF84>ZT-?BwMpmZAdS<4E#ztnk1}0Vp L1`Mn~|9}AiMHoFs diff --git a/v03_pipeline/var/test/reference_data/test_interval_1.ht/index/part-0-1224c3b3-ab5b-49d7-8d6d-6084ccbbc683.idx/.index.crc b/v03_pipeline/var/test/reference_data/test_interval_1.ht/index/part-0-1224c3b3-ab5b-49d7-8d6d-6084ccbbc683.idx/.index.crc new file mode 100644 index 0000000000000000000000000000000000000000..23324f542252c6a23a4e82c7ff3ae91d7e155c49 GIT binary patch literal 12 TcmYc;N@ieSU}9ME-1rm#6ZHeb literal 0 HcmV?d00001 diff --git a/v03_pipeline/var/test/reference_data/test_interval_1.ht/index/part-0-1224c3b3-ab5b-49d7-8d6d-6084ccbbc683.idx/.metadata.json.gz.crc b/v03_pipeline/var/test/reference_data/test_interval_1.ht/index/part-0-1224c3b3-ab5b-49d7-8d6d-6084ccbbc683.idx/.metadata.json.gz.crc new file mode 100644 index 0000000000000000000000000000000000000000..576d4ffd2f53ecd87bc049cfc67827d78f6adf26 GIT binary patch literal 12 TcmYc;N@ieSU}9*~ZOjA!5sCuX literal 0 HcmV?d00001 diff --git a/v03_pipeline/var/test/reference_data/test_interval_1.ht/index/part-0-1224c3b3-ab5b-49d7-8d6d-6084ccbbc683.idx/index b/v03_pipeline/var/test/reference_data/test_interval_1.ht/index/part-0-1224c3b3-ab5b-49d7-8d6d-6084ccbbc683.idx/index new file mode 100644 index 0000000000000000000000000000000000000000..952d782a33fc4e7095e476ee391088b2cd8ec83b GIT binary patch literal 79 zcmY#pU|3c@fHM&IR53K^;`rOhfRqM)d_h(j80n_#(-xyWGj}^ ze>aL3Zv$^0(?Mx#yr2)(5y*5l&n5zLuSgq9)kkbzn_h^*anbe0hS7U9Tb}=d^xV( eIo_9Lz*tXWUNL){H;nPp_5B;Mk&JZ40001?{7q2+ diff --git a/v03_pipeline/var/test/reference_data/test_interval_1.ht/metadata.json.gz b/v03_pipeline/var/test/reference_data/test_interval_1.ht/metadata.json.gz index 55f91078dda7c95904f86864cadeaf2b0e58569d..2654291a9fd4c6eafa6adafa6135c775e0d69833 100644 GIT binary patch literal 372 zcmV-)0gL`0iwFP!000000HspVPQxG+{TH4z!DVaJZM>NojnN0Q8DE;Fq4cgJQfPo~ zsZ0M|%F3obm?b{JJ@=f$fy*APxqxg7+HfTixt+$*45JO@;sm1cC>l-T@l_6Vd|k}h zbe`WJj5Mr4gJfVb4I-~!|c8nnNXBcRkX;Y zk{MG4m!(^zv6^yedyYe}VH!XVRfmU=|FHnsRA=jZPWR2BuQik|g1;rX)0(!)!8+|a zTu=+glRls&i8BJdr6R>z?Se%B$afXny$MWC!|VEzvr2Os#IL)GqlohLg{ zO||8g`x`bxhQZ)9v=+Ye+br;tN!t5pRfFTh4~HW-XnwK^ZY-C~iuM^=C=hW^2JTUC S(EGpZrQH`FU6-CT0{{Th8?w{@ literal 372 zcmV-)0gL`0iwFP!000000HspVYQrED{TH7$aHVZZT6`NF8)JJI?bu5Q!J6AtW;Cd2 z$wK~p#>RHt!zk=Y@44q3k6v4}cencXqA$fI~R zUTmTmVWeRT8YBagX%JaO?aqJTgfNS=pH@~T*EZk|@lU9`AO@in==GQ*Sx@+cIw@$8S7 z8PClS)g2g3UDD1?i>b0w5~MYVwCFe}zm>G?2e|rpSO&_v0_Oj~Y4CyHIaD34*LkuN zRa9GUxxZmOWEc$ILTl+ezs&+anWVjsRuwotTsWM-L37Ev{V#snw9xUH(!7n1PDs@>m1z${D01u@@EUAn zo2nG$zxO(UkOaEIp4RdL9N&9=?(w;aH$>6}kbsONhezOdzrK!_z}mzk;XSNLFd;%( z&<`{rQ^c6!z+40gHoMfH>m~5(VgaMTLAdg1IG_f`R{!Z zRIk0IGQtWa5gAF_X{R1Yer}48ELm|dp6AEr{@r%}UJ=qeRrl}L|VJlo+veEX( zORAjpHxa^>9%BP<20Qf zbNODVl1#ZsC3#L9iGs`ll^Th6;%XE@(&Rgu@!wpU?WeP+)e&E2V-Ut7fv|SQhwjdi zH}2%T2@UwfB;s#*1-$>-{VA}R_DD#v)1>bRFhNUYA(?` zH+WXYU5DJCMzhgsx4Wzpbs9A4MUCzizijlm&l`TT-(@U{nBVVYH>N*#RL)?aR+78? z4auzU`YA3}K?%$)jSSdaw~*S-(j@3#w%UFZ`QRA59(m;=w=N&&#jzzQu%Rmw)5Ji! dFZmxadM(f)sneHQt5H~V@iTJbe$31Y001twM6v(? literal 647 zcmV;20(ku&iwFP!000000NqwmZ`v>r{x5#ow9r9FX~^5?=!R5HQkw1YikZWN zq(I|J}gQWXy(nNx~DvrO1hSXoZ*)= zR#WVvzV%aVn9_jYRjR1%1(|1<29WhI4Kw5uTNEZ1+~DFTt?BA^LvWI1PPm7Sa81cZ zn;*B=iY?8%FE^L-ZuoxkcbW}?O?uA|D`-_oO_xamx^d{&!$HRn2fHPYu(?QiC1j8) zc88p5K~o2g#S;c}{fl0>pLYE#_oD`RFUV%nJCHZQE8BPE4D@zBu&4e8~@Fw`F=WoUL7!IJ_m6kQivOe zyl?LUdF56vn$UpHOn$iK6Wv|vTXXcju1!^&njY={LmfQB|4ohrigns+H1B<OZnX|-EE@3zA*;4Rt<{Z_#H(CYi0pcPD$V0zv;@3DTEUz&d3P&tEv zT1xKn*ChA8>!-Y1MHR5{G%{dw-9c(wPm?IT==i-h^1(THIkN8}H#Q#^#jzzQv7sxH h(9}S-EBPNWdMVH$nbVhAt5I0A@gFI@7D~?w004o|L7M;o diff --git a/v03_pipeline/var/test/reference_data/test_interval_1.ht/rows/parts/.part-0-1224c3b3-ab5b-49d7-8d6d-6084ccbbc683.crc b/v03_pipeline/var/test/reference_data/test_interval_1.ht/rows/parts/.part-0-1224c3b3-ab5b-49d7-8d6d-6084ccbbc683.crc new file mode 100644 index 0000000000000000000000000000000000000000..a4b13f78f85ea891e2fa7aecc6b2add89427b83c GIT binary patch literal 12 TcmYc;N@ieSU}D(e=gW#P literal 0 HcmV?d00001 diff --git a/v03_pipeline/var/test/reference_data/test_interval_1.ht/rows/parts/.part-0-2d30884d-a682-4d9e-9214-4bf4b5156c98.crc b/v03_pipeline/var/test/reference_data/test_interval_1.ht/rows/parts/.part-0-2d30884d-a682-4d9e-9214-4bf4b5156c98.crc deleted file mode 100644 index 95a7bb76c80838d69905252270d9189e2834e14c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}Bi#cfbPx5~2ew diff --git a/v03_pipeline/var/test/reference_data/test_interval_1.ht/rows/parts/part-0-1224c3b3-ab5b-49d7-8d6d-6084ccbbc683 b/v03_pipeline/var/test/reference_data/test_interval_1.ht/rows/parts/part-0-1224c3b3-ab5b-49d7-8d6d-6084ccbbc683 new file mode 100644 index 0000000000000000000000000000000000000000..1d5c3980168f154018f031c1c8b8b1b40a775acd GIT binary patch literal 60 zcmY#qU|^5}VvVi(e-)%IGB7Z*Bxe*E-kD|!rhiQ{WdzDP*fTOR@B$ei09MKX6b1m5 C%?w}w literal 0 HcmV?d00001 diff --git a/v03_pipeline/var/test/reference_data/test_interval_1.ht/rows/parts/part-0-2d30884d-a682-4d9e-9214-4bf4b5156c98 b/v03_pipeline/var/test/reference_data/test_interval_1.ht/rows/parts/part-0-2d30884d-a682-4d9e-9214-4bf4b5156c98 deleted file mode 100644 index 2fb6ca9cbf23ad4487a8a16ef41dbedd791e8324..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 47 tcmY#lU|^5~;)#qb$r(k4ccz(gFnnPAHO-U}C

C>=_vuSRrf%1_0fS2iX7s From 15ea14cb0f77fa897727854e80fefa1e605273cf Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Mon, 20 May 2024 13:43:47 -0400 Subject: [PATCH 05/14] more types --- .../test_combined_mito_1.ht/.README.txt.crc | Bin 12 -> 12 bytes .../.metadata.json.gz.crc | Bin 16 -> 16 bytes .../test_combined_mito_1.ht/README.txt | 4 ++-- .../.index.crc | Bin 12 -> 0 bytes .../.index.crc | Bin 0 -> 12 bytes .../.metadata.json.gz.crc | Bin .../index | Bin 130 -> 130 bytes .../metadata.json.gz | Bin .../test_combined_mito_1.ht/metadata.json.gz | Bin 560 -> 561 bytes .../rows/.metadata.json.gz.crc | Bin 16 -> 16 bytes .../rows/metadata.json.gz | Bin 851 -> 848 bytes ...t-0-262c4389-1ff9-432a-ab5c-4c9cfa547dc4.crc | Bin 12 -> 0 bytes ...t-0-4acd73d5-8e51-42cf-a64b-a3aec87f5fc2.crc | Bin 0 -> 12 bytes .../part-0-262c4389-1ff9-432a-ab5c-4c9cfa547dc4 | Bin 219 -> 0 bytes .../part-0-4acd73d5-8e51-42cf-a64b-a3aec87f5fc2 | Bin 0 -> 224 bytes 15 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/index/part-0-262c4389-1ff9-432a-ab5c-4c9cfa547dc4.idx/.index.crc create mode 100644 v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/index/part-0-4acd73d5-8e51-42cf-a64b-a3aec87f5fc2.idx/.index.crc rename v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/index/{part-0-262c4389-1ff9-432a-ab5c-4c9cfa547dc4.idx => part-0-4acd73d5-8e51-42cf-a64b-a3aec87f5fc2.idx}/.metadata.json.gz.crc (100%) rename v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/index/{part-0-262c4389-1ff9-432a-ab5c-4c9cfa547dc4.idx => part-0-4acd73d5-8e51-42cf-a64b-a3aec87f5fc2.idx}/index (61%) rename v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/index/{part-0-262c4389-1ff9-432a-ab5c-4c9cfa547dc4.idx => part-0-4acd73d5-8e51-42cf-a64b-a3aec87f5fc2.idx}/metadata.json.gz (100%) delete mode 100644 v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/rows/parts/.part-0-262c4389-1ff9-432a-ab5c-4c9cfa547dc4.crc create mode 100644 v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/rows/parts/.part-0-4acd73d5-8e51-42cf-a64b-a3aec87f5fc2.crc delete mode 100644 v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/rows/parts/part-0-262c4389-1ff9-432a-ab5c-4c9cfa547dc4 create mode 100644 v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/rows/parts/part-0-4acd73d5-8e51-42cf-a64b-a3aec87f5fc2 diff --git a/v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/.README.txt.crc b/v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/.README.txt.crc index c591777978c115f329ab4f2a6d06078660dade37..8280cdd4c7d557eac8d866a12934dce9f00b53eb 100644 GIT binary patch literal 12 TcmYc;N@ieSU}9k0diOK{5-0~b#KgqF7{chdkn^F$=d^7DS(I zk@CTbiwq}GMzydI@*k!b*X(nj@jALVutP_e_vNAiY%FHezvRrWBS_R=b zsK=K3jgjLl3?4v}0xV;Lfs$#KyOXNrSas>bU2o1v)=9G+Nz(KT3_YJ%}f)G{~fokj85%adoKe9nH-?!ZnHu za7w7Ih6M;K1=4Abgp5JIp^6VokWwMc7%)&rB*jZm^_lp)`@Z}7xXG5Q_klNBW6D#_ z+@*WFw!Ox5mZTSy9x3sTyvz3H)@EjqkfN6n_?i0p=4OFp`*Phy8B*lniKSqC?sZ zPqpTHlVs4G99nOE;I-~QqV>-P&+%)Lu@1L2BJb6Nz@Ui-@o!f!#?CDw@Xzo!SsyK}h2y9BNg=akdLS%V8p zLa_;SyvI|9)gh*2S~wtz8WVUv%_bSD< zA#|^sy2sMqY^)SI+#LhgF6K-;Amv9)OV6s3uUHT-PvJt1fHSO_*#1B|lv^qohAL%J zYk6FWIryxSW+!yf*V^4mPL>*}b!5J^o9s5AKvPw1<|I)Hl1-!O zOWiQiAm18hf`6r=-fYe#%0_2K6QS5^S7(MMC8tPakt~W4*8j{$HX;e)mWeMH^ObK$ zyVpC7Oz$g{fI2@`l$eAXx6=$e$0X3J5aY*E*MxXEt|z3nNg+%}1Q2G}O9HZvA66<#b_YOq+Sw9@%T zH8L1CyC9&dwu4{ICKuh&q;$Cs4jCtvkO^RW4|mK8+8E=K{eghtV9*;jm;FjK$(EkE z4n{yxRDWNYnm-W1%$ZfwusGSWBTP|lo(Hgh{`@5iH&!4f$PTI(onF;FB48@fK*TuAdADJt6e$#C@p#;ISs*?&z6bIaQjm)B4fHcZmq(U}ky@-6V1 zvg@MjlF!GqP0ABO-Xpm%v{$~1WHTtIV`oVnlRQRB&zbp9fNuwL(fQixBTO!PWY8JW z!9}Ov#X$!R`EOrSc9u7^;*> zt>sB2X6Lg?nhn=QUrTo_A~M%Vs$;-Y3z1S(8c|>?b!5J!t2Qf8psB1jbCf7J$)?ft zrEVB&kZ%oBVSgd1HB6_8 z-J6X@ruPL>pw3SuC1xhYeGO^Ewk^4aV?U(DoZaE{aGB-3=)1$)!WW~hA4Xp3f~6w1 zmX?>PyXG(r@;^(a=1F)wJPFn_aBY4+e7_rqy~~dU0hCo2IMtGg=XVETlwK82r zx(W0XJh$PTmubRWlTwmwISPy|;u-)q*=-@WJw=t-CaurF+C^2I6WH1XR{`?(50uyn8t+U9N*e7LiKG2#|e%yJZEfjB(EXK)}Ud(7R|Z`=vC= zmY%o{27f?NTwj@*KM=vpSyj_8JKD0NOVY?Z3t&Gx`<#UvE6@^T2h@vBt?KR&FsU@q zu+1N9m@^!X3NBjC4)Azyb>gM22m*E7r=ShQxyZ4#Yo_V4ZR06*R8&~Igz5g&U8?A+ z(NnFPcS0X*4;)3e0L|-Y%IZnV+xf|G^#c}BzkhbKn-u1jwc9M;k)tc{`w&7Pg7WZY-%$ TfV7sTNSgiy#oXTZQw;zB1Vx=& diff --git a/v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/rows/parts/.part-0-262c4389-1ff9-432a-ab5c-4c9cfa547dc4.crc b/v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/rows/parts/.part-0-262c4389-1ff9-432a-ab5c-4c9cfa547dc4.crc deleted file mode 100644 index bfecb511a09eb9ee5317ef47608e86d2b070266c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}C8Kb7?*R6srU! diff --git a/v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/rows/parts/.part-0-4acd73d5-8e51-42cf-a64b-a3aec87f5fc2.crc b/v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/rows/parts/.part-0-4acd73d5-8e51-42cf-a64b-a3aec87f5fc2.crc new file mode 100644 index 0000000000000000000000000000000000000000..f8028adef6aefd664500ebcaa873e0cc197a2ee4 GIT binary patch literal 12 TcmYc;N@ieSU}E^YY(_r-6(s}( literal 0 HcmV?d00001 diff --git a/v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/rows/parts/part-0-262c4389-1ff9-432a-ab5c-4c9cfa547dc4 b/v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/rows/parts/part-0-262c4389-1ff9-432a-ab5c-4c9cfa547dc4 deleted file mode 100644 index 7acd67edc2e1c969198c7b12555a85b22d0845d2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 219 zcmV<103`p$0000%0ssIgwJ-f(K>@`D0J4Z7DsXK&d=c=~EqAnz3PC(Rplw{wVf1@T zSpKZhiS1P1RQyfkKQe~WqD(1fa7fk=n9OE&1JTHtDXgYa;4wI>-o@QZRa63Q!ViL` zJUDE}jMA|Sd+;;4*a)lRqk5*>yL;ExuaIVT+7~G~Vi;yBZR>RL_c4(892XXaDfjK( zLn;#qRXq~`Fu3Hx#=tmNSO~?#W+@QZRBvMI60#}>@ht?mU;scF+(Q%vCUM0rndn7; VJPiN<0000004TLD{U87V000;tT}1!@ diff --git a/v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/rows/parts/part-0-4acd73d5-8e51-42cf-a64b-a3aec87f5fc2 b/v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/rows/parts/part-0-4acd73d5-8e51-42cf-a64b-a3aec87f5fc2 new file mode 100644 index 0000000000000000000000000000000000000000..f78553dbff70cac10c26a3a847bcbbc696081582 GIT binary patch literal 224 zcmX@jz`&r(#K54jRsU~-F5_EPh9w+~94yHhMZSzo42+J9&h-rPt=jesJLFjH8JQRv z`hIgWxYsin8ynj*yb0W47m~E#xF#bv0~1hn2%}>?LrlMuJwyK?Q+wy6{RbJD_!t{JmO*44JJ7`bLtkwf&Ys~3Nm?{(rlcLi z^~DAZA`BNS9!j|#*bx@NaelL!NQ{nFUYNJQx+?Z#yexYdm=6f=a28=y%RHKUX4?S| Tu>vMupgkY}@{R%nBLf2fOqV>) literal 0 HcmV?d00001 From b5b7e47646e10eb993c2bd1a859add6383c5b00f Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Mon, 20 May 2024 13:48:57 -0400 Subject: [PATCH 06/14] ugh --- .../test_combined_1.ht/.README.txt.crc | Bin 12 -> 12 bytes .../test_combined_1.ht/.metadata.json.gz.crc | Bin 16 -> 16 bytes .../test_combined_1.ht/README.txt | 2 +- .../.index.crc | Bin .../.metadata.json.gz.crc | Bin .../index | Bin .../metadata.json.gz | Bin .../test_combined_1.ht/metadata.json.gz | Bin 724 -> 725 bytes .../rows/.metadata.json.gz.crc | Bin 20 -> 20 bytes .../test_combined_1.ht/rows/metadata.json.gz | Bin 1062 -> 1063 bytes ...0-6a5a9d6a-4ded-424b-9735-922a5346e7ad.crc | Bin 0 -> 12 bytes ...0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.crc | Bin 12 -> 0 bytes ...art-0-6a5a9d6a-4ded-424b-9735-922a5346e7ad | Bin 0 -> 106 bytes ...art-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d | Bin 103 -> 0 bytes 14 files changed, 1 insertion(+), 1 deletion(-) rename v03_pipeline/var/test/reference_data/test_combined_1.ht/index/{part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.idx => part-0-6a5a9d6a-4ded-424b-9735-922a5346e7ad.idx}/.index.crc (100%) rename v03_pipeline/var/test/reference_data/test_combined_1.ht/index/{part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.idx => part-0-6a5a9d6a-4ded-424b-9735-922a5346e7ad.idx}/.metadata.json.gz.crc (100%) rename v03_pipeline/var/test/reference_data/test_combined_1.ht/index/{part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.idx => part-0-6a5a9d6a-4ded-424b-9735-922a5346e7ad.idx}/index (100%) rename v03_pipeline/var/test/reference_data/test_combined_1.ht/index/{part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.idx => part-0-6a5a9d6a-4ded-424b-9735-922a5346e7ad.idx}/metadata.json.gz (100%) create mode 100644 v03_pipeline/var/test/reference_data/test_combined_1.ht/rows/parts/.part-0-6a5a9d6a-4ded-424b-9735-922a5346e7ad.crc delete mode 100644 v03_pipeline/var/test/reference_data/test_combined_1.ht/rows/parts/.part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.crc create mode 100644 v03_pipeline/var/test/reference_data/test_combined_1.ht/rows/parts/part-0-6a5a9d6a-4ded-424b-9735-922a5346e7ad delete mode 100644 v03_pipeline/var/test/reference_data/test_combined_1.ht/rows/parts/part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d diff --git a/v03_pipeline/var/test/reference_data/test_combined_1.ht/.README.txt.crc b/v03_pipeline/var/test/reference_data/test_combined_1.ht/.README.txt.crc index 62a72838065f36b5d5539bfb643254f95da0b339..1c47b9a3c9eae1262a59c31f8c62067028259bc0 100644 GIT binary patch literal 12 TcmYc;N@ieSU}A9Nc8>-C526At literal 12 TcmYc;N@ieSU}A8;sbl~E5t{<7 diff --git a/v03_pipeline/var/test/reference_data/test_combined_1.ht/.metadata.json.gz.crc b/v03_pipeline/var/test/reference_data/test_combined_1.ht/.metadata.json.gz.crc index 11f6d03d5ee0f27eddfe86535d46c9d5282ac93b..db7a7824cd79229b8fa039920ffc63383e227bdd 100644 GIT binary patch literal 16 XcmYc;N@ieSU}E_5UO7Zq$#*IMB`gHA literal 16 XcmYc;N@ieSU}E_9{a$TCspWhCEV2eT diff --git a/v03_pipeline/var/test/reference_data/test_combined_1.ht/README.txt b/v03_pipeline/var/test/reference_data/test_combined_1.ht/README.txt index b427c10e6..e46de4296 100644 --- a/v03_pipeline/var/test/reference_data/test_combined_1.ht/README.txt +++ b/v03_pipeline/var/test/reference_data/test_combined_1.ht/README.txt @@ -1,3 +1,3 @@ This folder comprises a Hail (www.hail.is) native Table or MatrixTable. Written with version 0.2.130-bea04d9c79b5 - Created at 2024/05/20 12:56:42 \ No newline at end of file + Created at 2024/05/20 13:48:16 \ No newline at end of file diff --git a/v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.idx/.index.crc b/v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-6a5a9d6a-4ded-424b-9735-922a5346e7ad.idx/.index.crc similarity index 100% rename from v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.idx/.index.crc rename to v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-6a5a9d6a-4ded-424b-9735-922a5346e7ad.idx/.index.crc diff --git a/v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.idx/.metadata.json.gz.crc b/v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-6a5a9d6a-4ded-424b-9735-922a5346e7ad.idx/.metadata.json.gz.crc similarity index 100% rename from v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.idx/.metadata.json.gz.crc rename to v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-6a5a9d6a-4ded-424b-9735-922a5346e7ad.idx/.metadata.json.gz.crc diff --git a/v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.idx/index b/v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-6a5a9d6a-4ded-424b-9735-922a5346e7ad.idx/index similarity index 100% rename from v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.idx/index rename to v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-6a5a9d6a-4ded-424b-9735-922a5346e7ad.idx/index diff --git a/v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.idx/metadata.json.gz b/v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-6a5a9d6a-4ded-424b-9735-922a5346e7ad.idx/metadata.json.gz similarity index 100% rename from v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.idx/metadata.json.gz rename to v03_pipeline/var/test/reference_data/test_combined_1.ht/index/part-0-6a5a9d6a-4ded-424b-9735-922a5346e7ad.idx/metadata.json.gz diff --git a/v03_pipeline/var/test/reference_data/test_combined_1.ht/metadata.json.gz b/v03_pipeline/var/test/reference_data/test_combined_1.ht/metadata.json.gz index e3a900771bb206aeead61dd8347234d908b3d317..d005657568a8c488be1e47dce7e7e6e22410a1ad 100644 GIT binary patch literal 725 zcmV;`0xJCr{x5mjq^xS8Tid)b3T;(rBS2^pLY7Gm@aEW= zZQ3G~|2{j36DMhs(rnqR5R;*Nz z1S>$8SVX3jL@uCsnt)9%^UL1+hnWLAB%QR4oYlFNYMEW3b@FS#PKE_u=2$4rEDNlW zr-4c!(p`n$2QxsyKnQ$X6`8Lyr;;Ls(`Y+NTqi;zqK-|jkEH~m^cQQ#tiO&4F+C`3 zycSq05GWP;xF#XX{9kYkR+L8v>0?TPf?WH8zuFkNC777sPj|E7zJLAM`n8gByC2@q zMz_J3a;*BjPH6@IAr2mxHv76qTiqh5~VhzqD4I=dPJRTwqA{O82=I57|7Yc=A@&0O0l{3NmY@G8$f?;Mk|gdn Hj0gY#qKa;d literal 724 zcmV;_0xSI=iwFP!000000M%CCZ`v>r{$KL6NmetwHbntaoS##P3NcO+b0%!-$CR)aEUcKUncQSn0f-=~ z$s&@1^NSmxk4>J|3{EhPk#W58Z1bP G2mkfaj!6n@DGt7T%FPWe zksou(c1p%L5l1(PQzlu=Q8MSU*Uj9MZxoD#AjglT>c;g6rM9)@^=kSKMYAv_`DQ<4 z{0kAbR$*~YhlL9lV+ca#OU8FgBKPqQIP)z|de}xxN)To+WdUMN^J+EMG8Y)S8k&JmmBdT-0?@TC5j+y# zQX9>@<=ygT^g%1RqnU9%@kaDh89UxxgEa5r$Xbh-zP&XzX6}y15uqMD#*r$t6@cOG zwz5`UnJ&s$y}On7esMP*EvmgUTs3VO!D#nbk^jekl-O0jSx$_Hk_Z+N2_9jVQtHP7 z47D`DzHMkx2qp1s-BJ}>CWXHa!0}plVCKp=iBwmZRhU#sw$;O^Le&A8OSe1?5_{lQ z(}(He6p$WHcRioo*P~p<;W60*?|Rn{qg8$N<{*h`_j{7!KzIo@vP$AKa_+-)ecx}{ zvs8LDae%WJMc&OH3k5Ws{F*|RDT;kLQOx4kLjC{pel^+mE`QZ{D&`Lr(bfq~7L*!D zs>A3-7HJQy5HcCmB+xSeYU5Agrb(Y#f)#qKg_1ih$gzpjfQ`YL8fq-kSfH^vV_~M1 zGSz?^{jA5k>g(r+yEfa_Xt$H)Kk=z(Vt}NLBNjqh2+{(NZ?nTjM@0<{jYhszn)*p< zV6>p31r*=p#0!{cLZY1!FCoGr#P$#bAZo{<#dLhuy&P9duH)}nNDY-Svg9+`w3N0= zxMp8bVQ_xl8#K55nM#^(edRu^E|93cGgCiIf~f;wEr)q2%g=)XLWLugzj^a}uCAmZ zkRU(x{iCCx`9=tsNF_d$#{%AizKbVL>x@_F;>z#`9wRe14uTY^8=UvbG^_|+5$}@@ljLnNh`_5xxUW2MSww26L#uFms zjHz#L{2W&|owuC3{{s1lr%|F#Ytq9-27|(=M z?SFDoJPb;eSn*+x7ex=FSAQ?gFr2c7o#ExhS$A-iRUiKtzz8)a4nA@RIQ>`bFz$y3 h3hN>;%pA0As$3{Zrg9-t3gs!w{swU;>2|mf0092>{x|>t literal 1062 zcmV+>1lju^iwFP!000000Ns~cZ`w)_fd7l1&QVQIxD-;}fI@=~Nf1O))d*ROcMRLw zYoGN}g+~7GXV%8{uI(gE)r(XfkUcv)H_v=tKUkauU=6J~r*IE+b^C4|f!ncc%lQmP zYj|h*p?iq&p7=ia5L+8uazhHPQnZE-xrLi!1awj?pcW_*uDqI#NepW#4z7Gi%?%Ba z8*<5ZO2#-5M>`5*CRxZ)GUu|}N$itv6pVx*$Gcp0W4nY>)7tW4HGNlZb$!Oa5MkUq98)(}#g>WTzXNc*E;=x=Wf%pjEA%Q%$|RfWp;e*mfW)R-9($2FaI5L< zba4vkRbgs9pWRfWT!#KJ+5_)67q_ETb@lQfiE8&765~KP5jL_);xuyh?R0(9YuK|? zdKIyUvls-<<-ZGsGo1X2LXs$keK|4Apw~kE|Mh+~*>}(XF5r~SA1b1)6POGnRe)4Q z(X%8f>^CAvCs3U~kKm_?KAD?3dn)Ob;4v0*>NEhy&uG(7+A86i zeMN=-n>XElecQ)a(sb(!_hEH`1l661`e71G9sVjg%yU_K78Kws9HIR6>wi*pCG~-L z>51=~j)D67z+)no_)zXMboYBUo;Hp1U8##J!=HGBOzhC}Vvt27g!3?KAk#;ZY?@vP zWhz=C2CG#9FMWR$QPpCqMDvFVufOSwtDA1+IqVs{ni<6L#Nwo4+b5Yex`p~5x#>lo(ZYi z|6s*<7?dip?4uqpvK~gS?q0MpoVAy|N4}T0`gc=hIAGrmb{t!EiyZ(W~ gItvUlhe9@0E|eruxezIZ@{}b10}PQzf3y$)0O0xcrvLx| diff --git a/v03_pipeline/var/test/reference_data/test_combined_1.ht/rows/parts/.part-0-6a5a9d6a-4ded-424b-9735-922a5346e7ad.crc b/v03_pipeline/var/test/reference_data/test_combined_1.ht/rows/parts/.part-0-6a5a9d6a-4ded-424b-9735-922a5346e7ad.crc new file mode 100644 index 0000000000000000000000000000000000000000..dd555f5530818b965a42aefee5e49be3d67965d2 GIT binary patch literal 12 TcmYc;N@ieSU}8AWR@njo5<~+H literal 0 HcmV?d00001 diff --git a/v03_pipeline/var/test/reference_data/test_combined_1.ht/rows/parts/.part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.crc b/v03_pipeline/var/test/reference_data/test_combined_1.ht/rows/parts/.part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d.crc deleted file mode 100644 index fafabb335e3aadf5b534cbb2655fe1b6c458b46c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12 TcmYc;N@ieSU}DJl)T#;q6D0#b diff --git a/v03_pipeline/var/test/reference_data/test_combined_1.ht/rows/parts/part-0-6a5a9d6a-4ded-424b-9735-922a5346e7ad b/v03_pipeline/var/test/reference_data/test_combined_1.ht/rows/parts/part-0-6a5a9d6a-4ded-424b-9735-922a5346e7ad new file mode 100644 index 0000000000000000000000000000000000000000..446fb54911ffb22915fe6eeaa755c8b05942e6ed GIT binary patch literal 106 zcmWG#U|{e7VvVi(e-%6&nHU&%uq0;`89tq6%Eai%=nRx{sArhweA-??it)Oo@bgcW z#>NZ{=hzqyFtb~JyTK>V5PtHNMSJ973kHV%Ltkx}<&P>D*fKEi0(F7_*kA@m1_l5U CmK@0d literal 0 HcmV?d00001 diff --git a/v03_pipeline/var/test/reference_data/test_combined_1.ht/rows/parts/part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d b/v03_pipeline/var/test/reference_data/test_combined_1.ht/rows/parts/part-0-a11d3d0b-0773-484d-9ba6-0d1e2341772d deleted file mode 100644 index 58e2f027e56362f00860d923050811f9835cd1a1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 103 zcmWGyU|?_sVvVi(e-&IcnHU&%uq0;`89tq6%Eai%=*(2lFwOb2y@C|ubxYyrpDc}y z85+*9F&toKxBPa4Po5$CRWPt+VBiIs2m)YJ85kKD0N!#O AY5)KL From 314484babda658bffc81c1128bedca36f0e59b21 Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Mon, 20 May 2024 14:09:07 -0400 Subject: [PATCH 07/14] twiddle it back --- .../test_combined_mito_1.ht/.README.txt.crc | Bin 12 -> 12 bytes .../.metadata.json.gz.crc | Bin 16 -> 16 bytes .../test_combined_mito_1.ht/README.txt | 2 +- .../.index.crc | Bin .../.metadata.json.gz.crc | Bin .../index | Bin .../metadata.json.gz | Bin .../test_combined_mito_1.ht/metadata.json.gz | Bin 561 -> 564 bytes .../rows/.metadata.json.gz.crc | Bin 16 -> 16 bytes .../rows/metadata.json.gz | Bin 848 -> 855 bytes ...-0-2e25c2cc-84c0-4816-b8dc-5e8c19c4f1d2.crc} | Bin ...part-0-2e25c2cc-84c0-4816-b8dc-5e8c19c4f1d2} | Bin 12 files changed, 1 insertion(+), 1 deletion(-) rename v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/index/{part-0-4acd73d5-8e51-42cf-a64b-a3aec87f5fc2.idx => part-0-2e25c2cc-84c0-4816-b8dc-5e8c19c4f1d2.idx}/.index.crc (100%) rename v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/index/{part-0-4acd73d5-8e51-42cf-a64b-a3aec87f5fc2.idx => part-0-2e25c2cc-84c0-4816-b8dc-5e8c19c4f1d2.idx}/.metadata.json.gz.crc (100%) rename v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/index/{part-0-4acd73d5-8e51-42cf-a64b-a3aec87f5fc2.idx => part-0-2e25c2cc-84c0-4816-b8dc-5e8c19c4f1d2.idx}/index (100%) rename v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/index/{part-0-4acd73d5-8e51-42cf-a64b-a3aec87f5fc2.idx => part-0-2e25c2cc-84c0-4816-b8dc-5e8c19c4f1d2.idx}/metadata.json.gz (100%) rename v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/rows/parts/{.part-0-4acd73d5-8e51-42cf-a64b-a3aec87f5fc2.crc => .part-0-2e25c2cc-84c0-4816-b8dc-5e8c19c4f1d2.crc} (100%) rename v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/rows/parts/{part-0-4acd73d5-8e51-42cf-a64b-a3aec87f5fc2 => part-0-2e25c2cc-84c0-4816-b8dc-5e8c19c4f1d2} (100%) diff --git a/v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/.README.txt.crc b/v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/.README.txt.crc index 8280cdd4c7d557eac8d866a12934dce9f00b53eb..436531ab22aa29ccf30ce13b907c450e9fba8bf5 100644 GIT binary patch literal 12 TcmYc;N@ieSU}9j@e%}lL5M~0| literal 12 TcmYc;N@ieSU}9k0diOK{5-0LpRrj3!@JNvC)fR2o$xph*G3LQb`0u|6Y<3^|Dg5w_XJ8 z?CfwzF83T9DTC-4q@scc;qBdGevg6_Q`UjO>}q~>y_kK70ncud2Xglie+*C{;Rq5q z0ToGLQI2(L$Q^1Z8plRnmpPcs&Iacb6ESAtR?C9uvn^6SIDc`G;Uvna78auXgi>Ji zb%q#Mu|53 z$DUi-*SgqBM7#2W>-xVspp#(9R%5A}>>EYx)(zyBahU^BMQqHPGrri0;0e{P7e@y_ zykbR!gmBFSwwA{C-5Cl$3BdsKi4GV^Otiyxv))Am8h@JTkaojUt+`x9DKsaCmTMPy zt>upf&+u!MvJN*jBJ0(dptU*iB>s7g+nYbGQB`Jk5s~GB1J^1D&p|!5+%LzO9cNEJ zKd(Gxn*(tZu^oe+Ul~-q{C%Axo1z?>#&NCVILQ>?m31J}7 pf=$tol^{@M-U-pH{0b!`L1>N{?1T07Zj`&y>NgQZaM42s008V_3Ss~N delta 548 zcmV+<0^9wx1hE8=7=OEM65SjZE{r}9#6~WLAyCxXB1(}0NhJ{s{d-AD)XNIcLl3zK z+S%FRkX-ILI8p}DGe|`R55liEi}@`IQcPI~2D7XA)%9ZbDF!_IoZOL{yZB3h0trWu zzzL{G0*i92Q$y}hL(w=k^195yWOg<@*k!b*X(nj@jA zLVutP_e_vNA%DymFi;01#fw*UneeCme)#^lN*A{uJ#Vzel&6}xNx}`+i#Z~K9~mXu z@E>|^XVQsyC0mW9YO-$>wOcokU&dt)NENX$YtHy$D}pCfyIvd} z{P2nu5fZ{R6WCfB-*;yyycdE2<`W$-l9*_R?Pk4;27fd((IM@Ir&@Elic)A!4lUO% z@LJ2?4W8lGC}kaPYDCtnF+po{;z|7PHLA+YE+VpAaNt@6;W?D$Uv zwmA?t5!*57`ISM%%fHt-vMI{3X&l!&uJif=4?eYWnX#DX@2_VED>|3uJd^SC5Z)n literal 16 XcmYc;N@ieSU}9K(P~wLDw%5!6CdvhR diff --git a/v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/rows/metadata.json.gz b/v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/rows/metadata.json.gz index 907e1eb31d9c6e3e1db6e8bd0eb240c1b5f56947..29e15e9d80d58b8e943112439dbdfeaf3e22c342 100644 GIT binary patch delta 839 zcmV-N1GxOq2G<6EABzY800000007;WQE%EX5XZkuo;EFPg#u!E0|paPTTKP+VHJvu z?GUet9oeRWRr&5aCy>NRx|V8vX&+E?cfPy-pWl`V#XZM$ z{Cj$EygMfnc&FeRMG=jta@OF&f>3M%9q;j!VRc9-nHCOzh@!>>o=>w0Qgo^14(rD( zomaJB!8O}@5XCr>2R9LlX(l*W7F=KUQukQen~kMJhr46o+Qkv$4@mkk)55c=i)n-l-1t-}wn!eNxV-51HVJa*n^=9)(plozzG%-rNc6DZGLPQkt zERw|u!up^2$cC#RX_@$PF<<$1w0phN$n?HK3e@?Dq{PgmxUV6N*uEv#cc zA1_aN$2Wy9MteVu%93WOh^?jBWg%e8VIJf^79I9~pDmi2r{UrFG+58UmHGYf{dO8& zjy@Kg=QkPx0oNMkb-tg)yU5kbbrtI-)K4(ohIC%033N?LNweiBP`0RRK-}cFh2HiQ zRc@QaJ_BqMSDP7&m3pZ9ECddw|7oA?!JtAOIX&_>sKkG1QI3yKZw45Q}@!sm>OI;BJ>a<%y z8;Em}lU)KIe>jS60Gij)l+}}zxAT+X>IW>Me(h#ADa5tj2zig>!q8s%E|SfloQ|C(bxiUENj+!gLjk@W%thy-)1$os?qS>+ z_3=fgKk5!Un-RgC0UhD)SKJS}q?dhH{JuqeODFI;RYJ`AM@PYXh&ICVuMqerCm5>Y zP+mhjCxRWXo0cCbP$Nv|eHjQj+el_AP$&x8jTTM;0hgA`BY!kd1 z8yTlcR`I{@3<+SMYg4IvXdj|-cfPyt@xg{iN1%v0p0f~y?kTzb`E40f+;v>Xzo!Ss zyK}h2y9BNg=akdLS%V8pLa_;SyvI|9)gh*2S~wtz8WVUv%_bw^7i4Q&nx|BvA^IO{3{c-7wN1-x_9urJ~+!&Lzr5XGRmD*lSm3h9)JaNMw;LiV@cT z%ttmN3F4NCFBkKbZ%4b=JB>{5E0lmbKUS2OnH2Xmq@3+ra*d~cNQ*hUvro`#3x(_lRZSLXM__uFaM8+|M|&u=sY0wG_pcaf`=>nhewsGnfE4e7j06X=?hl4i?MplngsfVjzT3%%_rs@yh-eFoSjt~N6k zF%@1cT57OZsI=1gMl~`RH@hI9s_4h|V7m5>Qwdk=Ta3fdUslKp{z z;b71kHkbWMG|85pxei7^QB+@uAdADJt6e$#C@p#;ISs*-Z*_%i9u{*H9ET zOw!)bnGhE8E%2JM>!RzD&&RY)$`eA~Be^iNSH6p6GbpEHXGtBCJVr{-nfXwFZwGVH z`P%6tOfGw5&>7LeMW^4zK?e={n-1zBibt2hAb{=Kd{_LwMPf@Q@H#@w`bQ^$_YiG_ z{g4FCZ4j+Kr8 diff --git a/v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/rows/parts/.part-0-4acd73d5-8e51-42cf-a64b-a3aec87f5fc2.crc b/v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/rows/parts/.part-0-2e25c2cc-84c0-4816-b8dc-5e8c19c4f1d2.crc similarity index 100% rename from v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/rows/parts/.part-0-4acd73d5-8e51-42cf-a64b-a3aec87f5fc2.crc rename to v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/rows/parts/.part-0-2e25c2cc-84c0-4816-b8dc-5e8c19c4f1d2.crc diff --git a/v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/rows/parts/part-0-4acd73d5-8e51-42cf-a64b-a3aec87f5fc2 b/v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/rows/parts/part-0-2e25c2cc-84c0-4816-b8dc-5e8c19c4f1d2 similarity index 100% rename from v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/rows/parts/part-0-4acd73d5-8e51-42cf-a64b-a3aec87f5fc2 rename to v03_pipeline/var/test/reference_data/test_combined_mito_1.ht/rows/parts/part-0-2e25c2cc-84c0-4816-b8dc-5e8c19c4f1d2 From a2dd6952c56f7cd93be69dda09712476caebfc31 Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Mon, 20 May 2024 15:39:16 -0400 Subject: [PATCH 08/14] update type --- .../test_combined_37.ht/.README.txt.crc | Bin 12 -> 12 bytes .../test_combined_37.ht/.metadata.json.gz.crc | Bin 16 -> 16 bytes .../test_combined_37.ht/README.txt | 4 ++-- .../.index.crc | Bin 0 -> 12 bytes .../.metadata.json.gz.crc | Bin 0 -> 12 bytes .../index | Bin 0 -> 140 bytes .../metadata.json.gz | Bin 0 -> 187 bytes .../.index.crc | Bin 12 -> 0 bytes .../.metadata.json.gz.crc | Bin 12 -> 0 bytes .../index | Bin 137 -> 0 bytes .../metadata.json.gz | Bin 186 -> 0 bytes .../test_combined_37.ht/metadata.json.gz | Bin 703 -> 703 bytes .../rows/.metadata.json.gz.crc | Bin 20 -> 20 bytes .../test_combined_37.ht/rows/metadata.json.gz | Bin 1029 -> 1027 bytes ...0-ac85fcb0-1e7c-453f-be81-9cd356dc49ff.crc | Bin 0 -> 12 bytes ...0-da748236-79a9-461e-a62a-a9280e863d48.crc | Bin 12 -> 0 bytes ...art-0-ac85fcb0-1e7c-453f-be81-9cd356dc49ff | Bin 0 -> 210 bytes ...art-0-da748236-79a9-461e-a62a-a9280e863d48 | Bin 212 -> 0 bytes 18 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 v03_pipeline/var/test/reference_data/test_combined_37.ht/index/part-0-ac85fcb0-1e7c-453f-be81-9cd356dc49ff.idx/.index.crc create mode 100644 v03_pipeline/var/test/reference_data/test_combined_37.ht/index/part-0-ac85fcb0-1e7c-453f-be81-9cd356dc49ff.idx/.metadata.json.gz.crc create mode 100644 v03_pipeline/var/test/reference_data/test_combined_37.ht/index/part-0-ac85fcb0-1e7c-453f-be81-9cd356dc49ff.idx/index create mode 100644 v03_pipeline/var/test/reference_data/test_combined_37.ht/index/part-0-ac85fcb0-1e7c-453f-be81-9cd356dc49ff.idx/metadata.json.gz delete mode 100644 v03_pipeline/var/test/reference_data/test_combined_37.ht/index/part-0-da748236-79a9-461e-a62a-a9280e863d48.idx/.index.crc delete mode 100644 v03_pipeline/var/test/reference_data/test_combined_37.ht/index/part-0-da748236-79a9-461e-a62a-a9280e863d48.idx/.metadata.json.gz.crc delete mode 100644 v03_pipeline/var/test/reference_data/test_combined_37.ht/index/part-0-da748236-79a9-461e-a62a-a9280e863d48.idx/index delete mode 100644 v03_pipeline/var/test/reference_data/test_combined_37.ht/index/part-0-da748236-79a9-461e-a62a-a9280e863d48.idx/metadata.json.gz create mode 100644 v03_pipeline/var/test/reference_data/test_combined_37.ht/rows/parts/.part-0-ac85fcb0-1e7c-453f-be81-9cd356dc49ff.crc delete mode 100644 v03_pipeline/var/test/reference_data/test_combined_37.ht/rows/parts/.part-0-da748236-79a9-461e-a62a-a9280e863d48.crc create mode 100644 v03_pipeline/var/test/reference_data/test_combined_37.ht/rows/parts/part-0-ac85fcb0-1e7c-453f-be81-9cd356dc49ff delete mode 100644 v03_pipeline/var/test/reference_data/test_combined_37.ht/rows/parts/part-0-da748236-79a9-461e-a62a-a9280e863d48 diff --git a/v03_pipeline/var/test/reference_data/test_combined_37.ht/.README.txt.crc b/v03_pipeline/var/test/reference_data/test_combined_37.ht/.README.txt.crc index 80764f4193efa77482e95ab809e6fea8bf42f2fb..1b96b5393db01d08490ce1d23fd7ed5e96f1950b 100644 GIT binary patch literal 12 TcmYc;N@ieSU}D&F?e8W46`llq literal 12 TcmYc;N@ieSU}9J#*}ew=5*-6M diff --git a/v03_pipeline/var/test/reference_data/test_combined_37.ht/.metadata.json.gz.crc b/v03_pipeline/var/test/reference_data/test_combined_37.ht/.metadata.json.gz.crc index d542606ef0674e76f6773c168911494435e5aae9..82e0d403587f5270439f388cc565f7e12900c5f4 100644 GIT binary patch literal 16 XcmYc;N@ieSU}9MRJ9@2H-S-dxC|Cu| literal 16 XcmYc;N@ieSU}AWC#@PR7RAM^-C}0J# diff --git a/v03_pipeline/var/test/reference_data/test_combined_37.ht/README.txt b/v03_pipeline/var/test/reference_data/test_combined_37.ht/README.txt index e4ab633c1..e38d73d71 100644 --- a/v03_pipeline/var/test/reference_data/test_combined_37.ht/README.txt +++ b/v03_pipeline/var/test/reference_data/test_combined_37.ht/README.txt @@ -1,3 +1,3 @@ This folder comprises a Hail (www.hail.is) native Table or MatrixTable. - Written with version 0.2.128-eead8100a1c1 - Created at 2024/03/21 11:28:13 \ No newline at end of file + Written with version 0.2.130-bea04d9c79b5 + Created at 2024/05/20 15:38:26 \ No newline at end of file diff --git a/v03_pipeline/var/test/reference_data/test_combined_37.ht/index/part-0-ac85fcb0-1e7c-453f-be81-9cd356dc49ff.idx/.index.crc b/v03_pipeline/var/test/reference_data/test_combined_37.ht/index/part-0-ac85fcb0-1e7c-453f-be81-9cd356dc49ff.idx/.index.crc new file mode 100644 index 0000000000000000000000000000000000000000..26d303267bcec7b683e6e888f9c63277957dc30d GIT binary patch literal 12 TcmYc;N@ieSU}8vbF)9N95rP8F literal 0 HcmV?d00001 diff --git a/v03_pipeline/var/test/reference_data/test_combined_37.ht/index/part-0-ac85fcb0-1e7c-453f-be81-9cd356dc49ff.idx/.metadata.json.gz.crc b/v03_pipeline/var/test/reference_data/test_combined_37.ht/index/part-0-ac85fcb0-1e7c-453f-be81-9cd356dc49ff.idx/.metadata.json.gz.crc new file mode 100644 index 0000000000000000000000000000000000000000..0e401dc36c7de7707943d53d64b4b45ce2bde5d0 GIT binary patch literal 12 TcmYc;N@ieSU}AXgQ?3XA6AuFh literal 0 HcmV?d00001 diff --git a/v03_pipeline/var/test/reference_data/test_combined_37.ht/index/part-0-ac85fcb0-1e7c-453f-be81-9cd356dc49ff.idx/index b/v03_pipeline/var/test/reference_data/test_combined_37.ht/index/part-0-ac85fcb0-1e7c-453f-be81-9cd356dc49ff.idx/index new file mode 100644 index 0000000000000000000000000000000000000000..df93a68fe874c33238376176adcc0abbed9d5a6f GIT binary patch literal 140 zcmYdgU|?7V#2Q=m|0*my%EYjQnSp_ufzj~kG*c!9Mn@(`M+RpZcZ-k^M&|-X_w9^~ zA$J&=9Gw{*8=06Loq_aR7Dwk@tnQ4CH<)-B6gXCR8~4mSYT?SGz>>4*s$9b11|V#n W?JW%QVWB>pF diff --git a/v03_pipeline/var/test/reference_data/test_combined_37.ht/index/part-0-da748236-79a9-461e-a62a-a9280e863d48.idx/metadata.json.gz b/v03_pipeline/var/test/reference_data/test_combined_37.ht/index/part-0-da748236-79a9-461e-a62a-a9280e863d48.idx/metadata.json.gz deleted file mode 100644 index aaacf6d649dcba0682cabdbd0990b089b058efdf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 186 zcmV;r07d^FiwFP!0000009B5`3c@fDME_+^3OTe|s^%tw9uyQ6FXADs+a?$iNw#1p z{dc$CybKF7`(}D*jKv#6+dM z5-t?FntJQ&F3Q8Glihiv=ZmRO)}qBI*CD8M=O7aw0|-#gc~Zoc|B%shdrT2zo(nMU oiJy}q&57U5cJQ8#O%suKB;2e zYKvX#!SU;Wys=hsuFx$w+p&E;?;Z*r=uzx z2{iyQuEMXZyg)%HCC;B}<(tAeN!`LR+Fm7Lgiu0t-sk#QDUjNHv0=~d>y#~SGmXot z2dp$mluEN*pI{i~B(0l2ArGFIAer(pW57Udd?|k0epMz|mcQ?=RwtR`!v%`&qucPiP5a@Z zTh34z|7cX2;2N|Oh{Vk#)JAt5>5Y5tZgJN(X%zl>H0Ez_5(*y7cx_%5-gXAk?-{~z1^Ntabq=Ez2$_tKO{e)FZj+63fTRmRRY{@zW zQoWAtlW6M;gNomO&#Q_JDaB^ms(Gu%)ol>j)U3)3o7wuOv$IXvNQ$(aTMkB@!v(c2LU=mmr%oO!|d#o6@I_YRoQF6f<}Oiw1$ z%P#;pxtyGy;z>O5d{4j*1h5z+5agrFDu)QBs!d|ejZn9-^mst=H##5Xic;hL^K~O{7XqvUCOpp{S zK$zHw%qWT7fZ|ySmR#vK+Wd#if#vClwz12OTjMtDYH!4^7j_ud@rGmV^jX%i9#0c$ z0smAEECVxo#Z*2hwUQ2NA%J-gE}E$(uKP1Pfo z3Ixi8Uak+&2)&b5&0kOi&y>fx^f9GCL2i7(Uv0lC6D-a5)Aee0IJ@|4>jnhflZb9s z^Q&M%IaX)W(Kc)RZ4%rvo%Xd}J1v~jT_Q&S;;t}K2ayh*GgaH=#@}A(olM?IQcc2gMxx2)D+oUm^zLpz$SfT6X!?$qR;ubpTwCUFWXomjn zn$CRQMYB%o@uhaL;u+8BWd9wtBjn*^+ez zarHX3Pok|ab(FvUJ+3O&pbYD2tLCj5SGPfAQ?n{FY-a19CzFk7BzR^f7kVw)U<2w` lB}D3YY?RAn)wFd!9$*XGf3-9M72x22a3NkMGwB&@;z%{LgT7 zypK*8(}Tn_2}1}$Ivc#AF$1a<9q+mH@Ni6kT~`;V1zLnRZ>AF>VXe{wRnNIQ&y>(u zD8BPhiwlw--6WP=@mQc_F4XWWb5EhY={QZv@#9Hx<5I$yDlpzt_4#l>z()IQ#U;1n~c16t;Zp>y=P;pQMzmGj2k#UWcOlFCxy zB|{=5q`F{~)ivgK8&@8-5myRQGnumdSg^cP?Yqncj-jmPOtr*M_9-xmAT{T29$Y>`+i~GrVQQOY&*7Rj0Q-#Nl{6F!d#P0g-^4fSQNy;Olz$e^Nk~B`iP?HJH zZA*(msEBXtmZ~^168=7bz~^cLGgrk)1U8iFdb|5!bfKPr%w=04gTw~hYWg@`oDzCd zxmwR>56vu>ad=GjAkO@o$MLFpdwWnskNX3W2navHLDof-kjpgZd^vHB#WoB5|J#Sv z^?rEyd&MRNWKa=-f8r90Mp`V=Eh^$BPWjr?EXrZy24E~dtng9&!cR}Am zeFx|}@cwnU+i0(WUg5k|Z(U~fCM%40pt}RxUxoHJkZs_4dk6wZ8!N|RIvJc@P3kMx zp?e-OLuG;t@EOP4Qrc?an*W0e7o*YeqHXqRddX_kFfV2K z&Xq!_Nd)C@-~N%SYiS51$TzF+TDMH<4R|UQw(_Z9$%}Irw>pW3j>QIVyq~qIWo{e< z5>%B25h5<$xg9ghHeEbk*I`s)(W}n>wWMEV)yk=n%`Y;2h%G_FI~{ z@^SS89)jO^+3hJTBtLiJiWn&%)3EN0UO9wS`PSgI%Wf9kEcx|g+EvO6h5U+>i%t8% zwJz^L-5uLY=4j^$N!5(&FI)WB)rVfcN9ffkpxb_L0GG6PJ{ktSEnE$H?7BQ9uoV6ZO2V|`G`rrMd7WIAJ_RH}Vu*}v%0)A;-k005XO38er4 literal 1029 zcmV+g1p50QiwFP!000000Nqz@Z<{(4{x5vG)5w;WE=~ECbYzW`woy`5)14yZ;zQyM z48zXU+=~Bxk0F2!bXzCwUTOId;p6k}^E|SjEKVY@9BaWTJOW+czuzR_4=vmBzQECP zK3QSx9|W#R7(xib+TxZUQ}C6dZmwY<>`GkeG5h)@q6;|N3}0WI21?XCRM zLzKCCw`=di@^&^|*0yup)qQEnRN=8B|4;lVF}r@fy3`IONm)cBc!U{B!jDtX)I@@F z+tQ*CO5&Nir7Dh$fUgJO_+3q4X3IEK_i9sVR7HRUw59y?=qxd`?8*ntx*i8oiNlZlm8uq)O zZ=k*d^c{HrKHN>T*Fdju-m14QvpUHN;~nVk!1i~c{R3ocxPCYU0i=y9$I_h*hLc&n zWn2E9g;Z0SAp?BDF*lSRlyJlTL51~A4yZu&wy>6hcKJD1YcJL((G`!$}_| z;}JcZ!xMJKKdg*~Pp6qOF)%tZIXc!ea9vlnVP-z6U|`GO zyx2WN`2eFcgR!wRP-dF*X?q1J#_N{C&p%lj8#6SVV`DhL%x?MZ2A@1b_{mom?U9Eq z7#R8wef_}b&M@!2;D-=~o99HC9Gw|~x>=YVomm{6S>1sa*g`E}5Mgj?ZFs;Y%D3Q~ zfX<$Z1cSv4C%D-57p@U1I=Jt#A?ty(1uQJ=PHZ6#1(lUnjJ!Y(fB?t?3JgG>000xF BKBfQw literal 0 HcmV?d00001 diff --git a/v03_pipeline/var/test/reference_data/test_combined_37.ht/rows/parts/part-0-da748236-79a9-461e-a62a-a9280e863d48 b/v03_pipeline/var/test/reference_data/test_combined_37.ht/rows/parts/part-0-da748236-79a9-461e-a62a-a9280e863d48 deleted file mode 100644 index df40d6fa16824219422825070637a5072157f50c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 212 zcmdnbz`(GCk%2*DtN!1FB@Df+3|H70|FAL|KAmRD#K7ptz;#{OhM75QPswBn zmSy$~&Wqhcln*dEGZ-6717)Wvv!gSMqcf{J&=y-}1~G<&s~X7% zgj`LVH4Hzj<2zVT)_9EVfk@K6eYt1DPsFi2p!(XF^zofqf@5CC~W IfdS|j0Mh6}+5i9m From 3573cb7ff62117721df9178a1701f5d412ed9ac1 Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Mon, 20 May 2024 23:57:26 -0400 Subject: [PATCH 09/14] more tweaks --- .../lib/reference_data/compare_globals.py | 41 ++++++------------- .../reference_data/compare_globals_test.py | 25 ++--------- ...ble_with_updated_reference_dataset_test.py | 2 +- 3 files changed, 16 insertions(+), 52 deletions(-) diff --git a/v03_pipeline/lib/reference_data/compare_globals.py b/v03_pipeline/lib/reference_data/compare_globals.py index ca4578867..7a1a35140 100644 --- a/v03_pipeline/lib/reference_data/compare_globals.py +++ b/v03_pipeline/lib/reference_data/compare_globals.py @@ -87,40 +87,23 @@ def get_datasets_to_update( ) -> list[str]: datasets_to_update = set() for field in dataclasses.fields(Globals): - if field.name == 'selects': + if field.name == 'selects' and not validate_selects: continue datasets_to_update.update( ht1_globals[field.name].keys() ^ ht2_globals[field.name].keys(), ) for dataset in ht1_globals[field.name].keys() & ht2_globals[field.name].keys(): - if ht1_globals[field.name].get(dataset) != ht2_globals[field.name].get( - dataset, - ): - logger.info(f'{field.name} mismatch for {dataset}') - datasets_to_update.add(dataset) - - # Selects are a special case and are handled separately - if validate_selects: - ht1_selects, ht2_selects = ht1_globals['selects'], ht2_globals['selects'] - datasets_to_update.update(ht1_selects.keys() ^ ht2_globals[field.name].keys()) - for dataset in ht1_selects.keys() & ht2_selects.keys(): - # Special integrity check to ensure that fields do not change - # without an explicit configuration change. Because we check - # that the dataset isn't already "to_update", this must run - # after all other fields have been validated. - if ( - dataset not in datasets_to_update - and (ht1_selects.get(dataset).keys() == ht2_selects.get(dataset).keys()) - and (ht1_selects.get(dataset) != ht2_selects.get(dataset)) - ): - differing_items = ( - ht2_selects.get(dataset).items() - ht1_selects.get(dataset).items() - ) - msg = f'Unexpected field types detected in {dataset}: {differing_items}' - raise ValueError(msg) - if ht1_globals[field.name].get(dataset) != ht2_globals[field.name].get( - dataset, - ): + if field.name == 'selects': + # Assert that all shared annotations have identical types + shared_annotations = ht1_globals['selects'][dataset].keys() & ht2_globals['selects'].get(dataset).keys() + mismatched_annotations = [] + for annotation in shared_annotations: + if ht1_globals['selects'][dataset][annotation] != ht2_globals['selects'][dataset][annotation]: + mismatched_annotations.append((annotation, ht2_globals['selects'][dataset][annotation])) + if mismatched_annotations: + msg = f'Unexpected field types detected in {dataset}: {mismatched_annotations}' + raise ValueError(msg) + if ht1_globals[field.name][dataset] != ht2_globals[field.name][dataset]: logger.info(f'{field.name} mismatch for {dataset}') datasets_to_update.add(dataset) return sorted(datasets_to_update) diff --git a/v03_pipeline/lib/reference_data/compare_globals_test.py b/v03_pipeline/lib/reference_data/compare_globals_test.py index 39fcdaf8e..c6751a81a 100644 --- a/v03_pipeline/lib/reference_data/compare_globals_test.py +++ b/v03_pipeline/lib/reference_data/compare_globals_test.py @@ -286,14 +286,13 @@ def test_get_datasets_to_update_select_different(self): def test_get_datasets_to_update_select_type_validation(self): self.assertRaisesRegex( ValueError, - "Unexpected field types detected in a: {\\('field1', dtype\\('int32'\\)\\)}", + "Unexpected field types detected in a: \\[\\('field1', dtype\\('int32'\\)\\)\\]", get_datasets_to_update, ht1_globals=Globals( paths={'a': 'a_path'}, versions={'a': 'v1'}, enums={'a': {}}, selects={ - # field1 is an array in ht1 but an int in ht2. 'a': {'field1': hl.tarray(hl.tint32)}, }, ), @@ -301,27 +300,9 @@ def test_get_datasets_to_update_select_type_validation(self): paths={'a': 'a_path'}, versions={'a': 'v1'}, enums={'a': {}}, - selects={'a': {'field1': hl.tint32}}, - ), - ) - result = get_datasets_to_update( - ht1_globals=Globals( - paths={'a': 'a_path'}, - versions={'a': 'v1'}, - enums={'a': {}}, - selects={ - 'a': {'field1': hl.tarray(hl.tint32)}, - }, - ), - ht2_globals=Globals( - paths={'a': 'a_path'}, - versions={'a': 'v1'}, - enums={'a': {}}, - # additional field selects={'a': {'field1': hl.tint32, 'field2': hl.tint32}}, ), ) - self.assertTrue(result == ['a']) result = get_datasets_to_update( ht1_globals=Globals( paths={'a': 'a_path'}, @@ -333,9 +314,9 @@ def test_get_datasets_to_update_select_type_validation(self): ), ht2_globals=Globals( paths={'a': 'a_path'}, - versions={'a': 'v2'}, # version bump + versions={'a': 'v1'}, enums={'a': {}}, - selects={'a': {'field1': hl.tint32}}, + selects={'a': {'field1': hl.tarray(hl.tint32), 'field2': hl.tint32}}, ), ) self.assertTrue(result == ['a']) diff --git a/v03_pipeline/lib/tasks/reference_data/update_variant_annotations_table_with_updated_reference_dataset_test.py b/v03_pipeline/lib/tasks/reference_data/update_variant_annotations_table_with_updated_reference_dataset_test.py index f3a62969d..404ff93b1 100644 --- a/v03_pipeline/lib/tasks/reference_data/update_variant_annotations_table_with_updated_reference_dataset_test.py +++ b/v03_pipeline/lib/tasks/reference_data/update_variant_annotations_table_with_updated_reference_dataset_test.py @@ -760,7 +760,7 @@ def test_update_vat_with_updated_rdc_snv_indel_38( conditions=None, ), dbnsfp=hl.Struct( - REVEL_score=0.043, + REVEL_score=hl.eval(hl.float32(0.043)), SIFT_score=None, Polyphen2_HVAR_score=None, MutationTaster_pred_id=0, From 6a6373d5be6e658c2292a030e16cf1685463cfe4 Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Tue, 21 May 2024 00:09:57 -0400 Subject: [PATCH 10/14] lint --- .../lib/reference_data/compare_globals.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/v03_pipeline/lib/reference_data/compare_globals.py b/v03_pipeline/lib/reference_data/compare_globals.py index 7a1a35140..d6b91bc4e 100644 --- a/v03_pipeline/lib/reference_data/compare_globals.py +++ b/v03_pipeline/lib/reference_data/compare_globals.py @@ -95,11 +95,18 @@ def get_datasets_to_update( for dataset in ht1_globals[field.name].keys() & ht2_globals[field.name].keys(): if field.name == 'selects': # Assert that all shared annotations have identical types - shared_annotations = ht1_globals['selects'][dataset].keys() & ht2_globals['selects'].get(dataset).keys() - mismatched_annotations = [] - for annotation in shared_annotations: - if ht1_globals['selects'][dataset][annotation] != ht2_globals['selects'][dataset][annotation]: - mismatched_annotations.append((annotation, ht2_globals['selects'][dataset][annotation])) + shared_annotations = ( + ht1_globals['selects'][dataset].keys() + & ht2_globals['selects'].get(dataset).keys() + ) + mismatched_annotations = [ + (annotation, ht2_globals['selects'][dataset][annotation]) + for annotation in shared_annotations + if ( + ht1_globals['selects'][dataset][annotation] + != ht2_globals['selects'][dataset][annotation] + ) + ] if mismatched_annotations: msg = f'Unexpected field types detected in {dataset}: {mismatched_annotations}' raise ValueError(msg) From d04325ece5722b1dac73fe053f6fe3d659c5fd5e Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Tue, 21 May 2024 08:50:35 -0400 Subject: [PATCH 11/14] fix floats --- ...t_annotations_table_with_updated_reference_dataset_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v03_pipeline/lib/tasks/reference_data/update_variant_annotations_table_with_updated_reference_dataset_test.py b/v03_pipeline/lib/tasks/reference_data/update_variant_annotations_table_with_updated_reference_dataset_test.py index 404ff93b1..67a5492bf 100644 --- a/v03_pipeline/lib/tasks/reference_data/update_variant_annotations_table_with_updated_reference_dataset_test.py +++ b/v03_pipeline/lib/tasks/reference_data/update_variant_annotations_table_with_updated_reference_dataset_test.py @@ -760,7 +760,7 @@ def test_update_vat_with_updated_rdc_snv_indel_38( conditions=None, ), dbnsfp=hl.Struct( - REVEL_score=hl.eval(hl.float32(0.043)), + REVEL_score=0.0430000014603138, SIFT_score=None, Polyphen2_HVAR_score=None, MutationTaster_pred_id=0, @@ -1168,7 +1168,7 @@ def test_update_vat_with_updated_rdc_snv_indel_37( conditions=None, ), dbnsfp=hl.Struct( - REVEL_score=0.043, + REVEL_score=0.0430000014603138, SIFT_score=None, Polyphen2_HVAR_score=None, MutationTaster_pred_id=0, From d3ee4fa35e3310a4b2917236abf2924b17a82918 Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Tue, 21 May 2024 09:00:38 -0400 Subject: [PATCH 12/14] decompose --- .../lib/reference_data/compare_globals.py | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/v03_pipeline/lib/reference_data/compare_globals.py b/v03_pipeline/lib/reference_data/compare_globals.py index d6b91bc4e..5bfcf9ab9 100644 --- a/v03_pipeline/lib/reference_data/compare_globals.py +++ b/v03_pipeline/lib/reference_data/compare_globals.py @@ -80,6 +80,27 @@ def from_ht( return cls(paths, versions, enums, selects) +def validate_selects_types( + ht1_globals: Globals, ht2_globals: Globals, dataset: str +) -> None: + # Assert that all shared annotations have identical types + shared_selects = ( + ht1_globals['selects'][dataset].keys() + & ht2_globals['selects'].get(dataset).keys() + ) + mismatched_select_types = [ + (select, ht2_globals['selects'][dataset][select]) + for select in shared_selects + if ( + ht1_globals['selects'][dataset][select] + != ht2_globals['selects'][dataset][select] + ) + ] + if mismatched_select_types: + msg = f'Unexpected field types detected in {dataset}: {mismatched_select_types}' + raise ValueError(msg) + + def get_datasets_to_update( ht1_globals: Globals, ht2_globals: Globals, @@ -94,22 +115,7 @@ def get_datasets_to_update( ) for dataset in ht1_globals[field.name].keys() & ht2_globals[field.name].keys(): if field.name == 'selects': - # Assert that all shared annotations have identical types - shared_annotations = ( - ht1_globals['selects'][dataset].keys() - & ht2_globals['selects'].get(dataset).keys() - ) - mismatched_annotations = [ - (annotation, ht2_globals['selects'][dataset][annotation]) - for annotation in shared_annotations - if ( - ht1_globals['selects'][dataset][annotation] - != ht2_globals['selects'][dataset][annotation] - ) - ] - if mismatched_annotations: - msg = f'Unexpected field types detected in {dataset}: {mismatched_annotations}' - raise ValueError(msg) + validate_selects_types(ht1_globals, ht2_globals, dataset) if ht1_globals[field.name][dataset] != ht2_globals[field.name][dataset]: logger.info(f'{field.name} mismatch for {dataset}') datasets_to_update.add(dataset) From 9edfc4f872bf36e718e00a90d513739b9f2614f3 Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Tue, 21 May 2024 16:57:04 -0400 Subject: [PATCH 13/14] ruff formatg --- v03_pipeline/lib/reference_data/compare_globals.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/v03_pipeline/lib/reference_data/compare_globals.py b/v03_pipeline/lib/reference_data/compare_globals.py index 5bfcf9ab9..1feb0ac12 100644 --- a/v03_pipeline/lib/reference_data/compare_globals.py +++ b/v03_pipeline/lib/reference_data/compare_globals.py @@ -81,7 +81,9 @@ def from_ht( def validate_selects_types( - ht1_globals: Globals, ht2_globals: Globals, dataset: str + ht1_globals: Globals, + ht2_globals: Globals, + dataset: str, ) -> None: # Assert that all shared annotations have identical types shared_selects = ( From 093890ecee2b5b6e7ac697ac9cb45635752f83ca Mon Sep 17 00:00:00 2001 From: Benjamin Blankenmeister Date: Thu, 23 May 2024 10:36:51 -0400 Subject: [PATCH 14/14] Update compare_globals_test.py --- v03_pipeline/lib/reference_data/compare_globals_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/v03_pipeline/lib/reference_data/compare_globals_test.py b/v03_pipeline/lib/reference_data/compare_globals_test.py index c6751a81a..786964fcb 100644 --- a/v03_pipeline/lib/reference_data/compare_globals_test.py +++ b/v03_pipeline/lib/reference_data/compare_globals_test.py @@ -194,7 +194,6 @@ def test_from_rdc_or_annotations_ht(self): self.assertTrue( rdc_globals.enums == {'screen': {'region_type': ['C', 'D']}}, ) - print(rdc_globals.selects) self.assertTrue( rdc_globals.selects == {