Merge pull request #76 from databio/dev

Release 0.4.1

Author: khoroshevskyi

.gitignore

@@ -151,3 +151,8 @@ data/baa91c8f6e2780cfd8fd1040ff37f51c379947a2a4820d6c/fasta/default/baa91c8f6e27
 data/baa91c8f6e2780cfd8fd1040ff37f51c379947a2a4820d6c/fasta/default/baa91c8f6e2780cfd8fd1040ff37f51c379947a2a4820d6c.fa
 data/baa91c8f6e2780cfd8fd1040ff37f51c379947a2a4820d6c/fasta/default/baa91c8f6e2780cfd8fd1040ff37f51c379947a2a4820d6c.fa.fai
 test/Untitled.ipynb
+/scripts/ref_genome_validating/genome_folder/
+/scripts/ref_genome_validating/data/
+/scripts/ref_genome_validating/results/
+/scripts/ref_genome_validating/stats_results/results.yaml
+/scripts/ref_genome_validating/stats_results/results_backup_11sep2024.yaml

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 repos:
   # Using this mirror lets us use mypyc-compiled black, which is about 2x faster
   - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 24.1.1
+    rev: 24.8.0
     hooks:
       - id: black
         # It is recommended to specify the latest version of Python

MANIFEST.in

@@ -9,5 +9,8 @@ include bedboss/qdrant_index/*
 include bedboss/bedbuncher/*
 include bedboss/bedbuncher/tools/*
 include bedboss/bedclassifier/*
+include bedboss/refgenome_validator/*
 include bedboss/tokens/*
-include bedboss/bbuploader/*
+include bedboss/tokens/*
+include bedboss/bbuploader/*
+include bedboss/refgenome_validator/chrom_sizes/*

bedboss/_version.py

@@ -1 +1 @@
-__version__ = "0.4.0"
+__version__ = "0.5.0"

bedboss/bbuploader/cli.py

@@ -41,6 +41,9 @@ def upload_all(
         True, help="Run skipped projects. [Default: False]"
     ),
     run_failed: bool = typer.Option(True, help="Run failed projects. [Default: False]"),
+    standardize_pep: bool = typer.Option(
+        False, help="Standardize pep with BEDMESS. [Default: False]"
+    ),
 ):
     from .main import upload_all as upload_all_function
 
@@ -57,6 +60,7 @@ def upload_all(
         rerun=rerun,
         run_skipped=run_skipped,
         run_failed=run_failed,
+        standardize_pep=standardize_pep,
     )
 
 
@@ -79,6 +83,9 @@ def upload_gse(
         True, help="Run skipped projects. [Default: False]"
     ),
     run_failed: bool = typer.Option(True, help="Run failed projects. [Default: False]"),
+    standardize_pep: bool = typer.Option(
+        False, help="Standardize pep with BEDMESS. [Default: False]"
+    ),
 ):
     from .main import upload_gse as upload_gse_function
 
@@ -91,6 +98,7 @@ def upload_gse(
         rerun=rerun,
         run_skipped=run_skipped,
         run_failed=run_failed,
+        standardize_pep=standardize_pep,
     )
 
 

bedboss/bbuploader/main.py

@@ -6,6 +6,7 @@
 from bbconf import BedBaseAgent
 from bbconf.db_utils import GeoGseStatus, GeoGsmStatus
 from pephubclient import PEPHubClient
+from pephubclient.helpers import MessageHandler
 from pephubclient.models import SearchReturnModel
 from sqlalchemy import and_, select
 from sqlalchemy.orm import Session
@@ -19,7 +20,7 @@
 from bedboss.bedboss import run_all
 from bedboss.bedbuncher.bedbuncher import run_bedbuncher
 from bedboss.exceptions import BedBossException
-from bedboss.utils import standardize_genome_name
+from bedboss.utils import standardize_genome_name, standardize_pep as pep_standardizer
 
 _LOGGER = logging.getLogger(PKG_NAME)
 _LOGGER.setLevel(logging.DEBUG)
@@ -36,8 +37,9 @@ def upload_all(
     genome: str = None,
     create_bedset: bool = True,
     rerun: bool = False,
-    run_skipped=False,
-    run_failed=True,
+    run_skipped: bool = False,
+    run_failed: bool = True,
+    standardize_pep: bool = False,
 ):
     """
     This is main function that is responsible for processing bed files from PEPHub.
@@ -54,6 +56,7 @@ def upload_all(
     :param rerun: rerun processing of the series
     :param run_skipped: rerun files that were skipped
     :param run_failed: rerun failed files
+    :param standardize_pep: standardize pep metadata using BEDMS
     """
 
     phc = PEPHubClient()
@@ -76,10 +79,14 @@ def upload_all(
     _LOGGER.info(f"found {pep_annotation_list.count} projects")
 
     count = 0
+    total_projects = len(pep_annotation_list.results)
     for gse_pep in pep_annotation_list.results:
-
+        count += 1
         with Session(bbagent.config.db_engine.engine) as session:
-            _LOGGER.info(f"Processing: '{gse_pep.name}'")
+            MessageHandler.print_success(f"{'##' * 30}")
+            MessageHandler.print_success(
+                f"#### Processing: '{gse_pep.name}'. #### Processing {count} / {total_projects}. ####"
+            )
 
             gse_status = session.scalar(
                 select(GeoGseStatus).where(GeoGseStatus.gse == gse_pep.name)
@@ -122,6 +129,8 @@ def upload_all(
                     genome=genome,
                     sa_session=session,
                     gse_status_sa_model=gse_status,
+                    standardize_pep=standardize_pep,
+                    rerun=rerun,
                 )
             except Exception as err:
                 _LOGGER.error(
@@ -134,7 +143,6 @@ def upload_all(
             status_parser(gse_status, upload_result)
             session.commit()
 
-            count += 1
             if count >= download_limit:
                 break
 
@@ -244,8 +252,9 @@ def upload_gse(
     create_bedset: bool = True,
     genome: str = None,
     rerun: bool = False,
-    run_skipped=False,
-    run_failed=True,
+    run_skipped: bool = False,
+    run_failed: bool = True,
+    standardize_pep: bool = False,
 ):
     """
     Upload bed files from GEO series to BedBase
@@ -258,6 +267,7 @@ def upload_gse(
     :param rerun: rerun processing of the series
     :param run_skipped: rerun files that were skipped
     :param run_failed: rerun failed files
+    :param standardize_pep: standardize pep metadata using BEDMS
 
     :return: None
     """
@@ -302,6 +312,8 @@ def upload_gse(
                 genome=genome,
                 sa_session=session,
                 gse_status_sa_model=gse_status,
+                standardize_pep=standardize_pep,
+                rerun=rerun,
             )
         except Exception as e:
             _LOGGER.error(f"Processing of '{gse}' failed with error: {e}")
@@ -347,6 +359,8 @@ def _upload_gse(
     genome: str = None,
     sa_session: Session = None,
     gse_status_sa_model: GeoGseStatus = None,
+    standardize_pep: bool = False,
+    rerun: bool = False,
 ) -> ProjectProcessingStatus:
     """
     Upload bed files from GEO series to BedBase
@@ -358,6 +372,8 @@ def _upload_gse(
     :param genome: reference genome to upload to database. If None, all genomes will be processed
     :param sa_session: opened session to the database
     :param gse_status_sa_model: sqlalchemy model for project status
+    :param standardize_pep: standardize pep metadata using BEDMS
+    :param rerun: force overwrite data in the database
 
     :return: None
     """
@@ -371,12 +387,14 @@ def _upload_gse(
 
     project = phc.load_project(f"bedbase/{gse}:{DEFAULT_GEO_TAG}")
 
+    if standardize_pep:
+        project = pep_standardizer(project)
+
     project_status = ProjectProcessingStatus(number_of_samples=len(project.samples))
     uploaded_files = []
     gse_status_sa_model.number_of_files = len(project.samples)
     sa_session.commit()
     for project_sample in project.samples:
-
         sample_gsm = project_sample.get("sample_geo_accession", "").lower()
 
         required_metadata = process_pep_sample(
@@ -435,7 +453,7 @@ def _upload_gse(
                 upload_pephub=True,
                 upload_s3=True,
                 upload_qdrant=True,
-                force_overwrite=False,
+                force_overwrite=rerun,
             )
             uploaded_files.append(file_digest)
             sample_status.status = STATUS.SUCCESS
@@ -449,7 +467,6 @@ def _upload_gse(
         sa_session.commit()
 
     if create_bedset and uploaded_files:
-
         _LOGGER.info(f"Creating bedset for: '{gse}'")
         run_bedbuncher(
             bedbase_config=bedbase_config,
@@ -470,41 +487,3 @@ def _upload_gse(
 
     _LOGGER.info(f"Processing of '{gse}' is finished with success!")
     return project_status
-
-
-#
-# if __name__ == "__main__":
-#     # upload_gse(
-#     #     # gse="gse246900",
-#     #     # gse="gse247593",
-#     #     # gse="gse241222",
-#     #     #gse="gse266130",
-#     #     gse="gse99178",
-#     #     # gse="gse240325", # TODO: check if qc works
-#     #     # gse="gse229592", # mice
-#     #     bedbase_config="/home/bnt4me/virginia/repos/bbuploader/config_db_local.yaml",
-#     #     outfolder="/home/bnt4me/virginia/repos/bbuploader/data",
-#     #     # genome="HG38",
-#     #     # rerun=True,
-#     #     run_failed=True,
-#     #     run_skipped=True,
-#     # )
-#     upload_all(
-#         bedbase_config="/home/bnt4me/virginia/repos/bbuploader/config_db_local.yaml",
-#         outfolder="/home/bnt4me/virginia/repos/bbuploader/data",
-#         start_date="2024/01/21",
-#         end_date="2024/08/28",
-#         search_limit=2,
-#         search_offset=0,
-#         genome="GRCh38",
-#         rerun=True,
-#     )
-# # upload_all(
-# #     bedbase_config="/home/bnt4me/virginia/repos/bbuploader/config_db_local.yaml",
-# #     outfolder="/home/bnt4me/virginia/repos/bbuploader/data",
-# #     start_date="2024/01/01",
-# #     # end_date="2024/03/28",
-# #     search_limit=200,
-# #     search_offset=0,
-# #     genome="GRCh38",
-# # )

bedboss/bbuploader/models.py

@@ -5,12 +5,12 @@
 
 class BedBossMetadata(BaseModel):
     genome: str = Field(None, alias="ref_genome")
-    organism: Optional[str] = Field("", alias="sample_organism_ch1")
+    species_name: Optional[str] = Field("", alias="sample_organism_ch1")
     species_id: Optional[str] = Field("", alias="sample_taxid_ch1")
     cell_type: Optional[str] = ""
     cell_line: Optional[str] = ""
     genotype: Optional[str] = ""
-    exp_protocol: Optional[str] = Field("", alias="sample_library_strategy")
+    assay: Optional[str] = Field("", alias="sample_library_strategy")
     library_source: Optional[str] = Field("", alias="sample_library_source")
     target: Optional[str] = Field("")
     antibody: Optional[str] = Field("", alias="chip_antibody")

bedboss/bedboss.py

@@ -14,19 +14,26 @@
 from pephubclient.helpers import MessageHandler as m
 from pephubclient.helpers import is_registry_path
 
-from bedboss._version import __version__
 from bedboss.bedbuncher import run_bedbuncher
 from bedboss.bedmaker.bedmaker import make_all
 from bedboss.bedstat.bedstat import bedstat
 from bedboss.const import BEDBOSS_PEP_SCHEMA_PATH, PKG_NAME
-from bedboss.exceptions import BedBossException
 from bedboss.models import (
     BedClassificationUpload,
     FilesUpload,
     PlotsUpload,
     StatsUpload,
 )
-from bedboss.utils import get_genome_digest, standardize_genome_name
+from bedboss.refgenome_validator.main import ReferenceValidator
+
+from bedboss.utils import (
+    standardize_genome_name,
+    get_genome_digest,
+    standardize_pep as pep_standardizer,
+)
+from bedboss.exceptions import BedBossException
+from bedboss._version import __version__
+
 
 _LOGGER = logging.getLogger(PKG_NAME)
 
@@ -54,6 +61,7 @@ def run_all(
     rfg_config: str = None,
     narrowpeak: bool = False,
     check_qc: bool = True,
+    validate_reference: bool = True,
     chrom_sizes: str = None,
     open_signal_matrix: str = None,
     ensdb: str = None,
@@ -83,6 +91,7 @@ def run_all(
     :param bool narrowpeak: whether the regions are narrow. Used to create bed file from bedgraph or bigwig
         (transcription factor implies narrow, histone mark implies broad peaks) [optional]
     :param bool check_qc: set True to run quality control during badmaking [optional] (default: True)
+    :param bool validate_reference: set True to run genome reference validator
     :param str chrom_sizes: a full path to the chrom.sizes required for the bedtobigbed conversion [optional]
     :param str open_signal_matrix: a full path to the openSignalMatrix required for the tissue [optional]
     :param dict other_metadata: a dict containing all attributes from the sample
@@ -137,7 +146,7 @@ def run_all(
         pm=pm,
     )
     if not other_metadata:
-        other_metadata = {}
+        other_metadata = {"sample_name": name}
 
     statistics_dict = bedstat(
         bedfile=bed_metadata.bed_file,
@@ -187,13 +196,22 @@ def run_all(
         bed_format=bed_metadata.bed_format.value,
     )
 
+    if validate_reference:
+        _LOGGER.info("Validating reference genome")
+        ref_valid_stats = ReferenceValidator().determine_compatibility(
+            bedfile=bed_metadata.bed_file, concise=True
+        )
+    else:
+        ref_valid_stats = None
+
     bbagent.bed.add(
         identifier=bed_metadata.bed_digest,
         stats=stats.model_dump(exclude_unset=True),
         metadata=other_metadata,
         plots=plots.model_dump(exclude_unset=True),
         files=files.model_dump(exclude_unset=True),
         classification=classification.model_dump(exclude_unset=True),
+        ref_validation=ref_valid_stats,
         license_id=license_id,
         upload_qdrant=upload_qdrant,
         upload_pephub=upload_pephub,
@@ -235,6 +253,7 @@ def insert_pep(
     upload_pephub: bool = False,
     upload_qdrant: bool = False,
     no_fail: bool = False,
+    standardize_pep: bool = False,
     pm: pypiper.PipelineManager = None,
 ) -> None:
     """
@@ -260,6 +279,7 @@ def insert_pep(
     :param bool upload_pephub: whether to push bedfiles and metadata to pephub (default: False)
     :param bool upload_qdrant: whether to execute qdrant indexing
     :param bool no_fail: whether to raise an error if bedset was not added to the database
+    :param bool standardize_pep: whether to standardize the pep file before processing by using bedms. (default: False)
     :param pypiper.PipelineManager pm: pypiper object
     :return: None
     """
@@ -276,6 +296,9 @@ def insert_pep(
     else:
         raise BedBossException("Incorrect pep type. Exiting...")
 
+    if standardize_pep:
+        pep = pep_standardizer(pep)
+
     bbagent = BedBaseAgent(bedbase_config)
 
     validate_project(pep, BEDBOSS_PEP_SCHEMA_PATH)

bedboss/bedbuncher/tools/bedsetStat.R

@@ -67,7 +67,7 @@ if (is.null(opt$json)) {
 #'
 #' Calculates how many regionsets (bedfiles) overlap at least said percentage 
 #' of regions included in the universe. The universe is considered a union of 
-#' all regionsets (bedfiles) in the colection of 
+#' all regionsets (bedfiles) in the collection of 
 #' regionsets (bedset, or set of bedfiles)
 #'
 #' @param queryList GRangesList object with regionsets to be considered

bedboss/bedclassifier/__init__.py

@@ -1 +0,0 @@
-from bedboss.bedclassifier.bedclassifier import get_bed_type