From 22090a3fd3af5adb8906d08e08fdee3fb021ce4e Mon Sep 17 00:00:00 2001 From: Maxime Mulder Date: Thu, 1 Aug 2024 10:43:47 -0400 Subject: [PATCH 1/5] Update Python installation (#1145) * python install update * change version to 3.11 * fix oopsi * remove distutils * change description --------- Co-authored-by: Maxime Mulder --- .github/workflows/flake8_python_linter.yml | 12 +++--- README.md | 46 +++++++++++----------- install/imaging_install.sh | 2 +- 3 files changed, 29 insertions(+), 31 deletions(-) diff --git a/.github/workflows/flake8_python_linter.yml b/.github/workflows/flake8_python_linter.yml index 03f1e1344..2c4891a38 100644 --- a/.github/workflows/flake8_python_linter.yml +++ b/.github/workflows/flake8_python_linter.yml @@ -7,19 +7,19 @@ name: Flake8 Linter on: pull_request jobs: - + flake8-lint: runs-on: ubuntu-latest - + steps: - name: Check out source repository uses: actions/checkout@v2 - + - name: Set up Python environment - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: - python-version: "3.8" - + python-version: "3.11" + - name: Flake8 Lint uses: reviewdog/action-flake8@v3 with: diff --git a/README.md b/README.md index 49f6ecd2b..05aec1786 100644 --- a/README.md +++ b/README.md @@ -3,22 +3,22 @@ This Readme covers release 26.0.* of the LORIS Imaging Insertion Pipeline for Ub This repo accompanies the [LORIS neuroimaging data platform main repo](https://github.com/aces/Loris/releases), release 24.1.*.
For documentation and detailed setup information, please see the [LORIS-MRI documentation](docs/) for your installed version. -This repo can be installed on the same VM as the main LORIS codebase, or on a different machine such as a designated fileserver where large imaging filesets are to be stored. +This repo can be installed on the same VM as the main LORIS codebase, or on a different machine such as a designated fileserver where large imaging filesets are to be stored. # System Requirements * Perl -* Python 3 with pip3 and virtualenv (step 2 below) +* Python 3.11 with pip3 and venv (step 2 below) * MINC toolkit (step 3 below) * DICOM toolkit (step 4 below) * tpcclib (for HRRT PET only) (step 5 below) -On Ubuntu, DICOM toolkit will be installed by the imaging install script (step 4 below). This script will _apt-get install dcmtk_. +On Ubuntu, DICOM toolkit will be installed by the imaging install script (step 4 below). This script will _apt-get install dcmtk_. For CentOS: The [LORIS wiki](https://github.com/aces/Loris/wiki/Imaging-Database) Imaging setup page (see Section 1, installing codebase) includes links to older transcripts for [CentOS installation](https://github.com/aces/Loris/wiki/CentOS-Imaging-installation-transcript) and notes on dependencies including [DICOM toolkit](https://github.com/aces/Loris/wiki/CentOS-Imaging-installation-transcript#7-install-dicom-toolkit). The following installation should be run by the `$lorisadmin` user. `sudo` permission is required. -See [aces/Loris](https://github.com/aces/loris) README.md for further information. +See [aces/Loris](https://github.com/aces/loris) README.md for further information. # Dependencies @@ -54,21 +54,19 @@ sudo chown -R lorisadmin:lorisadmin /opt/$projectname cd /opt/$projectname/bin ``` -Get the code: Download the latest release from the -[releases page](https://github.com/aces/Loris-MRI/releases) +Get the code: Download the latest release from the +[releases page](https://github.com/aces/Loris-MRI/releases) and extract it to `/opt/$projectname/bin/mri` -#### 2. Install Python 3 with `pip` and `virtualenv` +#### 2. Install Python 3.11 with `pip` and `venv` ```bash -sudo apt install python3 -sudo apt install python3-dev +sudo apt install python3.11 python3.11-dev python3.11-venv sudo apt install python3-pip sudo apt install libmysqlclient-dev -sudo apt install virtualenv ``` -#### 3. Install MINC toolkit from http://bic-mni.github.io/ +#### 3. Install MINC toolkit from http://bic-mni.github.io/ - Install MINC dependencies: @@ -82,7 +80,7 @@ sudo apt-get install libgl1-mesa-glx libglu1-mesa - Download the MINC pre-compiled package for your operating system from http://bic-mni.github.io/. -- Then install the MINC toolkit package: +- Then install the MINC toolkit package: ```bash # main minc lib @@ -112,31 +110,31 @@ sudo dpkg -i beast-library-.deb #### 4. Run installer to set up directories, configure environment, install Perl libraries and DICOM toolkit: -```bash +```bash cd /opt/$projectname/bin/mri/install/ bash ./imaging_install.sh ``` -You will be asked for the following input: +You will be asked for the following input: * What is the database name? $dbname * What is the database host? $dbhost * What is the MySQL user? $lorisuser [Use the same mysql user from the Loris installation, i.e. _lorisuser_] -* What is the MySQL password? +* What is the MySQL password? * What is the Linux user which the installation will be based on? $lorisadmin * What is the project name? $projectname -* What is your email address? +* What is your email address? * What prod file name would you like to use? default: prod [leave blank] -If the imaging install script reports errors in creating directories -(due to `/data/` mount permissions), review and manually execute -`mkdir/chmod/chown` commands starting at +If the imaging install script reports errors in creating directories +(due to `/data/` mount permissions), review and manually execute +`mkdir/chmod/chown` commands starting at [imaging_install.sh:L97](https://github.com/aces/Loris-MRI/blob/main/install/imaging_install.sh#L97) -Note: The installer will allow Apache to write to the `/data/` and `/opt/` directories by -adding user `lorisadmin` to the Apache linux group. To ensure this change takes -effect, log out and log back into your terminal session before running the -imaging pipeline. The installer will also set Apache group ownership of certain +Note: The installer will allow Apache to write to the `/data/` and `/opt/` directories by +adding user `lorisadmin` to the Apache linux group. To ensure this change takes +effect, log out and log back into your terminal session before running the +imaging pipeline. The installer will also set Apache group ownership of certain `/data/` and `/opt/` subdirectories. #### 5. HRRT PET @@ -157,7 +155,7 @@ Then source the `.bashrc` file. **INSTALLATION COMPLETE!** -Please refer to the [Install](docs/02-Install.md) section in the +Please refer to the [Install](docs/02-Install.md) section in the [LORIS-MRI documentation](docs/) for your installed version for: - customizations and protocol configurations ([Section 2.2](docs/02-Install.md#configuration)). - verifying that certain fields were correctly populated by `imaging_install.sh` diff --git a/install/imaging_install.sh b/install/imaging_install.sh index fbaa7c1c5..adeee58af 100755 --- a/install/imaging_install.sh +++ b/install/imaging_install.sh @@ -115,7 +115,7 @@ echo echo "Creating loris-mri Python virtualenv in $mridir/python_virtualenvs/loris-mri-python/" # create a directory in $mridir that will store python 3 virtualenv sudo -S su $USER -c "mkdir -m 770 -p $mridir/python_virtualenvs/loris-mri-python" -virtualenv $mridir/python_virtualenvs/loris-mri-python -p `which python3` +python3.11 -m venv $mridir/python_virtualenvs/loris-mri-python source $mridir/python_virtualenvs/loris-mri-python/bin/activate echo "Installing the Python libraries into the loris-mri virtualenv..." pip3 install -r "$mridir/python/requirements.txt" From 8e7a8bc5b2c39dd45b93e0ec368499d5f575b28e Mon Sep 17 00:00:00 2001 From: Maxime Mulder Date: Wed, 14 Aug 2024 09:13:38 -0400 Subject: [PATCH 2/5] remove trailing whitespaces (#1149) Co-authored-by: Maxime Mulder --- .github/workflows/flake8_python_linter.yml | 6 +----- python/lib/__init__.py | 3 +-- python/lib/aws_s3.py | 4 ++-- python/lib/bidsreader.py | 6 +++--- python/lib/eeg.py | 5 ++--- python/lib/imaging.py | 2 +- python/lib/mri.py | 4 ++-- python/lib/physiological.py | 2 +- python/lib/point_3d.py | 2 +- python/lib/scanstsv.py | 2 +- python/lib/session.py | 2 +- python/react-series-data-viewer/chunking.py | 6 +++--- python/react-series-data-viewer/edf_to_chunks.py | 2 +- 13 files changed, 20 insertions(+), 26 deletions(-) diff --git a/.github/workflows/flake8_python_linter.yml b/.github/workflows/flake8_python_linter.yml index 2c4891a38..8814adc3c 100644 --- a/.github/workflows/flake8_python_linter.yml +++ b/.github/workflows/flake8_python_linter.yml @@ -26,17 +26,13 @@ jobs: # For more flake 8 arguments, visit the link below: # https://flake8.pycqa.org/en/latest/user/options.html # Currently, flake8 ignores the following; - # W291: Trailing whitespace - # W292: No newline at end of file - # W293: Blank line contains whitespace # E202: Whitespace before ')' # E203: Whitespace before ':' # E221: Multiple spaces before operator # E241: Multiple spaces after ',' # E251: Unexpected spaces around keyword / parameter equals # E272: Multiple spaces before keyword - flake8_args: "--ignore=W291,W292,W293,W503,E202,E203,E221,E241,E251,E272,E126,E131,E121,E111,E114,E127 + flake8_args: "--ignore=W503,E202,E203,E221,E241,E251,E272,E126,E131,E121,E111,E114,E127 --max-line-length 120 --exclude python/react-series-data-viewer/protocol_buffers/" github_token: ${{ secrets.GITHUB_TOKEN }} - diff --git a/python/lib/__init__.py b/python/lib/__init__.py index d7a393692..c5c77fab3 100644 --- a/python/lib/__init__.py +++ b/python/lib/__init__.py @@ -1,4 +1,3 @@ - """ From the documentation at https://docs.python.org/2/tutorial/modules.html#packages The __init__.py files are required to make Python treat the directories as @@ -7,4 +6,4 @@ the module search path. In the simplest case, __init__.py can just be an empty file, but it can also execute initialization code for the package or set the __all__ variable, described later. -""" \ No newline at end of file +""" diff --git a/python/lib/aws_s3.py b/python/lib/aws_s3.py index baa62d315..eae2a74be 100755 --- a/python/lib/aws_s3.py +++ b/python/lib/aws_s3.py @@ -82,9 +82,9 @@ def check_object_content_exists(self, file_path, key): etag = lib.utilities.compute_md5_hash(file_path) self.s3_client.head_object(Bucket=self.bucket_name, Key=key, IfMatch=etag) except ClientError: - """ + """ Per Boto3 documentation for S3.Client.head_object IfMatch will: - Return the object only if its entity tag (ETag) is the same as the one specified; + Return the object only if its entity tag (ETag) is the same as the one specified; otherwise, return a 412 (precondition failed) error. """ return False diff --git a/python/lib/bidsreader.py b/python/lib/bidsreader.py index 79ec93e82..688bf628e 100755 --- a/python/lib/bidsreader.py +++ b/python/lib/bidsreader.py @@ -97,8 +97,8 @@ def load_bids_data(self, validate): # BIDSLayoutIndexer is required for PyBIDS >= 0.12.1 # bids_pack_version = list(map(int, bids.__version__.split('.'))) # disabled until is a workaround for https://github.com/bids-standard/pybids/issues/760 is found - # [file] bids_import.py - # [function] read_and_insert_bids + # [file] bids_import.py + # [function] read_and_insert_bids # [line] for modality in row['modalities']: (row['modalities'] is empty) # if (bids_pack_version[0] > 0 # or bids_pack_version[1] > 12 @@ -147,7 +147,7 @@ def load_candidates_from_bids(self): else: bids_subjects = self.bids_layout.get_subjects() participants_info = [{'participant_id': sub_id} for sub_id in bids_subjects] - + if self.verbose: print('\t=> List of participants found:') for participant in participants_info: diff --git a/python/lib/eeg.py b/python/lib/eeg.py index 42caa19b1..efd5bf8e2 100755 --- a/python/lib/eeg.py +++ b/python/lib/eeg.py @@ -507,7 +507,7 @@ def fetch_and_insert_eeg_files(self, derivatives=False, detect=True): physio_file_id = physiological.insert_physiological_file( eeg_file_info, eeg_file_data ) - + if self.loris_bids_root_dir: # If we copy the file in assembly_bids and # if the EEG file was a set file, then update the filename for the .set @@ -767,7 +767,7 @@ def fetch_and_insert_event_files( event_metadata_path = self.copy_file_to_loris_bids_dir( event_metadata_file.path, derivatives, inheritance ) - + # load json data with open(event_metadata_file.path) as metadata_file: event_metadata = json.load(metadata_file) @@ -922,7 +922,6 @@ def create_and_insert_archive(self, files_to_archive, archive_rel_name, } physiological.insert_archive_file(archive_info) - def create_and_insert_event_archive(self, files_to_archive, archive_rel_name, eeg_file_id): """ Create an archive with all event files associated to a specific recording diff --git a/python/lib/imaging.py b/python/lib/imaging.py index 4a09b7950..3c3331437 100644 --- a/python/lib/imaging.py +++ b/python/lib/imaging.py @@ -1110,7 +1110,7 @@ def get_list_of_files_sorted_by_acq_time(self, files_list): sorted_files_list = sorted(new_files_list, key=lambda x: x['acq_time']) except TypeError: return None - + return sorted_files_list def modify_fmap_json_file_to_write_intended_for(self, sorted_fmap_files_list, s3_obj, tmp_dir): diff --git a/python/lib/mri.py b/python/lib/mri.py index 70ee1b533..7ebaabbb2 100644 --- a/python/lib/mri.py +++ b/python/lib/mri.py @@ -130,7 +130,7 @@ def __init__(self, bids_reader, bids_sub_id, bids_ses_id, bids_modality, db, if len(cohort_info) > 0: self.cohort_id = cohort_info[0]['CohortID'] break - + self.session_id = self.get_loris_session_id() # grep all the NIfTI files for the modality @@ -139,7 +139,7 @@ def __init__(self, bids_reader, bids_sub_id, bids_ses_id, bids_modality, db, # check if a tsv with acquisition dates or age is available for the subject self.scans_file = None if self.bids_layout.get(suffix='scans', subject=self.psc_id, return_type='filename'): - self.scans_file = self.bids_layout.get(suffix='scans', subject=self.psc_id, + self.scans_file = self.bids_layout.get(suffix='scans', subject=self.psc_id, return_type='filename', extension='tsv')[0] # loop through NIfTI files and register them in the DB diff --git a/python/lib/physiological.py b/python/lib/physiological.py index a560b77e2..0db6293b1 100755 --- a/python/lib/physiological.py +++ b/python/lib/physiological.py @@ -386,7 +386,7 @@ def insert_electrode_file(self, electrode_data, electrode_file, row.get('impedance'), electrode_file ) - + inserted_electrode_id = self.db.insert( table_name = 'physiological_electrode', column_names = electrode_fields, diff --git a/python/lib/point_3d.py b/python/lib/point_3d.py index 351b765dd..e1fc8c6ec 100644 --- a/python/lib/point_3d.py +++ b/python/lib/point_3d.py @@ -23,4 +23,4 @@ def __init__(self, pid: int, x: float, y: float, z: float): def __str__(self) -> str: """Prints out the point info""" - return f"Point [{self.id}] ({self.x}, {self.y}, {self.z})" \ No newline at end of file + return f"Point [{self.id}] ({self.x}, {self.y}, {self.z})" diff --git a/python/lib/scanstsv.py b/python/lib/scanstsv.py index 74d6b8456..c4505177a 100644 --- a/python/lib/scanstsv.py +++ b/python/lib/scanstsv.py @@ -81,7 +81,7 @@ def get_acquisition_time(self): else: print('More than one or no acquisition time has been found for ', self.acquisition_file) exit() - else: + else: eeg_acq_time = self.acquisition_data['acq_time'] if eeg_acq_time == 'n/a': diff --git a/python/lib/session.py b/python/lib/session.py index 22729e946..81fe021a0 100644 --- a/python/lib/session.py +++ b/python/lib/session.py @@ -22,7 +22,7 @@ class Session: db.connect() session = Session( - verbose, cand_id, visit_label, + verbose, cand_id, visit_label, center_id, project_id, cohort_id ) diff --git a/python/react-series-data-viewer/chunking.py b/python/react-series-data-viewer/chunking.py index 7dd94a1eb..596ea0c10 100755 --- a/python/react-series-data-viewer/chunking.py +++ b/python/react-series-data-viewer/chunking.py @@ -105,7 +105,7 @@ def write_index_json( ('channelMetadata', channel_metadata) ]) create_path_dirs(chunk_dir) - + data = None try: with open(os.path.join(chunk_dir, 'index.json'), 'r+') as index_json: @@ -206,14 +206,14 @@ def mne_file_to_chunks(path, chunk_size, loader, from_channel_name, channel_coun return channel_chunks_list, time_interval, signal_range, channel_names, channel_ranges, valid_samples_in_last_chunk -def write_chunk_directory(path, chunk_size, loader, from_channel_index=0, from_channel_name=None, +def write_chunk_directory(path, chunk_size, loader, from_channel_index=0, from_channel_name=None, channel_count=None, downsamplings=None, prefix=None, destination=None): chunk_dir = chunk_dir_path(path, prefix=prefix, destination=destination) channel_chunks_list, time_interval, signal_range, \ channel_names, channel_ranges, valid_samples_in_last_chunk = \ mne_file_to_chunks(path, chunk_size, loader, from_channel_name, channel_count) - + if downsamplings is not None: channel_chunks_list = channel_chunks_list[:downsamplings] diff --git a/python/react-series-data-viewer/edf_to_chunks.py b/python/react-series-data-viewer/edf_to_chunks.py index 58ef541f8..ccbdbb42d 100755 --- a/python/react-series-data-viewer/edf_to_chunks.py +++ b/python/react-series-data-viewer/edf_to_chunks.py @@ -60,7 +60,7 @@ def load_channels(exclude): # check if channel_index is a stim channel # to avoid a bug in mne.io.edf.edf - # (see issue https://github.com/mne-tools/mne-python/issues/9811) + # (see issue https://github.com/mne-tools/mne-python/issues/9811) stim_channel_idxs, _ = mne_edf._check_stim_channel( 'auto', [channel_names[channel_index]] ) From e84bd8c66c91cb94ab9c1a1334e9754258ae8497 Mon Sep 17 00:00:00 2001 From: Maxime Mulder Date: Tue, 13 Aug 2024 20:12:01 -0400 Subject: [PATCH 3/5] move database to dataclass namespace --- python/bids_import.py | 10 +++++----- python/extract_eeg_bids_archive.py | 2 +- python/ingest_eeg_bids_datasets.py | 10 +++++----- python/lib/candidate.py | 2 +- python/lib/database_lib/candidate_db.py | 2 +- python/lib/database_lib/config.py | 2 +- python/lib/database_lib/files.py | 2 +- python/lib/database_lib/mri_candidate_errors.py | 2 +- python/lib/database_lib/mri_protocol.py | 2 +- python/lib/database_lib/mri_protocol_checks.py | 2 +- python/lib/database_lib/mri_protocol_violated_scans.py | 2 +- python/lib/database_lib/mri_scan_type.py | 2 +- python/lib/database_lib/mri_scanner.py | 2 +- python/lib/database_lib/mri_upload_db.py | 2 +- python/lib/database_lib/mri_violations_log.py | 2 +- python/lib/database_lib/notification.py | 2 +- python/lib/database_lib/parameter_file.py | 2 +- python/lib/database_lib/parameter_type.py | 2 +- python/lib/database_lib/project_cohort_rel.py | 2 +- python/lib/database_lib/session_db.py | 2 +- python/lib/database_lib/site.py | 2 +- python/lib/database_lib/tarchive.py | 2 +- python/lib/database_lib/tarchive_series.py | 2 +- python/lib/database_lib/visit_windows.py | 2 +- python/lib/{ => dataclass}/database.py | 2 +- .../lib/dcm2bids_imaging_pipeline_lib/base_pipeline.py | 2 +- python/lib/dicom_archive.py | 2 +- python/lib/eeg.py | 4 ++-- python/lib/imaging.py | 2 +- python/lib/imaging_upload.py | 2 +- python/lib/lorisgetopt.py | 2 +- python/lib/mri.py | 4 ++-- python/lib/physiological.py | 2 +- python/lib/session.py | 4 ++-- python/mass_electrophysiology_chunking.py | 2 +- python/mass_nifti_pic.py | 2 +- tools/correct_blake2b_and_md5_hashes_in_database.py | 2 +- ...orrect_lists_incorrectly_saved_in_parameter_file.py | 2 +- 38 files changed, 49 insertions(+), 49 deletions(-) rename python/lib/{ => dataclass}/database.py (99%) diff --git a/python/bids_import.py b/python/bids_import.py index 5f3e73a9f..c8471a4f4 100755 --- a/python/bids_import.py +++ b/python/bids_import.py @@ -10,12 +10,12 @@ import lib.exitcode import lib.utilities import lib.physiological -from lib.database import Database -from lib.candidate import Candidate +from lib.dataclass.database import Database +from lib.candidate import Candidate from lib.bidsreader import BidsReader -from lib.session import Session -from lib.eeg import Eeg -from lib.mri import Mri +from lib.session import Session +from lib.eeg import Eeg +from lib.mri import Mri from lib.database_lib.config import Config __license__ = "GPLv3" diff --git a/python/extract_eeg_bids_archive.py b/python/extract_eeg_bids_archive.py index a5724c624..8ff5b499a 100755 --- a/python/extract_eeg_bids_archive.py +++ b/python/extract_eeg_bids_archive.py @@ -7,7 +7,7 @@ import re from lib.lorisgetopt import LorisGetOpt from lib.imaging_io import ImagingIO -from lib.database import Database +from lib.dataclass.database import Database from lib.database_lib.config import Config from lib.exitcode import SUCCESS, BAD_CONFIG_SETTING from lib.log import Log diff --git a/python/ingest_eeg_bids_datasets.py b/python/ingest_eeg_bids_datasets.py index e62e78d0c..8b2fecf88 100755 --- a/python/ingest_eeg_bids_datasets.py +++ b/python/ingest_eeg_bids_datasets.py @@ -5,7 +5,7 @@ import os import sys from lib.lorisgetopt import LorisGetOpt -from lib.database import Database +from lib.dataclass.database import Database from lib.database_lib.config import Config from lib.exitcode import SUCCESS, INVALID_ARG import subprocess @@ -112,7 +112,7 @@ def main(): if not session_data: print('Session ID ' + eeg_dataset['SessionID'] + ' associated with UploadID ' + uploadid + ' does not exist.') sys.exit(INVALID_ARG) - + candid = session_data[0]['CandID'] pscid = session_data[0]['PSCID'] visit = session_data[0]['Visit_label'] @@ -150,13 +150,13 @@ def main(): # Assume eeg and raw data for now eeg_path = os.path.join(path, 'eeg') command = 'python ' + script + ' -p ' + profile + ' -d ' + eeg_path + ' --nobidsvalidation --nocopy --type raw' - + try: result = subprocess.run(command, shell = True, capture_output=True) if result.stdout: print(result.stdout.decode('utf-8')) - + if result.stderr: print( f'ERROR: EEG Dataset with uploadID {uploadid} ingestion log:\n ' + result.stderr.decode('utf-8') @@ -177,7 +177,7 @@ def main(): "UPDATE electrophysiology_uploader SET Status = 'Failed Ingestion' WHERE UploadID = %s", (uploadid,) ) - + # TODO: reupload of archive after ingestion # Delete if already exist diff --git a/python/lib/candidate.py b/python/lib/candidate.py index b2bb20c72..37098b7aa 100644 --- a/python/lib/candidate.py +++ b/python/lib/candidate.py @@ -16,7 +16,7 @@ class Candidate: :Example: from lib.candidate import Candidate - from lib.database import Database + from lib.dataclass.database import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/candidate_db.py b/python/lib/database_lib/candidate_db.py index bbe4e92d3..abcf5f7d8 100644 --- a/python/lib/database_lib/candidate_db.py +++ b/python/lib/database_lib/candidate_db.py @@ -11,7 +11,7 @@ class CandidateDB: :Example: from lib.database_lib.candidate_db import CandidateDB - from lib.database import Database + from lib.dataclass.database import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/config.py b/python/lib/database_lib/config.py index a5030762c..32d31398a 100644 --- a/python/lib/database_lib/config.py +++ b/python/lib/database_lib/config.py @@ -11,7 +11,7 @@ class Config: :Example: from lib.database_lib.config import Config - from lib.database import Database + from lib.dataclass.database import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/files.py b/python/lib/database_lib/files.py index 4fab37e06..41ad93380 100644 --- a/python/lib/database_lib/files.py +++ b/python/lib/database_lib/files.py @@ -12,7 +12,7 @@ class Files: :Example: from lib.files import Files - from lib.database import Database + from lib.dataclass.database import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/mri_candidate_errors.py b/python/lib/database_lib/mri_candidate_errors.py index 25b70e5e7..5c105b99c 100644 --- a/python/lib/database_lib/mri_candidate_errors.py +++ b/python/lib/database_lib/mri_candidate_errors.py @@ -11,7 +11,7 @@ class MriCandidateErrors: :Example: from lib.mri_candidate_errors import MriCandidateErrors - from lib.database import Database + from lib.dataclass.database import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/mri_protocol.py b/python/lib/database_lib/mri_protocol.py index 86fa29035..3719877ee 100644 --- a/python/lib/database_lib/mri_protocol.py +++ b/python/lib/database_lib/mri_protocol.py @@ -11,7 +11,7 @@ class MriProtocol: :Example: from lib.mri_protocol import MriProtocol - from lib.database import Database + from lib.dataclass.database import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/mri_protocol_checks.py b/python/lib/database_lib/mri_protocol_checks.py index 1bdd02aac..47e74da41 100644 --- a/python/lib/database_lib/mri_protocol_checks.py +++ b/python/lib/database_lib/mri_protocol_checks.py @@ -11,7 +11,7 @@ class MriProtocolChecks: :Example: from lib.mri_protocol_checks import MriProtocolChecks - from lib.database import Database + from lib.dataclass.database import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/mri_protocol_violated_scans.py b/python/lib/database_lib/mri_protocol_violated_scans.py index fd65b55fa..21f9f434e 100644 --- a/python/lib/database_lib/mri_protocol_violated_scans.py +++ b/python/lib/database_lib/mri_protocol_violated_scans.py @@ -11,7 +11,7 @@ class MriProtocolViolatedScans: :Example: from lib.mri_protocol_violated_scans import MriProtocolViolatedScans - from lib.database import Database + from lib.dataclass.database import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/mri_scan_type.py b/python/lib/database_lib/mri_scan_type.py index 64f4759b6..02e9b7308 100644 --- a/python/lib/database_lib/mri_scan_type.py +++ b/python/lib/database_lib/mri_scan_type.py @@ -11,7 +11,7 @@ class MriScanType: :Example: from lib.mri_scan_type import MriScanType - from lib.database import Database + from lib.dataclass.database import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/mri_scanner.py b/python/lib/database_lib/mri_scanner.py index 7ea0cbc52..6ba60b67f 100644 --- a/python/lib/database_lib/mri_scanner.py +++ b/python/lib/database_lib/mri_scanner.py @@ -13,7 +13,7 @@ class MriScanner: :Example: from lib.mri_scanner import MriScanner - from lib.database import Database + from lib.dataclass.database import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/mri_upload_db.py b/python/lib/database_lib/mri_upload_db.py index c636429e2..1e830f5c4 100644 --- a/python/lib/database_lib/mri_upload_db.py +++ b/python/lib/database_lib/mri_upload_db.py @@ -10,7 +10,7 @@ class MriUploadDB: :Example: from lib.mri_upload import MriUploadDB - from lib.database import Database + from lib.dataclass.database import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/mri_violations_log.py b/python/lib/database_lib/mri_violations_log.py index 517c97a54..f8f0748ce 100644 --- a/python/lib/database_lib/mri_violations_log.py +++ b/python/lib/database_lib/mri_violations_log.py @@ -11,7 +11,7 @@ class MriViolationsLog: :Example: from lib.mri_violations_log import MriViolationsLog - from lib.database import Database + from lib.dataclass.database import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/notification.py b/python/lib/database_lib/notification.py index 6c873fa06..2dbb844ee 100644 --- a/python/lib/database_lib/notification.py +++ b/python/lib/database_lib/notification.py @@ -12,7 +12,7 @@ class Notification: :Example: from lib.notification import Notification - from lib.database import Database + from lib.dataclass.database import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/parameter_file.py b/python/lib/database_lib/parameter_file.py index 720431389..dce0de8eb 100644 --- a/python/lib/database_lib/parameter_file.py +++ b/python/lib/database_lib/parameter_file.py @@ -10,7 +10,7 @@ class ParameterFile: :Example: from lib.parameter_file import ParameterFile - from lib.database import Database + from lib.dataclass.database import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/parameter_type.py b/python/lib/database_lib/parameter_type.py index 421cae4a1..48e46255a 100644 --- a/python/lib/database_lib/parameter_type.py +++ b/python/lib/database_lib/parameter_type.py @@ -10,7 +10,7 @@ class ParameterType: :Example: from lib.parameter_type import ParameterType - from lib.database import Database + from lib.dataclass.database import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/project_cohort_rel.py b/python/lib/database_lib/project_cohort_rel.py index 06acb9197..325a1eb91 100644 --- a/python/lib/database_lib/project_cohort_rel.py +++ b/python/lib/database_lib/project_cohort_rel.py @@ -11,7 +11,7 @@ class ProjectCohortRel: :Example: from lib.database_lib.project_cohort_rel import ProjectCohortRel - from lib.database import Database + from lib.dataclass.database import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/session_db.py b/python/lib/database_lib/session_db.py index 764015bd1..74feff117 100644 --- a/python/lib/database_lib/session_db.py +++ b/python/lib/database_lib/session_db.py @@ -11,7 +11,7 @@ class SessionDB: :Example: from lib.database_lib.session_db import SessionDB - from lib.database import Database + from lib.dataclass.database import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/site.py b/python/lib/database_lib/site.py index 192befebd..5b5bdc962 100644 --- a/python/lib/database_lib/site.py +++ b/python/lib/database_lib/site.py @@ -11,7 +11,7 @@ class Site: :Example: from lib.site import Site - from lib.database import Database + from lib.dataclass.database import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/tarchive.py b/python/lib/database_lib/tarchive.py index ec22865a0..896448aa0 100644 --- a/python/lib/database_lib/tarchive.py +++ b/python/lib/database_lib/tarchive.py @@ -11,7 +11,7 @@ class Tarchive: :Example: from lib.tarchive import Tarchive - from lib.database import Database + from lib.dataclass.database import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/tarchive_series.py b/python/lib/database_lib/tarchive_series.py index d0f8c5ac0..f9a0faefb 100644 --- a/python/lib/database_lib/tarchive_series.py +++ b/python/lib/database_lib/tarchive_series.py @@ -11,7 +11,7 @@ class TarchiveSeries: :Example: from lib.tarchive_series import TarchiveSeries - from lib.database import Database + from lib.dataclass.database import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/visit_windows.py b/python/lib/database_lib/visit_windows.py index 14e1636ee..e00aa5872 100644 --- a/python/lib/database_lib/visit_windows.py +++ b/python/lib/database_lib/visit_windows.py @@ -11,7 +11,7 @@ class VisitWindows: :Example: from lib.visit_windows import VisitWindows - from lib.database import Database + from lib.dataclass.database import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database.py b/python/lib/dataclass/database.py similarity index 99% rename from python/lib/database.py rename to python/lib/dataclass/database.py index fa182d7f3..4ab0c2e32 100644 --- a/python/lib/database.py +++ b/python/lib/dataclass/database.py @@ -15,7 +15,7 @@ class Database: :Example: - from lib.database import Database + from lib.dataclass.database import Database db = Database(config.mysql, verbose) diff --git a/python/lib/dcm2bids_imaging_pipeline_lib/base_pipeline.py b/python/lib/dcm2bids_imaging_pipeline_lib/base_pipeline.py index 01176531c..d6b4ffc28 100644 --- a/python/lib/dcm2bids_imaging_pipeline_lib/base_pipeline.py +++ b/python/lib/dcm2bids_imaging_pipeline_lib/base_pipeline.py @@ -9,7 +9,7 @@ from lib.database_lib.candidate_db import CandidateDB from lib.database_lib.config import Config from lib.database_lib.visit_windows import VisitWindows -from lib.database import Database +from lib.dataclass.database import Database from lib.dicom_archive import DicomArchive from lib.imaging import Imaging from lib.log import Log diff --git a/python/lib/dicom_archive.py b/python/lib/dicom_archive.py index 47f3b87b4..f7868afb0 100644 --- a/python/lib/dicom_archive.py +++ b/python/lib/dicom_archive.py @@ -14,7 +14,7 @@ class DicomArchive: :Example: - from lib.database import Database + from lib.dataclass.database import Database from lib.dicom_archive import DicomArchive # database connection diff --git a/python/lib/eeg.py b/python/lib/eeg.py index efd5bf8e2..af9eb983e 100755 --- a/python/lib/eeg.py +++ b/python/lib/eeg.py @@ -28,8 +28,8 @@ class Eeg: :Example: from lib.bidsreader import BidsReader - from lib.eeg import Eeg - from lib.database import Database + from lib.eeg import Eeg + from lib.dataclass.database import Database from lib.database_lib.config import Config # database connection diff --git a/python/lib/imaging.py b/python/lib/imaging.py index 3c3331437..1ef61b261 100644 --- a/python/lib/imaging.py +++ b/python/lib/imaging.py @@ -32,7 +32,7 @@ class Imaging: :Example: from lib.imaging import Imaging - from lib.database import Database + from lib.dataclass.database import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/imaging_upload.py b/python/lib/imaging_upload.py index 00e5889cb..ed9a61f34 100644 --- a/python/lib/imaging_upload.py +++ b/python/lib/imaging_upload.py @@ -13,7 +13,7 @@ class ImagingUpload: :Example: from lib.imaging_upload import ImagingUpload - from lib.database import Database + from lib.dataclass.database import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/lorisgetopt.py b/python/lib/lorisgetopt.py index c71740bbb..b47699e07 100644 --- a/python/lib/lorisgetopt.py +++ b/python/lib/lorisgetopt.py @@ -7,7 +7,7 @@ import sys from lib.aws_s3 import AwsS3 -from lib.database import Database +from lib.dataclass.database import Database from lib.database_lib.config import Config diff --git a/python/lib/mri.py b/python/lib/mri.py index 7ebaabbb2..2cbcf5051 100644 --- a/python/lib/mri.py +++ b/python/lib/mri.py @@ -25,8 +25,8 @@ class Mri: :Example: from lib.bidsreader import BidsReader - from lib.mri import Mri - from lib.database import Database + from lib.mri import Mri + from lib.dataclass.database import Database # database connection db = Database(config_file.mysql, verbose) diff --git a/python/lib/physiological.py b/python/lib/physiological.py index 0db6293b1..c9cfe8ef8 100755 --- a/python/lib/physiological.py +++ b/python/lib/physiological.py @@ -32,7 +32,7 @@ class Physiological: :Example: from lib.physiological import Physiological - from lib.database import Database + from lib.dataclass.database import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/session.py b/python/lib/session.py index 81fe021a0..bc11f9b66 100644 --- a/python/lib/session.py +++ b/python/lib/session.py @@ -14,8 +14,8 @@ class Session: :Example: - from lib.session import Session - from lib.database import Database + from lib.session import Session + from lib.dataclass.database import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/mass_electrophysiology_chunking.py b/python/mass_electrophysiology_chunking.py index baee595b6..7b26cd5fa 100755 --- a/python/mass_electrophysiology_chunking.py +++ b/python/mass_electrophysiology_chunking.py @@ -6,7 +6,7 @@ import sys import getopt import lib.exitcode -from lib.database import Database +from lib.dataclass.database import Database from lib.physiological import Physiological from lib.database_lib.config import Config diff --git a/python/mass_nifti_pic.py b/python/mass_nifti_pic.py index 589c1ff47..f7eddcba1 100755 --- a/python/mass_nifti_pic.py +++ b/python/mass_nifti_pic.py @@ -7,7 +7,7 @@ import sys import getopt import lib.exitcode -from lib.database import Database +from lib.dataclass.database import Database from lib.imaging import Imaging from lib.database_lib.config import Config diff --git a/tools/correct_blake2b_and_md5_hashes_in_database.py b/tools/correct_blake2b_and_md5_hashes_in_database.py index 9b5e45ad4..8cf5c087d 100755 --- a/tools/correct_blake2b_and_md5_hashes_in_database.py +++ b/tools/correct_blake2b_and_md5_hashes_in_database.py @@ -6,7 +6,7 @@ import shutil import sys -from lib.database import Database +from lib.dataclass.database import Database from lib.database_lib.config import Config from lib.lorisgetopt import LorisGetOpt diff --git a/tools/correct_lists_incorrectly_saved_in_parameter_file.py b/tools/correct_lists_incorrectly_saved_in_parameter_file.py index 773d2cece..547a666f0 100755 --- a/tools/correct_lists_incorrectly_saved_in_parameter_file.py +++ b/tools/correct_lists_incorrectly_saved_in_parameter_file.py @@ -2,7 +2,7 @@ import os -from lib.database import Database +from lib.dataclass.database import Database from lib.lorisgetopt import LorisGetOpt From 5b78448a2a802d425979013f8d3c547fbeea67cc Mon Sep 17 00:00:00 2001 From: Maxime Mulder Date: Tue, 20 Aug 2024 12:44:30 -0400 Subject: [PATCH 4/5] change database file name --- python/bids_import.py | 2 +- python/extract_eeg_bids_archive.py | 2 +- python/ingest_eeg_bids_datasets.py | 2 +- python/lib/candidate.py | 2 +- python/lib/database_lib/candidate_db.py | 2 +- python/lib/database_lib/config.py | 2 +- python/lib/database_lib/files.py | 2 +- python/lib/database_lib/mri_candidate_errors.py | 2 +- python/lib/database_lib/mri_protocol.py | 2 +- python/lib/database_lib/mri_protocol_checks.py | 2 +- python/lib/database_lib/mri_protocol_violated_scans.py | 2 +- python/lib/database_lib/mri_scan_type.py | 2 +- python/lib/database_lib/mri_scanner.py | 2 +- python/lib/database_lib/mri_upload_db.py | 2 +- python/lib/database_lib/mri_violations_log.py | 2 +- python/lib/database_lib/notification.py | 2 +- python/lib/database_lib/parameter_file.py | 2 +- python/lib/database_lib/parameter_type.py | 2 +- python/lib/database_lib/project_cohort_rel.py | 2 +- python/lib/database_lib/session_db.py | 2 +- python/lib/database_lib/site.py | 2 +- python/lib/database_lib/tarchive.py | 2 +- python/lib/database_lib/tarchive_series.py | 2 +- python/lib/database_lib/visit_windows.py | 2 +- python/lib/{dataclass/database.py => database_mysqldb.py} | 2 +- python/lib/dcm2bids_imaging_pipeline_lib/base_pipeline.py | 2 +- python/lib/dicom_archive.py | 2 +- python/lib/eeg.py | 2 +- python/lib/imaging.py | 2 +- python/lib/imaging_upload.py | 2 +- python/lib/lorisgetopt.py | 2 +- python/lib/mri.py | 2 +- python/lib/physiological.py | 2 +- python/lib/session.py | 2 +- python/mass_electrophysiology_chunking.py | 2 +- python/mass_nifti_pic.py | 2 +- tools/correct_blake2b_and_md5_hashes_in_database.py | 2 +- tools/correct_lists_incorrectly_saved_in_parameter_file.py | 2 +- 38 files changed, 38 insertions(+), 38 deletions(-) rename python/lib/{dataclass/database.py => database_mysqldb.py} (99%) diff --git a/python/bids_import.py b/python/bids_import.py index c8471a4f4..462a9427c 100755 --- a/python/bids_import.py +++ b/python/bids_import.py @@ -10,7 +10,7 @@ import lib.exitcode import lib.utilities import lib.physiological -from lib.dataclass.database import Database +from lib.database_mysqldb import Database from lib.candidate import Candidate from lib.bidsreader import BidsReader from lib.session import Session diff --git a/python/extract_eeg_bids_archive.py b/python/extract_eeg_bids_archive.py index 8ff5b499a..a4c198c04 100755 --- a/python/extract_eeg_bids_archive.py +++ b/python/extract_eeg_bids_archive.py @@ -7,7 +7,7 @@ import re from lib.lorisgetopt import LorisGetOpt from lib.imaging_io import ImagingIO -from lib.dataclass.database import Database +from lib.database_mysqldb import Database from lib.database_lib.config import Config from lib.exitcode import SUCCESS, BAD_CONFIG_SETTING from lib.log import Log diff --git a/python/ingest_eeg_bids_datasets.py b/python/ingest_eeg_bids_datasets.py index 8b2fecf88..9c7218680 100755 --- a/python/ingest_eeg_bids_datasets.py +++ b/python/ingest_eeg_bids_datasets.py @@ -5,7 +5,7 @@ import os import sys from lib.lorisgetopt import LorisGetOpt -from lib.dataclass.database import Database +from lib.database_mysqldb import Database from lib.database_lib.config import Config from lib.exitcode import SUCCESS, INVALID_ARG import subprocess diff --git a/python/lib/candidate.py b/python/lib/candidate.py index 37098b7aa..10e70090c 100644 --- a/python/lib/candidate.py +++ b/python/lib/candidate.py @@ -16,7 +16,7 @@ class Candidate: :Example: from lib.candidate import Candidate - from lib.dataclass.database import Database + from lib.database_mysqldb import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/candidate_db.py b/python/lib/database_lib/candidate_db.py index abcf5f7d8..cd833a49b 100644 --- a/python/lib/database_lib/candidate_db.py +++ b/python/lib/database_lib/candidate_db.py @@ -11,7 +11,7 @@ class CandidateDB: :Example: from lib.database_lib.candidate_db import CandidateDB - from lib.dataclass.database import Database + from lib.database_mysqldb import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/config.py b/python/lib/database_lib/config.py index 32d31398a..3e84b2f06 100644 --- a/python/lib/database_lib/config.py +++ b/python/lib/database_lib/config.py @@ -11,7 +11,7 @@ class Config: :Example: from lib.database_lib.config import Config - from lib.dataclass.database import Database + from lib.database_mysqldb import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/files.py b/python/lib/database_lib/files.py index 41ad93380..fd2734561 100644 --- a/python/lib/database_lib/files.py +++ b/python/lib/database_lib/files.py @@ -12,7 +12,7 @@ class Files: :Example: from lib.files import Files - from lib.dataclass.database import Database + from lib.database_mysqldb import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/mri_candidate_errors.py b/python/lib/database_lib/mri_candidate_errors.py index 5c105b99c..145564e08 100644 --- a/python/lib/database_lib/mri_candidate_errors.py +++ b/python/lib/database_lib/mri_candidate_errors.py @@ -11,7 +11,7 @@ class MriCandidateErrors: :Example: from lib.mri_candidate_errors import MriCandidateErrors - from lib.dataclass.database import Database + from lib.database_mysqldb import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/mri_protocol.py b/python/lib/database_lib/mri_protocol.py index 3719877ee..dafcb8ae3 100644 --- a/python/lib/database_lib/mri_protocol.py +++ b/python/lib/database_lib/mri_protocol.py @@ -11,7 +11,7 @@ class MriProtocol: :Example: from lib.mri_protocol import MriProtocol - from lib.dataclass.database import Database + from lib.database_mysqldb import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/mri_protocol_checks.py b/python/lib/database_lib/mri_protocol_checks.py index 47e74da41..a3889b1a2 100644 --- a/python/lib/database_lib/mri_protocol_checks.py +++ b/python/lib/database_lib/mri_protocol_checks.py @@ -11,7 +11,7 @@ class MriProtocolChecks: :Example: from lib.mri_protocol_checks import MriProtocolChecks - from lib.dataclass.database import Database + from lib.database_mysqldb import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/mri_protocol_violated_scans.py b/python/lib/database_lib/mri_protocol_violated_scans.py index 21f9f434e..f9da01a27 100644 --- a/python/lib/database_lib/mri_protocol_violated_scans.py +++ b/python/lib/database_lib/mri_protocol_violated_scans.py @@ -11,7 +11,7 @@ class MriProtocolViolatedScans: :Example: from lib.mri_protocol_violated_scans import MriProtocolViolatedScans - from lib.dataclass.database import Database + from lib.database_mysqldb import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/mri_scan_type.py b/python/lib/database_lib/mri_scan_type.py index 02e9b7308..167e95ce0 100644 --- a/python/lib/database_lib/mri_scan_type.py +++ b/python/lib/database_lib/mri_scan_type.py @@ -11,7 +11,7 @@ class MriScanType: :Example: from lib.mri_scan_type import MriScanType - from lib.dataclass.database import Database + from lib.database_mysqldb import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/mri_scanner.py b/python/lib/database_lib/mri_scanner.py index 6ba60b67f..95b6c3754 100644 --- a/python/lib/database_lib/mri_scanner.py +++ b/python/lib/database_lib/mri_scanner.py @@ -13,7 +13,7 @@ class MriScanner: :Example: from lib.mri_scanner import MriScanner - from lib.dataclass.database import Database + from lib.database_mysqldb import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/mri_upload_db.py b/python/lib/database_lib/mri_upload_db.py index 1e830f5c4..5941af598 100644 --- a/python/lib/database_lib/mri_upload_db.py +++ b/python/lib/database_lib/mri_upload_db.py @@ -10,7 +10,7 @@ class MriUploadDB: :Example: from lib.mri_upload import MriUploadDB - from lib.dataclass.database import Database + from lib.database_mysqldb import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/mri_violations_log.py b/python/lib/database_lib/mri_violations_log.py index f8f0748ce..f08b990ef 100644 --- a/python/lib/database_lib/mri_violations_log.py +++ b/python/lib/database_lib/mri_violations_log.py @@ -11,7 +11,7 @@ class MriViolationsLog: :Example: from lib.mri_violations_log import MriViolationsLog - from lib.dataclass.database import Database + from lib.database_mysqldb import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/notification.py b/python/lib/database_lib/notification.py index 2dbb844ee..29f2ef20b 100644 --- a/python/lib/database_lib/notification.py +++ b/python/lib/database_lib/notification.py @@ -12,7 +12,7 @@ class Notification: :Example: from lib.notification import Notification - from lib.dataclass.database import Database + from lib.database_mysqldb import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/parameter_file.py b/python/lib/database_lib/parameter_file.py index dce0de8eb..356829313 100644 --- a/python/lib/database_lib/parameter_file.py +++ b/python/lib/database_lib/parameter_file.py @@ -10,7 +10,7 @@ class ParameterFile: :Example: from lib.parameter_file import ParameterFile - from lib.dataclass.database import Database + from lib.database_mysqldb import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/parameter_type.py b/python/lib/database_lib/parameter_type.py index 48e46255a..935432c6a 100644 --- a/python/lib/database_lib/parameter_type.py +++ b/python/lib/database_lib/parameter_type.py @@ -10,7 +10,7 @@ class ParameterType: :Example: from lib.parameter_type import ParameterType - from lib.dataclass.database import Database + from lib.database_mysqldb import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/project_cohort_rel.py b/python/lib/database_lib/project_cohort_rel.py index 325a1eb91..f6d776335 100644 --- a/python/lib/database_lib/project_cohort_rel.py +++ b/python/lib/database_lib/project_cohort_rel.py @@ -11,7 +11,7 @@ class ProjectCohortRel: :Example: from lib.database_lib.project_cohort_rel import ProjectCohortRel - from lib.dataclass.database import Database + from lib.database_mysqldb import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/session_db.py b/python/lib/database_lib/session_db.py index 74feff117..4516f400e 100644 --- a/python/lib/database_lib/session_db.py +++ b/python/lib/database_lib/session_db.py @@ -11,7 +11,7 @@ class SessionDB: :Example: from lib.database_lib.session_db import SessionDB - from lib.dataclass.database import Database + from lib.database_mysqldb import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/site.py b/python/lib/database_lib/site.py index 5b5bdc962..b8aca23e0 100644 --- a/python/lib/database_lib/site.py +++ b/python/lib/database_lib/site.py @@ -11,7 +11,7 @@ class Site: :Example: from lib.site import Site - from lib.dataclass.database import Database + from lib.database_mysqldb import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/tarchive.py b/python/lib/database_lib/tarchive.py index 896448aa0..3c3a52c94 100644 --- a/python/lib/database_lib/tarchive.py +++ b/python/lib/database_lib/tarchive.py @@ -11,7 +11,7 @@ class Tarchive: :Example: from lib.tarchive import Tarchive - from lib.dataclass.database import Database + from lib.database_mysqldb import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/tarchive_series.py b/python/lib/database_lib/tarchive_series.py index f9a0faefb..1a39cebec 100644 --- a/python/lib/database_lib/tarchive_series.py +++ b/python/lib/database_lib/tarchive_series.py @@ -11,7 +11,7 @@ class TarchiveSeries: :Example: from lib.tarchive_series import TarchiveSeries - from lib.dataclass.database import Database + from lib.database_mysqldb import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/visit_windows.py b/python/lib/database_lib/visit_windows.py index e00aa5872..5538857db 100644 --- a/python/lib/database_lib/visit_windows.py +++ b/python/lib/database_lib/visit_windows.py @@ -11,7 +11,7 @@ class VisitWindows: :Example: from lib.visit_windows import VisitWindows - from lib.dataclass.database import Database + from lib.database_mysqldb import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/dataclass/database.py b/python/lib/database_mysqldb.py similarity index 99% rename from python/lib/dataclass/database.py rename to python/lib/database_mysqldb.py index 4ab0c2e32..e35d9caf7 100644 --- a/python/lib/dataclass/database.py +++ b/python/lib/database_mysqldb.py @@ -15,7 +15,7 @@ class Database: :Example: - from lib.dataclass.database import Database + from lib.database_mysqldb import Database db = Database(config.mysql, verbose) diff --git a/python/lib/dcm2bids_imaging_pipeline_lib/base_pipeline.py b/python/lib/dcm2bids_imaging_pipeline_lib/base_pipeline.py index d6b4ffc28..c9523f238 100644 --- a/python/lib/dcm2bids_imaging_pipeline_lib/base_pipeline.py +++ b/python/lib/dcm2bids_imaging_pipeline_lib/base_pipeline.py @@ -9,7 +9,7 @@ from lib.database_lib.candidate_db import CandidateDB from lib.database_lib.config import Config from lib.database_lib.visit_windows import VisitWindows -from lib.dataclass.database import Database +from lib.database_mysqldb import Database from lib.dicom_archive import DicomArchive from lib.imaging import Imaging from lib.log import Log diff --git a/python/lib/dicom_archive.py b/python/lib/dicom_archive.py index f7868afb0..197a26bb1 100644 --- a/python/lib/dicom_archive.py +++ b/python/lib/dicom_archive.py @@ -14,7 +14,7 @@ class DicomArchive: :Example: - from lib.dataclass.database import Database + from lib.database_mysqldb import Database from lib.dicom_archive import DicomArchive # database connection diff --git a/python/lib/eeg.py b/python/lib/eeg.py index af9eb983e..667689443 100755 --- a/python/lib/eeg.py +++ b/python/lib/eeg.py @@ -29,7 +29,7 @@ class Eeg: from lib.bidsreader import BidsReader from lib.eeg import Eeg - from lib.dataclass.database import Database + from lib.database_mysqldb import Database from lib.database_lib.config import Config # database connection diff --git a/python/lib/imaging.py b/python/lib/imaging.py index 1ef61b261..476e1afa2 100644 --- a/python/lib/imaging.py +++ b/python/lib/imaging.py @@ -32,7 +32,7 @@ class Imaging: :Example: from lib.imaging import Imaging - from lib.dataclass.database import Database + from lib.database_mysqldb import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/imaging_upload.py b/python/lib/imaging_upload.py index ed9a61f34..cae29f5d9 100644 --- a/python/lib/imaging_upload.py +++ b/python/lib/imaging_upload.py @@ -13,7 +13,7 @@ class ImagingUpload: :Example: from lib.imaging_upload import ImagingUpload - from lib.dataclass.database import Database + from lib.database_mysqldb import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/lorisgetopt.py b/python/lib/lorisgetopt.py index b47699e07..ae62300f6 100644 --- a/python/lib/lorisgetopt.py +++ b/python/lib/lorisgetopt.py @@ -7,7 +7,7 @@ import sys from lib.aws_s3 import AwsS3 -from lib.dataclass.database import Database +from lib.database_mysqldb import Database from lib.database_lib.config import Config diff --git a/python/lib/mri.py b/python/lib/mri.py index 2cbcf5051..52cf17a75 100644 --- a/python/lib/mri.py +++ b/python/lib/mri.py @@ -26,7 +26,7 @@ class Mri: from lib.bidsreader import BidsReader from lib.mri import Mri - from lib.dataclass.database import Database + from lib.database_mysqldb import Database # database connection db = Database(config_file.mysql, verbose) diff --git a/python/lib/physiological.py b/python/lib/physiological.py index c9cfe8ef8..67f9d6420 100755 --- a/python/lib/physiological.py +++ b/python/lib/physiological.py @@ -32,7 +32,7 @@ class Physiological: :Example: from lib.physiological import Physiological - from lib.dataclass.database import Database + from lib.database_mysqldb import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/session.py b/python/lib/session.py index bc11f9b66..856e105db 100644 --- a/python/lib/session.py +++ b/python/lib/session.py @@ -15,7 +15,7 @@ class Session: :Example: from lib.session import Session - from lib.dataclass.database import Database + from lib.database_mysqldb import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/mass_electrophysiology_chunking.py b/python/mass_electrophysiology_chunking.py index 7b26cd5fa..3a3a3748b 100755 --- a/python/mass_electrophysiology_chunking.py +++ b/python/mass_electrophysiology_chunking.py @@ -6,7 +6,7 @@ import sys import getopt import lib.exitcode -from lib.dataclass.database import Database +from lib.database_mysqldb import Database from lib.physiological import Physiological from lib.database_lib.config import Config diff --git a/python/mass_nifti_pic.py b/python/mass_nifti_pic.py index f7eddcba1..92ce2df5c 100755 --- a/python/mass_nifti_pic.py +++ b/python/mass_nifti_pic.py @@ -7,7 +7,7 @@ import sys import getopt import lib.exitcode -from lib.dataclass.database import Database +from lib.database_mysqldb import Database from lib.imaging import Imaging from lib.database_lib.config import Config diff --git a/tools/correct_blake2b_and_md5_hashes_in_database.py b/tools/correct_blake2b_and_md5_hashes_in_database.py index 8cf5c087d..d6e22df45 100755 --- a/tools/correct_blake2b_and_md5_hashes_in_database.py +++ b/tools/correct_blake2b_and_md5_hashes_in_database.py @@ -6,7 +6,7 @@ import shutil import sys -from lib.dataclass.database import Database +from lib.database_mysqldb import Database from lib.database_lib.config import Config from lib.lorisgetopt import LorisGetOpt diff --git a/tools/correct_lists_incorrectly_saved_in_parameter_file.py b/tools/correct_lists_incorrectly_saved_in_parameter_file.py index 547a666f0..8c72c6b9e 100755 --- a/tools/correct_lists_incorrectly_saved_in_parameter_file.py +++ b/tools/correct_lists_incorrectly_saved_in_parameter_file.py @@ -2,7 +2,7 @@ import os -from lib.dataclass.database import Database +from lib.database_mysqldb import Database from lib.lorisgetopt import LorisGetOpt From 42aca330bd9a5db15dee85aba758293ddc8c6482 Mon Sep 17 00:00:00 2001 From: Maxime Mulder Date: Tue, 20 Aug 2024 12:56:29 -0400 Subject: [PATCH 5/5] use database_lib namespace --- python/bids_import.py | 2 +- python/extract_eeg_bids_archive.py | 2 +- python/ingest_eeg_bids_datasets.py | 2 +- python/lib/candidate.py | 2 +- python/lib/database_lib/__init__.py | 277 ++++++++++++++++++ python/lib/database_lib/candidate_db.py | 2 +- python/lib/database_lib/config.py | 2 +- python/lib/database_lib/files.py | 2 +- .../lib/database_lib/mri_candidate_errors.py | 2 +- python/lib/database_lib/mri_protocol.py | 2 +- .../lib/database_lib/mri_protocol_checks.py | 2 +- .../mri_protocol_violated_scans.py | 2 +- python/lib/database_lib/mri_scan_type.py | 2 +- python/lib/database_lib/mri_scanner.py | 2 +- python/lib/database_lib/mri_upload_db.py | 2 +- python/lib/database_lib/mri_violations_log.py | 2 +- python/lib/database_lib/notification.py | 2 +- python/lib/database_lib/parameter_file.py | 2 +- python/lib/database_lib/parameter_type.py | 2 +- python/lib/database_lib/project_cohort_rel.py | 2 +- python/lib/database_lib/session_db.py | 2 +- python/lib/database_lib/site.py | 2 +- python/lib/database_lib/tarchive.py | 2 +- python/lib/database_lib/tarchive_series.py | 2 +- python/lib/database_lib/visit_windows.py | 2 +- python/lib/database_mysqldb.py | 277 ------------------ .../base_pipeline.py | 2 +- python/lib/dicom_archive.py | 2 +- python/lib/eeg.py | 2 +- python/lib/imaging.py | 2 +- python/lib/imaging_upload.py | 2 +- python/lib/lorisgetopt.py | 2 +- python/lib/mri.py | 2 +- python/lib/physiological.py | 2 +- python/lib/session.py | 2 +- python/mass_electrophysiology_chunking.py | 2 +- python/mass_nifti_pic.py | 2 +- ...rect_blake2b_and_md5_hashes_in_database.py | 2 +- ...sts_incorrectly_saved_in_parameter_file.py | 2 +- 39 files changed, 314 insertions(+), 314 deletions(-) delete mode 100644 python/lib/database_mysqldb.py diff --git a/python/bids_import.py b/python/bids_import.py index 462a9427c..379aa4c6a 100755 --- a/python/bids_import.py +++ b/python/bids_import.py @@ -10,7 +10,7 @@ import lib.exitcode import lib.utilities import lib.physiological -from lib.database_mysqldb import Database +from lib.database_lib import Database from lib.candidate import Candidate from lib.bidsreader import BidsReader from lib.session import Session diff --git a/python/extract_eeg_bids_archive.py b/python/extract_eeg_bids_archive.py index a4c198c04..97a485647 100755 --- a/python/extract_eeg_bids_archive.py +++ b/python/extract_eeg_bids_archive.py @@ -7,7 +7,7 @@ import re from lib.lorisgetopt import LorisGetOpt from lib.imaging_io import ImagingIO -from lib.database_mysqldb import Database +from lib.database_lib import Database from lib.database_lib.config import Config from lib.exitcode import SUCCESS, BAD_CONFIG_SETTING from lib.log import Log diff --git a/python/ingest_eeg_bids_datasets.py b/python/ingest_eeg_bids_datasets.py index 9c7218680..a3922fbb6 100755 --- a/python/ingest_eeg_bids_datasets.py +++ b/python/ingest_eeg_bids_datasets.py @@ -5,7 +5,7 @@ import os import sys from lib.lorisgetopt import LorisGetOpt -from lib.database_mysqldb import Database +from lib.database_lib import Database from lib.database_lib.config import Config from lib.exitcode import SUCCESS, INVALID_ARG import subprocess diff --git a/python/lib/candidate.py b/python/lib/candidate.py index 10e70090c..e25fce319 100644 --- a/python/lib/candidate.py +++ b/python/lib/candidate.py @@ -16,7 +16,7 @@ class Candidate: :Example: from lib.candidate import Candidate - from lib.database_mysqldb import Database + from lib.database_lib import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/__init__.py b/python/lib/database_lib/__init__.py index e69de29bb..99dee85b4 100644 --- a/python/lib/database_lib/__init__.py +++ b/python/lib/database_lib/__init__.py @@ -0,0 +1,277 @@ +"""Allows LORIS database connectivity for LORIS-MRI python code base""" + +import MySQLdb +import sys +import lib.exitcode + + +__license__ = "GPLv3" + + +class Database: + """ + This class performs common tasks related to database connectivity between + the LORIS-MRI python code base and the LORIS backend database. + + :Example: + + from lib.database_lib import Database + + db = Database(config.mysql, verbose) + + db.connect() + + # to select data corresponding to specific parameters + results = db.pselect( + "SELECT CandID FROM candidate WHERE Active = %s AND Sex = %s", + ('Y', 'Male') + ) + + # to select data without any specific parameter + results = db.pselect( + "SELECT CandID FROM candidate" + ) # args is optional in db.pselect + + # to insert multiple rows + db.insert( + 'media', + ('session_id', 'file_name', 'data_dir'), + [ + ('6834', 'bla', 'bndjf'), + ('6834', 'blu', 'blui') + ] + ) + + # to insert one row and return the last inserted ID + last_id = db.insert( + 'media', + ('session_id', 'file_name', 'data_dir'), + [ + ('6834', 'bla', 'bndjf') + ], + True + ) # get_last_id is default to False in db.insert + + # to update data + db.update( + "UPDATE media SET file_name = %s WHERE ID = %s, + ('filename.txt', '1') + ) + + db.disconnect() + """ + + def __init__(self, credentials, verbose): + """ + Constructor method for the Database class. + + :param credentials: LORIS database credentials + :type credentials: dict + :param verbose : whether to be verbose or not + :type verbose : bool + """ + + self.verbose = verbose + + # grep database credentials + default_port = 3306 + self.db_name = credentials['database'] + self.user_name = credentials['username'] + self.password = credentials['passwd'] + self.host_name = credentials['host'] + port = credentials['port'] + + if not self.user_name: + raise Exception("\nUser name cannot be empty string.\n") + if not self.db_name: + raise Exception("\nDatabase name cannot be empty string.\n") + if not self.host_name: + raise Exception("\nDatabase host cannot be empty string.\n") + + self.port = int(port) if port else default_port + + def connect(self): + """ + Attempts to connect to the database using the connection parameters + passed at construction time. This method will throw a + DatabaseException if the connection could not be established. + """ + + connect_statement = "\nConnecting to:" \ + "\n\tdatabase: " + self.db_name + \ + "\n\tusername: " + self.user_name + \ + "\n\thostname: " + self.host_name + \ + "\n\tport : " + str(self.port) + '\n' + if self.verbose: + print(connect_statement) + + try: + self.con = MySQLdb.connect( + host=self.host_name, + user=self.user_name, + passwd=self.password, + port=self.port, + db=self.db_name + ) + # self.cnx.cursor = self.cnx.cursor(prepared=True) + except MySQLdb.Error as err: + raise Exception("Database connection failure: " + format(err)) + + def pselect(self, query, args=None): + """ + Executes a select query on the database. This method will first prepare + the statement passed as parameter before sending the request to the + database. + + :param query: select query to execute (containing the argument + placeholders if any + :type query: str + :param args: arguments to replace the placeholders with + :type args: tuple + + :return: list of dictionaries with MySQL column header name + :rtype: list of dict + """ + if self.verbose: + print("\nExecuting query:\n\t" + query + "\n") + if args: + print("With arguments:\n\t" + str(args) + "\n") + + try: + cursor = self.con.cursor(MySQLdb.cursors.DictCursor) + cursor.execute(query, args) if args else cursor.execute(query) + results = cursor.fetchall() + cursor.close() + except MySQLdb.Error as err: + raise Exception("Select query failure: " + format(err)) + + return results + + def insert(self, table_name, column_names, values, get_last_id=False): + """ + Inserts records in a given database table with the specified column + values. This method will raise an exception if the record cannot be + inserted. + + :param table_name : name of the database table to insert into + :type table_name : str + :param column_names: name of the table's columns to insert into + :type column_names: tuple + :param values : list of values to insert into the table + :type values : list + + :return: the last ID that have been inserted into the database + :rtype: int + """ + + placeholders = ','.join(map(lambda x: '%s', column_names)) + + query = "INSERT INTO %s (%s) VALUES (%s)" % ( + table_name, ', '.join(column_names), placeholders + ) + + if self.verbose: + print("\nExecuting query:\n\t" + query + "\n" + + "With arguments:\n\t" + str(values) + "\n") + + try: + cursor = self.con.cursor() + if isinstance(values, list): + # if values is a list, use cursor.executemany + # (to execute multiple inserts at once) + cursor.executemany(query, values) + else: + # else, values is a tuple and want to execute only one insert + cursor.execute(query, values) + self.con.commit() + last_id = cursor.lastrowid + cursor.close() + except MySQLdb.Error as err: + raise Exception("Insert query failure: " + format(err)) + + if get_last_id: + return last_id + + def update(self, query, args): + """ + Executes an update query on the database. This method will first prepare + the statement passed as parameter before sending the request to the + database. + + :param query: update query to be run + :type query: str + :param args : arguments to replace the placeholders with + :type args : tuple + """ + + if self.verbose: + print("\nExecuting query:\n\t" + query + "\n" + + "With arguments:\n\t" + str(args) + "\n") + + try: + cursor = self.con.cursor() + cursor.execute(query, args) + self.con.commit() + except MySQLdb.Error as err: + raise Exception("Update query failure: " + format(err)) + + def grep_id_from_lookup_table(self, id_field_name, table_name, where_field_name, + where_value, insert_if_not_found=None): + """ + Greps an ID from a given lookup table based on the provided value. + + :param id_field_name : name of the ID field in the table + :type id_field_name : str + :param table_name : name of the lookup table + :type table_name : str + :param where_field_name : name of the field to use to find the ID + :type where_field_name : str + :param where_value : value of the field to use to find the ID + :type where_value : str + :param insert_if_not_found: whether to insert a new row in the lookup table + if no entry was found for the provided value + :type insert_if_not_found: bool + + :return: ID found in the lookup table + :rtype: int + + """ + + query = "SELECT " + id_field_name + " " \ + "FROM " + table_name + " " \ + "WHERE " + where_field_name + " = %s" + + result = self.pselect(query=query, args=(where_value,)) + id = result[0][id_field_name] if result else None + + if not id and insert_if_not_found: + id = self.insert( + table_name = table_name, + column_names = (where_field_name,), + values = (where_value,), + get_last_id = True + ) + + if not id: + message = "\nERROR: " + where_value + " " + where_field_name + \ + " does not exist in " + table_name + " database table\n" + print(message) + sys.exit(lib.exitcode.SELECT_FAILURE) + + return id + + def disconnect(self): + """ + Terminates the connection previously instantiated to the database if a + connection was previously established. + """ + + if hasattr(self, 'cnx'): + if self.verbose: + print("\nDisconnecting from the database") + + try: + self.con.close() + except MySQLdb.Error as err: + message = "Database disconnection failure: " + format(err) + raise Exception(message) diff --git a/python/lib/database_lib/candidate_db.py b/python/lib/database_lib/candidate_db.py index cd833a49b..958caaf6d 100644 --- a/python/lib/database_lib/candidate_db.py +++ b/python/lib/database_lib/candidate_db.py @@ -11,7 +11,7 @@ class CandidateDB: :Example: from lib.database_lib.candidate_db import CandidateDB - from lib.database_mysqldb import Database + from lib.database_lib import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/config.py b/python/lib/database_lib/config.py index 3e84b2f06..22dd12348 100644 --- a/python/lib/database_lib/config.py +++ b/python/lib/database_lib/config.py @@ -11,7 +11,7 @@ class Config: :Example: from lib.database_lib.config import Config - from lib.database_mysqldb import Database + from lib.database_lib import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/files.py b/python/lib/database_lib/files.py index fd2734561..dd5a32e67 100644 --- a/python/lib/database_lib/files.py +++ b/python/lib/database_lib/files.py @@ -12,7 +12,7 @@ class Files: :Example: from lib.files import Files - from lib.database_mysqldb import Database + from lib.database_lib import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/mri_candidate_errors.py b/python/lib/database_lib/mri_candidate_errors.py index 145564e08..eec21fefc 100644 --- a/python/lib/database_lib/mri_candidate_errors.py +++ b/python/lib/database_lib/mri_candidate_errors.py @@ -11,7 +11,7 @@ class MriCandidateErrors: :Example: from lib.mri_candidate_errors import MriCandidateErrors - from lib.database_mysqldb import Database + from lib.database_lib import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/mri_protocol.py b/python/lib/database_lib/mri_protocol.py index dafcb8ae3..60230b013 100644 --- a/python/lib/database_lib/mri_protocol.py +++ b/python/lib/database_lib/mri_protocol.py @@ -11,7 +11,7 @@ class MriProtocol: :Example: from lib.mri_protocol import MriProtocol - from lib.database_mysqldb import Database + from lib.database_lib import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/mri_protocol_checks.py b/python/lib/database_lib/mri_protocol_checks.py index a3889b1a2..b6a8dd7c0 100644 --- a/python/lib/database_lib/mri_protocol_checks.py +++ b/python/lib/database_lib/mri_protocol_checks.py @@ -11,7 +11,7 @@ class MriProtocolChecks: :Example: from lib.mri_protocol_checks import MriProtocolChecks - from lib.database_mysqldb import Database + from lib.database_lib import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/mri_protocol_violated_scans.py b/python/lib/database_lib/mri_protocol_violated_scans.py index f9da01a27..87c9f734c 100644 --- a/python/lib/database_lib/mri_protocol_violated_scans.py +++ b/python/lib/database_lib/mri_protocol_violated_scans.py @@ -11,7 +11,7 @@ class MriProtocolViolatedScans: :Example: from lib.mri_protocol_violated_scans import MriProtocolViolatedScans - from lib.database_mysqldb import Database + from lib.database_lib import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/mri_scan_type.py b/python/lib/database_lib/mri_scan_type.py index 167e95ce0..bf14fb0ee 100644 --- a/python/lib/database_lib/mri_scan_type.py +++ b/python/lib/database_lib/mri_scan_type.py @@ -11,7 +11,7 @@ class MriScanType: :Example: from lib.mri_scan_type import MriScanType - from lib.database_mysqldb import Database + from lib.database_lib import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/mri_scanner.py b/python/lib/database_lib/mri_scanner.py index 95b6c3754..edc8a0d2d 100644 --- a/python/lib/database_lib/mri_scanner.py +++ b/python/lib/database_lib/mri_scanner.py @@ -13,7 +13,7 @@ class MriScanner: :Example: from lib.mri_scanner import MriScanner - from lib.database_mysqldb import Database + from lib.database_lib import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/mri_upload_db.py b/python/lib/database_lib/mri_upload_db.py index 5941af598..5bf4a24d3 100644 --- a/python/lib/database_lib/mri_upload_db.py +++ b/python/lib/database_lib/mri_upload_db.py @@ -10,7 +10,7 @@ class MriUploadDB: :Example: from lib.mri_upload import MriUploadDB - from lib.database_mysqldb import Database + from lib.database_lib import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/mri_violations_log.py b/python/lib/database_lib/mri_violations_log.py index f08b990ef..a52950533 100644 --- a/python/lib/database_lib/mri_violations_log.py +++ b/python/lib/database_lib/mri_violations_log.py @@ -11,7 +11,7 @@ class MriViolationsLog: :Example: from lib.mri_violations_log import MriViolationsLog - from lib.database_mysqldb import Database + from lib.database_lib import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/notification.py b/python/lib/database_lib/notification.py index 29f2ef20b..33892b6b1 100644 --- a/python/lib/database_lib/notification.py +++ b/python/lib/database_lib/notification.py @@ -12,7 +12,7 @@ class Notification: :Example: from lib.notification import Notification - from lib.database_mysqldb import Database + from lib.database_lib import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/parameter_file.py b/python/lib/database_lib/parameter_file.py index 356829313..603fca396 100644 --- a/python/lib/database_lib/parameter_file.py +++ b/python/lib/database_lib/parameter_file.py @@ -10,7 +10,7 @@ class ParameterFile: :Example: from lib.parameter_file import ParameterFile - from lib.database_mysqldb import Database + from lib.database_lib import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/parameter_type.py b/python/lib/database_lib/parameter_type.py index 935432c6a..a6c2f60b3 100644 --- a/python/lib/database_lib/parameter_type.py +++ b/python/lib/database_lib/parameter_type.py @@ -10,7 +10,7 @@ class ParameterType: :Example: from lib.parameter_type import ParameterType - from lib.database_mysqldb import Database + from lib.database_lib import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/project_cohort_rel.py b/python/lib/database_lib/project_cohort_rel.py index f6d776335..c3e3ccde4 100644 --- a/python/lib/database_lib/project_cohort_rel.py +++ b/python/lib/database_lib/project_cohort_rel.py @@ -11,7 +11,7 @@ class ProjectCohortRel: :Example: from lib.database_lib.project_cohort_rel import ProjectCohortRel - from lib.database_mysqldb import Database + from lib.database_lib import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/session_db.py b/python/lib/database_lib/session_db.py index 4516f400e..b95311585 100644 --- a/python/lib/database_lib/session_db.py +++ b/python/lib/database_lib/session_db.py @@ -11,7 +11,7 @@ class SessionDB: :Example: from lib.database_lib.session_db import SessionDB - from lib.database_mysqldb import Database + from lib.database_lib import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/site.py b/python/lib/database_lib/site.py index b8aca23e0..1af4367bb 100644 --- a/python/lib/database_lib/site.py +++ b/python/lib/database_lib/site.py @@ -11,7 +11,7 @@ class Site: :Example: from lib.site import Site - from lib.database_mysqldb import Database + from lib.database_lib import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/tarchive.py b/python/lib/database_lib/tarchive.py index 3c3a52c94..f881f6c98 100644 --- a/python/lib/database_lib/tarchive.py +++ b/python/lib/database_lib/tarchive.py @@ -11,7 +11,7 @@ class Tarchive: :Example: from lib.tarchive import Tarchive - from lib.database_mysqldb import Database + from lib.database_lib import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/tarchive_series.py b/python/lib/database_lib/tarchive_series.py index 1a39cebec..2586848c8 100644 --- a/python/lib/database_lib/tarchive_series.py +++ b/python/lib/database_lib/tarchive_series.py @@ -11,7 +11,7 @@ class TarchiveSeries: :Example: from lib.tarchive_series import TarchiveSeries - from lib.database_mysqldb import Database + from lib.database_lib import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_lib/visit_windows.py b/python/lib/database_lib/visit_windows.py index 5538857db..f16ae8930 100644 --- a/python/lib/database_lib/visit_windows.py +++ b/python/lib/database_lib/visit_windows.py @@ -11,7 +11,7 @@ class VisitWindows: :Example: from lib.visit_windows import VisitWindows - from lib.database_mysqldb import Database + from lib.database_lib import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/database_mysqldb.py b/python/lib/database_mysqldb.py deleted file mode 100644 index e35d9caf7..000000000 --- a/python/lib/database_mysqldb.py +++ /dev/null @@ -1,277 +0,0 @@ -"""Allows LORIS database connectivity for LORIS-MRI python code base""" - -import MySQLdb -import sys -import lib.exitcode - - -__license__ = "GPLv3" - - -class Database: - """ - This class performs common tasks related to database connectivity between - the LORIS-MRI python code base and the LORIS backend database. - - :Example: - - from lib.database_mysqldb import Database - - db = Database(config.mysql, verbose) - - db.connect() - - # to select data corresponding to specific parameters - results = db.pselect( - "SELECT CandID FROM candidate WHERE Active = %s AND Sex = %s", - ('Y', 'Male') - ) - - # to select data without any specific parameter - results = db.pselect( - "SELECT CandID FROM candidate" - ) # args is optional in db.pselect - - # to insert multiple rows - db.insert( - 'media', - ('session_id', 'file_name', 'data_dir'), - [ - ('6834', 'bla', 'bndjf'), - ('6834', 'blu', 'blui') - ] - ) - - # to insert one row and return the last inserted ID - last_id = db.insert( - 'media', - ('session_id', 'file_name', 'data_dir'), - [ - ('6834', 'bla', 'bndjf') - ], - True - ) # get_last_id is default to False in db.insert - - # to update data - db.update( - "UPDATE media SET file_name = %s WHERE ID = %s, - ('filename.txt', '1') - ) - - db.disconnect() - """ - - def __init__(self, credentials, verbose): - """ - Constructor method for the Database class. - - :param credentials: LORIS database credentials - :type credentials: dict - :param verbose : whether to be verbose or not - :type verbose : bool - """ - - self.verbose = verbose - - # grep database credentials - default_port = 3306 - self.db_name = credentials['database'] - self.user_name = credentials['username'] - self.password = credentials['passwd'] - self.host_name = credentials['host'] - port = credentials['port'] - - if not self.user_name: - raise Exception("\nUser name cannot be empty string.\n") - if not self.db_name: - raise Exception("\nDatabase name cannot be empty string.\n") - if not self.host_name: - raise Exception("\nDatabase host cannot be empty string.\n") - - self.port = int(port) if port else default_port - - def connect(self): - """ - Attempts to connect to the database using the connection parameters - passed at construction time. This method will throw a - DatabaseException if the connection could not be established. - """ - - connect_statement = "\nConnecting to:" \ - "\n\tdatabase: " + self.db_name + \ - "\n\tusername: " + self.user_name + \ - "\n\thostname: " + self.host_name + \ - "\n\tport : " + str(self.port) + '\n' - if self.verbose: - print(connect_statement) - - try: - self.con = MySQLdb.connect( - host=self.host_name, - user=self.user_name, - passwd=self.password, - port=self.port, - db=self.db_name - ) - # self.cnx.cursor = self.cnx.cursor(prepared=True) - except MySQLdb.Error as err: - raise Exception("Database connection failure: " + format(err)) - - def pselect(self, query, args=None): - """ - Executes a select query on the database. This method will first prepare - the statement passed as parameter before sending the request to the - database. - - :param query: select query to execute (containing the argument - placeholders if any - :type query: str - :param args: arguments to replace the placeholders with - :type args: tuple - - :return: list of dictionaries with MySQL column header name - :rtype: list of dict - """ - if self.verbose: - print("\nExecuting query:\n\t" + query + "\n") - if args: - print("With arguments:\n\t" + str(args) + "\n") - - try: - cursor = self.con.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query, args) if args else cursor.execute(query) - results = cursor.fetchall() - cursor.close() - except MySQLdb.Error as err: - raise Exception("Select query failure: " + format(err)) - - return results - - def insert(self, table_name, column_names, values, get_last_id=False): - """ - Inserts records in a given database table with the specified column - values. This method will raise an exception if the record cannot be - inserted. - - :param table_name : name of the database table to insert into - :type table_name : str - :param column_names: name of the table's columns to insert into - :type column_names: tuple - :param values : list of values to insert into the table - :type values : list - - :return: the last ID that have been inserted into the database - :rtype: int - """ - - placeholders = ','.join(map(lambda x: '%s', column_names)) - - query = "INSERT INTO %s (%s) VALUES (%s)" % ( - table_name, ', '.join(column_names), placeholders - ) - - if self.verbose: - print("\nExecuting query:\n\t" + query + "\n" - + "With arguments:\n\t" + str(values) + "\n") - - try: - cursor = self.con.cursor() - if isinstance(values, list): - # if values is a list, use cursor.executemany - # (to execute multiple inserts at once) - cursor.executemany(query, values) - else: - # else, values is a tuple and want to execute only one insert - cursor.execute(query, values) - self.con.commit() - last_id = cursor.lastrowid - cursor.close() - except MySQLdb.Error as err: - raise Exception("Insert query failure: " + format(err)) - - if get_last_id: - return last_id - - def update(self, query, args): - """ - Executes an update query on the database. This method will first prepare - the statement passed as parameter before sending the request to the - database. - - :param query: update query to be run - :type query: str - :param args : arguments to replace the placeholders with - :type args : tuple - """ - - if self.verbose: - print("\nExecuting query:\n\t" + query + "\n" - + "With arguments:\n\t" + str(args) + "\n") - - try: - cursor = self.con.cursor() - cursor.execute(query, args) - self.con.commit() - except MySQLdb.Error as err: - raise Exception("Update query failure: " + format(err)) - - def grep_id_from_lookup_table(self, id_field_name, table_name, where_field_name, - where_value, insert_if_not_found=None): - """ - Greps an ID from a given lookup table based on the provided value. - - :param id_field_name : name of the ID field in the table - :type id_field_name : str - :param table_name : name of the lookup table - :type table_name : str - :param where_field_name : name of the field to use to find the ID - :type where_field_name : str - :param where_value : value of the field to use to find the ID - :type where_value : str - :param insert_if_not_found: whether to insert a new row in the lookup table - if no entry was found for the provided value - :type insert_if_not_found: bool - - :return: ID found in the lookup table - :rtype: int - - """ - - query = "SELECT " + id_field_name + " " \ - "FROM " + table_name + " " \ - "WHERE " + where_field_name + " = %s" - - result = self.pselect(query=query, args=(where_value,)) - id = result[0][id_field_name] if result else None - - if not id and insert_if_not_found: - id = self.insert( - table_name = table_name, - column_names = (where_field_name,), - values = (where_value,), - get_last_id = True - ) - - if not id: - message = "\nERROR: " + where_value + " " + where_field_name + \ - " does not exist in " + table_name + " database table\n" - print(message) - sys.exit(lib.exitcode.SELECT_FAILURE) - - return id - - def disconnect(self): - """ - Terminates the connection previously instantiated to the database if a - connection was previously established. - """ - - if hasattr(self, 'cnx'): - if self.verbose: - print("\nDisconnecting from the database") - - try: - self.con.close() - except MySQLdb.Error as err: - message = "Database disconnection failure: " + format(err) - raise Exception(message) diff --git a/python/lib/dcm2bids_imaging_pipeline_lib/base_pipeline.py b/python/lib/dcm2bids_imaging_pipeline_lib/base_pipeline.py index c9523f238..70c3a82c4 100644 --- a/python/lib/dcm2bids_imaging_pipeline_lib/base_pipeline.py +++ b/python/lib/dcm2bids_imaging_pipeline_lib/base_pipeline.py @@ -9,7 +9,7 @@ from lib.database_lib.candidate_db import CandidateDB from lib.database_lib.config import Config from lib.database_lib.visit_windows import VisitWindows -from lib.database_mysqldb import Database +from lib.database_lib import Database from lib.dicom_archive import DicomArchive from lib.imaging import Imaging from lib.log import Log diff --git a/python/lib/dicom_archive.py b/python/lib/dicom_archive.py index 197a26bb1..8497d473c 100644 --- a/python/lib/dicom_archive.py +++ b/python/lib/dicom_archive.py @@ -14,7 +14,7 @@ class DicomArchive: :Example: - from lib.database_mysqldb import Database + from lib.database_lib import Database from lib.dicom_archive import DicomArchive # database connection diff --git a/python/lib/eeg.py b/python/lib/eeg.py index 667689443..5baea46d3 100755 --- a/python/lib/eeg.py +++ b/python/lib/eeg.py @@ -29,7 +29,7 @@ class Eeg: from lib.bidsreader import BidsReader from lib.eeg import Eeg - from lib.database_mysqldb import Database + from lib.database_lib import Database from lib.database_lib.config import Config # database connection diff --git a/python/lib/imaging.py b/python/lib/imaging.py index 476e1afa2..420417f9e 100644 --- a/python/lib/imaging.py +++ b/python/lib/imaging.py @@ -32,7 +32,7 @@ class Imaging: :Example: from lib.imaging import Imaging - from lib.database_mysqldb import Database + from lib.database_lib import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/imaging_upload.py b/python/lib/imaging_upload.py index cae29f5d9..f9d28fadf 100644 --- a/python/lib/imaging_upload.py +++ b/python/lib/imaging_upload.py @@ -13,7 +13,7 @@ class ImagingUpload: :Example: from lib.imaging_upload import ImagingUpload - from lib.database_mysqldb import Database + from lib.database_lib import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/lorisgetopt.py b/python/lib/lorisgetopt.py index ae62300f6..081f1c18a 100644 --- a/python/lib/lorisgetopt.py +++ b/python/lib/lorisgetopt.py @@ -7,7 +7,7 @@ import sys from lib.aws_s3 import AwsS3 -from lib.database_mysqldb import Database +from lib.database_lib import Database from lib.database_lib.config import Config diff --git a/python/lib/mri.py b/python/lib/mri.py index 52cf17a75..84fb13353 100644 --- a/python/lib/mri.py +++ b/python/lib/mri.py @@ -26,7 +26,7 @@ class Mri: from lib.bidsreader import BidsReader from lib.mri import Mri - from lib.database_mysqldb import Database + from lib.database_lib import Database # database connection db = Database(config_file.mysql, verbose) diff --git a/python/lib/physiological.py b/python/lib/physiological.py index 67f9d6420..4b2cbc87a 100755 --- a/python/lib/physiological.py +++ b/python/lib/physiological.py @@ -32,7 +32,7 @@ class Physiological: :Example: from lib.physiological import Physiological - from lib.database_mysqldb import Database + from lib.database_lib import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/lib/session.py b/python/lib/session.py index 856e105db..7c91e2c68 100644 --- a/python/lib/session.py +++ b/python/lib/session.py @@ -15,7 +15,7 @@ class Session: :Example: from lib.session import Session - from lib.database_mysqldb import Database + from lib.database_lib import Database # database connection db = Database(config.mysql, verbose) diff --git a/python/mass_electrophysiology_chunking.py b/python/mass_electrophysiology_chunking.py index 3a3a3748b..e93f7f8a8 100755 --- a/python/mass_electrophysiology_chunking.py +++ b/python/mass_electrophysiology_chunking.py @@ -6,7 +6,7 @@ import sys import getopt import lib.exitcode -from lib.database_mysqldb import Database +from lib.database_lib import Database from lib.physiological import Physiological from lib.database_lib.config import Config diff --git a/python/mass_nifti_pic.py b/python/mass_nifti_pic.py index 92ce2df5c..cd8e8b023 100755 --- a/python/mass_nifti_pic.py +++ b/python/mass_nifti_pic.py @@ -7,7 +7,7 @@ import sys import getopt import lib.exitcode -from lib.database_mysqldb import Database +from lib.database_lib import Database from lib.imaging import Imaging from lib.database_lib.config import Config diff --git a/tools/correct_blake2b_and_md5_hashes_in_database.py b/tools/correct_blake2b_and_md5_hashes_in_database.py index d6e22df45..1ac534bcd 100755 --- a/tools/correct_blake2b_and_md5_hashes_in_database.py +++ b/tools/correct_blake2b_and_md5_hashes_in_database.py @@ -6,7 +6,7 @@ import shutil import sys -from lib.database_mysqldb import Database +from lib.database_lib import Database from lib.database_lib.config import Config from lib.lorisgetopt import LorisGetOpt diff --git a/tools/correct_lists_incorrectly_saved_in_parameter_file.py b/tools/correct_lists_incorrectly_saved_in_parameter_file.py index 8c72c6b9e..d9fc98602 100755 --- a/tools/correct_lists_incorrectly_saved_in_parameter_file.py +++ b/tools/correct_lists_incorrectly_saved_in_parameter_file.py @@ -2,7 +2,7 @@ import os -from lib.database_mysqldb import Database +from lib.database_lib import Database from lib.lorisgetopt import LorisGetOpt