Integration test to check that the ORM definitions match the LORIS SQL database

.github/workflows/integration.yml

@@ -18,7 +18,7 @@ jobs:
     - name: Clone the LORIS core repository
       run: git clone https://github.com/aces/Loris.git /tmp/Loris
 
-    - name: Import database files 
+    - name: Import database files
       run: |
         mkdir -p ./test/database
         mkdir -p ./test/database/SQL
@@ -39,8 +39,6 @@ jobs:
       with:
         context: .
         file: ./test/db.Dockerfile
-        build-args: |
-          BASE_DIR=/app/
         tags: loris-db
         load: true
         cache-from: type=gha,scope=loris-db

python/lib/db/model/candidate.py

@@ -1,4 +1,4 @@
-from datetime import date
+from datetime import date, datetime
 from typing import Optional
 
 from sqlalchemy import ForeignKey
@@ -35,7 +35,7 @@ class DbCandidate(Base):
     flagged_reason          : Mapped[Optional[int]]  = mapped_column('flagged_reason')
     flagged_other           : Mapped[Optional[str]]  = mapped_column('flagged_other')
     flagged_other_status    : Mapped[Optional[str]]  = mapped_column('flagged_other_status')
-    test_date               : Mapped[int]            = mapped_column('Testdate')
+    test_date               : Mapped[datetime]       = mapped_column('Testdate')
     entity_type             : Mapped[str]            = mapped_column('Entity_type')
     proband_sex             : Mapped[Optional[str]]  = mapped_column('ProbandSex')
     proband_sate_of_birth   : Mapped[Optional[date]] = mapped_column('ProbandDoB')

python/lib/db/model/dicom_archive.py

@@ -22,7 +22,7 @@ class DbDicomArchive(Base):
     patient_sex              : Mapped[Optional[str]]      = mapped_column('PatientSex')
     neuro_db_center_name     : Mapped[Optional[str]]      = mapped_column('neurodbCenterName')
     center_name              : Mapped[str]                = mapped_column('CenterName')
-    last_update              : Mapped[Optional[date]]     = mapped_column('LastUpdate')
+    last_update              : Mapped[Optional[datetime]] = mapped_column('LastUpdate')
     date_acquired            : Mapped[Optional[date]]     = mapped_column('DateAcquired')
     date_first_archived      : Mapped[Optional[datetime]] = mapped_column('DateFirstArchived')
     date_last_archived       : Mapped[Optional[datetime]] = mapped_column('DateLastArchived')
@@ -45,7 +45,7 @@ class DbDicomArchive(Base):
     create_info              : Mapped[Optional[str]]      = mapped_column('CreateInfo')
     acquisition_metadata     : Mapped[str]                = mapped_column('AcquisitionMetadata')
     date_sent                : Mapped[Optional[datetime]] = mapped_column('DateSent')
-    pending_transfer         : Mapped[int]                = mapped_column('PendingTransfer')
+    pending_transfer         : Mapped[bool]               = mapped_column('PendingTransfer')
 
     series  : Mapped[list['db_dicom_archive_series.DbDicomArchiveSeries']] \
         = relationship('DbDicomArchiveSeries', back_populates='archive')

python/lib/db/model/session.py

@@ -20,12 +20,12 @@ class DbSession(Base):
     project_id               : Mapped[int]                = mapped_column('ProjectID', ForeignKey('Project.ProjectID'))
     visit_number             : Mapped[Optional[int]]      = mapped_column('VisitNo')
     visit_label              : Mapped[str]                = mapped_column('Visit_label')
-    cohort_id                : Mapped[int]                = mapped_column('CohortID')
+    cohort_id                : Mapped[Optional[int]]      = mapped_column('CohortID')
     submitted                : Mapped[bool]               = mapped_column('Submitted', YNBool)
     current_stage            : Mapped[str]                = mapped_column('Current_stage')
     date_stage_change        : Mapped[Optional[date]]     = mapped_column('Date_stage_change')
     screening                : Mapped[Optional[str]]      = mapped_column('Screening')
-    date_screening           : Mapped[date]               = mapped_column('Date_screening')
+    date_screening           : Mapped[Optional[date]]     = mapped_column('Date_screening')
     visit                    : Mapped[Optional[str]]      = mapped_column('Visit')
     date_visit               : Mapped[Optional[date]]     = mapped_column('Date_visit')
     date_status_change       : Mapped[Optional[date]]     = mapped_column('Date_status_change')
@@ -36,17 +36,17 @@ class DbSession(Base):
     registered_by            : Mapped[Optional[str]]      = mapped_column('RegisteredBy')
     user_id                  : Mapped[str]                = mapped_column('UserID')
     date_registered          : Mapped[Optional[date]]     = mapped_column('Date_registered')
-    test_date                : Mapped[int]                = mapped_column('Testdate')
+    test_date                : Mapped[datetime]           = mapped_column('Testdate')
     hardcopy_request         : Mapped[str]                = mapped_column('Hardcopy_request')
-    bvl_qc_status            : Mapped[Optional[str]]      = mapped_column('BVLQCStaus')
+    bvl_qc_status            : Mapped[Optional[str]]      = mapped_column('BVLQCStatus')
     bvl_qc_type              : Mapped[Optional[str]]      = mapped_column('BVLQCType')
     bvl_qc_exclusion         : Mapped[Optional[str]]      = mapped_column('BVLQCExclusion')
     qcd                      : Mapped[Optional[str]]      = mapped_column('QCd')
     scan_done                : Mapped[Optional[bool]]     = mapped_column('Scan_done', YNBool)
     mri_qc_status            : Mapped[str]                = mapped_column('MRIQCStatus')
     mri_qc_pending           : Mapped[bool]               = mapped_column('MRIQCPending', YNBool)
-    mri_qc_first_change_time : Mapped[Optional[datetime]] = mapped_column('MRIQCFirstChange')
-    mri_qc_last_change_time  : Mapped[Optional[datetime]] = mapped_column('MRIQCLastChange')
+    mri_qc_first_change_time : Mapped[Optional[datetime]] = mapped_column('MRIQCFirstChangeTime')
+    mri_qc_last_change_time  : Mapped[Optional[datetime]] = mapped_column('MRIQCLastChangeTime')
     mri_caveat               : Mapped[str]                = mapped_column('MRICaveat')
     language_id              : Mapped[Optional[int]]      = mapped_column('languageID')
 

python/lib/db/model/visit_window.py

@@ -9,7 +9,7 @@ class DbVisitWindow(Base):
     __tablename__  = 'Visit_Windows'
 
     id                   : Mapped[int]           = mapped_column('ID', primary_key=True)
-    visit_label          : Mapped[str]           = mapped_column('Visit_label')
+    visit_label          : Mapped[Optional[str]] = mapped_column('Visit_label')
     window_min_days      : Mapped[Optional[int]] = mapped_column('WindowMinDays')
     window_max_days      : Mapped[Optional[int]] = mapped_column('WindowMaxDays')
     optimum_min_days     : Mapped[Optional[int]] = mapped_column('OptimumMinDays')

python/tests/integration/test_orm_sql_sync.py

@@ -0,0 +1,68 @@
+import importlib
+from pathlib import Path
+from typing import Any
+
+from sqlalchemy import MetaData
+from sqlalchemy.dialects.mysql.types import DOUBLE, TINYINT
+from sqlalchemy.types import TypeDecorator, TypeEngine
+
+from lib.db.base import Base
+from tests.util.database import get_integration_database_engine
+
+
+def test_orm_sql_sync():
+    """
+    Test that the SQLAlchemy ORM definitions are in sync with the existing SQL database.
+    """
+
+    # Load all the ORM table definitions in the SQLAlchemy schema
+    for file in Path('python/lib/db/model').glob('*.py'):
+        importlib.import_module(f'lib.db.model.{file.name[:-3]}')
+
+    orm_metadata = Base.metadata
+
+    engine = get_integration_database_engine()
+    sql_metadata = MetaData()
+    sql_metadata.reflect(engine)
+
+    for orm_table in orm_metadata.sorted_tables:
+        print(f'test table: {orm_table.name}')
+
+        # Check that each ORM table has a corresponding SQL table
+        sql_table = sql_metadata.tables.get(orm_table.name)
+        assert sql_table is not None
+
+        # Check that the ORM and SQL tables have the same columns
+        orm_column_names = orm_table.columns.keys().sort()
+        sql_column_names = sql_table.columns.keys().sort()
+        assert orm_column_names == sql_column_names
+
+        for orm_column in orm_table.columns:
+            print(f'  test column: {orm_column.name}')
+
+            # Check that the types of the ORM and SQL column are equal
+            # The type comparison is not exact, minor differences are ignored
+            sql_column = sql_table.columns[orm_column.name]
+            orm_column_python_type = get_orm_python_type(orm_column.type)
+            sql_column_python_type = get_sql_python_type(sql_column.type)
+            assert orm_column_python_type == sql_column_python_type
+            assert orm_column.nullable == sql_column.nullable
+
+        print()
+
+
+def get_orm_python_type(orm_type: TypeEngine[Any]):
+    if isinstance(orm_type, TypeDecorator):
+        return orm_type.impl.python_type
+
+    return orm_type.python_type
+
+
+def get_sql_python_type(sql_type: TypeEngine[Any]):
+    if isinstance(sql_type, TINYINT) and sql_type.display_width == 1:  # type: ignore
+        return bool
+
+    if isinstance(sql_type, DOUBLE):
+        return float
+
+    return sql_type.python_type

python/tests/unit/db/query/test_candidate.py

@@ -1,4 +1,5 @@
 from dataclasses import dataclass
+from datetime import datetime
 
 import pytest
 from sqlalchemy.orm import Session as Database
@@ -26,7 +27,7 @@ def setup():
         registration_project_id = 1,
         active = True,
         user_id = 'admin',
-        test_date = 0,
+        test_date = datetime.now(),
         entity_type = 'human',
     )
 
@@ -37,7 +38,7 @@ def setup():
         registration_project_id = 1,
         active = True,
         user_id = 'admin',
-        test_date = 0,
+        test_date = datetime.now(),
         entity_type = 'human',
     )
 

python/tests/unit/db/query/test_dicom_archive.py

@@ -44,7 +44,7 @@ def setup():
         scanner_software_version  = 'Test scanner software version',
         upload_attempt            = 0,
         acquisition_metadata      = '',
-        pending_transfer          = 0,
+        pending_transfer          = False,
     )
 
     dicom_archive_2 = DbDicomArchive(
@@ -65,7 +65,7 @@ def setup():
         scanner_software_version  = 'Test scanner software version',
         upload_attempt            = 0,
         acquisition_metadata      = '',
-        pending_transfer          = 0,
+        pending_transfer          = False,
     )
 
     db.add(dicom_archive_1)

python/tests/util/database.py

@@ -1,7 +1,12 @@
-from sqlalchemy import create_engine
+import os
+import sys
+from typing import cast
+
+from sqlalchemy import Engine, create_engine
 from sqlalchemy.orm import Session
 
 from lib.db.base import Base
+from lib.db.connect import connect_to_database
 
 
 def create_test_database():
@@ -12,3 +17,16 @@ def create_test_database():
     engine = create_engine('sqlite:///:memory:')
     Base.metadata.create_all(engine)
     return Session(engine)
+
+
+def get_integration_database_engine():
+    """
+    Get an SQLAlchemy engine for the integration testing database using the configuration from the
+    Python configuration file.
+    """
+
+    config_file = os.path.join(os.environ['LORIS_CONFIG'], '.loris_mri', 'database_config.py')
+    sys.path.append(os.path.dirname(config_file))
+    config = __import__(os.path.basename(config_file[:-3]))
+    session = connect_to_database(config.mysql)
+    return cast(Engine, session.get_bind())

test/db.Dockerfile

@@ -1,7 +1,5 @@
 FROM mariadb:latest
 
-ARG BASE_DIR
-
 COPY test/database/SQL/0000-00-00-schema.sql /0000-00-00-schema.sql
 COPY test/database/SQL/0000-00-01-Modules.sql /0000-00-01-Modules.sql
 COPY test/database/SQL/0000-00-02-Permission.sql /0000-00-02-Permission.sql
@@ -16,7 +14,7 @@ COPY test/database/raisinbread/instruments/instrument_sql/mri_parameter_form.sql
 COPY test/database/raisinbread/instruments/instrument_sql/radiology_review.sql  /radiology_review.sql
 COPY test/database/test/test_instrument/testtest.sql /test_instrument.sql
 
-RUN echo "Use LorisTest;" | cat - \
+RUN echo "CREATE DATABASE LorisTest; USE LorisTest;" | cat - \
     0000-00-00-schema.sql \
     0000-00-01-Modules.sql \
     0000-00-02-Permission.sql \
@@ -31,11 +29,22 @@ RUN echo "Use LorisTest;" | cat - \
     test_instrument.sql \
     RB_files/*.sql > /docker-entrypoint-initdb.d/0000-compiled.sql
 
-RUN echo "Use LorisTest;" >> /docker-entrypoint-initdb.d/0001-paths.sql
-RUN echo "UPDATE Config SET Value='${BASE_DIR}/' WHERE ConfigID=(SELECT ID FROM ConfigSettings WHERE Name='base');" >> /docker-entrypoint-initdb.d/0001-paths.sql
-RUN echo "GRANT UPDATE,INSERT,SELECT,DELETE,DROP,CREATE TEMPORARY TABLES ON LorisTest.* TO 'SQLTestUser'@'%' IDENTIFIED BY 'TestPassword' WITH GRANT OPTION;" >> /docker-entrypoint-initdb.d/0004-sql-user.sql
+RUN echo "CREATE USER 'SQLTestUser'@'%' IDENTIFIED BY 'TestPassword';" >> /docker-entrypoint-initdb.d/0000-compiled.sql
+RUN echo "CREATE USER 'SQLTestUser'@'localhost' IDENTIFIED BY 'TestPassword';" >> /docker-entrypoint-initdb.d/0000-compiled.sql
+RUN echo "GRANT SELECT,INSERT,UPDATE,DELETE,DROP,CREATE TEMPORARY TABLES ON LorisTest.* TO 'SQLTestUser'@'%';" >> /docker-entrypoint-initdb.d/0000-compiled.sql
 
 # Run the LORIS-MRI database installation script
 COPY install/install_database.sql /tmp/install_database.sql
-RUN echo "SET @email := 'root@localhost'; SET @project := 'loris'; SET @minc_dir = '/opt/minc/1.9.18';" >> 0001-paths.sql
-RUN cat /tmp/install_database.sql >> /docker-entrypoint-initdb.d/0001-paths.sql
+RUN echo "SET @email := 'root@localhost'; SET @project := 'loris'; SET @minc_dir = '/opt/minc/1.9.18';" >> 0000-compiled.sql
+RUN cat /tmp/install_database.sql >> /docker-entrypoint-initdb.d/0000-compiled.sql
+
+# By default, MariaDB runs the scripts in /docker-entrypoint-initdb.d/ at the time of the first
+# startup to initialize the database. However, we want to populate the database at build time so
+# that the database is part of the final image (and can be cached for CI).
+RUN mariadb-install-db
+RUN docker-entrypoint.sh mariadbd & \
+    sleep 30 && \
+    mariadb < /docker-entrypoint-initdb.d/0000-compiled.sql && \
+    killall mariadbd
+
+CMD ["mariadbd", "--user=root"]

test/docker-compose.yml

@@ -3,15 +3,14 @@ services:
     image: loris-db
     volumes:
       - ./test/mysql-config:/etc/mysql/conf.d
-    environment:
-      MYSQL_DATABASE: LorisTest
-      MYSQL_RANDOM_ROOT_PASSWORD: yes
-    ports:
-      - "3306:3306"
+    healthcheck:
+      test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"]
+      start_period: 10s
+      interval: 10s
+      timeout: 5s
+      retries: 5
   mri:
     image: loris-mri
-    volumes:
-      - ../:/opt/loris/bin/mri
     depends_on:
-      - db
-    command: pytest
+      db:
+        condition: service_healthy