refactor tests suite in order to run it locally outside docker (#102)

* refactor tests suite in order to run it locally outside docker

* lint

* remove service

* try pg install

* use python 3.9 for pre-commit to avoid deprecation issue form pydocstyle

* revert

---------

Co-authored-by: Jonathan Healy <jonathan.d.healy@gmail.com>

Author: vincentsarago

.github/workflows/cicd.yaml

@@ -5,38 +5,15 @@ on:
   pull_request:
     branches: [main]
 
+
 jobs:
   test:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
     timeout-minutes: 20
 
-    services:
-      db_service:
-        image: ghcr.io/stac-utils/pgstac:v0.7.10
-        env:
-          POSTGRES_USER: username
-          POSTGRES_PASSWORD: password
-          POSTGRES_DB: postgis
-          POSTGRES_HOST: localhost
-          POSTGRES_PORT: 5432
-          PGUSER: username
-          PGPASSWORD: password
-          PGDATABASE: postgis
-          ALLOW_IP_RANGE: 0.0.0.0/0
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 10s
-          --health-retries 10
-          --log-driver none
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
-
     steps:
       - name: Check out repository code
         uses: actions/checkout@v4
@@ -55,20 +32,16 @@ jobs:
           python -m pip install pre-commit
           pre-commit run --all-files
 
-      - name: Install
+      - name: install lib postgres
+        uses: nyurik/action-setup-postgis@v1
+
+      - name: Install dependencies
         run: |
-          pip install .[dev,server]
+          python -m pip install --upgrade pip
+          python -m pip install .[dev,server]
 
       - name: Run test suite
-        run: make test
-        env:
-          ENVIRONMENT: testing
-          POSTGRES_USER: username
-          POSTGRES_PASS: password
-          POSTGRES_DBNAME: postgis
-          POSTGRES_HOST_READER: localhost
-          POSTGRES_HOST_WRITER: localhost
-          POSTGRES_PORT: 5432
+        run:  python -m pytest --cov stac_fastapi.pgstac --cov-report xml --cov-report term-missing
 
   validate:
     runs-on: ubuntu-latest
@@ -90,17 +63,23 @@ jobs:
           --log-driver none
         ports:
           - 5432:5432
+
     steps:
       - name: Check out repository code
         uses: actions/checkout@v4
+
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
           python-version: "3.10"
           cache: pip
           cache-dependency-path: setup.py
+
       - name: Install stac-fastapi and stac-api-validator
-        run: pip install .[server] stac-api-validator==0.4.1
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install .[server] stac-api-validator==0.4.1
+
       - name: Load data and validate
         run: python -m stac_fastapi.pgstac.app & ./scripts/wait-for-it.sh localhost:8080 && python ./scripts/ingest_joplin.py http://localhost:8080 && ./scripts/validate http://localhost:8080
         env:
@@ -120,15 +99,21 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
           python-version: "3.10"
           cache: pip
           cache-dependency-path: setup.py
+
       - name: Install with documentation dependencies
-        run: pip install .[docs,dev,server]
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install .[docs,dev,server]
+
       - name: Generate API docs
         run: make docs
+
       - name: Build documentation
         run: mkdocs build --strict

.pre-commit-config.yaml

@@ -3,18 +3,20 @@ repos:
     rev: 5.12.0
     hooks:
       - id: isort
-        language_version: python3.8
+        language_version: python
+
   - repo: https://github.com/psf/black
     rev: 22.12.0
     hooks:
       - id: black
         args: ["--safe"]
-        language_version: python3.8
+        language_version: python
+
   - repo: https://github.com/pycqa/flake8
     rev: 6.0.0
     hooks:
       - id: flake8
-        language_version: python3.8
+        language_version: python
         args: [
             # E501 let black handle all line length decisions
             # W503 black conflicts with "line break before operator" rule
@@ -26,7 +28,7 @@ repos:
     rev: v2.1.1
     hooks:
       - id: pydocstyle
-        language_version: python3.8
+        language_version: python
         exclude: ".*(test|scripts).*"
         args:
           [
@@ -42,11 +44,12 @@ repos:
   #            -   id: mypy
   #                language_version: python3.8
   #                args: [--no-strict-optional, --ignore-missing-imports]
+
   - repo: https://github.com/PyCQA/pydocstyle
     rev: 6.3.0
     hooks:
       - id: pydocstyle
-        language_version: python3.8
+        language_version: python
         exclude: ".*(test|scripts).*"
         #args: [
         # Don't require docstrings for tests

Dockerfile

@@ -1,4 +1,6 @@
-FROM python:3.8-slim as base
+ARG PYTHON_VERSION=3.11
+
+FROM python:${PYTHON_VERSION}-slim as base
 
 # Any python libraries that require system libraries to be installed will likely
 # need the following packages in order to build
@@ -16,6 +18,6 @@ WORKDIR /app
 
 COPY . /app
 
-RUN pip install -e .[dev,server]
+RUN python -m pip install -e .[server]
 
-CMD ["uvicorn", "stac_fastapi.pgstac.app:app", "--host", "0.0.0.0", "--port", "8080"]
+CMD ["uvicorn", "stac_fastapi.pgstac.app:app", "--host", "0.0.0.0", "--port", "8080"]

Dockerfile.tests

@@ -0,0 +1,19 @@
+ARG PYTHON_VERSION=3.11
+
+FROM python:${PYTHON_VERSION}-slim as base
+
+# Any python libraries that require system libraries to be installed will likely
+# need the following packages in order to build
+RUN apt-get update && \
+    apt-get -y upgrade && \
+    apt-get install -y build-essential git libpq-dev postgresql-15-postgis-3  && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN useradd -ms /bin/bash newuser
+USER newuser
+
+WORKDIR /app
+COPY . /app
+
+RUN python -m pip install -e .[dev,server] --user

Makefile

@@ -10,6 +10,8 @@ run = docker-compose run --rm \
 				-e APP_PORT=${APP_PORT} \
 				app
 
+runtests = docker-compose run --rm tests
+
 .PHONY: image
 image:
 	docker-compose build
@@ -28,7 +30,7 @@ docker-shell:
 
 .PHONY: test
 test:
-	$(run) /bin/bash -c 'export && ./scripts/wait-for-it.sh database:5432 && cd /app/tests/ && pytest -vvv --log-cli-level $(LOG_LEVEL)'
+	$(runtests) /bin/bash -c 'export && python -m pytest /app/tests/api/test_api.py --log-cli-level $(LOG_LEVEL)'
 
 .PHONY: run-database
 run-database:

docker-compose.yml

@@ -3,7 +3,6 @@ services:
   app:
     container_name: stac-fastapi-pgstac
     image: stac-utils/stac-fastapi-pgstac
-    platform: linux/amd64
     build: .
     environment:
       - APP_HOST=0.0.0.0
@@ -31,6 +30,19 @@ services:
       - database
     command: bash -c "./scripts/wait-for-it.sh database:5432 && python -m stac_fastapi.pgstac.app"
 
+  tests:
+    container_name: stac-fastapi-pgstac-test
+    image: stac-utils/stac-fastapi-pgstac-test
+    build:
+      context: .
+      dockerfile: Dockerfile.tests
+    environment:
+      - ENVIRONMENT=local
+      - DB_MIN_CONN_SIZE=1
+      - DB_MAX_CONN_SIZE=1
+      - USE_API_HYDRATE=${USE_API_HYDRATE:-false}
+    command: bash -c "python -m pytest -s -vv"
+
   database:
     container_name: stac-db
     image: ghcr.io/stac-utils/pgstac:v0.7.10

setup.py

@@ -24,6 +24,7 @@
     "dev": [
         "pystac[validation]",
         "pypgstac[psycopg]==0.7.*",
+        "pytest-postgresql",
         "pytest",
         "pytest-cov",
         "pytest-asyncio>=0.17,<0.23.0",

tests/api/test_api.py

@@ -636,7 +636,7 @@ async def search(query: Dict[str, Any]) -> List[Item]:
 
 
 @pytest.mark.asyncio
-async def test_wrapped_function(load_test_data) -> None:
+async def test_wrapped_function(load_test_data, database) -> None:
     # Ensure wrappers, e.g. Planetary Computer's rate limiting, work.
     # https://github.com/gadomski/planetary-computer-apis/blob/2719ccf6ead3e06de0784c39a2918d4d1811368b/pccommon/pccommon/redis.py#L205-L238
 
@@ -672,7 +672,16 @@ async def get_collection(
                 collection_id, request=request, **kwargs
             )
 
-    settings = Settings(testing=True)
+    settings = Settings(
+        postgres_user=database.user,
+        postgres_pass=database.password,
+        postgres_host_reader=database.host,
+        postgres_host_writer=database.host,
+        postgres_port=database.port,
+        postgres_dbname=database.dbname,
+        testing=True,
+    )
+
     extensions = [
         TransactionExtension(client=TransactionsClient(), settings=settings),
         FieldsExtension(),