Merge pull request #67 from odissei-data/development

Development

Author: FjodorvRijsselberg

.github/workflows/docker-image.yml

@@ -7,35 +7,43 @@ on:
 
 env:
   TAG: $GITHUB_REF_NAME
-  DOCKER_IMAGE_NAME: "dans-prefect"
   DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
   DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
-  ORG: "dansknaw"
+  ORG: "odissei"
 
 jobs:
   build:
     runs-on: ubuntu-latest
 
     steps:
-      -
-        name: Checkout
+      - name: Checkout
         uses: actions/checkout@v4
         with:
           submodules: recursive
-      -
-        name: Login to Docker Hub
+
+      - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-      -
-        name: Set up Docker Buildx
+
+      - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
-      -
-        name: Build and push
+
+      # Build and push the Prefect server image
+      - name: Build and push Prefect Server Image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: Dockerfile.server
+          push: true
+          tags: ${{ env.ORG }}/prefect:${{ github.ref_name }}
+
+      # Build and push the Prefect worker image
+      - name: Build and push Prefect Worker Image
         uses: docker/build-push-action@v5
         with:
           context: .
-          file: Dockerfile
+          file: Dockerfile.worker
           push: true
-          tags: dansknaw/dans-prefect:${{ github.ref_name }}
+          tags: ${{ env.ORG }}/prefect-worker:${{ github.ref_name }}

Base

@@ -1,4 +1,4 @@
-FROM python:3.11-slim
+FROM python:3.12-slim
 
 ENV PYTHONPATH "${PYTHONPATH}:/app/scripts/"
 ENV PYTHONDONTWRITEBYTECODE 1

Dockerfile

@@ -0,0 +1,31 @@
+FROM python:3.11-slim
+
+ENV PYTHONPATH="${PYTHONPATH}:/app/scripts/" \
+    PYTHONDONTWRITEBYTECODE=1
+
+EXPOSE 4200
+
+RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/*
+
+
+RUN useradd -m -u 1000 -s /bin/bash prefect
+
+WORKDIR /app
+
+COPY pyproject.toml .
+RUN pip install poetry \
+    && poetry config virtualenvs.create false \
+    && poetry install --no-root
+
+COPY entrypoint.sh /app/
+RUN chmod +x  /app/entrypoint.sh
+
+COPY scripts ./scripts
+COPY resources ./resources
+
+RUN chown -R prefect:prefect /app
+
+WORKDIR /app/scripts
+USER prefect
+
+ENTRYPOINT ["/app/entrypoint.sh"]

Dockerfile.server

@@ -0,0 +1,32 @@
+FROM python:3.11-slim
+
+ENV PYTHONPATH="${PYTHONPATH}:/app/scripts/" \
+    PYTHONDONTWRITEBYTECODE=1
+
+EXPOSE 4200
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN useradd -m -u 1000 -s /bin/bash prefect
+
+WORKDIR /app
+
+COPY pyproject.toml .
+RUN pip install poetry \
+    && poetry config virtualenvs.create false \
+    && poetry install --no-root
+
+COPY entrypoint-worker.sh /app/
+RUN chmod +x  /app/entrypoint-worker.sh
+
+COPY scripts ./scripts
+COPY resources ./resources
+
+RUN chown -R prefect:prefect /app
+
+WORKDIR /app/scripts
+USER prefect
+
+ENTRYPOINT ["/app/entrypoint-worker.sh"]

Dockerfile.worker

@@ -1,10 +1,8 @@
-version: '3.7'
-
 services:
   prefect:
     build:
       context: .
-      dockerfile: Dockerfile
+      dockerfile: Dockerfile.server
     container_name: 'prefect'
     volumes:
       - "/tmp:/tmp"
@@ -16,12 +14,29 @@ services:
       - db
     networks:
       - prefect
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:4200/api/health" ]
+      interval: 15s
+      timeout: 10s
+      retries: 5
+
+  prefect-worker:
+    container_name: 'prefect-worker'
+    build:
+      context: .
+      dockerfile: Dockerfile.worker
+    env_file: .env
+    depends_on:
+      prefect:
+        condition: service_healthy
+    networks:
+      - prefect
 
   db:
     image: postgres:latest
     container_name: db
     volumes:
-      - ./db:/var/lib/postgresql/data
+      - prefectdb:/var/lib/postgresql/data
     environment:
       - POSTGRES_USER=postgres
       - POSTGRES_PASSWORD=lolgres
@@ -31,3 +46,6 @@ services:
 
 networks:
   prefect:
+
+volumes:
+  prefectdb:

docker-compose.yml

@@ -1,16 +1,15 @@
 HOSTNAME="localhost"
 PROJECT_CONTAINER_NAME="prefect"
 # This python path is added to avoid having to do relative imports.
-PYTHONPATH="/scripts/"
+PYTHONPATH="/app/scripts/"
 GITHUB_USERNAME="odissei-data"
 DOCKERHUB_USERNAME="fjodorvr"
-VERSION="v2.4.3"
+VERSION="v3.0.1"
 ENV_FOR_DYNACONF=development
 PREFECT_SERVER_API_HOST='0.0.0.0'
 PREFECT_API_DATABASE_CONNECTION_URL="postgresql+asyncpg://username:password@db:5432/prefect"
-
 POSTGRES_USER=postgres
 POSTGRES_PASSWORD=postgres
 POSTGRES_DB=prefect
 POSTGRES_PORT=5432
-POSTGRES_DB_URL=postgresql://postgres:postgres@database:5432/prefect
+API_URL=http://0.0.0.0:4200/api

dot_env_example

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+
+prefect config set PREFECT_API_URL="http://prefect:4200/api"
+prefect worker start --pool default

entrypoint-worker.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env sh
+
+prefect config set PREFECT_API_URL="${API_URL}"
+prefect config set PREFECT_UI_STATIC_DIRECTORY="/app/"
+prefect server start --host 0.0.0.0