prepare for prod prof docker images and sample (#68)

* prepare for prod prof docker images and sample

* rework docker-compose and add doc

* Update README.md

Co-authored-by: Arnaud Morvan <arnaud.morvan@camptocamp.com>

* Update README.md

Co-authored-by: Arnaud Morvan <arnaud.morvan@camptocamp.com>

* Rework composition files (#69)

* Update README.md

Co-authored-by: Moritz Kirmse <61978301+mki-c2c@users.noreply.github.com>

* Update README.md

Co-authored-by: Moritz Kirmse <61978301+mki-c2c@users.noreply.github.com>

* Update README.md

Co-authored-by: Moritz Kirmse <61978301+mki-c2c@users.noreply.github.com>

* correct fr mistake

* make harvested false by default

---------

Co-authored-by: Arnaud Morvan <arnaud.morvan@camptocamp.com>
Co-authored-by: Moritz Kirmse <61978301+mki-c2c@users.noreply.github.com>

Author: 3 people

.github/workflows/ci.yml

@@ -12,13 +12,13 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: build
-        run: docker compose build check
+        run: docker compose -f docker-compose-dev.yml build check
 
       - name: check
-        run: docker compose run --rm check
+        run: docker compose -f docker-compose-dev.yml run --rm check
 
       - name: test
-        run: docker compose run --rm check pytest --cov=maelstro tests/
+        run: docker compose -f docker-compose-dev.yml run --rm check pytest --cov=maelstro tests/
 
   frontend-check:
     runs-on: ubuntu-22.04
@@ -62,7 +62,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Build
-        run: docker compose build maelstro-back maelstro-front
+        run: docker compose -f docker-compose-dev.yml build maelstro-back maelstro-front
 
       - name: Login to Docker Hub
         uses: docker/login-action@v3

.gitignore

@@ -172,3 +172,5 @@ cython_debug/
 
 # Custom for this repo
 .envs-database
+docker-compose.override.yml
+docker-compose.yml

README.md

@@ -2,9 +2,36 @@
 
 geOrchestra Maelstro is an application which helps synchronise geonetwork and geoserver instances
 
-## Quick start with geOrchestra
+## Docker deploy
 
-Before starting developement you need to setup some geOrchestra configurations.
+### Solo quick start (without included geOrchestra composition)
+
+Maelstro can be used outside geOrchestra.
+
+First select the `docker-compose-solo.yml` file as current composition:
+
+```bash
+ln -s docker-compose-solo.yml docker-compose.yml
+```
+
+Then start the compositon:
+
+```bash
+docker compose up
+```
+
+Open : http://127.0.0.1:8080/maelstro/
+There is no authentication to access the page but if needed it can be done with basic auth in the [nginx config](./config/nginx-solo.conf) (or another way)
+
+### Quick start with geOrchestra
+
+First select the `docker-compose-geOrchestra.yml` file as current composition:
+
+```bash
+ln -s docker-compose-geOrchestra.yml docker-compose.yml
+```
+
+Before starting development you need to setup some geOrchestra configurations.
 
 See commands documented here: [georchestra/README.md](georchestra/README.md)
 
@@ -24,16 +51,26 @@ With credentials:
 - testadmin:testadmin
 - tmaelstro:tmaelstro
 
-## Solo quick start
-Maelstro can be used outside geOrchestra.
+### Development
 
+For development purpose, you may need to use `docker-compose-dev.yml` instead:
+
+```bash
+rm -f docker-compose.yml
+ln -s docker-compose-dev.yml docker-compose.yml
 ```
-docker compose -f docker-compose-solo.yml up 
+
+And maybe add overrides for debugging:
+
+```bash
+cp docker-compose.override.sampl.yml docker-compose.override.yml
 ```
-Open : http://127.0.0.1:8080/maelstro/ 
-There is no authentication to access the page but if needed it can be done with basic auth in the [nginx config](./config/nginx-solo.conf) (or another way)
 
-## Frontend developement
+## Kubernetes deployment
+
+The [helm](helm/) folder contains a helm chart which can be used to deploy the app.
+
+## Frontend development
 
 The folder [frontend](frontend) contains the SPA written with VueJS.
 
@@ -50,6 +87,7 @@ The configuration is based on a YAML file containing connection information abou
 For dev use of the platform, there is a sample config in the backend folder: [dev_config.yaml](backend/dev_config.yaml). This config is used by default in the docker compo.
 
 The file has 4 distict parts:
+
 - sources
 - destinations
 - db_logging
@@ -66,6 +104,7 @@ The file has 4 distict parts:
 #### DB logging
 
 The section db_logging contains all connection information to reach a writable postgres DB to use for writing and reading operation logs:
+
 - host (default: database)
 - port (default: 5432)
 - login (default: georchestra)
@@ -81,13 +120,16 @@ Substitution of credentials (login and password) can be done for the DB configur
 The `transformations` section conatains a list of xsl transformations which can be applied to the xml metadata of source or destination servers.
 
 Each named `transformation` item must conatain
+
 - `xslt_path`: local path on the server to an xsl file in which the trasformation is defind (typically in the datadir)
 - `description`: Details of the transformation content to easily identify which transformation is applied
 
 For each geonetwork item of the source and destination servers, a specific key is added to the configuration file:
+
 - transformations: list of xslt keys to be applied to the metadata of the corresponding geonetwork server
 
 If a list of transformations is defined for both the source and destination server, the copy operation is executed in the way described below:
+
 - read metadata from source
 - apply all source transformations
 - apply all destination transformations
@@ -111,7 +153,6 @@ __Nota__:
 
 At each level where login/password are specified, one can add the option `verify: false` to deactivate verification of HTTPS certificates. By default the HTTPS certificate must be valid, which is equivalent to `verify: true`
 
-
 #### Example
 
 (see [doc_sample_config.yaml](backend/tests/doc_sample_config.yaml)):
@@ -158,23 +199,23 @@ Automatic code quality checks are implemented in the CI.
 The code test can be launched manually via the docker command below.
 
 ```bash
-docker compose run --rm  check
+docker compose -f docker-compose-dev.yml run --rm  check
 ```
 
 In case formatting issues are found, the code can be auto-fixed with:
 
 ```bash
-docker compose run --rm  check /app/maelstro/scripts/code_fix.sh
+docker compose -f docker-compose-dev.yml run --rm  check /app/maelstro/scripts/code_fix.sh
 ```
 
 To launch the tests locally, use the command as in github CI:
 
 ```bash
-docker compose run --rm  check pytest --cov=maelstro tests/
+docker compose -f docker-compose-dev.yml run --rm  check pytest --cov=maelstro tests/
 ```
 
 or
 
 ```bash
-docker compose run --rm  check pytest --cov=maelstro --cov-report=html tests/
+docker compose -f docker-compose-ci.yml run --rm  check pytest --cov=maelstro --cov-report=html tests/
 ```

docker-compose-dev.yml

@@ -0,0 +1,42 @@
+# This composition add minimal development overrides over geOrchestra composition
+
+include:
+  - georchestra/geor-compose.yml
+  - georchestra/geor-compose.override.yml
+
+services:
+  maelstro-front:
+    extends:
+      file: docker-compose-geOrchestra.yml
+      service: maelstro-front
+    build:
+      context: ./frontend
+
+  maelstro-back:
+    extends:
+      file: docker-compose-geOrchestra.yml
+      service: maelstro-back
+    build:
+      context: ./backend
+      target: server
+    volumes:
+      - ./backend:/app
+      - ./config/:/etc/maelstro/:ro
+    environment:
+      MAELSTRO_CONFIG: /app/dev_config.yaml
+      DEBUG: true
+    ports:
+      - 8000:8000 # Used for frontend development
+
+  check:
+    profiles:
+      - check
+    build:
+      context: ./backend
+      target: check
+    environment:
+      MAELSTRO_CONFIG: /app/tests/test_config.yaml
+      LOCAL_LOGIN: testadmin
+      LOCAL_PASSWORD: testadmin
+    volumes:
+      - ./backend:/app

docker-compose.yml renamed to docker-compose-geOrchestra.yml

@@ -1,3 +1,5 @@
+# This is an example of production ready composition for geOrchestra
+
 include:
   - georchestra/geor-compose.yml
   - georchestra/geor-compose.override.yml
@@ -22,13 +24,12 @@ services:
       file: docker-compose-solo.yml
       service: maelstro-back
     volumes:
-      - ./backend:/app
       - georchestra_datadir:/etc/georchestra
     depends_on:
       envsubst:
         condition: service_completed_successfully
     environment:
-      MAELSTRO_CONFIG: /app/dev_config.yaml
+      MAELSTRO_CONFIG: /etc/georchestra/maelstro/config.yaml
       DEMO_LOGIN: testadmin
       DEMO_PASSWORD: testadmin
       LOCAL_LOGIN: testadmin
@@ -38,8 +39,3 @@ services:
       PGDATABASE: "georchestra"
       PGUSER: "georchestra"
       PGPASSWORD: "georchestra"
-
-  check:
-    extends:
-      file: docker-compose-solo.yml
-      service: check

docker-compose-solo.yml

@@ -1,6 +1,7 @@
+# This is an example of production ready standalone composition (without included geOrchestra)
 
 volumes:
-    postgresql_data:
+  postgresql_data:
 
 services:
   database:
@@ -14,8 +15,7 @@ services:
     restart: no
 
   maelstro-front:
-    build:
-      context: ./frontend
+    image: georchestra/maelstro-frontend:latest
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8080/maelstro/"]
       interval: 10s
@@ -30,17 +30,14 @@ services:
       - "8080:8080"
 
   maelstro-back:
-    build:
-      context: ./backend
-      target: server
+    image: georchestra/maelstro-backend:latest
     healthcheck:
       test: "health_check"
       interval: 10s
       retries: 5
       start_period: 10s
       timeout: 10s
     volumes:
-      - ./backend:/app
       - ./config/:/etc/maelstro/:ro
     env_file:
       - .envs-database
@@ -51,16 +48,3 @@ services:
       DEMO_CRD: "test12"
     command:
       - serve_docker_dev
-
-  check:
-    profiles:
-      - check
-    build:
-      context: ./backend
-      target: check
-    environment:
-      MAELSTRO_CONFIG: /app/tests/test_config.yaml
-      LOCAL_LOGIN: testadmin
-      LOCAL_PASSWORD: testadmin
-    volumes:
-      - ./backend:/app

docker-compose.override.sample.yaml

@@ -0,0 +1,12 @@
+# Here are example of some overrides for debugging
+
+services:
+  maelstro-back:
+    {}
+    # For debugging with VS Code
+    # command: >
+    #   python -m debugpy --listen 0.0.0.0:5678
+    #   /usr/local/bin/fastapi dev --host 0.0.0.0 --port 8000 --reload
+    #   maelstro/main.py
+    # ports:
+    #   - 5678:5678

frontend/src/locales/fr.json

@@ -33,5 +33,5 @@
   "Destination": "Destination",
   "Source title": "Titre de la source",
   "Destination title": "Titre de la destination",
-  "Include harvested datasets": "Inclure les jeux de données moissonés"
+  "Include harvested datasets": "Inclure les jeux de données moissonnés"
 }

frontend/src/views/FormView.vue

@@ -24,7 +24,7 @@ import {
 import { computed, ref } from 'vue'
 import { useI18n } from 'vue-i18n'
 
-const includeHarvested = ref(true)
+const includeHarvested = ref(false)
 const confirmation = ref(false)
 
 const configStore = useConfigStore()