Use QJazz in tests

Author: dmarteau

.github/workflows/e2e_tests.yml

@@ -30,12 +30,12 @@ jobs:
           - name: "LEGACY"
             php: "8.2"
             pg-postgis: "14-3"
-            qgis-server: "3.40"
+            qgis-server: "qgis-ltr-eager"
             update-projects: "FALSE"
           - name: "BLEEDING_EDGE"
             php: "8.4"
             pg-postgis: "17-3"
-            qgis-server: "3.42"
+            qgis-server: "qgis-release-eager"
             update-projects: "TRUE"
     env:
       CYPRESS_CI: TRUE
@@ -315,6 +315,7 @@ jobs:
           mkdir -p /tmp/e2e/lwc
           mkdir -p /tmp/e2e/docker
           docker logs lizmap${{ env.LZMBRANCH }}_test_qgis &> /tmp/e2e/docker/qgis-server.log | true
+          docker logs lizmap${{ env.LZMBRANCH }}_test_qgis_rpc &> /tmp/e2e/docker/qgis-server-rpc.log | true
           docker logs lizmap${{ env.LZMBRANCH }}_test_php &> /tmp/e2e/docker/php.log | true
           docker logs lizmap${{ env.LZMBRANCH }}_test_nginx &> /tmp/e2e/docker/nginx.log | true
           cp -r ../lizmap/var/log /tmp/e2e/lwc/

tests/Makefile

@@ -15,10 +15,11 @@ LIZMAP_GROUP_ID ?= $(shell id -g)
 # Note that GH Action E2E has its own version config for PHP, PG and QGIS
 PHP_VERSION ?= 8.2
 LZMPOSTGISVERSION ?= 14-3
-LZMQGSRVVERSION ?= 3.40
+LZMQGSRVVERSION ?= qgis-ltr-eager
 QGSRVAPIPORT ?= 8134
 LZMPGPORT ?= 8132
 LZMQGSRVPORT ?= 8131
+LZMQGSADMINPORT ?= 9876
 LZMWEBPORT ?= 8130
 LIZMAP_ADMIN_LOGIN ?= admin
 LIZMAP_ADMIN_EMAIL ?= admin@localhost.local
@@ -37,6 +38,7 @@ env:
 	LZMBRANCH=$(LZMBRANCH)
 	LZMPGPORT=$(LZMPGPORT)
 	LZMQGSRVPORT=$(LZMQGSRVPORT)
+	LZMQGSADMINPORT=$(LZMQGSADMINPORT)
 	LZMWEBPORT=$(LZMWEBPORT)
 	QGSRVAPIPORT=$(QGSRVAPIPORT)
 	LIZMAP_ADMIN_LOGIN=$(LIZMAP_ADMIN_LOGIN)
@@ -50,32 +52,24 @@ env:
 
 
 build-plugins:
-	docker run \
-	    -u $(LIZMAP_USER_ID):$(LIZMAP_GROUP_ID) \
-	    --rm -i \
-	    -e QGSRV_SERVER_PLUGINPATH=/srv/plugins \
-	    -e QGIS_PLUGIN_MANAGER_SOURCES_FILE=/tmp/sources-plugin-manager.list \
-	    -e QGIS_PLUGIN_MANAGER_CACHE_DIR=/tmp/cache-plugin-manager \
-	    -v $(shell pwd)/qgis-server-plugins:/srv/plugins \
-	    -v $(shell pwd)/:/src \
-        --entrypoint /src/add_server_plugins.sh \
-	    3liz/qgis-map-server:${LZMQGSRVVERSION} \
+	echo "Plugins are auto installed at qgis worker startup"
 
 show-qgis-server-versions:
 	docker run \
 	    --rm -i \
-	    --entrypoint qgisserver \
-	    3liz/qgis-map-server:${LZMQGSRVVERSION} \
-	    --version
+	    --entrypoint qjazz-config \
+	    3liz/qjazz:${LZMQGSRVVERSION} \
+	    version
 	docker run \
 	    -u $(LIZMAP_USER_ID):$(LIZMAP_GROUP_ID) \
 	    --rm -i \
-	    -e QGSRV_SERVER_PLUGINPATH=/srv/plugins \
+	    -e QGIS_PLUGINPATH=/srv/plugins \
 	    -e QGIS_PLUGIN_MANAGER_SKIP_SOURCES_FILE=True \
 	    -v $(shell pwd)/qgis-server-plugins:/srv/plugins \
 	    -v $(shell pwd)/:/src \
+	    --workdir /srv/plugins \
 	    --entrypoint qgis-plugin-manager \
-	    3liz/qgis-map-server:${LZMQGSRVVERSION} \
+	    3liz/qjazz:${LZMQGSRVVERSION} \
 	    list
 
 upgrade-projects:
@@ -86,14 +80,14 @@ upgrade-projects:
 	    -v $(shell pwd)/qgis-server-plugins:/srv/plugins \
 	    -v $(shell pwd)/qgis-projects/tests:/tmp/qgis-projects \
 	    --entrypoint /srv/plugins/upgrade_projects.py \
-	    3liz/qgis-map-server:${LZMQGSRVVERSION} \
+	    3liz/qjazz:${LZMQGSRVVERSION} \
 
 up: env
-	docker compose up  -V --force-recreate -d
+	docker compose --profile=swagger --profile=webdav up  -V --force-recreate -d
 
 stop:
-	docker compose stop
+	docker compose --profile=swagger --profile=webdav  stop
 
 reset:
-	docker compose down -v --remove-orphans
+	docker compose --profile=swagger --profile=webdav  down -v --remove-orphans
 	./lizmap-ctl clean

tests/add_server_plugins.sh

@@ -67,32 +67,40 @@ services:
       - ${LZMWEBPORT}:80
     depends_on:
       - lizmap
-  map:
-    container_name: "lizmap${LZMBRANCH}_test_qgis"
-    image: 3liz/qgis-map-server:${LZMQGSRVVERSION}
+  # QGIS RPC backend
+  map-rpc:
+    container_name: "lizmap${LZMBRANCH}_test_qgis_rpc"
+    image: 3liz/qjazz:${LZMQGSRVVERSION}
+    command: ["qjazz-rpc", "serve", "-C", "/srv/etc/qgis-server.toml"]
     environment:
-      PGSERVICEFILE: /srv/etc/pg_service.conf
+      CONF_LOGGING__LEVEL: trace
+      CONF_WORKER__NUM_PROCESSES: 4
+      CONF_DISPLAY_XVFB: ON
+      CONF_USER: ${LIZMAP_USER_ID}:${LIZMAP_GROUP_ID}
+      QGIS_PLUGINPATH: /srv/qgis-server-plugins
       QGIS_SERVER_LIZMAP_REVEAL_SETTINGS: yes
-      QGSRV_API_ENABLED_LIZMAP: yes
-      QGSRV_CACHE_ROOTDIR: /srv/projects
-      QGSRV_CACHE_SIZE: '20'
-      # For testing purpose, I think it's better to have a strict check of layers
-      QGSRV_CACHE_STRICT_CHECK: yes
-      QGSRV_LOGGING_LEVEL: DEBUG
-      QGSRV_SERVER_PLUGINPATH: /srv/qgis-server-plugins
-      QGSRV_MANAGEMENT_ENABLED: yes
-      QGSRV_MANAGEMENT_INTERFACES: "0.0.0.0"
-      QGSRV_SERVER_WORKERS: 4
-      QGSRV_USER: ${LIZMAP_USER_ID}:${LIZMAP_GROUP_ID}
-      ROUTER_HOST: map
+      QGIS_PLUGIN_MANAGER_SOURCES_FILE: /srv/etc/plugin-sources.list
+      PGSERVICEFILE: /srv/etc/pg_service.conf
     volumes:
       - { type: bind, source: ./qgis-projects, target: /srv/projects }
       - { type: bind, source: ./qgis-server-plugins, target: /srv/qgis-server-plugins }
-      - { type: bind, source: ./wps-data, target: /srv/data }
       - { type: bind, source: ./docker-conf/pg_service.conf, target: /srv/etc/pg_service.conf }
+      - { type: bind, source: ./docker-conf/qgis-server.toml, target: /srv/etc/qgis-server.toml }
+      - type: bind
+        source: ./docker-conf/unstable-plugin-sources.list
+        target: /srv/etc/plugin-sources.list
+  # QGIS workers frontend
+  map:
+    container_name: "lizmap${LZMBRANCH}_test_qgis"
+    image: 3liz/qjazz:${LZMQGSRVVERSION}
+    command: ["qjazz-map", "serve", "-C", "/srv/etc/qgis-server.toml"]
+    environment:
+        CONF_LOGGING__LEVEL: trace
+    volumes:
+      - { type: bind, source: ./docker-conf/qgis-server.toml, target: /srv/etc/qgis-server.toml }
     ports:
-      - ${LZMQGSRVPORT}:8080
-      - ${QGSRVAPIPORT}:19876
+      - ${LZMQGSRVPORT}:9080
+      - ${LZMQGSADMINPORT}:9876
   webdav:
     container_name: lizmap${LZMBRANCH}_test_webdav
     image: ugeek/webdav:amd64
@@ -104,6 +112,8 @@ services:
       - UID=${LIZMAP_USER_ID}
       - GID=${LIZMAP_GROUP_ID}
       - TZ=Europe/Rome
+    profiles:
+      - webdav
   swagger:
     container_name: lizmap${LZMBRANCH}_swagger_ui
     image: docker.swagger.io/swaggerapi/swagger-ui

tests/docker-compose.yml

@@ -5,11 +5,11 @@
 ;list the different map services (servers, generic parameters, etc.)
 [services]
 ;URL to QGIS Server for OGC web services
-wmsServerURL="http://map:8080/ows/"
+wmsServerURL="http://map:9080/"
 ;WMS subdomain URLs list (optional)
 wmsPublicUrlList=
 ;URL to the API exposed by the Lizmap plugin for QGIS Server
-lizmapPluginAPIURL="http://map:8080/lizmap/"
+lizmapPluginAPIURL="http://map:9080/lizmap/"
 
 onlyMaps=off
 defaultRepository=testsrepository

tests/docker-conf/phpfpm/lizmapConfig.ini.php

@@ -0,0 +1,40 @@
+#
+# Worker backend configuration
+#
+
+
+[worker]
+name = "Server"
+
+[worker.qgis]
+max_projects = 20
+
+[worker.qgis.projects]
+ignore_bad_layers = false
+
+[worker.qgis.projects.search_paths]
+'/' = "/srv/projects"
+
+# Plugins allowed to install from plugin paths
+[worker.qgis.plugins]
+install = [
+    "Lizmap server",
+    "atlasprint",
+    "wfsOutputExtension",
+]
+# Auto install plugins from embedded plugin manager
+install_mode = "auto"
+
+#
+# HTTP Front end configuration
+#
+
+[backends.root]
+title = "Root backend"
+host = "map-rpc"
+route = "/"
+
+[[backends.root.api]]
+endpoint = "lizmap"
+name = "Lizmap"
+

tests/docker-conf/qgis-server.toml

@@ -0,0 +1 @@
+https://packages.3liz.org/pub/server-plugins-repository/unstable/plugins.[VERSION].xml

tests/docker-conf/unstable-plugin-sources.list

@@ -4,7 +4,7 @@ export default defineConfig({
     testDir: './playwright',
     snapshotPathTemplate: '{testDir}/__screenshots__/{testFilePath}/{arg}{ext}',
     /* Maximum time one test can run for. */
-    timeout: 30 * 1000,
+    timeout: 10 * 1000,
     expect: {
         /**
          * Maximum time expect() should wait for the condition to be met.
@@ -17,7 +17,8 @@ export default defineConfig({
     /* Fail the build on CI if you accidentally left test.only in the source code. */
     forbidOnly: !!process.env.CI,
     /* Retry on CI only */
-    retries: process.env.CI ? 2 : 0,
+    /* retries: process.env.CI ? 2 : 0, */
+    retries: 0,
     /* Opt out of parallel tests on CI. */
     workers: process.env.CI ? 2 : undefined,
     /* Reporter to use.  */

tests/end2end/playwright.config.ts

@@ -18,9 +18,7 @@ if [ "$CMD" == "reset" ]; then
     exit 0
 elif [ "$CMD" == "build" ]; then
     # install or update base images, to be sure we have the latest images
-    docker compose pull
-    # install or update plugins, to be sure we have the latest plugins
-    make build-plugins
+    docker compose --profile=webdav --profile=swagger pull
 fi
 
-docker compose $CMD "$@"
+docker compose --profile=webdav --profile=swagger $CMD "$@"