Add Mediawiki Docker containers for unit tests

.github/workflows/python-lint.yaml

@@ -13,7 +13,7 @@ on:
       - 'test/**.py'
 
 jobs:
-  build:
+  lint:
     runs-on: ubuntu-latest
 
     steps:

.github/workflows/python-pytest.yaml

@@ -18,8 +18,8 @@ on:
       - 'poetry.lock'
 
 jobs:
-  build:
-    name: pytest ${{ matrix.python-version }}
+  pytest:
+    name: pytest ${{ matrix.python-version }}/${{ matrix.mediawiki-version }}
     runs-on: ubuntu-latest
     container: ubuntu
 
@@ -32,6 +32,7 @@ jobs:
     strategy:
       matrix:
         python-version: [ '3.9', '3.10', '3.11', '3.12', '3.13', '3.14-dev' ]
+        mediawiki-version: [ '1.39.7-wmde.18', '1.40.3-wmde.19', '1.41.1-wmde.20' ]
 
     steps:
       - uses: actions/checkout@v5
@@ -49,6 +50,13 @@ jobs:
       - name: Install poetry
         run: pipx install poetry
 
+      - name: Set up Docker containers with v${{ matrix.mediawiki-version }}
+        run: |
+          docker compose -f ./docker-compose.yml -f ./docker-compose.extra.yml up -d
+        working-directory: test/docker-compose
+        env:
+          WIKIBASE_BUNDLE_IMAGE_NAME: wikibase/wikibase-bundle:${{ matrix.mediawiki-version }}
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v6.0.0
         with:

pyproject.toml

@@ -52,6 +52,7 @@ optional = true
 
 [tool.poetry.group.dev.dependencies]
 pytest = "*"
+pytest-order = "*"
 pylint = "*"
 pylint-exit = "*"
 mypy = "*"

test/docker-compose/.env

@@ -0,0 +1,39 @@
+## Example / Template .env file for Wikibase release pipeline docker-compose example
+# WARNING: Do not add comments on the same line as env vars, as in some environments they will be included in the var!
+
+## Image Configuration
+WDQS_IMAGE_NAME=wikibase/wdqs:0.3.135-wmde.18
+WDQS_FRONTEND_IMAGE_NAME=wikibase/wdqs-frontend:wmde.18
+WDQS_PROXY_IMAGE_NAME=wikibase/wdqs-proxy:wmde.18
+MYSQL_IMAGE_NAME=mariadb:10.11
+
+## Mediawiki Configuration
+## Admin password
+## Passwords must be at least 10 characters.
+## Your password must be different from your username.
+## Your password must not appear within your username.
+## The password must not be in a list of very commonly used passwords. Please choose a unique password.
+MW_ADMIN_PASS=change-this-password
+MW_ADMIN_NAME=admin
+MW_ADMIN_EMAIL=admin@example.com
+MW_SECRET_KEY=some-secret-key
+MW_WG_ENABLE_UPLOADS=false
+
+## Jobrunner Configuration
+MAX_JOBS=1
+
+## Database Configuration
+DB_NAME=my_wiki
+DB_USER=sqluser
+DB_PASS=change-this-sqlpassword
+
+## Wikibase Configuration
+WIKIBASE_PINGBACK=false
+# wikibase.svc is the internal docker hostname, change this value to the public hostname
+WIKIBASE_HOST=wikibase.svc
+WIKIBASE_PORT=80
+
+## WDQS-frontend Configuration
+# wdqs-frontend.svc is the internal docker hostname, change this value to the public hostname
+WDQS_FRONTEND_HOST=wdqs-frontend.svc
+WDQS_FRONTEND_PORT=8834

test/docker-compose/README.md

@@ -0,0 +1 @@
+Come from https://github.com/wmde/wikibase-release-pipeline/tree/main/example

test/docker-compose/docker-compose.extra.yml

@@ -0,0 +1,69 @@
+# Additional services example
+version: '3.4'
+
+services:
+
+  wdqs-frontend:
+    image: "${WDQS_FRONTEND_IMAGE_NAME}"
+    restart: unless-stopped
+    ports:
+      - "${WDQS_FRONTEND_PORT}:80"
+    depends_on:
+      - wdqs-proxy
+    networks:
+      default:
+        aliases:
+          - ${WDQS_FRONTEND_HOST}
+    environment:
+      - WIKIBASE_HOST=${WIKIBASE_HOST}
+      - WDQS_HOST=wdqs-proxy.svc
+  wdqs:
+    image: "${WDQS_IMAGE_NAME}"
+    restart: unless-stopped
+    command: /runBlazegraph.sh
+    volumes:
+      - query-service-data:/wdqs/data
+    networks:
+      default:
+        aliases:
+          - wdqs.svc
+    environment:
+      - WIKIBASE_HOST=${WIKIBASE_HOST}
+      - WDQS_HOST=wdqs.svc
+      - WDQS_PORT=9999
+    expose:
+      - 9999
+
+  wdqs-proxy:
+    image: "${WDQS_PROXY_IMAGE_NAME}"
+    restart: unless-stopped
+    environment:
+      - PROXY_PASS_HOST=wdqs.svc:9999
+    depends_on:
+      - wdqs
+    networks:
+      default:
+        aliases:
+          - wdqs-proxy.svc
+
+  wdqs-updater:
+    image: "${WDQS_IMAGE_NAME}"
+    restart: unless-stopped
+    command: /runUpdate.sh
+    depends_on:
+      - wdqs
+      - wikibase
+    networks:
+      default:
+        aliases:
+          - wdqs-updater.svc
+    environment:
+      - WIKIBASE_HOST=${WIKIBASE_HOST}
+      - WDQS_HOST=wdqs.svc
+      - WDQS_PORT=9999
+      # CONFIG - WIKIBASE_SCHEME can be set to 'https' if the updater should expect https concept uris
+
+volumes:
+  LocalSettings:
+  query-service-data:
+  mediawiki-mysql-data:

test/docker-compose/docker-compose.yml

@@ -0,0 +1,84 @@
+# Example Wikibase docker-compose setup
+version: '3.4'
+
+x-common-variables: &wikibase_variables
+  DB_SERVER: mysql.svc:3306
+  MW_ADMIN_NAME: ${MW_ADMIN_NAME}
+  MW_ADMIN_PASS: ${MW_ADMIN_PASS}
+  MW_ADMIN_EMAIL: ${MW_ADMIN_EMAIL}
+  MW_WG_SECRET_KEY: ${MW_SECRET_KEY}
+  # Disable jobs running after requests when wikibase_jobrunner is defined
+  MW_WG_JOB_RUN_RATE: 0
+  DB_USER: ${DB_USER}
+  DB_PASS: ${DB_PASS}
+  DB_NAME: ${DB_NAME}
+  WIKIBASE_HOST: ${WIKIBASE_HOST}
+
+services:
+  wikibase:
+    image: "${WIKIBASE_BUNDLE_IMAGE_NAME}"
+    links:
+      - mysql
+    depends_on:
+      - mysql
+    restart: unless-stopped
+    ports:
+      - "${WIKIBASE_PORT}:80"
+    volumes:
+      ## This shares the configuration with jobrunner
+      - shared:/var/www/html/:rw
+      ## Uncomment this next line to mount your own LocalSettings.php file
+      ## also uncomment the same line in the wikibase_jobrunner service!
+      #- ./LocalSettings.php:/var/www/html/LocalSettings.d/LocalSettings.override.php
+    networks:
+      default:
+        aliases:
+          - ${WIKIBASE_HOST}
+          - wikibase-docker.svc
+    environment:
+      <<: *wikibase_variables
+      WIKIBASE_PINGBACK:
+      MW_WG_ENABLE_UPLOADS:
+
+  wikibase_jobrunner:
+    image: "${WIKIBASE_BUNDLE_IMAGE_NAME}"
+    entrypoint: /bin/bash
+    command:  /jobrunner-entrypoint.sh
+    links:
+      - mysql
+    depends_on:
+      - mysql
+    restart: always
+    volumes:
+      - shared:/shared/:ro
+      - ./jobrunner-entrypoint.sh:/jobrunner-entrypoint.sh
+      ## Uncomment this next line to mount your own LocalSettings.php file
+      ## also uncomment the same line in the wikibase service
+      #- ./LocalSettings.php:/var/www/html/LocalSettings.d/LocalSettings.override.php
+    networks:
+      default:
+        aliases:
+          - wikibase-jobrunner.svc
+    environment:
+      <<: *wikibase_variables
+      MAX_JOBS: ${MAX_JOBS}
+
+  mysql:
+    image: "${MYSQL_IMAGE_NAME}"
+    restart: unless-stopped
+    volumes:
+      - mediawiki-mysql-data:/var/lib/mysql
+    environment:
+      # CONFIG - Change the default values below (should match values passed to wikibase)
+      MYSQL_DATABASE: ${DB_NAME}
+      MYSQL_USER: ${DB_USER}
+      MYSQL_PASSWORD: ${DB_PASS}
+      MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
+    networks:
+      default:
+        aliases:
+          - mysql.svc
+
+volumes:
+  shared:
+  mediawiki-mysql-data:

test/docker-compose/jobrunner-entrypoint.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Originally inspired by Brennen Bearnes jobrunner entrypoint
+# https://gerrit.wikimedia.org/r/plugins/gitiles/releng/dev-images/+/refs/heads/master/common/jobrunner/entrypoint.sh
+
+# Wait for the db to come up
+/wait-for-it.sh "$DB_SERVER" -t 300
+# Sometimes it appears to come up and then go back down meaning MW install fails
+# So wait for a second and double check!
+sleep 1
+/wait-for-it.sh "$DB_SERVER" -t 300
+
+kill_runner() {
+	kill "$PID" 2> /dev/null
+}
+trap kill_runner SIGTERM
+
+while true; do
+	if [ -e /shared/LocalSettings.php ]; then
+		php maintenance/runJobs.php --wait --maxjobs="$MAX_JOBS" --conf /shared/LocalSettings.php &
+		PID=$!
+		wait "$PID"
+	else
+		echo "LocalSettings.php not shared yet - waiting for 10 seconds."
+		sleep 10
+	fi
+done