MCP: Add software tests

Author: amotl

.github/workflows/framework-mcp.yml

@@ -0,0 +1,84 @@
+name: mcp
+
+on:
+  pull_request:
+    paths:
+    - '.github/workflows/framework-mcp.yml'
+    - 'framework/mcp/**'
+    - '/requirements.txt'
+  push:
+    branches: [ main ]
+    paths:
+    - '.github/workflows/framework-mcp.yml'
+    - 'framework/mcp/**'
+    - '/requirements.txt'
+
+  # Allow job to be triggered manually.
+  workflow_dispatch:
+
+  # Run job each night after CrateDB nightly has been published.
+  schedule:
+    - cron: '0 3 * * *'
+
+# Cancel in-progress jobs when pushing to the same branch.
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.ref }}
+
+jobs:
+  test:
+    name: "
+     Python: ${{ matrix.python-version }}
+     CrateDB: ${{ matrix.cratedb-version }}
+     on ${{ matrix.os }}"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ 'ubuntu-latest' ]
+        python-version: [ '3.10', '3.13' ]
+        cratedb-version: [ 'nightly' ]
+
+    services:
+      cratedb:
+        image: crate/crate:${{ matrix.cratedb-version }}
+        ports:
+          - 4200:4200
+          - 5432:5432
+        env:
+          CRATE_HEAP_SIZE: 4g
+
+    env:
+      UV_SYSTEM_PYTHON: true
+
+    steps:
+
+      - name: Acquire sources
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+          cache: 'pip'
+          cache-dependency-path: |
+            framework/mcp/requirements*.txt
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Install JBang
+        uses: jbangdev/setup-jbang@main
+
+      - name: Configure JBang
+        run: |
+          jbang trust add https://github.com/quarkiverse
+
+      - name: Install utilities
+        run: |
+          uv pip install -r requirements.txt
+
+      - name: Validate framework/mcp
+        run: |
+          ngr test --accept-no-venv framework/mcp

framework/mcp/README.md

@@ -74,7 +74,7 @@ Acquire sources, set up sandbox, and install packages.
 ```bash
 git clone https://github.com/crate/cratedb-examples
 cd cratedb-examples/framework/mcp
-uv pip install -r requirements.txt
+uv pip install -r requirements.txt -r requirements-test.txt
 ```
 
 ## Synopsis
@@ -89,6 +89,13 @@ uv run example_dbhub.py
 uv run example_jdbc.py
 ```
 
+## Tests
+
+Run integration tests.
+```bash
+pytest
+```
+
 ## Development
 
 `ctk tail` is a handy command to follow the progress of CrateDB's `sys.jobs_log`,

framework/mcp/pyproject.toml

@@ -0,0 +1,12 @@
+[tool.pytest.ini_options]
+minversion = "2.0"
+addopts = """
+  -rfEXs -p pytester --strict-markers --verbosity=3
+  --capture=no
+  """
+log_level = "DEBUG"
+log_cli_level = "DEBUG"
+testpaths = ["*.py"]
+xfail_strict = true
+markers = [
+]

framework/mcp/requirements-test.txt

@@ -0,0 +1,2 @@
+cratedb-toolkit
+pytest<9

framework/mcp/test.py

@@ -0,0 +1,103 @@
+import dataclasses
+import re
+import shlex
+import subprocess
+import sys
+
+from cratedb_toolkit.util import DatabaseAdapter
+
+
+@dataclasses.dataclass
+class Process:
+    """
+    Manage outputs of a process.
+    """
+    proc: subprocess.Popen
+    stdout: bytes
+    stderr: bytes
+
+    @property
+    def returncode(self) -> int:
+        return self.proc.returncode
+
+
+def run(command: str, timeout: int = 30) -> Process:
+    """
+    Invoke a command in a subprocess.
+    """
+    proc = subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = proc.communicate(timeout=timeout)
+    return Process(proc, stdout, stderr)
+
+
+def test_builtin():
+    """
+    Validate the basic built-in MCP server for PostgreSQL works well.
+
+    It is written in TypeScript.
+    https://www.npmjs.com/package/@modelcontextprotocol/server-postgres
+    """
+    p = run(f"{sys.executable} example_builtin.py")
+    assert p.returncode == 0
+
+    # Validate output specific to the MCP server.
+    assert b"Could not roll back transaction: error: line 1:1: mismatched input 'ROLLBACK' expecting" in p.stderr
+
+    # Validate output specific to CrateDB.
+    assert b"Calling tool: read_query" in p.stdout
+    assert b'"mountain": "Mont Blanc"' in p.stdout
+
+
+def test_jdbc():
+    """
+    Validate the Quarkus MCP server for JDBC works well.
+
+    It is written in Java.
+    https://github.com/quarkiverse/quarkus-mcp-servers/tree/main/jdbc#readme
+    """
+    p = run(f"{sys.executable} example_jdbc.py")
+    assert p.returncode == 0
+
+    # Validate output specific to the MCP server.
+    assert re.match(br".*\[io.quarkus\] \(main\) mcp-server-jdbc 999-SNAPSHOT on JVM.*", p.stderr, re.DOTALL)
+
+    # Validate output specific to CrateDB.
+    assert b"Calling tool: database_info" in p.stdout
+    assert b'{"database_product_name":"PostgreSQL","driver_name":"PostgreSQL JDBC Driver"' in p.stdout
+    assert b"Calling tool: describe_table" in p.stdout
+    assert b"Failed to describe table: The column name current_database was not found in this ResultSet." in p.stdout
+    assert b"Calling tool: read_query" in p.stdout
+    assert b'"mountain":"Mont Blanc"' in p.stdout
+
+    # Validate database content.
+    db = DatabaseAdapter("crate://crate@localhost:4200/")
+    records = db.refresh_table("doc.testdrive")
+    records = db.run_sql("SELECT * FROM doc.testdrive", records=True)
+    assert len(records) >= 1
+    assert records[0] == {"id": 42, "data": "foobar"}
+
+
+def test_dbhub():
+    """
+    Validate the DBHub MCP server works well.
+
+    DBHub is a universal database gateway implementing the Model Context
+    Protocol (MCP) server interface. This gateway allows MCP-compatible
+    clients to connect to and explore different databases.
+
+    It is written in TypeScript.
+    https://github.com/bytebase/dbhub
+    """
+    p = run(f"{sys.executable} example_dbhub.py")
+    assert p.returncode == 0
+
+    # Validate output specific to the MCP server.
+    assert b"Successfully connected to PostgreSQL database" in p.stderr
+    assert b"Universal Database MCP Server" in p.stderr
+
+    # Validate output specific to CrateDB.
+    assert b"Calling tool: run_query" in p.stdout
+    assert b'"mountain": "Mont Blanc"' in p.stdout
+
+    assert b"Calling tool: list_connectors" in p.stdout
+    assert b' "dsn": "postgres://' in p.stdout