ConnectorX: Hello, World!

Author: amotl

.github/dependabot.yml

@@ -78,6 +78,11 @@ updates:
     schedule:
       interval: "daily"
 
+  - directory: "/by-language/python-connectorx"
+    package-ecosystem: "pip"
+    schedule:
+      interval: "daily"
+
   - directory: "/by-language/python-dbapi"
     package-ecosystem: "pip"
     schedule:

.github/workflows/lang-python-connectorx.yml

@@ -0,0 +1,79 @@
+name: Python ConnectorX
+
+on:
+  pull_request:
+    branches: ~
+    paths:
+    - '.github/workflows/lang-python-connectorx.yml'
+    - 'by-language/python-connectorx/**'
+    - '/requirements.txt'
+  push:
+    branches: [ main ]
+    paths:
+    - '.github/workflows/lang-python-connectorx.yml'
+    - 'by-language/python-connectorx/**'
+    - '/requirements.txt'
+
+  # Allow job to be triggered manually.
+  workflow_dispatch:
+
+  # Run job each night after CrateDB nightly has been published.
+  schedule:
+    - cron: '0 3 * * *'
+
+# Cancel in-progress jobs when pushing to the same branch.
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.ref }}
+
+jobs:
+  test:
+    name: "
+     Python: ${{ matrix.python-version }}
+     CrateDB: ${{ matrix.cratedb-version }}
+     on ${{ matrix.os }}"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ 'ubuntu-22.04' ]
+        python-version: [
+          '3.10',
+          '3.13',
+        ]
+        cratedb-version: [ 'nightly' ]
+
+    services:
+      cratedb:
+        image: crate/crate:${{ matrix.cratedb-version }}
+        ports:
+          - 4200:4200
+          - 5432:5432
+        env:
+          CRATE_HEAP_SIZE: 4g
+
+    steps:
+
+      - name: Acquire sources
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+          cache: 'pip'
+          cache-dependency-path: |
+            requirements.txt
+            by-language/python-connectorx/requirements*.txt
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Install utilities
+        run: |
+          uv pip install --system -r requirements.txt
+
+      - name: Validate by-language/python-connectorx
+        run: |
+          ngr test --accept-no-venv by-language/python-connectorx

by-language/python-connectorx/README.md

@@ -0,0 +1,37 @@
+# Connect to CrateDB and CrateDB Cloud using ConnectorX
+
+## About
+
+[ConnectorX] enables you to load data from databases into Python in the
+fastest and most memory efficient way.
+
+## Usage
+
+You can optionally accelerate the data loading using parallelism by
+specifying a partition column.
+```python
+import connectorx as cx
+
+cx.read_sql(
+    "postgresql://username:password@server:port/database", 
+    "SELECT * FROM lineitem", 
+    partition_on="l_orderkey", 
+    partition_num=10,
+)
+```
+Please find more [usage examples] on the [ConnectorX documentation].
+
+## Status
+
+The example program `demo.py` in this folder demonstrates basic use
+by reading from the built-in `sys.summits` table.
+
+Please note many special data types have not been verified to work
+with CrateDB yet, or have been identified to not work yet. We are
+enumerating them within the [backlog] file.
+
+
+[backlog]: ./backlog.md
+[ConnectorX]: https://pypi.org/project/connectorx/
+[ConnectorX documentation]: https://sfu-db.github.io/connector-x/
+[usage examples]: https://sfu-db.github.io/connector-x/api.html#examples

by-language/python-connectorx/backlog.md

@@ -0,0 +1,3 @@
+SELECT * FROM sys.summits ORDER BY height DESC LIMIT 10
+pyo3_runtime.PanicException: not implemented: point
+https://github.com/sfu-db/connector-x/issues/723

by-language/python-connectorx/demo.py

@@ -0,0 +1,32 @@
+"""
+Example program connecting to CrateDB [1] using ConnectorX [2],
+using its PostgreSQL interface [3].
+
+[1] https://cratedb.com/database
+[2] https://sfu-db.github.io/connector-x/
+[3] https://sfu-db.github.io/connector-x/databases/postgres.html
+"""
+
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "connectorx<0.5",
+#     "pandas>2,<3",
+# ]
+# ///
+
+from pprint import pprint
+import connectorx as cx
+
+
+def main():
+    data = cx.read_sql(
+        "postgresql://crate@localhost/",
+        "SELECT country, height, mountain, range, region FROM sys.summits ORDER BY height DESC LIMIT 10",
+        protocol="cursor",
+    )
+    pprint(data)
+
+
+if __name__ == "__main__":
+    main()

by-language/python-connectorx/pyproject.toml

@@ -0,0 +1,15 @@
+[tool.pytest.ini_options]
+minversion = "2.0"
+addopts = """
+  -rfEX -p pytester --strict-markers --verbosity=3
+  --capture=no
+  """
+log_level = "DEBUG"
+log_cli_level = "DEBUG"
+testpaths = ["*.py"]
+xfail_strict = true
+markers = [
+]
+
+[tool.ruff]
+line-length = 100

by-language/python-connectorx/requirements-dev.txt

@@ -0,0 +1 @@
+pytest<9

by-language/python-connectorx/requirements.txt

@@ -0,0 +1,2 @@
+connectorx<0.5
+pandas>2,<3

by-language/python-connectorx/test.py

@@ -0,0 +1,11 @@
+import shlex
+import subprocess
+
+
+def run(command: str):
+    subprocess.check_call(shlex.split(command))
+
+
+def test_demo():
+    cmd = "time python3 demo.py"
+    run(cmd)