Merge pull request #79 from cipherstash/operators

Ensure EQL works with operators

Author: auxesis

.github/workflows/test-eql.yml

@@ -6,13 +6,17 @@ on:
     paths:
       - ".github/workflows/test-eql.yml"
       - "sql/*.sql"
+      - "tests/**/*"
+      - "tasks/**/*"
 
   pull_request:
     branches:
       - main
     paths:
       - ".github/workflows/test-eql.yml"
       - "sql/*.sql"
+      - "tests/**/*"
+      - "tasks/**/*"
 
   workflow_dispatch:
 
@@ -23,29 +27,29 @@ defaults:
 jobs:
   test:
     name: "Test EQL SQL components"
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-latest-m
 
     strategy:
       fail-fast: false
       matrix:
         postgres-version: [17, 16, 15, 14]
 
     env:
-      CS_DATABASE__PASSWORD:
-      CS_DATABASE__PORT: 5432
-      CS_DATABASE__NAME: test
+      POSTGRES_VERSION: ${{ matrix.postgres-version }}
 
     steps:
       - uses: actions/checkout@v4
 
-      - uses: extractions/setup-just@v1
-
-      - uses: ankane/setup-postgres@v1
+      - uses: jdx/mise-action@v2
         with:
-          postgres-version: ${{ matrix.postgres-version }}
-          database: ${{ env.CS_DATABASE__NAME }}
+          version: 2025.1.6 # [default: latest] mise version to install
+          install: true # [default: true] run `mise install`
+          cache: true # [default: true] cache mise using GitHub's cache
 
-      - name: Test EQL
+      - name: Setup database (Postgres ${{ matrix.postgres-version }})
         run: |
-          just build test
+          mise run postgres:up postgres-${POSTGRES_VERSION} --extra-args "--detach --wait"
 
+      - name: Test EQL for Postgres ${{ matrix.postgres-version }}
+        run: |
+          mise run --output prefix test --postgres ${POSTGRES_VERSION}

.tool-versions

@@ -392,3 +392,49 @@ To cut a [release](https://github.com/cipherstash/encrypt-query-language/release
 1. Click `Publish release`.
 
 This will trigger the [Release EQL](https://github.com/cipherstash/encrypt-query-language/actions/workflows/release-eql.yml) workflow, which will build and attach artifacts to [the release](https://github.com/cipherstash/encrypt-query-language/releases/).
+
+## Testing
+
+There are tests for EQL for PostgreSQL versions 14–17.
+
+They easiest way to run them is in [GitHub Actions](https://github.com/cipherstash/encrypt-query-language/actions/workflows/test-eql.yml).
+
+### Running tests locally
+
+> [!IMPORTANT]
+> **Before you run the tests** you need to have this software installed:
+>  - [mise](https://mise.jdx.dev/) — see the [installing mise](#installing-mise) instructions
+>  - [Docker](https://www.docker.com/) — see Docker's [documentation for installing](https://docs.docker.com/get-started/get-docker/)
+
+To run tests locally:
+
+``` shell
+# Clone the repo
+git clone https://github.com/cipherstash/encrypt-query-language
+cd encrypt-query-language
+
+# Install dependencies
+mise trust --yes
+
+# Start a postgres instance
+mise run postgres:up postgres-17 --extra-args "--detach --wait"
+
+# Run the tests
+mise run test --postgres 17
+
+# Stop and remove all containers and networks
+mise run postgres:down
+```
+
+You can run the same tasks for Postgres 14, 15, 16, and 17.
+
+The configuration for the Postgres containers in `tests/docker-compose.yml`.
+
+Limitations:
+
+- **Volumes for Postgres containers are not persistent.**
+  If you need to look at data in the container, uncomment a volume in
+  `tests/docker-compose.yml`
+- **You can't run multiple Postgres containers at the same time.**
+  All the containers bind to the same port (`7543`). If you want to run
+  multiple containers at the same time, you'll have to change the ports.

README.md

@@ -0,0 +1,20 @@
+[settings]
+# Config for test environments
+# Can be invoked with: mise --env tcp run <task>
+# trusted_config_paths = [
+#     "./tests/mise.toml",
+#     "./tests/mise.tcp.toml",
+#     "./tests/mise.tls.toml",
+# ]
+[task_config]
+includes = [
+  "tasks",
+  "tasks/postgres.toml"
+]
+
+[env]
+POSTGRES_DB = "cipherstash"
+POSTGRES_USER = "cipherstash"
+POSTGRES_PASSWORD = "password"
+POSTGRES_HOST = "localhost"
+POSTGRES_PORT = "7432"

justfile

@@ -0,0 +1,159 @@
+DROP DOMAIN IF EXISTS cs_match_index_v1;
+CREATE DOMAIN cs_match_index_v1 AS smallint[];
+
+DROP DOMAIN IF EXISTS cs_unique_index_v1;
+CREATE DOMAIN cs_unique_index_v1 AS text;
+
+
+-- cs_encrypted_v1 is a column type and cannot be dropped if in use
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'cs_encrypted_v1') THEN
+      CREATE DOMAIN cs_encrypted_v1 AS JSONB;
+	  END IF;
+END
+$$;
+
+
+-- Should include a kind field
+DROP FUNCTION IF EXISTS _cs_encrypted_check_k(jsonb);
+CREATE FUNCTION _cs_encrypted_check_k(val jsonb)
+  RETURNS boolean
+AS $$
+	BEGIN
+    IF (val->>'k' = ANY('{ct, sv}')) THEN
+      RETURN true;
+    END IF;
+    RAISE 'Invalid kind (%) in Encrypted column. Kind should be one of {ct, sv}', val;
+  END;
+$$ LANGUAGE plpgsql;
+
+
+--
+-- CT payload should include a c field
+--
+DROP FUNCTION IF EXISTS _cs_encrypted_check_k_ct(jsonb);
+CREATE FUNCTION _cs_encrypted_check_k_ct(val jsonb)
+  RETURNS boolean
+AS $$
+	BEGIN
+    IF (val->>'k' = 'ct') THEN
+      IF (val ? 'c') THEN
+        RETURN true;
+      END IF;
+      RAISE 'Encrypted column kind (k) of "ct" missing data field (c):  %', val;
+    END IF;
+    RETURN true;
+  END;
+$$ LANGUAGE plpgsql;
+
+
+--
+-- SV payload should include an sv field
+--
+DROP FUNCTION IF EXISTS _cs_encrypted_check_k_sv(jsonb);
+CREATE FUNCTION _cs_encrypted_check_k_sv(val jsonb)
+  RETURNS boolean
+AS $$
+	BEGIN
+    IF (val->>'k' = 'sv') THEN
+      IF (val ? 'sv') THEN
+        RETURN true;
+      END IF;
+      RAISE 'Encrypted column kind (k) of "sv" missing data field (sv):  %', val;
+    END IF;
+    RETURN true;
+  END;
+$$ LANGUAGE plpgsql;
+
+
+-- Plaintext field should never be present in an encrypted column
+DROP FUNCTION IF EXISTS _cs_encrypted_check_p(jsonb);
+CREATE FUNCTION _cs_encrypted_check_p(val jsonb)
+  RETURNS boolean
+AS $$
+	BEGIN
+    IF NOT val ? 'p' THEN
+      RETURN true;
+    END IF;
+    RAISE 'Encrypted column includes plaintext (p) field: %', val;
+  END;
+$$ LANGUAGE plpgsql;
+
+-- Should include an ident field
+DROP FUNCTION IF EXISTS _cs_encrypted_check_i(jsonb);
+CREATE FUNCTION _cs_encrypted_check_i(val jsonb)
+  RETURNS boolean
+AS $$
+	BEGIN
+    IF val ? 'i' THEN
+      RETURN true;
+    END IF;
+    RAISE 'Encrypted column missing ident (i) field: %', val;
+  END;
+$$ LANGUAGE plpgsql;
+
+-- Query field should never be present in an encrypted column
+DROP FUNCTION IF EXISTS _cs_encrypted_check_q(jsonb);
+CREATE FUNCTION _cs_encrypted_check_q(val jsonb)
+  RETURNS boolean
+AS $$
+	BEGIN
+    IF val ? 'q' THEN
+      RAISE 'Encrypted column includes query (q) field: %', val;
+    END IF;
+    RETURN true;
+  END;
+$$ LANGUAGE plpgsql;
+
+-- Ident field should include table and column
+DROP FUNCTION IF EXISTS _cs_encrypted_check_i_ct(jsonb);
+CREATE FUNCTION _cs_encrypted_check_i_ct(val jsonb)
+  RETURNS boolean
+AS $$
+	BEGIN
+    IF (val->'i' ?& array['t', 'c']) THEN
+      RETURN true;
+    END IF;
+    RAISE 'Encrypted column ident (i) missing table (t) or column (c) fields: %', val;
+  END;
+$$ LANGUAGE plpgsql;
+
+-- Should include a version field
+DROP FUNCTION IF EXISTS _cs_encrypted_check_v(jsonb);
+CREATE FUNCTION _cs_encrypted_check_v(val jsonb)
+  RETURNS boolean
+AS $$
+	BEGIN
+    IF (val ? 'v') THEN
+      RETURN true;
+    END IF;
+    RAISE 'Encrypted column missing version (v) field: %', val;
+  END;
+$$ LANGUAGE plpgsql;
+
+
+DROP FUNCTION IF EXISTS cs_check_encrypted_v1(val jsonb);
+
+CREATE FUNCTION cs_check_encrypted_v1(val jsonb)
+  RETURNS BOOLEAN
+LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE
+BEGIN ATOMIC
+    RETURN (
+      _cs_encrypted_check_v(val) AND
+      _cs_encrypted_check_i(val) AND
+      _cs_encrypted_check_k(val) AND
+      _cs_encrypted_check_k_ct(val) AND
+      _cs_encrypted_check_k_sv(val) AND
+      _cs_encrypted_check_q(val) AND
+      _cs_encrypted_check_p(val)
+    );
+END;
+
+ALTER DOMAIN cs_encrypted_v1 DROP CONSTRAINT IF EXISTS cs_encrypted_v1_check;
+
+ALTER DOMAIN cs_encrypted_v1
+  ADD CONSTRAINT cs_encrypted_v1_check CHECK (
+   cs_check_encrypted_v1(VALUE)
+);
+