Add Support for CQL JSON to Stac FastAPI PGStac Backend (#209)

* wait for 5432 to be available before running apps in docker

* allow filter item in search for CQL search

* bump required pgstac version to v0.3.0

* update pgstac dependency to 0.3.1

* Update docker to pgstac 0.3.1. Fix #179 (enforce limit range). Tests for CQL

* Update docker to pgstac 0.3.1. Fix #179 (enforce limit range). Tests for CQL

* test for cql search

* make tests wait for database to be running

* bump pgstac version to 0.3.3

* update pgstac test_item_search_get_filter_cql to reflect 200 with empty features array rather than 404

Co-authored-by: Rob Emanuele <rdemanuele@gmail.com>

Author: bitner

Makefile

@@ -36,11 +36,11 @@ docker-shell-pgstac:
 
 .PHONY: test-sqlalchemy
 test-sqlalchemy: run-joplin-sqlalchemy
-	$(run_docker) /bin/bash -c 'export && cd /app/stac_fastapi/sqlalchemy/tests/ && pytest'
+	$(run_docker) /bin/bash -c 'export && ./scripts/wait-for-it.sh database:5432 && cd /app/stac_fastapi/sqlalchemy/tests/ && pytest'
 
 .PHONY: test-pgstac
 test-pgstac:
-	$(run_pgstac) /bin/bash -c 'export && cd /app/stac_fastapi/pgstac/tests/ && pytest'
+	$(run_pgstac) /bin/bash -c 'export && ./scripts/wait-for-it.sh database:5432 && cd /app/stac_fastapi/pgstac/tests/ && pytest'
 
 .PHONY: run-database
 run-database:
@@ -50,10 +50,6 @@ run-database:
 run-joplin-sqlalchemy:
 	docker-compose run --rm loadjoplin-sqlalchemy
 
-.PHONY: run-joplin-sqlalchemy
-run-joplin-sqlalchemy:
-	docker-compose run --rm loadjoplin-sqlalchemy
-
 .PHONY: test
 test: test-sqlalchemy test-pgstac
 

docker-compose.yml

@@ -71,7 +71,7 @@ services:
 
   database:
     container_name: stac-db
-    image: ghcr.io/stac-utils/pgstac:v0.3.2
+    image: ghcr.io/stac-utils/pgstac:v0.3.3
     environment:
       - POSTGRES_USER=username
       - POSTGRES_PASSWORD=password

stac_fastapi/pgstac/setup.py

@@ -26,7 +26,7 @@
         "pytest-asyncio",
         "pre-commit",
         "requests",
-        "pypgstac==0.3.2",
+        "pypgstac==0.3.3",
         "httpx",
         "shapely",
     ],

stac_fastapi/pgstac/stac_fastapi/pgstac/types/search.py

@@ -5,7 +5,7 @@
 from types import DynamicClassAttribute
 from typing import Any, Callable, Dict, List, Optional, Set, Union
 
-from pydantic import Field, root_validator, validator
+from pydantic import Field, conint, root_validator, validator
 from stac_pydantic.api import Search
 from stac_pydantic.api.extensions.fields import FieldsExtension as FieldsBase
 from stac_pydantic.utils import AutoValueEnum
@@ -96,9 +96,11 @@ class PgstacSearch(Search):
     fields: FieldsExtension = Field(FieldsExtension())
     # Override query extension with supported operators
     query: Optional[Dict[str, Dict[Operator, Any]]]
+    filter: Optional[Dict]
     token: Optional[str] = None
     datetime: Optional[str] = None
     sortby: Any
+    limit: Optional[conint(ge=0, le=10000)] = 10
 
     @root_validator(pre=True)
     def validate_query_fields(cls, values: Dict) -> Dict:

stac_fastapi/pgstac/tests/api/test_api.py

@@ -65,6 +65,50 @@ async def test_app_query_extension(load_test_data, app_client, load_test_collect
     assert len(resp_json["features"]) == 1
 
 
+@pytest.mark.asyncio
+async def test_app_query_extension_limit_1(
+    load_test_data, app_client, load_test_collection
+):
+    coll = load_test_collection
+    item = load_test_data("test_item.json")
+    resp = await app_client.post(f"/collections/{coll.id}/items", json=item)
+    assert resp.status_code == 200
+
+    params = {"limit": 1}
+    resp = await app_client.post("/search", json=params)
+    assert resp.status_code == 200
+    resp_json = resp.json()
+    assert len(resp_json["features"]) == 1
+
+
+@pytest.mark.asyncio
+async def test_app_query_extension_limit_lt0(
+    load_test_data, app_client, load_test_collection
+):
+    coll = load_test_collection
+    item = load_test_data("test_item.json")
+    resp = await app_client.post(f"/collections/{coll.id}/items", json=item)
+    assert resp.status_code == 200
+
+    params = {"limit": -1}
+    resp = await app_client.post("/search", json=params)
+    assert resp.status_code == 400
+
+
+@pytest.mark.asyncio
+async def test_app_query_extension_limit_gt10000(
+    load_test_data, app_client, load_test_collection
+):
+    coll = load_test_collection
+    item = load_test_data("test_item.json")
+    resp = await app_client.post(f"/collections/{coll.id}/items", json=item)
+    assert resp.status_code == 200
+
+    params = {"limit": 10001}
+    resp = await app_client.post("/search", json=params)
+    assert resp.status_code == 400
+
+
 @pytest.mark.asyncio
 async def test_app_sort_extension(load_test_data, app_client, load_test_collection):
     coll = load_test_collection

stac_fastapi/pgstac/tests/resources/test_item.py

@@ -682,6 +682,51 @@ async def test_item_search_get_query_extension(
     )
 
 
+@pytest.mark.asyncio
+async def test_item_search_get_filter_extension_cql(
+    app_client, load_test_data, load_test_collection
+):
+    """Test GET search with JSONB query (cql json filter extension)"""
+    test_item = load_test_data("test_item.json")
+    resp = await app_client.post(
+        f"/collections/{test_item['collection']}/items", json=test_item
+    )
+    assert resp.status_code == 200
+
+    # EPSG is a JSONB key
+    params = {
+        "collections": [test_item["collection"]],
+        "filter": {
+            "gt": [
+                {"property": "proj:epsg"},
+                test_item["properties"]["proj:epsg"] + 1,
+            ]
+        },
+    }
+    resp = await app_client.post("/search", json=params)
+    resp_json = resp.json()
+
+    assert resp.status_code == 200
+    assert len(resp_json.get("features")) == 0
+
+    params = {
+        "collections": [test_item["collection"]],
+        "filter": {
+            "eq": [
+                {"property": "proj:epsg"},
+                test_item["properties"]["proj:epsg"],
+            ]
+        },
+    }
+    resp = await app_client.post("/search", json=params)
+    resp_json = resp.json()
+    assert len(resp.json()["features"]) == 1
+    assert (
+        resp_json["features"][0]["properties"]["proj:epsg"]
+        == test_item["properties"]["proj:epsg"]
+    )
+
+
 @pytest.mark.asyncio
 async def test_get_missing_item_collection(app_client):
     """Test reading a collection which does not exist"""
@@ -929,14 +974,6 @@ async def test_get_missing_item(app_client, load_test_data):
     assert resp.status_code == 404
 
 
-@pytest.mark.skip
-@pytest.mark.asyncio
-async def test_search_invalid_query_field(app_client):
-    body = {"query": {"gsd": {"lt": 100}, "invalid-field": {"eq": 50}}}
-    resp = await app_client.post("/search", json=body)
-    assert resp.status_code == 400
-
-
 @pytest.mark.asyncio
 async def test_relative_link_construction():
     req = Request(