diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 91b8d30af..373414f6c 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -24,6 +24,15 @@ jobs:
- python-version: '3.10'
env:
PYGEOAPI_CONFIG: "$(pwd)/pygeoapi-config.yml"
+
+ services:
+ postgres:
+ image: postgis/postgis:14-3.2
+ ports:
+ - 5432:5432
+ env:
+ POSTGRES_DB: test
+ POSTGRES_PASSWORD: ${{ secrets.DatabasePassword || 'postgres' }}
steps:
- name: Pre-pull Docker Images
@@ -32,7 +41,6 @@ jobs:
docker pull appropriate/curl:latest &
docker pull elasticsearch:8.17.0 &
docker pull opensearchproject/opensearch:2.18.0 &
- docker pull mdillon/postgis:latest &
docker pull mongo:8.0.4 &
docker pull ghcr.io/cgs-earth/sensorthings-action:0.1.0 &
docker pull postgis/postgis:14-3.2 &
@@ -58,12 +66,6 @@ jobs:
sudo sysctl -w vm.swappiness=1
sudo sysctl -w fs.file-max=262144
sudo sysctl -w vm.max_map_count=262144
- - name: Install and run PostgreSQL/PostGIS 📦
- uses: huaxk/postgis-action@v1
- with:
- postgresql password: ${{ secrets.DatabasePassword || 'postgres' }}
- postgresql db: 'test'
-
- name: "Install and run MySQL 📦"
uses: mirromutth/mysql-action@v1.1
with:
@@ -164,6 +166,7 @@ jobs:
pytest tests/test_oracle_provider.py
pytest tests/test_parquet_provider.py
pytest tests/test_postgresql_provider.py
+ pytest tests/test_postgresql_mvt_provider.py
pytest tests/test_mysql_provider.py
pytest tests/test_rasterio_provider.py
pytest tests/test_sensorthings_edr_provider.py
diff --git a/docs/source/data-publishing/ogcapi-tiles.rst b/docs/source/data-publishing/ogcapi-tiles.rst
index 66ab76c91..173d08057 100644
--- a/docs/source/data-publishing/ogcapi-tiles.rst
+++ b/docs/source/data-publishing/ogcapi-tiles.rst
@@ -140,9 +140,6 @@ MVT-postgresql
.. note::
Must have PostGIS installed with protobuf-c support
-.. note::
- Geometry must be using EPSG:4326
-
This provider gives support to serving tiles generated using `PostgreSQL `_ with `PostGIS `_.
The tiles are rendered on-the-fly using `ST_AsMVT `_ and related methods.
@@ -152,24 +149,28 @@ This code block shows how to configure pygeoapi to render Mapbox vector tiles fr
providers:
- type: tile
- name: MVT-postgresql
- data:
- host: 127.0.0.1
- port: 3010 # Default 5432 if not provided
- dbname: test
- user: postgres
- password: postgres
- search_path: [osm, public]
- id_field: osm_id
- table: hotosm_bdi_waterways
- geom_field: foo_geom
- options:
- zoom:
- min: 0
- max: 15
- format:
- name: pbf
- mimetype: application/vnd.mapbox-vector-tile
+ name: MVT-postgresql
+ data:
+ host: 127.0.0.1
+ port: 3010 # Default 5432 if not provided
+ dbname: test
+ user: postgres
+ password: postgres
+ search_path: [osm, public]
+ id_field: osm_id
+ table: hotosm_bdi_waterways
+ geom_field: foo_geom
+ storage_crs: http://www.opengis.net/def/crs/EPSG/0/4326
+ options:
+ zoom:
+ min: 0
+ max: 15
+ format:
+ name: pbf
+ mimetype: application/vnd.mapbox-vector-tile
+
+.. tip::
+ Geometry must have correctly defined :ref:`storage_crs`
PostgreSQL-related connection options can also be added to `options`. Please refer to the :ref:`PostgreSQL OGC Features Provider` documentation for more information.
diff --git a/pygeoapi/api/tiles.py b/pygeoapi/api/tiles.py
index 0d457e596..cb957eee8 100644
--- a/pygeoapi/api/tiles.py
+++ b/pygeoapi/api/tiles.py
@@ -246,7 +246,7 @@ def get_collection_tiles_data(
if content is None:
msg = 'identifier not found'
return api.get_exception(
- HTTPStatus.NO_CONTENT, headers, format_, 'NocContent', msg)
+ HTTPStatus.NO_CONTENT, headers, format_, 'NoContent', msg)
else:
return headers, HTTPStatus.OK, content
diff --git a/pygeoapi/provider/mvt_postgresql.py b/pygeoapi/provider/mvt_postgresql.py
index c8fa2fc1a..aa791ec45 100644
--- a/pygeoapi/provider/mvt_postgresql.py
+++ b/pygeoapi/provider/mvt_postgresql.py
@@ -3,10 +3,12 @@
# Authors: Prajwal Amaravati
# Tanvi Prasad
# Bryan Robert
+# Benjamin Webb
#
# Copyright (c) 2025 Prajwal Amaravati
# Copyright (c) 2025 Tanvi Prasad
# Copyright (c) 2025 Bryan Robert
+# Copyright (c) 2025 Benjamin Webb
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
@@ -31,10 +33,13 @@
#
# =================================================================
-from copy import deepcopy
import logging
-from sqlalchemy.sql import text
+from geoalchemy2.functions import (ST_TileEnvelope, ST_Transform, ST_AsMVTGeom,
+ ST_AsMVT, ST_CurveToLine, ST_MakeEnvelope)
+
+from sqlalchemy.sql import select
+from sqlalchemy.orm import Session
from pygeoapi.models.provider.base import (
TileSetMetadata, TileMatrixSetEnum, LinkType)
@@ -42,12 +47,12 @@
from pygeoapi.provider.base_mvt import BaseMVTProvider
from pygeoapi.provider.sql import PostgreSQLProvider
from pygeoapi.provider.tile import ProviderTileNotFoundError
-from pygeoapi.util import url_join
+from pygeoapi.util import url_join, get_crs_from_uri
LOGGER = logging.getLogger(__name__)
-class MVTPostgreSQLProvider(BaseMVTProvider):
+class MVTPostgreSQLProvider(BaseMVTProvider, PostgreSQLProvider):
"""
MVT PostgreSQL Provider
Provider for serving tiles rendered on-the-fly from
@@ -62,48 +67,42 @@ def __init__(self, provider_def):
:returns: pygeoapi.provider.MVT.MVTPostgreSQLProvider
"""
+ PostgreSQLProvider.__init__(self, provider_def)
+ BaseMVTProvider.__init__(self, provider_def)
- super().__init__(provider_def)
-
- pg_def = deepcopy(provider_def)
- # delete the zoom option before initializing the PostgreSQL provider
- # that provider breaks otherwise
- del pg_def["options"]["zoom"]
- self.postgres = PostgreSQLProvider(pg_def)
-
- self.layer_name = provider_def["table"]
- self.table = provider_def['table']
- self.id_field = provider_def['id_field']
- self.geom = provider_def.get('geom_field', 'geom')
-
- LOGGER.debug(f'DB connection: {repr(self.postgres._engine.url)}')
+ def get_fields(self):
+ """
+ Get Postgres columns
- def __repr__(self):
- return f' {self.data}'
+ :returns: `list` of columns
+ """
+ if not self._fields:
+ for column in self.table_model.__table__.columns:
+ LOGGER.debug(f'Testing {column.name}')
+ if column.name == self.geom:
+ continue
- @property
- def service_url(self):
- return self._service_url
+ self._fields[str(column.name)] = (
+ column.label('id')
+ if column.name == self.id_field else
+ column
+ )
- @property
- def service_metadata_url(self):
- return self._service_metadata_url
+ return self._fields
def get_layer(self):
"""
- Extracts layer name from url
+ Use table name as layer name
- :returns: layer name
+ :returns: `str` of layer name
"""
-
- return self.layer_name
+ return self.table
def get_tiling_schemes(self):
-
return [
- TileMatrixSetEnum.WEBMERCATORQUAD.value,
- TileMatrixSetEnum.WORLDCRS84QUAD.value
- ]
+ TileMatrixSetEnum.WEBMERCATORQUAD.value,
+ TileMatrixSetEnum.WORLDCRS84QUAD.value
+ ]
def get_tiles_service(self, baseurl=None, servicepath=None,
dirpath=None, tile_type=None):
@@ -118,13 +117,14 @@ def get_tiles_service(self, baseurl=None, servicepath=None,
:returns: `dict` of item tile service
"""
- super().get_tiles_service(baseurl, servicepath,
- dirpath, tile_type)
+ BaseMVTProvider.get_tiles_service(self,
+ baseurl, servicepath,
+ dirpath, tile_type)
self._service_url = servicepath
return self.get_tms_links()
- def get_tiles(self, layer=None, tileset=None,
+ def get_tiles(self, layer='default', tileset=None,
z=None, y=None, x=None, format_=None):
"""
Gets tile
@@ -138,67 +138,56 @@ def get_tiles(self, layer=None, tileset=None,
:returns: an encoded mvt tile
"""
- if format_ == 'mvt':
- format_ = self.format_type
-
- fields_arr = self.postgres.get_fields().keys()
- fields = ', '.join(['"' + f + '"' for f in fields_arr])
- if len(fields) != 0:
- fields = ',' + fields
-
- query = ''
- if tileset == TileMatrixSetEnum.WEBMERCATORQUAD.value.tileMatrixSet:
- if not self.is_in_limits(TileMatrixSetEnum.WEBMERCATORQUAD.value, z, x, y): # noqa
- raise ProviderTileNotFoundError
-
- query = text("""
- WITH
- bounds AS (
- SELECT ST_TileEnvelope(:z, :x, :y) AS boundgeom
- ),
- mvtgeom AS (
- SELECT ST_AsMVTGeom(ST_Transform(ST_CurveToLine({geom}), 3857), bounds.boundgeom) AS geom {fields}
- FROM "{table}", bounds
- WHERE ST_Intersects({geom}, ST_Transform(bounds.boundgeom, 4326))
- )
- SELECT ST_AsMVT(mvtgeom, 'default') FROM mvtgeom;
- """.format(geom=self.geom, table=self.table, fields=fields)) # noqa
-
- if tileset == TileMatrixSetEnum.WORLDCRS84QUAD.value.tileMatrixSet:
- if not self.is_in_limits(TileMatrixSetEnum.WORLDCRS84QUAD.value, z, x, y): # noqa
- raise ProviderTileNotFoundError
-
- query = text("""
- WITH
- bounds AS (
- SELECT ST_TileEnvelope(:z, :x, :y,
- 'SRID=4326;POLYGON((-180 -90,-180 90,180 90,180 -90,-180 -90))'::geometry) AS boundgeom
- ),
- mvtgeom AS (
- SELECT ST_AsMVTGeom(ST_CurveToLine({geom}), bounds.boundgeom) AS geom {fields}
- FROM "{table}", bounds
- WHERE ST_Intersects({geom}, bounds.boundgeom)
- )
- SELECT ST_AsMVT(mvtgeom, 'default') FROM mvtgeom;
- """.format(geom=self.geom, table=self.table, fields=fields)) # noqa
-
- with self.postgres._engine.connect() as session:
- result = session.execute(query, {
- 'z': z,
- 'y': y,
- 'x': x
- }).fetchone()
-
- if len(bytes(result[0])) == 0:
- return None
- return bytes(result[0])
+ z, y, x = map(int, [z, y, x])
+
+ [tileset_schema] = [
+ schema for schema in self.get_tiling_schemes()
+ if tileset == schema.tileMatrixSet
+ ]
+ if not self.is_in_limits(tileset_schema, z, x, y):
+ LOGGER.warning(f'Tile {z}/{x}/{y} not found')
+ return ProviderTileNotFoundError
+
+ storage_srid = get_crs_from_uri(self.storage_crs).to_string()
+ out_srid = get_crs_from_uri(tileset_schema.crs).to_string()
+ envelope = self.get_envelope(z, y, x, tileset)
+
+ geom_column = getattr(self.table_model, self.geom)
+ geom_filter = geom_column.intersects(
+ ST_Transform(envelope, storage_srid)
+ )
+
+ mvtgeom = (
+ ST_AsMVTGeom(
+ ST_Transform(ST_CurveToLine(geom_column), out_srid),
+ ST_Transform(envelope, out_srid))
+ .label('mvtgeom')
+ )
+
+ mvtrow = (
+ select(mvtgeom, *self.fields.values())
+ .filter(geom_filter)
+ .cte('mvtrow')
+ )
+
+ mvtquery = select(
+ ST_AsMVT(mvtrow.table_valued(), layer)
+ )
+
+ with Session(self._engine) as session:
+ result = bytes(
+ session.execute(mvtquery).scalar()
+ ) or None
+
+ return result
def get_html_metadata(self, dataset, server_url, layer, tileset,
title, description, keywords, **kwargs):
service_url = url_join(
server_url,
- f'collections/{dataset}/tiles/{tileset}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}?f=mvt') # noqa
+ f'collections/{dataset}/tiles/{tileset}',
+ '{tileMatrix}/{tileRow}/{tileCol}?f=mvt')
metadata_url = url_join(
server_url,
f'collections/{dataset}/tiles/{tileset}/metadata')
@@ -217,9 +206,10 @@ def get_default_metadata(self, dataset, server_url, layer, tileset,
service_url = url_join(
server_url,
- f'collections/{dataset}/tiles/{tileset}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}?f=mvt') # noqa
+ f'collections/{dataset}/tiles/{tileset}',
+ '{tileMatrix}/{tileRow}/{tileCol}?f=mvt'
+ )
- content = {}
tiling_schemes = self.get_tiling_schemes()
# Default values
tileMatrixSetURI = tiling_schemes[0].tileMatrixSetURI
@@ -231,17 +221,18 @@ def get_default_metadata(self, dataset, server_url, layer, tileset,
tileMatrixSetURI = schema.tileMatrixSetURI
tiling_scheme_url = url_join(
- server_url, f'/TileMatrixSets/{schema.tileMatrixSet}')
- tiling_scheme_url_type = "application/json"
+ server_url, 'TileMatrixSets', schema.tileMatrixSet)
+ tiling_scheme_url_type = 'application/json'
tiling_scheme_url_title = f'{schema.tileMatrixSet} tile matrix set definition' # noqa
- tiling_scheme = LinkType(href=tiling_scheme_url,
- rel="http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", # noqa
- type_=tiling_scheme_url_type,
- title=tiling_scheme_url_title)
+ tiling_scheme = LinkType(
+ href=tiling_scheme_url,
+ rel='http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme',
+ type_=tiling_scheme_url_type,
+ title=tiling_scheme_url_title)
if tiling_scheme is None:
- msg = f'Could not identify a valid tiling schema' # noqa
+ msg = 'Could not identify a valid tiling schema'
LOGGER.error(msg)
raise ProviderConnectionError(msg)
@@ -250,9 +241,9 @@ def get_default_metadata(self, dataset, server_url, layer, tileset,
tileMatrixSetURI=tileMatrixSetURI)
links = []
- service_url_link_type = "application/vnd.mapbox-vector-tile"
+ service_url_link_type = 'application/vnd.mapbox-vector-tile'
service_url_link_title = f'{tileset} vector tiles for {layer}'
- service_url_link = LinkType(href=service_url, rel="item",
+ service_url_link = LinkType(href=service_url, rel='item',
type_=service_url_link_type,
title=service_url_link_title)
@@ -261,4 +252,42 @@ def get_default_metadata(self, dataset, server_url, layer, tileset,
content.links = links
- return content.dict(exclude_none=True)
+ return content.model_dump(exclude_none=True)
+
+ @staticmethod
+ def get_envelope(z, y, x, tileset):
+ """
+ Calculate the Tile bounding box of a tile at zoom z, y, x.
+
+ WorldCRS84Quad tiles have:
+ - origin top-left (y=0 is north)
+ - full lon: -180 to 180
+
+ :param tileset: mvt tileset
+ :param z: z index
+ :param y: y index
+ :param x: x index
+
+ :returns: SQL Alchemy Tile Envelope
+ """
+
+ if tileset == TileMatrixSetEnum.WORLDCRS84QUAD.value.tileMatrixSet:
+
+ tile_size = 180 / 2 ** z
+
+ xmin = tile_size * x - 180
+ ymax = tile_size * -y + 90
+
+ # getting bottom-right coordinates of the tile
+ xmax = xmin + tile_size
+ ymin = ymax - tile_size
+
+ envelope = ST_MakeEnvelope(xmin, ymin, xmax, ymax, 4326)
+
+ else:
+ envelope = ST_TileEnvelope(z, x, y)
+
+ return envelope.label('bounds')
+
+ def __repr__(self):
+ return f' {self.data}'
diff --git a/pygeoapi/provider/sql.py b/pygeoapi/provider/sql.py
index 2775e705b..cdc90d2c7 100644
--- a/pygeoapi/provider/sql.py
+++ b/pygeoapi/provider/sql.py
@@ -107,7 +107,7 @@ def __init__(
self,
provider_def: dict,
driver_name: str,
- extra_conn_args: Optional[dict]
+ extra_conn_args: Optional[dict] = {}
):
"""
GenericSQLProvider Class constructor
@@ -143,9 +143,7 @@ def __init__(
LOGGER.debug(f'Configured Storage CRS: {self.storage_crs}')
# Read table information from database
- options = None
- if provider_def.get('options'):
- options = provider_def['options']
+ options = provider_def.get('options', {})
self._store_db_parameters(provider_def['data'], options)
self._engine = get_engine(
driver_name,
@@ -154,7 +152,7 @@ def __init__(
self.db_name,
self.db_user,
self._db_password,
- **(self.db_options or {}) | (extra_conn_args or {})
+ **self.db_options | extra_conn_args
)
self.table_model = get_table_model(
self.table, self.id_field, self.db_search_path, self._engine
@@ -443,7 +441,11 @@ def _store_db_parameters(self, parameters, options):
# reflecting the table definition from the DB
self.db_search_path = tuple(parameters.get('search_path', ['public']))
self._db_password = parameters.get('password')
- self.db_options = options
+ self.db_options = {
+ k: v
+ for k, v in options.items()
+ if not isinstance(v, dict)
+ }
def _sqlalchemy_to_feature(self, item, crs_transform_out=None):
feature = {'type': 'Feature'}
@@ -608,7 +610,7 @@ def get_engine(
database: str,
user: str,
password: str,
- **connection_options
+ **connect_args
):
"""Create SQL Alchemy engine."""
conn_str = URL.create(
@@ -619,11 +621,8 @@ def get_engine(
port=int(port),
database=database
)
- conn_args = {
- **connection_options
- }
engine = create_engine(
- conn_str, connect_args=conn_args, pool_pre_ping=True
+ conn_str, connect_args=connect_args, pool_pre_ping=True
)
return engine
diff --git a/pygeoapi/provider/tile.py b/pygeoapi/provider/tile.py
index 7e6352ae3..a57488db1 100644
--- a/pygeoapi/provider/tile.py
+++ b/pygeoapi/provider/tile.py
@@ -56,7 +56,6 @@ def __init__(self, provider_def):
self.mimetype = provider_def['format']['mimetype']
self.options = provider_def.get('options')
self.tile_type = None
- self.fields = {}
def get_layer(self):
"""
diff --git a/pygeoapi/templates/collections/tiles/index.html b/pygeoapi/templates/collections/tiles/index.html
index a9bd56cd9..25e8248d7 100644
--- a/pygeoapi/templates/collections/tiles/index.html
+++ b/pygeoapi/templates/collections/tiles/index.html
@@ -69,6 +69,7 @@ Tiles
{% block extrafoot %}