diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 91b8d30af..373414f6c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,6 +24,15 @@ jobs: - python-version: '3.10' env: PYGEOAPI_CONFIG: "$(pwd)/pygeoapi-config.yml" + + services: + postgres: + image: postgis/postgis:14-3.2 + ports: + - 5432:5432 + env: + POSTGRES_DB: test + POSTGRES_PASSWORD: ${{ secrets.DatabasePassword || 'postgres' }} steps: - name: Pre-pull Docker Images @@ -32,7 +41,6 @@ jobs: docker pull appropriate/curl:latest & docker pull elasticsearch:8.17.0 & docker pull opensearchproject/opensearch:2.18.0 & - docker pull mdillon/postgis:latest & docker pull mongo:8.0.4 & docker pull ghcr.io/cgs-earth/sensorthings-action:0.1.0 & docker pull postgis/postgis:14-3.2 & @@ -58,12 +66,6 @@ jobs: sudo sysctl -w vm.swappiness=1 sudo sysctl -w fs.file-max=262144 sudo sysctl -w vm.max_map_count=262144 - - name: Install and run PostgreSQL/PostGIS 📦 - uses: huaxk/postgis-action@v1 - with: - postgresql password: ${{ secrets.DatabasePassword || 'postgres' }} - postgresql db: 'test' - - name: "Install and run MySQL 📦" uses: mirromutth/mysql-action@v1.1 with: @@ -164,6 +166,7 @@ jobs: pytest tests/test_oracle_provider.py pytest tests/test_parquet_provider.py pytest tests/test_postgresql_provider.py + pytest tests/test_postgresql_mvt_provider.py pytest tests/test_mysql_provider.py pytest tests/test_rasterio_provider.py pytest tests/test_sensorthings_edr_provider.py diff --git a/docs/source/data-publishing/ogcapi-tiles.rst b/docs/source/data-publishing/ogcapi-tiles.rst index 66ab76c91..173d08057 100644 --- a/docs/source/data-publishing/ogcapi-tiles.rst +++ b/docs/source/data-publishing/ogcapi-tiles.rst @@ -140,9 +140,6 @@ MVT-postgresql .. note:: Must have PostGIS installed with protobuf-c support -.. note:: - Geometry must be using EPSG:4326 - This provider gives support to serving tiles generated using `PostgreSQL `_ with `PostGIS `_. The tiles are rendered on-the-fly using `ST_AsMVT `_ and related methods. @@ -152,24 +149,28 @@ This code block shows how to configure pygeoapi to render Mapbox vector tiles fr providers: - type: tile - name: MVT-postgresql - data: - host: 127.0.0.1 - port: 3010 # Default 5432 if not provided - dbname: test - user: postgres - password: postgres - search_path: [osm, public] - id_field: osm_id - table: hotosm_bdi_waterways - geom_field: foo_geom - options: - zoom: - min: 0 - max: 15 - format: - name: pbf - mimetype: application/vnd.mapbox-vector-tile + name: MVT-postgresql + data: + host: 127.0.0.1 + port: 3010 # Default 5432 if not provided + dbname: test + user: postgres + password: postgres + search_path: [osm, public] + id_field: osm_id + table: hotosm_bdi_waterways + geom_field: foo_geom + storage_crs: http://www.opengis.net/def/crs/EPSG/0/4326 + options: + zoom: + min: 0 + max: 15 + format: + name: pbf + mimetype: application/vnd.mapbox-vector-tile + +.. tip:: + Geometry must have correctly defined :ref:`storage_crs` PostgreSQL-related connection options can also be added to `options`. Please refer to the :ref:`PostgreSQL OGC Features Provider` documentation for more information. diff --git a/pygeoapi/api/tiles.py b/pygeoapi/api/tiles.py index 0d457e596..cb957eee8 100644 --- a/pygeoapi/api/tiles.py +++ b/pygeoapi/api/tiles.py @@ -246,7 +246,7 @@ def get_collection_tiles_data( if content is None: msg = 'identifier not found' return api.get_exception( - HTTPStatus.NO_CONTENT, headers, format_, 'NocContent', msg) + HTTPStatus.NO_CONTENT, headers, format_, 'NoContent', msg) else: return headers, HTTPStatus.OK, content diff --git a/pygeoapi/provider/mvt_postgresql.py b/pygeoapi/provider/mvt_postgresql.py index c8fa2fc1a..aa791ec45 100644 --- a/pygeoapi/provider/mvt_postgresql.py +++ b/pygeoapi/provider/mvt_postgresql.py @@ -3,10 +3,12 @@ # Authors: Prajwal Amaravati # Tanvi Prasad # Bryan Robert +# Benjamin Webb # # Copyright (c) 2025 Prajwal Amaravati # Copyright (c) 2025 Tanvi Prasad # Copyright (c) 2025 Bryan Robert +# Copyright (c) 2025 Benjamin Webb # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -31,10 +33,13 @@ # # ================================================================= -from copy import deepcopy import logging -from sqlalchemy.sql import text +from geoalchemy2.functions import (ST_TileEnvelope, ST_Transform, ST_AsMVTGeom, + ST_AsMVT, ST_CurveToLine, ST_MakeEnvelope) + +from sqlalchemy.sql import select +from sqlalchemy.orm import Session from pygeoapi.models.provider.base import ( TileSetMetadata, TileMatrixSetEnum, LinkType) @@ -42,12 +47,12 @@ from pygeoapi.provider.base_mvt import BaseMVTProvider from pygeoapi.provider.sql import PostgreSQLProvider from pygeoapi.provider.tile import ProviderTileNotFoundError -from pygeoapi.util import url_join +from pygeoapi.util import url_join, get_crs_from_uri LOGGER = logging.getLogger(__name__) -class MVTPostgreSQLProvider(BaseMVTProvider): +class MVTPostgreSQLProvider(BaseMVTProvider, PostgreSQLProvider): """ MVT PostgreSQL Provider Provider for serving tiles rendered on-the-fly from @@ -62,48 +67,42 @@ def __init__(self, provider_def): :returns: pygeoapi.provider.MVT.MVTPostgreSQLProvider """ + PostgreSQLProvider.__init__(self, provider_def) + BaseMVTProvider.__init__(self, provider_def) - super().__init__(provider_def) - - pg_def = deepcopy(provider_def) - # delete the zoom option before initializing the PostgreSQL provider - # that provider breaks otherwise - del pg_def["options"]["zoom"] - self.postgres = PostgreSQLProvider(pg_def) - - self.layer_name = provider_def["table"] - self.table = provider_def['table'] - self.id_field = provider_def['id_field'] - self.geom = provider_def.get('geom_field', 'geom') - - LOGGER.debug(f'DB connection: {repr(self.postgres._engine.url)}') + def get_fields(self): + """ + Get Postgres columns - def __repr__(self): - return f' {self.data}' + :returns: `list` of columns + """ + if not self._fields: + for column in self.table_model.__table__.columns: + LOGGER.debug(f'Testing {column.name}') + if column.name == self.geom: + continue - @property - def service_url(self): - return self._service_url + self._fields[str(column.name)] = ( + column.label('id') + if column.name == self.id_field else + column + ) - @property - def service_metadata_url(self): - return self._service_metadata_url + return self._fields def get_layer(self): """ - Extracts layer name from url + Use table name as layer name - :returns: layer name + :returns: `str` of layer name """ - - return self.layer_name + return self.table def get_tiling_schemes(self): - return [ - TileMatrixSetEnum.WEBMERCATORQUAD.value, - TileMatrixSetEnum.WORLDCRS84QUAD.value - ] + TileMatrixSetEnum.WEBMERCATORQUAD.value, + TileMatrixSetEnum.WORLDCRS84QUAD.value + ] def get_tiles_service(self, baseurl=None, servicepath=None, dirpath=None, tile_type=None): @@ -118,13 +117,14 @@ def get_tiles_service(self, baseurl=None, servicepath=None, :returns: `dict` of item tile service """ - super().get_tiles_service(baseurl, servicepath, - dirpath, tile_type) + BaseMVTProvider.get_tiles_service(self, + baseurl, servicepath, + dirpath, tile_type) self._service_url = servicepath return self.get_tms_links() - def get_tiles(self, layer=None, tileset=None, + def get_tiles(self, layer='default', tileset=None, z=None, y=None, x=None, format_=None): """ Gets tile @@ -138,67 +138,56 @@ def get_tiles(self, layer=None, tileset=None, :returns: an encoded mvt tile """ - if format_ == 'mvt': - format_ = self.format_type - - fields_arr = self.postgres.get_fields().keys() - fields = ', '.join(['"' + f + '"' for f in fields_arr]) - if len(fields) != 0: - fields = ',' + fields - - query = '' - if tileset == TileMatrixSetEnum.WEBMERCATORQUAD.value.tileMatrixSet: - if not self.is_in_limits(TileMatrixSetEnum.WEBMERCATORQUAD.value, z, x, y): # noqa - raise ProviderTileNotFoundError - - query = text(""" - WITH - bounds AS ( - SELECT ST_TileEnvelope(:z, :x, :y) AS boundgeom - ), - mvtgeom AS ( - SELECT ST_AsMVTGeom(ST_Transform(ST_CurveToLine({geom}), 3857), bounds.boundgeom) AS geom {fields} - FROM "{table}", bounds - WHERE ST_Intersects({geom}, ST_Transform(bounds.boundgeom, 4326)) - ) - SELECT ST_AsMVT(mvtgeom, 'default') FROM mvtgeom; - """.format(geom=self.geom, table=self.table, fields=fields)) # noqa - - if tileset == TileMatrixSetEnum.WORLDCRS84QUAD.value.tileMatrixSet: - if not self.is_in_limits(TileMatrixSetEnum.WORLDCRS84QUAD.value, z, x, y): # noqa - raise ProviderTileNotFoundError - - query = text(""" - WITH - bounds AS ( - SELECT ST_TileEnvelope(:z, :x, :y, - 'SRID=4326;POLYGON((-180 -90,-180 90,180 90,180 -90,-180 -90))'::geometry) AS boundgeom - ), - mvtgeom AS ( - SELECT ST_AsMVTGeom(ST_CurveToLine({geom}), bounds.boundgeom) AS geom {fields} - FROM "{table}", bounds - WHERE ST_Intersects({geom}, bounds.boundgeom) - ) - SELECT ST_AsMVT(mvtgeom, 'default') FROM mvtgeom; - """.format(geom=self.geom, table=self.table, fields=fields)) # noqa - - with self.postgres._engine.connect() as session: - result = session.execute(query, { - 'z': z, - 'y': y, - 'x': x - }).fetchone() - - if len(bytes(result[0])) == 0: - return None - return bytes(result[0]) + z, y, x = map(int, [z, y, x]) + + [tileset_schema] = [ + schema for schema in self.get_tiling_schemes() + if tileset == schema.tileMatrixSet + ] + if not self.is_in_limits(tileset_schema, z, x, y): + LOGGER.warning(f'Tile {z}/{x}/{y} not found') + return ProviderTileNotFoundError + + storage_srid = get_crs_from_uri(self.storage_crs).to_string() + out_srid = get_crs_from_uri(tileset_schema.crs).to_string() + envelope = self.get_envelope(z, y, x, tileset) + + geom_column = getattr(self.table_model, self.geom) + geom_filter = geom_column.intersects( + ST_Transform(envelope, storage_srid) + ) + + mvtgeom = ( + ST_AsMVTGeom( + ST_Transform(ST_CurveToLine(geom_column), out_srid), + ST_Transform(envelope, out_srid)) + .label('mvtgeom') + ) + + mvtrow = ( + select(mvtgeom, *self.fields.values()) + .filter(geom_filter) + .cte('mvtrow') + ) + + mvtquery = select( + ST_AsMVT(mvtrow.table_valued(), layer) + ) + + with Session(self._engine) as session: + result = bytes( + session.execute(mvtquery).scalar() + ) or None + + return result def get_html_metadata(self, dataset, server_url, layer, tileset, title, description, keywords, **kwargs): service_url = url_join( server_url, - f'collections/{dataset}/tiles/{tileset}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}?f=mvt') # noqa + f'collections/{dataset}/tiles/{tileset}', + '{tileMatrix}/{tileRow}/{tileCol}?f=mvt') metadata_url = url_join( server_url, f'collections/{dataset}/tiles/{tileset}/metadata') @@ -217,9 +206,10 @@ def get_default_metadata(self, dataset, server_url, layer, tileset, service_url = url_join( server_url, - f'collections/{dataset}/tiles/{tileset}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}?f=mvt') # noqa + f'collections/{dataset}/tiles/{tileset}', + '{tileMatrix}/{tileRow}/{tileCol}?f=mvt' + ) - content = {} tiling_schemes = self.get_tiling_schemes() # Default values tileMatrixSetURI = tiling_schemes[0].tileMatrixSetURI @@ -231,17 +221,18 @@ def get_default_metadata(self, dataset, server_url, layer, tileset, tileMatrixSetURI = schema.tileMatrixSetURI tiling_scheme_url = url_join( - server_url, f'/TileMatrixSets/{schema.tileMatrixSet}') - tiling_scheme_url_type = "application/json" + server_url, 'TileMatrixSets', schema.tileMatrixSet) + tiling_scheme_url_type = 'application/json' tiling_scheme_url_title = f'{schema.tileMatrixSet} tile matrix set definition' # noqa - tiling_scheme = LinkType(href=tiling_scheme_url, - rel="http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", # noqa - type_=tiling_scheme_url_type, - title=tiling_scheme_url_title) + tiling_scheme = LinkType( + href=tiling_scheme_url, + rel='http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme', + type_=tiling_scheme_url_type, + title=tiling_scheme_url_title) if tiling_scheme is None: - msg = f'Could not identify a valid tiling schema' # noqa + msg = 'Could not identify a valid tiling schema' LOGGER.error(msg) raise ProviderConnectionError(msg) @@ -250,9 +241,9 @@ def get_default_metadata(self, dataset, server_url, layer, tileset, tileMatrixSetURI=tileMatrixSetURI) links = [] - service_url_link_type = "application/vnd.mapbox-vector-tile" + service_url_link_type = 'application/vnd.mapbox-vector-tile' service_url_link_title = f'{tileset} vector tiles for {layer}' - service_url_link = LinkType(href=service_url, rel="item", + service_url_link = LinkType(href=service_url, rel='item', type_=service_url_link_type, title=service_url_link_title) @@ -261,4 +252,42 @@ def get_default_metadata(self, dataset, server_url, layer, tileset, content.links = links - return content.dict(exclude_none=True) + return content.model_dump(exclude_none=True) + + @staticmethod + def get_envelope(z, y, x, tileset): + """ + Calculate the Tile bounding box of a tile at zoom z, y, x. + + WorldCRS84Quad tiles have: + - origin top-left (y=0 is north) + - full lon: -180 to 180 + + :param tileset: mvt tileset + :param z: z index + :param y: y index + :param x: x index + + :returns: SQL Alchemy Tile Envelope + """ + + if tileset == TileMatrixSetEnum.WORLDCRS84QUAD.value.tileMatrixSet: + + tile_size = 180 / 2 ** z + + xmin = tile_size * x - 180 + ymax = tile_size * -y + 90 + + # getting bottom-right coordinates of the tile + xmax = xmin + tile_size + ymin = ymax - tile_size + + envelope = ST_MakeEnvelope(xmin, ymin, xmax, ymax, 4326) + + else: + envelope = ST_TileEnvelope(z, x, y) + + return envelope.label('bounds') + + def __repr__(self): + return f' {self.data}' diff --git a/pygeoapi/provider/sql.py b/pygeoapi/provider/sql.py index 2775e705b..cdc90d2c7 100644 --- a/pygeoapi/provider/sql.py +++ b/pygeoapi/provider/sql.py @@ -107,7 +107,7 @@ def __init__( self, provider_def: dict, driver_name: str, - extra_conn_args: Optional[dict] + extra_conn_args: Optional[dict] = {} ): """ GenericSQLProvider Class constructor @@ -143,9 +143,7 @@ def __init__( LOGGER.debug(f'Configured Storage CRS: {self.storage_crs}') # Read table information from database - options = None - if provider_def.get('options'): - options = provider_def['options'] + options = provider_def.get('options', {}) self._store_db_parameters(provider_def['data'], options) self._engine = get_engine( driver_name, @@ -154,7 +152,7 @@ def __init__( self.db_name, self.db_user, self._db_password, - **(self.db_options or {}) | (extra_conn_args or {}) + **self.db_options | extra_conn_args ) self.table_model = get_table_model( self.table, self.id_field, self.db_search_path, self._engine @@ -443,7 +441,11 @@ def _store_db_parameters(self, parameters, options): # reflecting the table definition from the DB self.db_search_path = tuple(parameters.get('search_path', ['public'])) self._db_password = parameters.get('password') - self.db_options = options + self.db_options = { + k: v + for k, v in options.items() + if not isinstance(v, dict) + } def _sqlalchemy_to_feature(self, item, crs_transform_out=None): feature = {'type': 'Feature'} @@ -608,7 +610,7 @@ def get_engine( database: str, user: str, password: str, - **connection_options + **connect_args ): """Create SQL Alchemy engine.""" conn_str = URL.create( @@ -619,11 +621,8 @@ def get_engine( port=int(port), database=database ) - conn_args = { - **connection_options - } engine = create_engine( - conn_str, connect_args=conn_args, pool_pre_ping=True + conn_str, connect_args=connect_args, pool_pre_ping=True ) return engine diff --git a/pygeoapi/provider/tile.py b/pygeoapi/provider/tile.py index 7e6352ae3..a57488db1 100644 --- a/pygeoapi/provider/tile.py +++ b/pygeoapi/provider/tile.py @@ -56,7 +56,6 @@ def __init__(self, provider_def): self.mimetype = provider_def['format']['mimetype'] self.options = provider_def.get('options') self.tile_type = None - self.fields = {} def get_layer(self): """ diff --git a/pygeoapi/templates/collections/tiles/index.html b/pygeoapi/templates/collections/tiles/index.html index a9bd56cd9..25e8248d7 100644 --- a/pygeoapi/templates/collections/tiles/index.html +++ b/pygeoapi/templates/collections/tiles/index.html @@ -69,6 +69,7 @@

Tiles

{% block extrafoot %}