From b6f869f665d61b362e4d02962414b04361eba7d3 Mon Sep 17 00:00:00 2001 From: Tyler <31015976+tylanderson@users.noreply.github.com> Date: Mon, 2 Jun 2025 10:37:37 -0400 Subject: [PATCH 1/7] update storage extension to v2.0.0 --- pystac/extensions/ext.py | 24 +- pystac/extensions/storage.py | 475 +++++++++++++----- tests/data-files/storage/collection-naip.json | 63 ++- tests/data-files/storage/item-naip.json | 65 ++- .../test_storage/test_refs_apply.yaml | 332 ++++++++++++ .../test_storage/test_validate_storage.yaml | 330 ++++++++++++ tests/extensions/test_storage.py | 323 +++++------- 7 files changed, 1237 insertions(+), 375 deletions(-) create mode 100644 tests/extensions/cassettes/test_storage/test_refs_apply.yaml diff --git a/pystac/extensions/ext.py b/pystac/extensions/ext.py index 84f60c39d..db487e93c 100644 --- a/pystac/extensions/ext.py +++ b/pystac/extensions/ext.py @@ -31,7 +31,7 @@ from pystac.extensions.sar import SarExtension from pystac.extensions.sat import SatExtension from pystac.extensions.scientific import ScientificExtension -from pystac.extensions.storage import StorageExtension +from pystac.extensions.storage import StorageRefsExtension, StorageSchemesExtension from pystac.extensions.table import TableExtension from pystac.extensions.timestamps import TimestampsExtension from pystac.extensions.version import BaseVersionExtension, VersionExtension @@ -85,7 +85,7 @@ SarExtension.name: SarExtension, SatExtension.name: SatExtension, ScientificExtension.name: ScientificExtension, - StorageExtension.name: StorageExtension, + StorageSchemesExtension.name: StorageSchemesExtension, TableExtension.name: TableExtension, TimestampsExtension.name: TimestampsExtension, VersionExtension.name: VersionExtension, @@ -172,6 +172,10 @@ def render(self) -> dict[str, Render]: def sci(self) -> ScientificExtension[Collection]: return ScientificExtension.ext(self.stac_object) + @property + def storage(self) -> StorageSchemesExtension[Collection]: + return StorageSchemesExtension.ext(self.stac_object) + @property def table(self) -> TableExtension[Collection]: return TableExtension.ext(self.stac_object) @@ -265,8 +269,8 @@ def sci(self) -> ScientificExtension[Item]: return ScientificExtension.ext(self.stac_object) @property - def storage(self) -> StorageExtension[Item]: - return StorageExtension.ext(self.stac_object) + def storage(self) -> StorageSchemesExtension[Item]: + return StorageSchemesExtension.ext(self.stac_object) @property def table(self) -> TableExtension[Item]: @@ -376,8 +380,8 @@ def sat(self) -> SatExtension[U]: return SatExtension.ext(self.stac_object) @property - def storage(self) -> StorageExtension[U]: - return StorageExtension.ext(self.stac_object) + def storage(self) -> StorageRefsExtension[U]: + return StorageRefsExtension.ext(self.stac_object) @property def table(self) -> TableExtension[U]: @@ -432,6 +436,10 @@ class ItemAssetExt(_AssetExt[ItemAssetDefinition]): def mlm(self) -> MLMExtension[ItemAssetDefinition]: return MLMExtension.ext(self.stac_object) + @property + def storage(self) -> StorageRefsExtension[ItemAssetDefinition]: + return StorageRefsExtension.ext(self.stac_object) + @dataclass class LinkExt(_AssetsExt[Link]): @@ -444,3 +452,7 @@ class LinkExt(_AssetsExt[Link]): @property def file(self) -> FileExtension[Link]: return FileExtension.ext(self.stac_object) + + @property + def storage(self) -> StorageRefsExtension[Link]: + return StorageRefsExtension.ext(self.stac_object) diff --git a/pystac/extensions/storage.py b/pystac/extensions/storage.py index 4270a9dc3..3a6a3c523 100644 --- a/pystac/extensions/storage.py +++ b/pystac/extensions/storage.py @@ -5,7 +5,7 @@ from __future__ import annotations -from collections.abc import Iterable +from abc import ABC from typing import ( Any, Generic, @@ -15,134 +15,253 @@ ) import pystac +from pystac.errors import RequiredPropertyMissing from pystac.extensions.base import ( ExtensionManagementMixin, PropertiesExtension, SummariesExtension, ) from pystac.extensions.hooks import ExtensionHooks -from pystac.utils import StringEnum +from pystac.utils import StringEnum, get_required, map_opt #: Generalized version of :class:`~pystac.Item`, :class:`~pystac.Asset` or #: :class:`~pystac.ItemAssetDefinition` -T = TypeVar("T", pystac.Item, pystac.Asset, pystac.ItemAssetDefinition) +T = TypeVar("T", pystac.Item, pystac.Catalog, pystac.Collection) +U = TypeVar("U", pystac.Asset, pystac.Link, pystac.ItemAssetDefinition) -SCHEMA_URI: str = "https://stac-extensions.github.io/storage/v1.0.0/schema.json" +SCHEMA_URI: str = "https://stac-extensions.github.io/storage/v2.0.0/schema.json" PREFIX: str = "storage:" # Field names -PLATFORM_PROP: str = PREFIX + "platform" -REGION_PROP: str = PREFIX + "region" -REQUESTER_PAYS_PROP: str = PREFIX + "requester_pays" -TIER_PROP: str = PREFIX + "tier" +REFS_PROP: str = PREFIX + "refs" +SCHEMES_PROP: str = PREFIX + "schemes" +# Storage scheme object names +TYPE_PROP: str = "type" +PLATFORM_PROP: str = "platform" +REGION_PROP: str = "region" +REQUESTER_PAYS_PROP: str = "requester_pays" -class CloudPlatform(StringEnum): - ALIBABA = "ALIBABA" - AWS = "AWS" - AZURE = "AZURE" - GCP = "GCP" - IBM = "IBM" - ORACLE = "ORACLE" - OTHER = "OTHER" +class StorageSchemeType(StringEnum): + AWS_S3 = "aws-s3" + CUSTOM_S3 = "custom-s3" + AZURE = "ms-azure" -class StorageExtension( - Generic[T], - PropertiesExtension, - ExtensionManagementMixin[pystac.Item | pystac.Collection], -): - """An abstract class that can be used to extend the properties of an - :class:`~pystac.Item` or :class:`~pystac.Asset` with properties from the - :stac-ext:`Storage Extension `. This class is generic over the type of - STAC Object to be extended (e.g. :class:`~pystac.Item`, - :class:`~pystac.Asset`). - To create a concrete instance of :class:`StorageExtension`, use the - :meth:`StorageExtension.ext` method. For example: +class StorageScheme: + properties: dict[str, Any] - .. code-block:: python + def __init__(self, properties: dict[str, Any]): + super().__setattr__("properties", properties) - >>> item: pystac.Item = ... - >>> storage_ext = StorageExtension.ext(item) - """ + def __eq__(self, other: object) -> bool: + if not isinstance(other, StorageScheme): + raise NotImplementedError + return self.properties == other.properties - name: Literal["storage"] = "storage" + def __getattr__(self, name: str) -> Any: + if name in self.properties: + return self.properties[name] + raise AttributeError(f"StorageScheme does not have attribute '{name}'") + + def __setattr__(self, name: str, value: Any) -> None: + self.properties[name] = value def apply( self, - platform: CloudPlatform | None = None, + type: str, + platform: str, region: str | None = None, requester_pays: bool | None = None, - tier: str | None = None, + **kwargs: dict[str, Any], ) -> None: - """Applies Storage Extension properties to the extended :class:`~pystac.Item` or - :class:`~pystac.Asset`. - - Args: - platform (str, CloudPlatform) : The cloud provider where data is stored. - region (str) : The region where the data is stored. Relevant to speed of - access and inter-region egress costs (as defined by PaaS provider). - requester_pays (bool) : Is the data requester pays or is it data - manager/cloud provider pays. - tier (str) : The title for the tier type (as defined by PaaS provider). - """ + self.type = type self.platform = platform self.region = region self.requester_pays = requester_pays - self.tier = tier + self.properties.update(kwargs) - @property - def platform(self) -> CloudPlatform | None: - """Get or sets the cloud provider where data is stored. + @classmethod + def create( + cls, + type: str, + platform: str, + region: str | None = None, + requester_pays: bool | None = None, + **kwargs: dict[str, Any], + ) -> StorageScheme: + """Set the properties for a new StorageScheme object. + + Additional properties can be set through kwargs to fulfill + any additional variables in a templated uri. + + Args: + type (str): Type identifier for the platform. + platform (str): The cloud provider where data is stored as URI or URI + template to the API. + region (str | None, optional): The region where the data is stored. + Defaults to None. + requester_pays (bool | None, optional): requester pays or data manager/cloud + provider pays. Defaults to None. + kwargs (dict[str | Any]): Additional properties to set on scheme Returns: - str or None + StorageScheme: storage scheme """ - return self._get_property(PLATFORM_PROP, CloudPlatform) + c = cls({}) + c.apply( + type=type, + platform=platform, + region=region, + requester_pays=requester_pays, + **kwargs, + ) + return c + + @property + def type(self) -> str: + """ + Get or set the required type property + """ + return get_required( + self.properties.get(TYPE_PROP), + self, + TYPE_PROP, + ) + + @type.setter + def type(self, v: str) -> None: + self.properties[TYPE_PROP] = v + + @property + def platform(self) -> str: + """ + Get or set the required platform property + """ + return get_required( + self.properties.get(PLATFORM_PROP), + self, + PLATFORM_PROP, + ) @platform.setter - def platform(self, v: CloudPlatform | None) -> None: - self._set_property(PLATFORM_PROP, v) + def platform(self, v: str) -> None: + self.properties[PLATFORM_PROP] = v @property def region(self) -> str | None: - """Gets or sets the region where the data is stored. Relevant to speed of - access and inter-region egress costs (as defined by PaaS provider).""" - return self._get_property(REGION_PROP, str) + """ + Get or set the optional region property + """ + return self.properties.get(REGION_PROP) @region.setter - def region(self, v: str | None) -> None: - self._set_property(REGION_PROP, v) + def region(self, v: str) -> None: + if v is not None: + self.properties[REGION_PROP] = v + else: + self.properties.pop(REGION_PROP, None) @property def requester_pays(self) -> bool | None: - # This value "defaults to false", according to the extension spec. - return self._get_property(REQUESTER_PAYS_PROP, bool) + """ + Get or set the optional requester_pays property + """ + return self.properties.get(REQUESTER_PAYS_PROP) @requester_pays.setter - def requester_pays(self, v: bool | None) -> None: - self._set_property(REQUESTER_PAYS_PROP, v) + def requester_pays(self, v: bool) -> None: + if v is not None: + self.properties[REQUESTER_PAYS_PROP] = v + else: + self.properties.pop(REQUESTER_PAYS_PROP, None) + + def to_dict(self) -> dict[str, Any]: + """ + Returns the dictionary encoding of this object + + Returns: + dict[str, Any + """ + return self.properties - @property - def tier(self) -> str | None: - return self._get_property(TIER_PROP, str) - @tier.setter - def tier(self, v: str | None) -> None: - self._set_property(TIER_PROP, v) +class _StorageExtension(ABC): + name: Literal["storage"] = "storage" @classmethod def get_schema_uri(cls) -> str: return SCHEMA_URI + +class StorageSchemesExtension( + _StorageExtension, + Generic[T], + PropertiesExtension, + ExtensionManagementMixin[pystac.Item | pystac.Collection | pystac.Catalog], +): + """An abstract class that can be used to extend the properties of an + :class:`~pystac.Collection`, :class:`~pystac.Catalog`, or :class:`~pystac.Item` + with properties from the :stac-ext:`Storage Extension `. + This class is generic over the type of STAC Object to be extended (e.g. + :class:`~pystac.Item`, :class:`~pystac.Collection`). + + To create a concrete instance of :class:`StorageExtension`, use the + :meth:`StorageExtension.ext` method. For example: + + .. code-block:: python + + >>> item: pystac.Item = ... + >>> storage_ext = StorageExtension.ext(item) + """ + + def apply( + self, + schemes: dict[str, StorageScheme], + ) -> None: + """Applies Storage Extension properties to the extended + :class:`~pystac.Catalog`, :class:`~pystac.Collection`, + or :class:`~pystac.Item`. + + Args: + schemes (dict[str, StorageScheme]): Storage schemes used by Assets and Links + in the STAC Item, Catalog or Collection. + """ + self.schemes = schemes + + @property + def schemes(self) -> dict[str, StorageScheme]: + """Get or sets the schemes used by Assets and Links. + + Returns: + dict[str, StorageScheme]: storage schemes + """ + schemes: dict[str, dict[str, Any]] = get_required( + self.properties.get(SCHEMES_PROP), + self, + SCHEMES_PROP, + ) + return {k: StorageScheme(v) for k, v in schemes.items()} + + @schemes.setter + def schemes(self, v: dict[str, StorageScheme]) -> None: + v_trans = {k: c.to_dict() for k, c in v.items()} + self._set_property(SCHEMES_PROP, v_trans) + + def add_scheme(self, key: str, scheme: StorageScheme) -> None: + try: + self.schemes = {**self.schemes, **{key: scheme}} + except RequiredPropertyMissing: + self.schemes = {key: scheme} + @classmethod - def ext(cls, obj: T, add_if_missing: bool = False) -> StorageExtension[T]: + def ext(cls, obj: T, add_if_missing: bool = False) -> StorageSchemesExtension[T]: """Extends the given STAC Object with properties from the :stac-ext:`Storage Extension `. - This extension can be applied to instances of :class:`~pystac.Item` or - :class:`~pystac.Asset`. + This extension can be applied to instances of :class:`~pystac.Catalog`, + :class:`~pystac.Collection`, or :class:`~pystac.Item`. Raises: @@ -150,13 +269,13 @@ def ext(cls, obj: T, add_if_missing: bool = False) -> StorageExtension[T]: """ if isinstance(obj, pystac.Item): cls.ensure_has_extension(obj, add_if_missing) - return cast(StorageExtension[T], ItemStorageExtension(obj)) - elif isinstance(obj, pystac.Asset): - cls.ensure_owner_has_extension(obj, add_if_missing) - return cast(StorageExtension[T], AssetStorageExtension(obj)) - elif isinstance(obj, pystac.ItemAssetDefinition): - cls.ensure_owner_has_extension(obj, add_if_missing) - return cast(StorageExtension[T], ItemAssetsStorageExtension(obj)) + return cast(StorageSchemesExtension[T], ItemStorageExtension(obj)) + elif isinstance(obj, pystac.Collection): + cls.ensure_has_extension(obj, add_if_missing) + return cast(StorageSchemesExtension[T], CollectionStorageExtension(obj)) + elif isinstance(obj, pystac.Catalog): + cls.ensure_has_extension(obj, add_if_missing) + return cast(StorageSchemesExtension[T], CatalogStorageExtension(obj)) else: raise pystac.ExtensionTypeError(cls._ext_error_message(obj)) @@ -169,13 +288,13 @@ def summaries( return SummariesStorageExtension(obj) -class ItemStorageExtension(StorageExtension[pystac.Item]): - """A concrete implementation of :class:`StorageExtension` on an +class ItemStorageExtension(StorageSchemesExtension[pystac.Item]): + """A concrete implementation of :class:`StorageSchemesExtension` on an :class:`~pystac.Item` that extends the properties of the Item to include properties defined in the :stac-ext:`Storage Extension `. This class should generally not be instantiated directly. Instead, call - :meth:`StorageExtension.ext` on an :class:`~pystac.Item` to extend it. + :meth:`StorageSchemesExtension.ext` on an :class:`~pystac.Item` to extend it. """ item: pystac.Item @@ -192,25 +311,122 @@ def __repr__(self) -> str: return f"" -class AssetStorageExtension(StorageExtension[pystac.Asset]): - """A concrete implementation of :class:`StorageExtension` on an - :class:`~pystac.Asset` that extends the Asset fields to include properties defined - in the :stac-ext:`Storage Extension `. +class CollectionStorageExtension(StorageSchemesExtension[pystac.Collection]): + """A concrete implementation of :class:`StorageSchemesExtension` on an + :class:`~pystac.Collection` that extends the properties of the Collection to include + properties defined in the :stac-ext:`Storage Extension `. This class should generally not be instantiated directly. Instead, call - :meth:`StorageExtension.ext` on an :class:`~pystac.Asset` to extend it. + :meth:`StorageSchemesExtension.ext` on an :class:`~pystac.Collection` to extend it. """ - asset_href: str - """The ``href`` value of the :class:`~pystac.Asset` being extended.""" + collection: pystac.Collection + """The :class:`~pystac.Collection` being extended.""" properties: dict[str, Any] - """The :class:`~pystac.Asset` fields, including extension properties.""" + """The :class:`~pystac.Collection` properties, including extension properties.""" + + def __init__(self, collection: pystac.Collection): + self.collection = collection + self.properties = collection.extra_fields + + def __repr__(self) -> str: + return f"" + + +class CatalogStorageExtension(StorageSchemesExtension[pystac.Catalog]): + """A concrete implementation of :class:`StorageSchemesExtension` on an + :class:`~pystac.Catalog` that extends the properties of the Catalog to include + properties defined in the :stac-ext:`Storage Extension `. + + This class should generally not be instantiated directly. Instead, call + :meth:`StorageSchemesExtension.ext` on an :class:`~pystac.Catalog` to extend it. + """ + + catalog: pystac.Catalog + """The :class:`~pystac.Catalog` being extended.""" + + properties: dict[str, Any] + """The :class:`~pystac.Catalog` properties, including extension properties.""" + + def __init__(self, catalog: pystac.Catalog): + self.catalog = catalog + self.properties = catalog.extra_fields + + def __repr__(self) -> str: + return f"" + + +class StorageRefsExtension( + _StorageExtension, + Generic[U], + PropertiesExtension, + ExtensionManagementMixin[pystac.Item | pystac.Collection | pystac.Catalog], +): + def apply( + self, + refs: list[str], + ) -> None: + """Applies Storage Extension properties to the extended :class:`~pystac.Asset`, + :class:`~pystac.Link`, or :class:`~pystac.ItemAssetDefinition`. - additional_read_properties: Iterable[dict[str, Any]] | None = None - """If present, this will be a list containing 1 dictionary representing the - properties of the owning :class:`~pystac.Item`.""" + Args: + refs (list[str]): specifies which schemes in storage:schemes may be used to + access an Asset or Link. Each value must be one of the keys defined in + storage:schemes. + """ + self.refs = refs + + @property + def refs(self) -> list[str]: + """Get or sets the keys of the schemes that may be used to access an Asset + or Link. + + Returns: + list[str] + """ + return get_required( + self.properties.get(REFS_PROP), + self, + REFS_PROP, + ) + + @refs.setter + def refs(self, v: list[str]) -> None: + self._set_property(REFS_PROP, v) + + def add_ref(self, ref: str) -> None: + try: + self.refs.append(ref) + except RequiredPropertyMissing: + self.refs = [ref] + + @classmethod + def ext(cls, obj: U, add_if_missing: bool = False) -> StorageRefsExtension[U]: + """Extends the given STAC Object with properties from the :stac-ext:`Storage + Extension `. + This extension can be applied to instances of :class:`~pystac.Item` or + :class:`~pystac.Asset`. + + Raises: + + pystac.ExtensionTypeError : If an invalid object type is passed. + """ + if isinstance(obj, pystac.Asset): + cls.ensure_owner_has_extension(obj, add_if_missing) + return AssetStorageExtension(obj) + if isinstance(obj, pystac.Link): + cls.ensure_owner_has_extension(obj, add_if_missing) + return LinkStorageExtension(obj) + if isinstance(obj, pystac.ItemAssetDefinition): + cls.ensure_owner_has_extension(obj, add_if_missing) + return ItemAssetsStorageExtension(obj) + else: + raise pystac.ExtensionTypeError(cls._ext_error_message(obj)) + + +class AssetStorageExtension(StorageRefsExtension[pystac.Asset]): def __init__(self, asset: pystac.Asset): self.asset_href = asset.href self.properties = asset.extra_fields @@ -221,7 +437,7 @@ def __repr__(self) -> str: return f"" -class ItemAssetsStorageExtension(StorageExtension[pystac.ItemAssetDefinition]): +class ItemAssetsStorageExtension(StorageRefsExtension[pystac.ItemAssetDefinition]): properties: dict[str, Any] asset_defn: pystac.ItemAssetDefinition @@ -230,6 +446,15 @@ def __init__(self, item_asset: pystac.ItemAssetDefinition): self.properties = item_asset.properties +class LinkStorageExtension(StorageRefsExtension[pystac.Link]): + properties: dict[str, Any] + link: pystac.Link + + def __init__(self, link: pystac.Link): + self.link = link + self.properties = link.extra_fields + + class SummariesStorageExtension(SummariesExtension): """A concrete implementation of :class:`~pystac.extensions.base.SummariesExtension` that extends the ``summaries`` field of a :class:`~pystac.Collection` to include @@ -237,54 +462,38 @@ class SummariesStorageExtension(SummariesExtension): """ @property - def platform(self) -> list[CloudPlatform] | None: + def schemes(self) -> list[dict[str, StorageScheme]] | None: """Get or sets the summary of :attr:`StorageExtension.platform` values for this Collection. """ - return self.summaries.get_list(PLATFORM_PROP) - - @platform.setter - def platform(self, v: list[CloudPlatform] | None) -> None: - self._set_summary(PLATFORM_PROP, v) - - @property - def region(self) -> list[str] | None: - """Get or sets the summary of :attr:`StorageExtension.region` values - for this Collection. - """ - return self.summaries.get_list(REGION_PROP) - - @region.setter - def region(self, v: list[str] | None) -> None: - self._set_summary(REGION_PROP, v) - - @property - def requester_pays(self) -> list[bool] | None: - """Get or sets the summary of :attr:`StorageExtension.requester_pays` values - for this Collection. - """ - return self.summaries.get_list(REQUESTER_PAYS_PROP) - - @requester_pays.setter - def requester_pays(self, v: list[bool] | None) -> None: - self._set_summary(REQUESTER_PAYS_PROP, v) - - @property - def tier(self) -> list[str] | None: - """Get or sets the summary of :attr:`StorageExtension.tier` values - for this Collection. - """ - return self.summaries.get_list(TIER_PROP) - - @tier.setter - def tier(self, v: list[str] | None) -> None: - self._set_summary(TIER_PROP, v) + v = map_opt( + lambda schemes: [ + {k: StorageScheme(v) for k, v in x.items()} for x in schemes + ], + self.summaries.get_list(SCHEMES_PROP), + ) + + print(v) + return v + + @schemes.setter + def schemes(self, v: list[dict[str, StorageScheme]] | None) -> None: + self._set_summary( + SCHEMES_PROP, + map_opt( + lambda schemes: [ + {k: c.to_dict() for k, c in x.items()} for x in schemes + ], + v, + ), + ) class StorageExtensionHooks(ExtensionHooks): schema_uri: str = SCHEMA_URI prev_extension_ids: set[str] = set() stac_object_types = { + pystac.STACObjectType.CATALOG, pystac.STACObjectType.COLLECTION, pystac.STACObjectType.ITEM, } diff --git a/tests/data-files/storage/collection-naip.json b/tests/data-files/storage/collection-naip.json index c24f38276..c3afac3da 100644 --- a/tests/data-files/storage/collection-naip.json +++ b/tests/data-files/storage/collection-naip.json @@ -16,29 +16,50 @@ } ], "stac_extensions": [ - "https://stac-extensions.github.io/storage/v1.0.0/schema.json" + "https://stac-extensions.github.io/storage/v2.0.0/schema.json" ], "summaries": { - "storage:platform": [ - "AZURE", - "GCP", - "AWS" - ], - "storage:region": [ - "westus2", - "us-central1", - "us-west-2", - "eastus" - ], - "storage:requester_pays": [ - true, - false - ], - "storage:tier": [ - "archive", - "COLDLINE", - "Standard", - "hot" + "storage:schemes": [ + { + "naip-azure-nsl": { + "type": "ms-azure", + "platform": "https://{account}.blob.core.windows.net", + "account": "naip-nsl" + } + }, + { + "naip-azure-eu": { + "type": "ms-azure", + "platform": "https://{account}.blob.core.windows.net", + "account": "naipeuwest", + "requester_pays": false + } + }, + { + "naip-azure-blobs": { + "type": "ms-azure", + "platform": "https://{account}.blob.core.windows.net", + "account": "naipblobs", + "region": "eastus" + } + }, + { + "naip-gcs": { + "type": "gcp", + "platform": "https://storage.googleapis.com/{bucket}", + "bucket": "naip-nsl", + "requester_pays": true + } + }, + { + "naip-aws": { + "type": "gcp", + "platform": "https://{bucket}.s3.{region}.amazonaws.com", + "bucket": "naip-visualization", + "region": "us-west-2", + "requester_pays": true + } + } ] }, "extent": { diff --git a/tests/data-files/storage/item-naip.json b/tests/data-files/storage/item-naip.json index 22069111d..59ebefb14 100644 --- a/tests/data-files/storage/item-naip.json +++ b/tests/data-files/storage/item-naip.json @@ -1,7 +1,7 @@ { "stac_version": "1.1.0", "stac_extensions": [ - "https://stac-extensions.github.io/storage/v1.0.0/schema.json" + "https://stac-extensions.github.io/storage/v2.0.0/schema.json" ], "id": "m_3009743_sw_14_1_20160928_20161129", "collection": "NAIP_MOSAIC", @@ -43,55 +43,70 @@ "datetime": "2016-09-28T00:00:00+00:00", "mission": "NAIP", "platform": "UNKNOWN_PLATFORM", - "gsd": 1 + "gsd": 1, + "storage:schemes": { + "naip-azure-nsl": { + "type": "ms-azure", + "platform": "https://{account}.blob.core.windows.net", + "account": "naip-nsl" + }, + "naip-azure-eu": { + "type": "ms-azure", + "platform": "https://{account}.blob.core.windows.net", + "account": "naipeuwest", + "requester_pays": false + }, + "naip-azure-blobs": { + "type": "ms-azure", + "platform": "https://{account}.blob.core.windows.net", + "account": "naipblobs", + "region": "eastus" + }, + "naip-gcs": { + "type": "gcp", + "platform": "https://storage.googleapis.com/{bucket}", + "bucket": "naip-nsl", + "requester_pays": true + }, + "naip-aws": { + "type": "gcp", + "platform": "https://{bucket}.s3.{region}.amazonaws.com", + "bucket": "naip-visualization", + "region": "us-west-2", + "requester_pays": true + } + } }, "assets": { "GEOTIFF_AZURE_RGBIR": { "href": "https://naip-nsl.blob.core.windows.net/tx/2016/100cm/rgb/30097/m_3009743_sw_14_1_20160928.tif", "type": "image/vnd.stac.geotiff", - "storage:platform": "AZURE", - "storage:region": "westus2", - "storage:tier": "archive" + "storage:refs": ["naip-azure-nsl"] }, "CO_GEOTIFF_GCP_RGB": { "href": "gs://naip-data/tx/2016/100cm/rgb/30097/m_3009743_sw_14_1_20160928.tif", "type": "image/vnd.stac.geotiff; cloud-optimized=true", - "storage:platform": "GCP", - "storage:region": "us-central1", - "storage:requester_pays": true, - "storage:tier": "COLDLINE" + "storage:refs": ["naip-gcs"] }, "CO_GEOTIFF_AWS_RGB": { "href": "s3://naip-visualization/tx/2016/100cm/rgb/30097/m_3009743_sw_14_1_20160928.tif", "type": "image/vnd.stac.geotiff; cloud-optimized=true", - "storage:platform": "AWS", - "storage:region": "us-west-2", - "storage:requester_pays": true, - "storage:tier": "Standard" + "storage:refs": ["naip-aws"] }, "CO_GEOTIFF_AZURE_RGB": { "href": "https://naipeuwest.blob.core.windows.net/naip/v002/tx/2016/tx_100cm_2016/30097/m_3009743_sw_14_1_20160928.tif", "type": "image/vnd.stac.geotiff; cloud-optimized=true", - "storage:platform": "AZURE", - "storage:region": "westeurope", - "storage:requester_pays": false, - "storage:tier": "hot" + "storage:refs": ["naip-azure-eu"] }, "CO_GEOTIFF_AZURE_RGB_DEPRECATED": { "href": "https://naipblobs.blob.core.windows.net/naip/v002/tx/2016/tx_100cm_2016/30097/m_3009743_sw_14_1_20160928.tif", "type": "image/vnd.stac.geotiff; cloud-optimized=true", - "storage:platform": "AZURE", - "storage:region": "eastus", - "storage:requester_pays": false, - "storage:tier": "hot" + "storage:refs": ["naip-azure-blobs"] }, "THUMBNAIL_AZURE_DEPRECATED": { "href": "https://naipblobs.blob.core.windows.net/naip/v002/tx/2016/tx_100cm_2016/30097/m_3009743_sw_14_1_20160928.200.jpg", "type": "image/jpeg", - "storage:platform": "AZURE", - "storage:region": "eastus", - "storage:requester_pays": false, - "storage:tier": "hot" + "storage:refs": ["naip-azure-blobs"] } }, "links": [ diff --git a/tests/extensions/cassettes/test_storage/test_refs_apply.yaml b/tests/extensions/cassettes/test_storage/test_refs_apply.yaml new file mode 100644 index 000000000..e5afbfe61 --- /dev/null +++ b/tests/extensions/cassettes/test_storage/test_refs_apply.yaml @@ -0,0 +1,332 @@ +interactions: +- request: + body: null + headers: + Connection: + - close + Host: + - stac-extensions.github.io + User-Agent: + - Python-urllib/3.10 + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/schema.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/schema.json\",\n \"title\": + \"STAC Storage Extension\",\n \"type\": \"object\",\n \"required\": [\n + \ \"stac_extensions\"\n ],\n \"properties\": {\n \"stac_extensions\": + {\n \"type\": \"array\",\n \"contains\": {\n \"const\": \"https://stac-extensions.github.io/storage/v2.0.0/schema.json\"\n + \ }\n }\n },\n \"oneOf\": [\n {\n \"$comment\": \"This is + the schema for STAC Items.\",\n \"type\": \"object\",\n \"required\": + [\n \"type\",\n \"properties\"\n ],\n \"properties\": + {\n \"type\": {\n \"const\": \"Feature\"\n },\n \"properties\": + {\n \"$ref\": \"#/definitions/schemes_field\"\n },\n \"assets\": + {\n \"$ref\": \"#/definitions/assets\"\n },\n \"links\": + {\n \"$ref\": \"#/definitions/links\"\n }\n }\n },\n + \ {\n \"$comment\": \"This is the schema for STAC Collections\",\n + \ \"type\": \"object\",\n \"required\": [\n \"type\"\n ],\n + \ \"properties\": {\n \"type\": {\n \"const\": \"Collection\"\n + \ },\n \"assets\": {\n \"$ref\": \"#/definitions/assets\"\n + \ },\n \"item_assets\": {\n \"$ref\": \"#/definitions/assets\"\n + \ },\n \"links\": {\n \"$ref\": \"#/definitions/links\"\n + \ }\n },\n \"allOf\": [\n {\n \"$ref\": \"#/definitions/schemes_field\"\n + \ }\n ]\n },\n {\n \"$comment\": \"This is the schema + for STAC Catalogs\",\n \"type\": \"object\",\n \"required\": [\n + \ \"type\"\n ],\n \"properties\": {\n \"type\": {\n + \ \"const\": \"Catalog\"\n },\n \"links\": {\n \"$ref\": + \"#/definitions/links\"\n }\n },\n \"allOf\": [\n {\n + \ \"$ref\": \"#/definitions/schemes_field\"\n }\n ]\n + \ }\n ], \n \"definitions\": {\n \"schemes_field\": {\n \"type\": + \"object\",\n \"required\": [\n \"storage:schemes\"\n ],\n + \ \"properties\": {\n \"storage:schemes\": {\n \"type\": + \"object\",\n \"patternProperties\": {\n \"^.{1,}$\": + {\n \"required\": [\n \"type\",\n \"platform\"\n + \ ],\n \"properties\": {\n \"type\": + {\n \"title\": \"Type identifier\",\n \"type\": + \"string\"\n },\n \"platform\": {\n \"title\": + \"Platform\",\n \"type\": \"string\",\n \"format\": + \"uri-template\",\n \"pattern\": \"^[\\\\w\\\\+.-]+://\"\n + \ },\n \"region\": {\n \"title\": + \"Region\",\n \"type\": \"string\"\n },\n + \ \"requester_pays\": {\n \"type\": \"boolean\",\n + \ \"title\": \"Requester pays\",\n \"default\": + false\n }\n },\n \"allOf\": [\n {\n + \ \"$ref\": \"./platforms/aws-s3.json\"\n },\n + \ {\n \"$ref\": \"./platforms/custom-s3.json\"\n + \ },\n {\n \"$ref\": \"./platforms/ms-azure.json\"\n + \ }\n ],\n \"additionalProperties\": + true\n }\n },\n \"additionalProperties\": false\n + \ }\n },\n \"patternProperties\": {\n \"^(?!storage:)\": + {}\n },\n \"additionalProperties\": false\n },\n \"refs_field\": + {\n \"type\": \"object\",\n \"properties\": {\n \"storage:refs\": + {\n \"type\": \"array\",\n \"items\": {\n \"type\": + \"string\",\n \"minLength\": 1\n }\n }\n },\n + \ \"patternProperties\": {\n \"^(?!storage:)\": {}\n },\n + \ \"additionalProperties\": false\n },\n \"assets\": {\n \"type\": + \"object\",\n \"additionalProperties\": {\n \"allOf\": [\n {\n + \ \"$ref\": \"#/definitions/refs_field\"\n },\n {\n + \ \"type\": \"object\",\n \"properties\": {\n \"alternate\": + {\n \"$ref\": \"#/definitions/refs_field\"\n }\n + \ }\n }\n ]\n }\n },\n \"links\": {\n + \ \"type\": \"array\",\n \"items\": {\n \"$ref\": \"#/definitions/refs_field\"\n + \ }\n }\n }\n}\n" + headers: + Accept-Ranges: + - bytes + Access-Control-Allow-Origin: + - '*' + Age: + - '0' + Cache-Control: + - max-age=600 + Connection: + - close + Content-Length: + - '4259' + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 01 Jun 2025 22:16:48 GMT + ETag: + - '"6718ce4f-10a3"' + Last-Modified: + - Wed, 23 Oct 2024 10:22:07 GMT + Server: + - GitHub.com + Strict-Transport-Security: + - max-age=31556952 + Vary: + - Accept-Encoding + Via: + - 1.1 varnish + X-Cache: + - HIT + X-Cache-Hits: + - '1' + X-Fastly-Request-ID: + - af64a991591f50737fadd2e89371cc12c8a2bb7d + X-GitHub-Request-Id: + - 591F:3B4DB0:4904A90:4C8204C:683CAF52 + X-Served-By: + - cache-iad-kcgs7200048-IAD + X-Timer: + - S1748816209.717952,VS0,VE1 + expires: + - Sun, 01 Jun 2025 20:01:47 GMT + x-proxy-cache: + - MISS + status: + code: 200 + message: OK +- request: + body: null + headers: + Connection: + - close + Host: + - stac-extensions.github.io + User-Agent: + - Python-urllib/3.10 + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/aws-s3.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/platforms/aws-s3.json\",\n + \ \"title\": \"AWS S3\",\n \"type\": \"object\",\n \"if\": {\n \"properties\": + {\n \"type\": {\n \"const\": \"aws-s3\"\n }\n }\n },\n + \ \"then\": {\n \"properties\": {\n \"platform\": {\n \"const\": + \"https://{bucket}.s3.{region}.amazonaws.com\"\n },\n \"bucket\": + {\n \"$comment\": \"See https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html\",\n + \ \"type\": \"string\",\n \"pattern\": \"^[a-z0-9][a-z0-9-.]{1,61}[a-z0-9]$\"\n + \ },\n \"region\": {\n \"type\": \"string\",\n \"pattern\": + \"^[a-z0-9-]+$\"\n }\n }\n }\n}" + headers: + Accept-Ranges: + - bytes + Access-Control-Allow-Origin: + - '*' + Age: + - '0' + Cache-Control: + - max-age=600 + Connection: + - close + Content-Length: + - '706' + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 01 Jun 2025 22:16:48 GMT + ETag: + - '"6718ce4f-2c2"' + Last-Modified: + - Wed, 23 Oct 2024 10:22:07 GMT + Server: + - GitHub.com + Strict-Transport-Security: + - max-age=31556952 + Vary: + - Accept-Encoding + Via: + - 1.1 varnish + X-Cache: + - HIT + X-Cache-Hits: + - '1' + X-Fastly-Request-ID: + - 43d2b7092055b40e12b92793975aae512e94642f + X-GitHub-Request-Id: + - 87AB:3BCE73:4C312A6:4FAEA0D:683CAF53 + X-Served-By: + - cache-iad-kiad7000035-IAD + X-Timer: + - S1748816209.780679,VS0,VE3 + expires: + - Sun, 01 Jun 2025 20:01:47 GMT + x-proxy-cache: + - MISS + status: + code: 200 + message: OK +- request: + body: null + headers: + Connection: + - close + Host: + - stac-extensions.github.io + User-Agent: + - Python-urllib/3.10 + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/custom-s3.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/platforms/custom-s3.json\",\n + \ \"title\": \"Generic S3\",\n \"type\": \"object\",\n \"if\": {\n \"properties\": + {\n \"type\": {\n \"const\": \"custom-s3\"\n }\n }\n },\n + \ \"then\": {\n \"$comment\": \"No specific validation rules apply\"\n + \ }\n}" + headers: + Accept-Ranges: + - bytes + Access-Control-Allow-Origin: + - '*' + Age: + - '0' + Cache-Control: + - max-age=600 + Connection: + - close + Content-Length: + - '353' + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 01 Jun 2025 22:16:48 GMT + ETag: + - '"6718ce4f-161"' + Last-Modified: + - Wed, 23 Oct 2024 10:22:07 GMT + Server: + - GitHub.com + Strict-Transport-Security: + - max-age=31556952 + Vary: + - Accept-Encoding + Via: + - 1.1 varnish + X-Cache: + - HIT + X-Cache-Hits: + - '1' + X-Fastly-Request-ID: + - c0e4c57aae853bd12f2127008255fb0d13a187fd + X-GitHub-Request-Id: + - 3461:2BF8CE:4BDFA81:4F5D0D4:683CAF53 + X-Served-By: + - cache-iad-kiad7000157-IAD + X-Timer: + - S1748816209.846702,VS0,VE4 + expires: + - Sun, 01 Jun 2025 20:01:47 GMT + x-origin-cache: + - HIT + x-proxy-cache: + - MISS + status: + code: 200 + message: OK +- request: + body: null + headers: + Connection: + - close + Host: + - stac-extensions.github.io + User-Agent: + - Python-urllib/3.10 + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/ms-azure.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/platforms/ms-azure.json\",\n + \ \"title\": \"Microsoft Azure\",\n \"type\": \"object\",\n \"if\": {\n + \ \"properties\": {\n \"type\": {\n \"const\": \"ms-azure\"\n + \ }\n }\n },\n \"then\": {\n \"properties\": {\n \"platform\": + {\n \"const\": \"https://{account}.blob.core.windows.net\"\n },\n + \ \"account\": {\n \"type\": \"string\"\n }\n }\n }\n}" + headers: + Accept-Ranges: + - bytes + Access-Control-Allow-Origin: + - '*' + Age: + - '0' + Cache-Control: + - max-age=600 + Connection: + - close + Content-Length: + - '469' + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 01 Jun 2025 22:16:48 GMT + ETag: + - '"6718ce4f-1d5"' + Last-Modified: + - Wed, 23 Oct 2024 10:22:07 GMT + Server: + - GitHub.com + Strict-Transport-Security: + - max-age=31556952 + Vary: + - Accept-Encoding + Via: + - 1.1 varnish + X-Cache: + - HIT + X-Cache-Hits: + - '1' + X-Fastly-Request-ID: + - ea7c52a0de24dedf14c95d765216ad65b6bfa746 + X-GitHub-Request-Id: + - D40E:3B1629:4B5EDFA:4EDC767:683CAF53 + X-Served-By: + - cache-iad-kiad7000135-IAD + X-Timer: + - S1748816209.915699,VS0,VE2 + expires: + - Sun, 01 Jun 2025 20:01:47 GMT + x-origin-cache: + - HIT + x-proxy-cache: + - MISS + status: + code: 200 + message: OK +version: 1 diff --git a/tests/extensions/cassettes/test_storage/test_validate_storage.yaml b/tests/extensions/cassettes/test_storage/test_validate_storage.yaml index 4c3b65fb3..57ddd7c13 100644 --- a/tests/extensions/cassettes/test_storage/test_validate_storage.yaml +++ b/tests/extensions/cassettes/test_storage/test_validate_storage.yaml @@ -101,4 +101,334 @@ interactions: status: code: 200 message: OK +- request: + body: null + headers: + Connection: + - close + Host: + - stac-extensions.github.io + User-Agent: + - Python-urllib/3.10 + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/schema.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/schema.json\",\n \"title\": + \"STAC Storage Extension\",\n \"type\": \"object\",\n \"required\": [\n + \ \"stac_extensions\"\n ],\n \"properties\": {\n \"stac_extensions\": + {\n \"type\": \"array\",\n \"contains\": {\n \"const\": \"https://stac-extensions.github.io/storage/v2.0.0/schema.json\"\n + \ }\n }\n },\n \"oneOf\": [\n {\n \"$comment\": \"This is + the schema for STAC Items.\",\n \"type\": \"object\",\n \"required\": + [\n \"type\",\n \"properties\"\n ],\n \"properties\": + {\n \"type\": {\n \"const\": \"Feature\"\n },\n \"properties\": + {\n \"$ref\": \"#/definitions/schemes_field\"\n },\n \"assets\": + {\n \"$ref\": \"#/definitions/assets\"\n },\n \"links\": + {\n \"$ref\": \"#/definitions/links\"\n }\n }\n },\n + \ {\n \"$comment\": \"This is the schema for STAC Collections\",\n + \ \"type\": \"object\",\n \"required\": [\n \"type\"\n ],\n + \ \"properties\": {\n \"type\": {\n \"const\": \"Collection\"\n + \ },\n \"assets\": {\n \"$ref\": \"#/definitions/assets\"\n + \ },\n \"item_assets\": {\n \"$ref\": \"#/definitions/assets\"\n + \ },\n \"links\": {\n \"$ref\": \"#/definitions/links\"\n + \ }\n },\n \"allOf\": [\n {\n \"$ref\": \"#/definitions/schemes_field\"\n + \ }\n ]\n },\n {\n \"$comment\": \"This is the schema + for STAC Catalogs\",\n \"type\": \"object\",\n \"required\": [\n + \ \"type\"\n ],\n \"properties\": {\n \"type\": {\n + \ \"const\": \"Catalog\"\n },\n \"links\": {\n \"$ref\": + \"#/definitions/links\"\n }\n },\n \"allOf\": [\n {\n + \ \"$ref\": \"#/definitions/schemes_field\"\n }\n ]\n + \ }\n ], \n \"definitions\": {\n \"schemes_field\": {\n \"type\": + \"object\",\n \"required\": [\n \"storage:schemes\"\n ],\n + \ \"properties\": {\n \"storage:schemes\": {\n \"type\": + \"object\",\n \"patternProperties\": {\n \"^.{1,}$\": + {\n \"required\": [\n \"type\",\n \"platform\"\n + \ ],\n \"properties\": {\n \"type\": + {\n \"title\": \"Type identifier\",\n \"type\": + \"string\"\n },\n \"platform\": {\n \"title\": + \"Platform\",\n \"type\": \"string\",\n \"format\": + \"uri-template\",\n \"pattern\": \"^[\\\\w\\\\+.-]+://\"\n + \ },\n \"region\": {\n \"title\": + \"Region\",\n \"type\": \"string\"\n },\n + \ \"requester_pays\": {\n \"type\": \"boolean\",\n + \ \"title\": \"Requester pays\",\n \"default\": + false\n }\n },\n \"allOf\": [\n {\n + \ \"$ref\": \"./platforms/aws-s3.json\"\n },\n + \ {\n \"$ref\": \"./platforms/custom-s3.json\"\n + \ },\n {\n \"$ref\": \"./platforms/ms-azure.json\"\n + \ }\n ],\n \"additionalProperties\": + true\n }\n },\n \"additionalProperties\": false\n + \ }\n },\n \"patternProperties\": {\n \"^(?!storage:)\": + {}\n },\n \"additionalProperties\": false\n },\n \"refs_field\": + {\n \"type\": \"object\",\n \"properties\": {\n \"storage:refs\": + {\n \"type\": \"array\",\n \"items\": {\n \"type\": + \"string\",\n \"minLength\": 1\n }\n }\n },\n + \ \"patternProperties\": {\n \"^(?!storage:)\": {}\n },\n + \ \"additionalProperties\": false\n },\n \"assets\": {\n \"type\": + \"object\",\n \"additionalProperties\": {\n \"allOf\": [\n {\n + \ \"$ref\": \"#/definitions/refs_field\"\n },\n {\n + \ \"type\": \"object\",\n \"properties\": {\n \"alternate\": + {\n \"$ref\": \"#/definitions/refs_field\"\n }\n + \ }\n }\n ]\n }\n },\n \"links\": {\n + \ \"type\": \"array\",\n \"items\": {\n \"$ref\": \"#/definitions/refs_field\"\n + \ }\n }\n }\n}\n" + headers: + Accept-Ranges: + - bytes + Access-Control-Allow-Origin: + - '*' + Age: + - '0' + Cache-Control: + - max-age=600 + Connection: + - close + Content-Length: + - '4259' + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 01 Jun 2025 22:16:48 GMT + ETag: + - '"6718ce4f-10a3"' + Last-Modified: + - Wed, 23 Oct 2024 10:22:07 GMT + Server: + - GitHub.com + Strict-Transport-Security: + - max-age=31556952 + Vary: + - Accept-Encoding + Via: + - 1.1 varnish + X-Cache: + - HIT + X-Cache-Hits: + - '0' + X-Fastly-Request-ID: + - 855a50e2b2d2411c04eaca800019a8413b70b9a8 + X-GitHub-Request-Id: + - 591F:3B4DB0:4904A90:4C8204C:683CAF52 + X-Served-By: + - cache-iad-kiad7000152-IAD + X-Timer: + - S1748816208.373228,VS0,VE14 + expires: + - Sun, 01 Jun 2025 20:01:47 GMT + x-proxy-cache: + - MISS + status: + code: 200 + message: OK +- request: + body: null + headers: + Connection: + - close + Host: + - stac-extensions.github.io + User-Agent: + - Python-urllib/3.10 + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/aws-s3.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/platforms/aws-s3.json\",\n + \ \"title\": \"AWS S3\",\n \"type\": \"object\",\n \"if\": {\n \"properties\": + {\n \"type\": {\n \"const\": \"aws-s3\"\n }\n }\n },\n + \ \"then\": {\n \"properties\": {\n \"platform\": {\n \"const\": + \"https://{bucket}.s3.{region}.amazonaws.com\"\n },\n \"bucket\": + {\n \"$comment\": \"See https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html\",\n + \ \"type\": \"string\",\n \"pattern\": \"^[a-z0-9][a-z0-9-.]{1,61}[a-z0-9]$\"\n + \ },\n \"region\": {\n \"type\": \"string\",\n \"pattern\": + \"^[a-z0-9-]+$\"\n }\n }\n }\n}" + headers: + Accept-Ranges: + - bytes + Access-Control-Allow-Origin: + - '*' + Age: + - '0' + Cache-Control: + - max-age=600 + Connection: + - close + Content-Length: + - '706' + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 01 Jun 2025 22:16:48 GMT + ETag: + - '"6718ce4f-2c2"' + Last-Modified: + - Wed, 23 Oct 2024 10:22:07 GMT + Server: + - GitHub.com + Strict-Transport-Security: + - max-age=31556952 + Vary: + - Accept-Encoding + Via: + - 1.1 varnish + X-Cache: + - HIT + X-Cache-Hits: + - '0' + X-Fastly-Request-ID: + - 278480db61c132221d05f66f4d67dd50f9669b9a + X-GitHub-Request-Id: + - 87AB:3BCE73:4C312A6:4FAEA0D:683CAF53 + X-Served-By: + - cache-iad-kiad7000064-IAD + X-Timer: + - S1748816208.451944,VS0,VE18 + expires: + - Sun, 01 Jun 2025 20:01:47 GMT + x-proxy-cache: + - MISS + status: + code: 200 + message: OK +- request: + body: null + headers: + Connection: + - close + Host: + - stac-extensions.github.io + User-Agent: + - Python-urllib/3.10 + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/custom-s3.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/platforms/custom-s3.json\",\n + \ \"title\": \"Generic S3\",\n \"type\": \"object\",\n \"if\": {\n \"properties\": + {\n \"type\": {\n \"const\": \"custom-s3\"\n }\n }\n },\n + \ \"then\": {\n \"$comment\": \"No specific validation rules apply\"\n + \ }\n}" + headers: + Accept-Ranges: + - bytes + Access-Control-Allow-Origin: + - '*' + Age: + - '0' + Cache-Control: + - max-age=600 + Connection: + - close + Content-Length: + - '353' + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 01 Jun 2025 22:16:48 GMT + ETag: + - '"6718ce4f-161"' + Last-Modified: + - Wed, 23 Oct 2024 10:22:07 GMT + Server: + - GitHub.com + Strict-Transport-Security: + - max-age=31556952 + Vary: + - Accept-Encoding + Via: + - 1.1 varnish + X-Cache: + - HIT + X-Cache-Hits: + - '0' + X-Fastly-Request-ID: + - 17cf88621af9473909a54f09b7280d23a8d03f5b + X-GitHub-Request-Id: + - 3461:2BF8CE:4BDFA81:4F5D0D4:683CAF53 + X-Served-By: + - cache-iad-kiad7000129-IAD + X-Timer: + - S1748816209.531423,VS0,VE21 + expires: + - Sun, 01 Jun 2025 20:01:47 GMT + x-origin-cache: + - HIT + x-proxy-cache: + - MISS + status: + code: 200 + message: OK +- request: + body: null + headers: + Connection: + - close + Host: + - stac-extensions.github.io + User-Agent: + - Python-urllib/3.10 + method: GET + uri: https://stac-extensions.github.io/storage/v2.0.0/platforms/ms-azure.json + response: + body: + string: "{\n \"$schema\": \"http://json-schema.org/draft-07/schema#\",\n \"$id\": + \"https://stac-extensions.github.io/storage/v2.0.0/platforms/ms-azure.json\",\n + \ \"title\": \"Microsoft Azure\",\n \"type\": \"object\",\n \"if\": {\n + \ \"properties\": {\n \"type\": {\n \"const\": \"ms-azure\"\n + \ }\n }\n },\n \"then\": {\n \"properties\": {\n \"platform\": + {\n \"const\": \"https://{account}.blob.core.windows.net\"\n },\n + \ \"account\": {\n \"type\": \"string\"\n }\n }\n }\n}" + headers: + Accept-Ranges: + - bytes + Access-Control-Allow-Origin: + - '*' + Age: + - '0' + Cache-Control: + - max-age=600 + Connection: + - close + Content-Length: + - '469' + Content-Type: + - application/json; charset=utf-8 + Date: + - Sun, 01 Jun 2025 22:16:48 GMT + ETag: + - '"6718ce4f-1d5"' + Last-Modified: + - Wed, 23 Oct 2024 10:22:07 GMT + Server: + - GitHub.com + Strict-Transport-Security: + - max-age=31556952 + Vary: + - Accept-Encoding + Via: + - 1.1 varnish + X-Cache: + - HIT + X-Cache-Hits: + - '0' + X-Fastly-Request-ID: + - ff0d43db7da3291d7786a451b605ce5d2e96e1c4 + X-GitHub-Request-Id: + - D40E:3B1629:4B5EDFA:4EDC767:683CAF53 + X-Served-By: + - cache-iad-kiad7000060-IAD + X-Timer: + - S1748816209.609643,VS0,VE15 + expires: + - Sun, 01 Jun 2025 20:01:47 GMT + x-origin-cache: + - HIT + x-proxy-cache: + - MISS + status: + code: 200 + message: OK version: 1 diff --git a/tests/extensions/test_storage.py b/tests/extensions/test_storage.py index 4c1ba2322..e843ca0b9 100644 --- a/tests/extensions/test_storage.py +++ b/tests/extensions/test_storage.py @@ -7,7 +7,12 @@ import pystac from pystac import ExtensionTypeError, Item from pystac.collection import Collection -from pystac.extensions.storage import CloudPlatform, StorageExtension +from pystac.extensions.storage import ( + StorageRefsExtension, + StorageScheme, + StorageSchemesExtension, + StorageSchemeType, +) from tests.utils import TestCases, assert_to_from_dict NAIP_EXAMPLE_URI = TestCases.get_path("data-files/storage/item-naip.json") @@ -31,20 +36,20 @@ def test_to_from_dict() -> None: def test_add_to(sample_item: Item) -> None: - assert StorageExtension.get_schema_uri() not in sample_item.stac_extensions + assert StorageSchemesExtension.get_schema_uri() not in sample_item.stac_extensions # Check that the URI gets added to stac_extensions - StorageExtension.add_to(sample_item) - assert StorageExtension.get_schema_uri() in sample_item.stac_extensions + StorageSchemesExtension.add_to(sample_item) + assert StorageSchemesExtension.get_schema_uri() in sample_item.stac_extensions # Check that the URI only gets added once, regardless of how many times add_to # is called. - StorageExtension.add_to(sample_item) - StorageExtension.add_to(sample_item) + StorageSchemesExtension.add_to(sample_item) + StorageSchemesExtension.add_to(sample_item) eo_uris = [ uri for uri in sample_item.stac_extensions - if uri == StorageExtension.get_schema_uri() + if uri == StorageSchemesExtension.get_schema_uri() ] assert len(eo_uris) == 1 @@ -58,40 +63,60 @@ def test_extend_invalid_object() -> None: link = pystac.Link("child", "https://some-domain.com/some/path/to.json") with pytest.raises(pystac.ExtensionTypeError): - StorageExtension.ext(link) # type: ignore + StorageSchemesExtension.ext(link) # type: ignore def test_extension_not_implemented(sample_item: Item) -> None: # Should raise exception if Item does not include extension URI with pytest.raises(pystac.ExtensionNotImplemented): - _ = StorageExtension.ext(sample_item) + _ = StorageSchemesExtension.ext(sample_item) # Should raise exception if owning Item does not include extension URI asset = sample_item.assets["thumbnail"] with pytest.raises(pystac.ExtensionNotImplemented): - _ = StorageExtension.ext(asset) + _ = StorageRefsExtension.ext(asset) # Should succeed if Asset has no owner ownerless_asset = pystac.Asset.from_dict(asset.to_dict()) - _ = StorageExtension.ext(ownerless_asset) + _ = StorageRefsExtension.ext(ownerless_asset) + + +def test_collection_ext_add_to(naip_collection: Collection) -> None: + naip_collection.stac_extensions = [] + assert ( + StorageSchemesExtension.get_schema_uri() not in naip_collection.stac_extensions + ) + + _ = StorageSchemesExtension.ext(naip_collection, add_if_missing=True) + + assert StorageSchemesExtension.get_schema_uri() in naip_collection.stac_extensions def test_item_ext_add_to(sample_item: Item) -> None: - assert StorageExtension.get_schema_uri() not in sample_item.stac_extensions + assert StorageSchemesExtension.get_schema_uri() not in sample_item.stac_extensions - _ = StorageExtension.ext(sample_item, add_if_missing=True) + _ = StorageSchemesExtension.ext(sample_item, add_if_missing=True) - assert StorageExtension.get_schema_uri() in sample_item.stac_extensions + assert StorageSchemesExtension.get_schema_uri() in sample_item.stac_extensions def test_asset_ext_add_to(sample_item: Item) -> None: - assert StorageExtension.get_schema_uri() not in sample_item.stac_extensions + assert StorageSchemesExtension.get_schema_uri() not in sample_item.stac_extensions asset = sample_item.assets["thumbnail"] - _ = StorageExtension.ext(asset, add_if_missing=True) + _ = StorageRefsExtension.ext(asset, add_if_missing=True) + + assert StorageSchemesExtension.get_schema_uri() in sample_item.stac_extensions + - assert StorageExtension.get_schema_uri() in sample_item.stac_extensions +def test_link_ext_add_to(sample_item: Item) -> None: + assert StorageSchemesExtension.get_schema_uri() not in sample_item.stac_extensions + asset = sample_item.links[0] + + _ = StorageRefsExtension.ext(asset, add_if_missing=True) + + assert StorageSchemesExtension.get_schema_uri() in sample_item.stac_extensions def test_asset_ext_add_to_ownerless_asset(sample_item: Item) -> None: @@ -99,84 +124,44 @@ def test_asset_ext_add_to_ownerless_asset(sample_item: Item) -> None: asset = pystac.Asset.from_dict(asset_dict) with pytest.raises(pystac.STACError): - _ = StorageExtension.ext(asset, add_if_missing=True) + _ = StorageRefsExtension.ext(asset, add_if_missing=True) def test_should_raise_exception_when_passing_invalid_extension_object() -> None: with pytest.raises( - ExtensionTypeError, match=r"^StorageExtension does not apply to type 'object'$" + ExtensionTypeError, + match=r"^StorageRefsExtension does not apply to type 'object'$", ): # calling it wrong purposely so ---------v - StorageExtension.ext(object()) # type: ignore - - -def test_summaries_platform(naip_collection: Collection) -> None: - col_dict = naip_collection.to_dict() - storage_summaries = StorageExtension.summaries(naip_collection) - - # Get - assert storage_summaries.platform == col_dict["summaries"]["storage:platform"] - # Set - new_platform_summary = [random.choice([v for v in CloudPlatform])] - assert storage_summaries.platform != new_platform_summary - storage_summaries.platform = new_platform_summary - assert storage_summaries.platform == new_platform_summary - - col_dict = naip_collection.to_dict() - assert col_dict["summaries"]["storage:platform"] == new_platform_summary - - -def test_summaries_region(naip_collection: Collection) -> None: - col_dict = naip_collection.to_dict() - storage_summaries = StorageExtension.summaries(naip_collection) - - # Get - assert storage_summaries.region == col_dict["summaries"]["storage:region"] - # Set - new_region_summary = [random.choice(ascii_letters)] - assert storage_summaries.region != new_region_summary - storage_summaries.region = new_region_summary - assert storage_summaries.region == new_region_summary - - col_dict = naip_collection.to_dict() - assert col_dict["summaries"]["storage:region"] == new_region_summary + StorageRefsExtension.ext(object()) # type: ignore -def test_summaries_requester_pays(naip_collection: Collection) -> None: +def test_summaries_schemes(naip_collection: Collection) -> None: col_dict = naip_collection.to_dict() - storage_summaries = StorageExtension.summaries(naip_collection) - + storage_summaries = StorageSchemesExtension.summaries(naip_collection) + print(naip_collection.summaries) # Get assert ( - storage_summaries.requester_pays - == col_dict["summaries"]["storage:requester_pays"] + list( + map( + lambda x: {k: c.to_dict() for k, c in x.items()}, + storage_summaries.schemes or [], + ) + ) + == col_dict["summaries"]["storage:schemes"] ) - # Set - new_requester_pays_summary = [True] - assert storage_summaries.requester_pays != new_requester_pays_summary - storage_summaries.requester_pays = new_requester_pays_summary - assert storage_summaries.requester_pays == new_requester_pays_summary - - col_dict = naip_collection.to_dict() - assert col_dict["summaries"]["storage:requester_pays"] == new_requester_pays_summary - - -def test_summaries_tier(naip_collection: Collection) -> None: - col_dict = naip_collection.to_dict() - storage_summaries = StorageExtension.summaries(naip_collection) - - # Get - assert storage_summaries.tier == col_dict["summaries"]["storage:tier"] - - # Set - new_tier_summary = [random.choice(ascii_letters)] - assert storage_summaries.tier != new_tier_summary - storage_summaries.tier = new_tier_summary - assert storage_summaries.tier == new_tier_summary + new_schemes_summary = [ + {"key": StorageScheme.create("aws-s3", "https://a.platform.example.com")} + ] + assert storage_summaries.schemes != new_schemes_summary + storage_summaries.schemes = new_schemes_summary + assert storage_summaries.schemes == new_schemes_summary col_dict = naip_collection.to_dict() - assert col_dict["summaries"]["storage:tier"] == new_tier_summary + assert col_dict["summaries"]["storage:schemes"] == [ + {k: c.to_dict() for k, c in x.items()} for x in new_schemes_summary + ] def test_summaries_adds_uri(naip_collection: Collection) -> None: @@ -185,152 +170,110 @@ def test_summaries_adds_uri(naip_collection: Collection) -> None: pystac.ExtensionNotImplemented, match="Extension 'storage' is not implemented", ): - StorageExtension.summaries(naip_collection, add_if_missing=False) - - _ = StorageExtension.summaries(naip_collection, add_if_missing=True) + StorageSchemesExtension.summaries(naip_collection, add_if_missing=False) - assert StorageExtension.get_schema_uri() in naip_collection.stac_extensions + _ = StorageSchemesExtension.summaries(naip_collection, add_if_missing=True) - StorageExtension.remove_from(naip_collection) - assert StorageExtension.get_schema_uri() not in naip_collection.stac_extensions + assert StorageSchemesExtension.get_schema_uri() in naip_collection.stac_extensions + StorageSchemesExtension.remove_from(naip_collection) + assert ( + StorageSchemesExtension.get_schema_uri() not in naip_collection.stac_extensions + ) -def test_item_apply(naip_item: Item) -> None: - asset = random.choice(list(naip_item.assets.values())) - - storage_ext = StorageExtension.ext(asset) - new_platform = random.choice( - [v for v in CloudPlatform if v != storage_ext.platform] - ) +def test_schemes_apply(naip_item: Item) -> None: + storage_ext = StorageSchemesExtension.ext(naip_item) + new_key = random.choice(ascii_letters) + new_type = random.choice(ascii_letters) + new_platform = random.choice(ascii_letters) new_region = random.choice(ascii_letters) - new_requestor_pays = random.choice( - [v for v in {True, False} if v != storage_ext.requester_pays] - ) - new_tier = random.choice(ascii_letters) + new_requestor_pays = random.choice([v for v in {True, False}]) storage_ext.apply( - platform=new_platform, - region=new_region, - requester_pays=new_requestor_pays, - tier=new_tier, + schemes={ + new_key: StorageScheme.create( + new_type, new_platform, new_region, new_requestor_pays + ), + } ) - assert storage_ext.platform == new_platform - assert storage_ext.region == new_region - assert storage_ext.requester_pays == new_requestor_pays - assert storage_ext.tier == new_tier + applied_schemes = storage_ext.schemes + assert list(applied_schemes) == [new_key] + assert applied_schemes[new_key].type == new_type + assert applied_schemes[new_key].platform == new_platform + assert applied_schemes[new_key].region == new_region + assert applied_schemes[new_key].requester_pays == new_requestor_pays @pytest.mark.vcr() -def test_asset_platform(naip_item: Item) -> None: +def test_refs_apply(naip_item: Item) -> None: # Grab a random asset with the platform property asset = random.choice( [ _asset for _asset in naip_item.assets.values() - if "storage:platform" in _asset.to_dict() + if "storage:refs" in _asset.to_dict() ] ) - storage_ext = StorageExtension.ext(asset) + storage_ext = StorageRefsExtension.ext(asset) # Get - assert storage_ext.platform == asset.extra_fields.get("storage:platform") + assert storage_ext.refs == asset.extra_fields.get("storage:refs") # Set - new_platform = random.choice( - [val for val in CloudPlatform if val != storage_ext.platform] - ) - storage_ext.platform = new_platform - assert storage_ext.platform == new_platform + new_refs = [random.choice(ascii_letters)] + storage_ext.refs = new_refs + assert storage_ext.refs == new_refs naip_item.validate() -@pytest.mark.vcr() -def test_asset_region(naip_item: Item) -> None: - # Grab a random asset with the platform property - asset = random.choice( - [ - _asset - for _asset in naip_item.assets.values() - if "storage:region" in _asset.to_dict() - ] - ) +def test_add_storage_scheme(naip_item: Item) -> None: + storage_ext = naip_item.ext.storage + storage_ext.add_scheme("new_scheme", StorageScheme.create("type", "platform")) + assert "new_scheme" in storage_ext.schemes - storage_ext = StorageExtension.ext(asset) + storage_ext.properties.pop("storage:schemes") + storage_ext.add_scheme("new_scheme", StorageScheme.create("type", "platform")) + assert len(storage_ext.schemes) == 1 + assert "new_scheme" in storage_ext.schemes - # Get - assert storage_ext.region == asset.extra_fields.get("storage:region") - # Set - new_region = random.choice( - [val for val in CloudPlatform if val != storage_ext.region] - ) - storage_ext.region = new_region - assert storage_ext.region == new_region +def test_add_refs(naip_item: Item) -> None: + scheme_name = random.choice(ascii_letters) + asset = naip_item.assets["GEOTIFF_AZURE_RGBIR"] + storage_ext = asset.ext.storage + storage_ext.add_ref(scheme_name) + assert scheme_name in storage_ext.refs - naip_item.validate() - - # Set to None - storage_ext.region = None - assert "storage:region" not in asset.extra_fields + storage_ext.properties.pop("storage:refs") + scheme_name_2 = random.choice(ascii_letters) + storage_ext.add_ref(scheme_name_2) + assert len(storage_ext.refs) == 1 + assert scheme_name_2 in storage_ext.refs -@pytest.mark.vcr() -def test_asset_requester_pays(naip_item: Item) -> None: - # Grab a random asset with the platform property - asset = random.choice( - [ - _asset - for _asset in naip_item.assets.values() - if "storage:requester_pays" in _asset.to_dict() - ] +def test_storage_scheme_create() -> None: + scheme = StorageScheme.create( + type=StorageSchemeType.AWS_S3, + platform="https://{bucket}.s3.{region}.amazonaws.com", + region="us-west-2", + requester_pays=True, ) - storage_ext = StorageExtension.ext(asset) + assert scheme.type == StorageSchemeType.AWS_S3 + assert scheme.platform == "https://{bucket}.s3.{region}.amazonaws.com" + assert scheme.region == "us-west-2" + assert scheme.requester_pays is True - # Get - assert storage_ext.requester_pays == asset.extra_fields.get( - "storage:requester_pays" - ) - - # Set - new_requester_pays = True if not storage_ext.requester_pays else False - storage_ext.requester_pays = new_requester_pays - assert storage_ext.requester_pays == new_requester_pays - - naip_item.validate() - - # Set to None - storage_ext.requester_pays = None - assert "storage:requester_pays" not in asset.extra_fields - - -@pytest.mark.vcr() -def test_asset_tier(naip_item: Item) -> None: - # Grab a random asset with the platform property - asset = random.choice( - [ - _asset - for _asset in naip_item.assets.values() - if "storage:tier" in _asset.to_dict() - ] - ) - - storage_ext = StorageExtension.ext(asset) - - # Get - assert storage_ext.tier == asset.extra_fields.get("storage:tier") - - # Set - new_tier = random.choice(ascii_letters) - storage_ext.tier = new_tier - assert storage_ext.tier == new_tier - - naip_item.validate() + scheme.type = StorageSchemeType.AZURE + scheme.platform = "https://{account}.blob.core.windows.net" + scheme.region = "eastus" + scheme.requester_pays = False - # Set to None - storage_ext.tier = None - assert "storage:tier" not in asset.extra_fields + assert scheme.type == StorageSchemeType.AZURE + assert scheme.platform == "https://{account}.blob.core.windows.net" + assert scheme.region == "eastus" + assert scheme.requester_pays is False From d534a0fb379d7e86de7852858a8531f74ff950be Mon Sep 17 00:00:00 2001 From: Tyler <31015976+tylanderson@users.noreply.github.com> Date: Mon, 2 Jun 2025 10:59:14 -0400 Subject: [PATCH 2/7] update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ada31eb9..494c741da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ media type value for these types and new media types COPC and VND_PMTILES ([#1554](https://github.com/stac-utils/pystac/pull/1554)) +### Changed + +- Updated storage extension to v2.0.0 + ### Fixed - More permissive collection extent deserialization ([#1559](https://github.com/stac-utils/pystac/pull/1559)) From d6c166ef033609206e62fefa88b34bf7bdeb7b90 Mon Sep 17 00:00:00 2001 From: Tyler <31015976+tylanderson@users.noreply.github.com> Date: Mon, 2 Jun 2025 20:10:23 -0400 Subject: [PATCH 3/7] improve equality and setting/getting attribute --- pystac/extensions/storage.py | 48 +++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/pystac/extensions/storage.py b/pystac/extensions/storage.py index 3a6a3c523..a0e443639 100644 --- a/pystac/extensions/storage.py +++ b/pystac/extensions/storage.py @@ -10,6 +10,7 @@ Any, Generic, Literal, + SupportsIndex, TypeVar, cast, ) @@ -50,23 +51,31 @@ class StorageSchemeType(StringEnum): class StorageScheme: - properties: dict[str, Any] + _properties: dict[str, Any] def __init__(self, properties: dict[str, Any]): - super().__setattr__("properties", properties) + super().__setattr__("_properties", properties) def __eq__(self, other: object) -> bool: if not isinstance(other, StorageScheme): - raise NotImplementedError - return self.properties == other.properties + return NotImplemented + + return bool(self.__dict__["_properties"] == other.__dict__["_properties"]) def __getattr__(self, name: str) -> Any: - if name in self.properties: - return self.properties[name] + properties = self.__dict__["_properties"] + if name in properties: + return properties[name] raise AttributeError(f"StorageScheme does not have attribute '{name}'") def __setattr__(self, name: str, value: Any) -> None: - self.properties[name] = value + if name.startswith("_") or hasattr(type(self), name): + super().__setattr__(name, value) + else: + self._properties[name] = value + + def __reduce_ex__(self, protocol: SupportsIndex) -> Any: + return (self.__class__, (self.__dict__["_properties"],), None) def apply( self, @@ -80,7 +89,7 @@ def apply( self.platform = platform self.region = region self.requester_pays = requester_pays - self.properties.update(kwargs) + self._properties.update(kwargs) @classmethod def create( @@ -125,14 +134,14 @@ def type(self) -> str: Get or set the required type property """ return get_required( - self.properties.get(TYPE_PROP), + self._properties.get(TYPE_PROP), self, TYPE_PROP, ) @type.setter def type(self, v: str) -> None: - self.properties[TYPE_PROP] = v + self._properties[TYPE_PROP] = v @property def platform(self) -> str: @@ -140,42 +149,42 @@ def platform(self) -> str: Get or set the required platform property """ return get_required( - self.properties.get(PLATFORM_PROP), + self._properties.get(PLATFORM_PROP), self, PLATFORM_PROP, ) @platform.setter def platform(self, v: str) -> None: - self.properties[PLATFORM_PROP] = v + self._properties[PLATFORM_PROP] = v @property def region(self) -> str | None: """ Get or set the optional region property """ - return self.properties.get(REGION_PROP) + return self._properties.get(REGION_PROP) @region.setter def region(self, v: str) -> None: if v is not None: - self.properties[REGION_PROP] = v + self._properties[REGION_PROP] = v else: - self.properties.pop(REGION_PROP, None) + self._properties.pop(REGION_PROP, None) @property def requester_pays(self) -> bool | None: """ Get or set the optional requester_pays property """ - return self.properties.get(REQUESTER_PAYS_PROP) + return self._properties.get(REQUESTER_PAYS_PROP) @requester_pays.setter def requester_pays(self, v: bool) -> None: if v is not None: - self.properties[REQUESTER_PAYS_PROP] = v + self._properties[REQUESTER_PAYS_PROP] = v else: - self.properties.pop(REQUESTER_PAYS_PROP, None) + self._properties.pop(REQUESTER_PAYS_PROP, None) def to_dict(self) -> dict[str, Any]: """ @@ -184,7 +193,7 @@ def to_dict(self) -> dict[str, Any]: Returns: dict[str, Any """ - return self.properties + return self._properties class _StorageExtension(ABC): @@ -473,7 +482,6 @@ def schemes(self) -> list[dict[str, StorageScheme]] | None: self.summaries.get_list(SCHEMES_PROP), ) - print(v) return v @schemes.setter From 4082d79f841bf82e0a8e5a92fcab4debbe27942a Mon Sep 17 00:00:00 2001 From: Tyler <31015976+tylanderson@users.noreply.github.com> Date: Mon, 2 Jun 2025 22:38:28 -0400 Subject: [PATCH 4/7] increase test coverage --- tests/extensions/test_storage.py | 95 +++++++++++++++++++++----------- 1 file changed, 63 insertions(+), 32 deletions(-) diff --git a/tests/extensions/test_storage.py b/tests/extensions/test_storage.py index e843ca0b9..d176e6f54 100644 --- a/tests/extensions/test_storage.py +++ b/tests/extensions/test_storage.py @@ -1,5 +1,6 @@ import json import random +from copy import deepcopy from string import ascii_letters import pytest @@ -29,6 +30,28 @@ def naip_collection() -> Collection: return Collection.from_file(NAIP_COLLECTION_URI) +@pytest.fixture +def sample_scheme() -> StorageScheme: + return StorageScheme.create( + type=StorageSchemeType.AWS_S3, + platform="https://{bucket}.s3.{region}.amazonaws.com", + region="us-west-2", + requester_pays=True, + ) + + +@pytest.fixture +def naip_asset(naip_item: Item) -> pystac.Asset: + # Grab a random asset with the platform property + return random.choice( + [ + _asset + for _asset in naip_item.assets.values() + if "storage:refs" in _asset.to_dict() + ] + ) + + def test_to_from_dict() -> None: with open(NAIP_EXAMPLE_URI) as f: item_dict = json.load(f) @@ -101,6 +124,16 @@ def test_item_ext_add_to(sample_item: Item) -> None: assert StorageSchemesExtension.get_schema_uri() in sample_item.stac_extensions +def test_catalog_ext_add_to() -> None: + catalog = pystac.Catalog("stac", "a catalog") + + assert StorageSchemesExtension.get_schema_uri() not in catalog.stac_extensions + + _ = StorageSchemesExtension.ext(catalog, add_if_missing=True) + + assert StorageSchemesExtension.get_schema_uri() in catalog.stac_extensions + + def test_asset_ext_add_to(sample_item: Item) -> None: assert StorageSchemesExtension.get_schema_uri() not in sample_item.stac_extensions asset = sample_item.assets["thumbnail"] @@ -207,28 +240,21 @@ def test_schemes_apply(naip_item: Item) -> None: @pytest.mark.vcr() -def test_refs_apply(naip_item: Item) -> None: - # Grab a random asset with the platform property - asset = random.choice( - [ - _asset - for _asset in naip_item.assets.values() - if "storage:refs" in _asset.to_dict() - ] - ) +def test_refs_apply(naip_asset: pystac.Asset) -> None: + test_refs = ["a_ref", "b_ref"] + + storage_ext = StorageRefsExtension.ext(naip_asset) - storage_ext = StorageRefsExtension.ext(asset) + storage_ext.apply(test_refs) # Get - assert storage_ext.refs == asset.extra_fields.get("storage:refs") + assert storage_ext.refs == test_refs # Set new_refs = [random.choice(ascii_letters)] storage_ext.refs = new_refs assert storage_ext.refs == new_refs - naip_item.validate() - def test_add_storage_scheme(naip_item: Item) -> None: storage_ext = naip_item.ext.storage @@ -255,25 +281,30 @@ def test_add_refs(naip_item: Item) -> None: assert scheme_name_2 in storage_ext.refs -def test_storage_scheme_create() -> None: - scheme = StorageScheme.create( - type=StorageSchemeType.AWS_S3, - platform="https://{bucket}.s3.{region}.amazonaws.com", - region="us-west-2", - requester_pays=True, - ) +def test_storage_scheme_create(sample_scheme: StorageScheme) -> None: + assert sample_scheme.type == StorageSchemeType.AWS_S3 + assert sample_scheme.platform == "https://{bucket}.s3.{region}.amazonaws.com" + assert sample_scheme.region == "us-west-2" + assert sample_scheme.requester_pays is True + + sample_scheme.type = StorageSchemeType.AZURE + sample_scheme.platform = "https://{account}.blob.core.windows.net" + sample_scheme.region = "eastus" + sample_scheme.account = "account" + sample_scheme.requester_pays = False + + assert sample_scheme.type == StorageSchemeType.AZURE + assert sample_scheme.platform == "https://{account}.blob.core.windows.net" + assert sample_scheme.region == "eastus" + assert sample_scheme.account == "account" + assert sample_scheme.requester_pays is False + - assert scheme.type == StorageSchemeType.AWS_S3 - assert scheme.platform == "https://{bucket}.s3.{region}.amazonaws.com" - assert scheme.region == "us-west-2" - assert scheme.requester_pays is True +def test_storage_scheme_equality(sample_scheme: StorageScheme) -> None: + other = deepcopy(sample_scheme) + assert sample_scheme == other - scheme.type = StorageSchemeType.AZURE - scheme.platform = "https://{account}.blob.core.windows.net" - scheme.region = "eastus" - scheme.requester_pays = False + other.requester_pays = False + assert sample_scheme != other - assert scheme.type == StorageSchemeType.AZURE - assert scheme.platform == "https://{account}.blob.core.windows.net" - assert scheme.region == "eastus" - assert scheme.requester_pays is False + assert sample_scheme != object() From f5277590574d0356de1842272a2a574890222db8 Mon Sep 17 00:00:00 2001 From: Tyler <31015976+tylanderson@users.noreply.github.com> Date: Mon, 2 Jun 2025 23:40:38 -0400 Subject: [PATCH 5/7] documentation fixes --- docs/api/extensions.rst | 3 ++- pystac/extensions/storage.py | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/docs/api/extensions.rst b/docs/api/extensions.rst index ce37d3215..19d17c84a 100644 --- a/docs/api/extensions.rst +++ b/docs/api/extensions.rst @@ -30,7 +30,8 @@ pystac.extensions sar.SarExtension sat.SatExtension scientific.ScientificExtension - storage.StorageExtension + storage.StorageSchemesExtension + storage.StorageRefsExtension table.TableExtension timestamps.TimestampsExtension version.VersionExtension diff --git a/pystac/extensions/storage.py b/pystac/extensions/storage.py index a0e443639..15f0ff37b 100644 --- a/pystac/extensions/storage.py +++ b/pystac/extensions/storage.py @@ -25,9 +25,11 @@ from pystac.extensions.hooks import ExtensionHooks from pystac.utils import StringEnum, get_required, map_opt -#: Generalized version of :class:`~pystac.Item`, :class:`~pystac.Asset` or -#: :class:`~pystac.ItemAssetDefinition` +#: Generalized version of :class:`~pystac.Item`, :class:`~pystac.Catalog` or +#: :class:`~pystac.Collection` T = TypeVar("T", pystac.Item, pystac.Catalog, pystac.Collection) +#: Generalized version of :class:`~pystac.Asset`, :class:`~pystac.Link` or +#: :class:`~pystac.ItemAssetDefinition` U = TypeVar("U", pystac.Asset, pystac.Link, pystac.ItemAssetDefinition) SCHEMA_URI: str = "https://stac-extensions.github.io/storage/v2.0.0/schema.json" @@ -109,9 +111,9 @@ def create( type (str): Type identifier for the platform. platform (str): The cloud provider where data is stored as URI or URI template to the API. - region (str | None, optional): The region where the data is stored. + region (str | None): The region where the data is stored. Defaults to None. - requester_pays (bool | None, optional): requester pays or data manager/cloud + requester_pays (bool | None): requester pays or data manager/cloud provider pays. Defaults to None. kwargs (dict[str | Any]): Additional properties to set on scheme @@ -216,13 +218,13 @@ class StorageSchemesExtension( This class is generic over the type of STAC Object to be extended (e.g. :class:`~pystac.Item`, :class:`~pystac.Collection`). - To create a concrete instance of :class:`StorageExtension`, use the - :meth:`StorageExtension.ext` method. For example: + To create a concrete instance of :class:`StorageSchemesExtension`, use the + :meth:`StorageSchemesExtension.ext` method. For example: .. code-block:: python >>> item: pystac.Item = ... - >>> storage_ext = StorageExtension.ext(item) + >>> storage_ext = StorageSchemesExtension.ext(item) """ def apply( @@ -472,7 +474,7 @@ class SummariesStorageExtension(SummariesExtension): @property def schemes(self) -> list[dict[str, StorageScheme]] | None: - """Get or sets the summary of :attr:`StorageExtension.platform` values + """Get or sets the summary of :attr:`StorageScheme.platform` values for this Collection. """ v = map_opt( From 469eb7d23292a061588698f77de4b5a022b70824 Mon Sep 17 00:00:00 2001 From: Tyler <31015976+tylanderson@users.noreply.github.com> Date: Wed, 18 Jun 2025 15:35:24 -0400 Subject: [PATCH 6/7] increase coverage, fix naming --- tests/extensions/test_storage.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/extensions/test_storage.py b/tests/extensions/test_storage.py index d176e6f54..4974bbfe0 100644 --- a/tests/extensions/test_storage.py +++ b/tests/extensions/test_storage.py @@ -6,7 +6,7 @@ import pytest import pystac -from pystac import ExtensionTypeError, Item +from pystac import ExtensionTypeError, Item, ItemAssetDefinition from pystac.collection import Collection from pystac.extensions.storage import ( StorageRefsExtension, @@ -69,12 +69,12 @@ def test_add_to(sample_item: Item) -> None: StorageSchemesExtension.add_to(sample_item) StorageSchemesExtension.add_to(sample_item) - eo_uris = [ + uris = [ uri for uri in sample_item.stac_extensions if uri == StorageSchemesExtension.get_schema_uri() ] - assert len(eo_uris) == 1 + assert len(uris) == 1 @pytest.mark.vcr() @@ -271,6 +271,8 @@ def test_add_refs(naip_item: Item) -> None: scheme_name = random.choice(ascii_letters) asset = naip_item.assets["GEOTIFF_AZURE_RGBIR"] storage_ext = asset.ext.storage + assert isinstance(storage_ext, StorageRefsExtension) + storage_ext.add_ref(scheme_name) assert scheme_name in storage_ext.refs @@ -308,3 +310,10 @@ def test_storage_scheme_equality(sample_scheme: StorageScheme) -> None: assert sample_scheme != other assert sample_scheme != object() + + +def test_item_asset_accessor() -> None: + item_asset = ItemAssetDefinition.create( + title="title", description="desc", media_type="media", roles=["a_role"] + ) + assert isinstance(item_asset.ext.storage, StorageRefsExtension) From 65f3f83e37b341c1edb95fab012a77e890cfa0b7 Mon Sep 17 00:00:00 2001 From: Tyler <31015976+tylanderson@users.noreply.github.com> Date: Tue, 24 Jun 2025 17:20:29 -0400 Subject: [PATCH 7/7] retrigger ci --- tests/extensions/test_storage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/extensions/test_storage.py b/tests/extensions/test_storage.py index 4974bbfe0..6f696c60d 100644 --- a/tests/extensions/test_storage.py +++ b/tests/extensions/test_storage.py @@ -244,7 +244,6 @@ def test_refs_apply(naip_asset: pystac.Asset) -> None: test_refs = ["a_ref", "b_ref"] storage_ext = StorageRefsExtension.ext(naip_asset) - storage_ext.apply(test_refs) # Get