Update I/O docs

Author: duckontheweb

docs/api.rst

@@ -171,6 +171,20 @@ StacIO
    :members:
    :undoc-members:
 
+DefaultStacIO
+~~~~~~~~~~~~~
+
+.. autoclass:: pystac.stac_io.DefaultStacIO
+   :members:
+   :show-inheritance:
+
+DuplicateKeyReportingMixin
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: pystac.stac_io.DuplicateKeyReportingMixin
+   :members:
+   :show-inheritance:
+
 STAC_IO
 ~~~~~~~
 
@@ -213,11 +227,47 @@ STACError
 
 .. autoclass:: pystac.STACError
 
+STACTypeError
+~~~~~~~~~~~~~
+
+.. autoclass:: pystac.STACTypeError
+
+DuplicateObjectKeyError
+~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: pystac.DuplicateObjectKeyError
+
+ExtensionAlreadyExistsError
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: pystac.ExtensionAlreadyExistsError
+
+ExtensionTypeError
+~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: pystac.ExtensionTypeError
+
+ExtensionNotImplemented
+~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: pystac.ExtensionNotImplemented
+
 ExtensionTypeError
 ~~~~~~~~~~~~~~~~~~
 
 .. autoclass:: pystac.ExtensionTypeError
 
+RequiredPropertyMissing
+~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: pystac.RequiredPropertyMissing
+
+STACValidationError
+~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: pystac.STACValidationError
+
+
 Extensions
 ----------
 

docs/concepts.rst

@@ -225,72 +225,96 @@ written (e.g. if you are working with self-contained catalogs).
 
 .. _using stac_io:
 
-Using STAC_IO
+I/O in PySTAC
 =============
 
-The :class:`~pystac.STAC_IO` class is the way PySTAC reads and writes text from file
-locations. Since PySTAC aims to be dependency-free, there is no default mechanisms to
-read and write from anything but the local file system. However, users of PySTAC may
-want to read and write from other file systems, such as HTTP or cloud object storage.
-STAC_IO allows users to hook into PySTAC and define their own reading and writing
-primitives to allow for those use cases.
-
-To enable reading from other types of file systems, it is recommended that in the
-`__init__.py` of the client module, or at the beginning of the script using PySTAC, you
-overwrite the :func:`STAC_IO.read_text_method <pystac.STAC_IO.read_text_method>` and
-:func:`STAC_IO.write_text_method <pystac.STAC_IO.write_text_method>` members of STAC_IO
-with functions that read and write however you need. For example, this code will allow
+The :class:`pystac.StacIO` class defines fundamental methods for I/O
+operations within PySTAC, including serialization and deserialization to and from
+JSON files and conversion to and from Python dictionaries. This is an abstract class
+and should not be instantiated directly. However, PySTAC provides a
+:class:`pystac.stac_io.DefaultStacIO` class with minimal implementations of these
+methods. This default implementation provides support for reading and writing files
+from the local filesystem as well as HTTP URIs (using ``urllib``). This class is
+created automatically by all of the object-specific I/O methods (e.g.
+:meth:`pystac.Catalog.from_file`), so most users will not need to instantiate this
+class themselves.
+
+If you require custom logic for I/O operations or would like to use a 3rd-party library
+for I/O operations (e.g. ``requests``), you can create a sub-class of
+:class:`pystac.StacIO` (or :class:`pystac.DefaultStacIO`) and customize the methods as
+you see fit. You can then pass instances of this custom sub-class into the ``stac_io``
+argument of most object-specific I/O methods. You can also use
+:meth:`pystac.StacIO.set_default` in your client's ``__init__.py`` file to make this
+sub-class the default :class:`pystac.StacIO` implementation throughout the library.
+
+For example, this code will allow
 for reading from AWS's S3 cloud object storage using `boto3
-<https://boto3.amazonaws.com/v1/documentation/api/latest/index.html>`_:
+<https://boto3.amazonaws.com/v1/documentation/api/latest/index.html>`__:
 
 .. code-block:: python
 
    from urllib.parse import urlparse
    import boto3
-   from pystac import STAC_IO
-
-   def my_read_method(uri):
-       parsed = urlparse(uri)
-       if parsed.scheme == 's3':
-           bucket = parsed.netloc
-           key = parsed.path[1:]
-           s3 = boto3.resource('s3')
-           obj = s3.Object(bucket, key)
-           return obj.get()['Body'].read().decode('utf-8')
-       else:
-           return STAC_IO.default_read_text_method(uri)
-
-   def my_write_method(uri, txt):
-       parsed = urlparse(uri)
-       if parsed.scheme == 's3':
-           bucket = parsed.netloc
-           key = parsed.path[1:]
-           s3 = boto3.resource("s3")
-           s3.Object(bucket, key).put(Body=txt)
-       else:
-           STAC_IO.default_write_text_method(uri, txt)
-
-   STAC_IO.read_text_method = my_read_method
-   STAC_IO.write_text_method = my_write_method
-
-If you are only going to read from another source, e.g. HTTP, you could only replace the
-read method. For example, using the `requests library
-<https://requests.kennethreitz.org/en/master>`_:
+   from pystac import Link
+   from pystac.stac_io import DefaultStacIO, StacIO
+
+   class CustomStacIO(DefaultStacIO):
+      def __init__():
+         self.s3 = boto3.resource("s3")
+
+      def read_text(
+         self, source: Union[str, Link], *args: Any, **kwargs: Any
+      ) -> str:
+         parsed = urlparse(uri)
+         if parsed.scheme == "s3":
+            bucket = parsed.netloc
+            key = parsed.path[1:]
+            
+            obj = self.s3.Object(bucket, key)
+            return obj.get()["Body"].read().decode("utf-8")
+         else:
+            return super().read_text(source, *args, **kwargs)
+
+      def write_text(
+         self, dest: Union[str, Link], txt: str, *args: Any, **kwargs: Any
+      ) -> None:
+         parsed = urlparse(uri)
+         if parsed.scheme == "s3":
+            bucket = parsed.netloc
+            key = parsed.path[1:]
+            s3 = boto3.resource("s3")
+            s3.Object(bucket, key).put(Body=txt, ContentEncoding="utf-8")
+         else:
+            super().write_text(dest, txt, *args, **kwargs)
+
+   StacIO.set_default(CustomStacIO)
+   
+
+If you only need to customize read operations you can inherit from
+:class:`~pystac.stac_io.DefaultStacIO` and only overwrite the read method. For example,
+to take advantage of connection pooling using a `requests.Session
+<https://requests.kennethreitz.org/en/master>`__:
 
 .. code-block:: python
 
    from urllib.parse import urlparse
    import requests
-   from pystac import STAC_IO
-
-   def my_read_method(uri):
-       parsed = urlparse(uri)
-       if parsed.scheme.startswith('http'):
-           return requests.get(uri).text
-       else:
-           return STAC_IO.default_read_text_method(uri)
-
-   STAC_IO.read_text_method = my_read_method
+   from pystac.stac_io import DefaultStacIO, StacIO
+
+   class ConnectionPoolingIO(DefaultStacIO):
+      def __init__():
+         self.session = requests.Session()
+      
+      def read_text(
+         self, source: Union[str, Link], *args: Any, **kwargs: Any
+      ) -> str:
+         parsed = urlparse(uri)
+         if parsed.scheme.startswith("http"):
+            return self.session.get(uri).text
+         else:
+            return super().read_text(source, *args, **kwargs)
+   
+   StacIO.set_default(ConnectionPoolingIO)
 
 Validation
 ==========

pystac/stac_io.py

@@ -46,14 +46,19 @@ def read_text(
     ) -> str:
         """Read text from the given URI.
 
-        The source to read from can be specified
-        as a string or a Link. If it's a string, it's the URL of the HREF from which to
-        read. When reading links, PySTAC will pass in the entire link body.
-        This enables implementations to utilize additional link information,
-        e.g. the "post" information in a pagination link from a STAC API search.
+        The source to read from can be specified as a string or a
+        :class:`~pystac.Link`. If it is a string, it must be a URI or local path from
+        which to read. Using a :class:`~pystac.Link` enables implementations to use
+        additional link information, such as paging information contained in the
+        extended links described in the `STAC API spec 
+        <https://github.com/radiantearth/stac-api-spec/tree/master/item-search#paging>`__.
 
         Args:
             source : The source to read from.
+            *args : Arbitrary positional arguments that may be utilized by the concrete
+                implementation.
+            **kwargs : Arbitrary keyword arguments that may be utilized by the concrete
+                implementation.
 
         Returns:
             str: The text contained in the file at the location specified by the uri.
@@ -66,10 +71,10 @@ def write_text(
     ) -> None:
         """Write the given text to a file at the given URI.
 
-        The destination to write to from can be specified
-        as a string or a Link. If it's a string, it's the URL of the HREF from which to
-        read. When writing based on links links, PySTAC will pass in the entire
-        link body.
+        The destination to write to from can be specified as a string or a
+        :class:`~pystac.Link`. If it is a string, it must be a URI or local path from
+        which to read. Using a :class:`~pystac.Link` enables implementations to use
+        additional link information.
 
         Args:
             dest : The destination to write to.
@@ -122,6 +127,21 @@ def stac_object_from_dict(
         root: Optional["Catalog_Type"] = None,
         preserve_dict: bool = True,
     ) -> "STACObject_Type":
+        """Deserializes a :class:`~pystac.STACObject` sub-class instance from a
+        dictionary.
+
+        Args:
+
+            d : The dictionary to deserialize
+            href : Optional href to associate with the STAC object
+            root : Optional root :class:`~pystac.Catalog` to associate with the
+                STAC object.
+            preserve_dict: If ``False``, the dict parameter ``d`` may be modified
+                during this method call. Otherwise the dict is not mutated.
+                Defaults to ``True``, which results results in a deepcopy of the
+                parameter. Set to ``False`` when possible to avoid the performance
+                hit of a deepcopy.
+        """
         if identify_stac_object_type(d) == pystac.STACObjectType.ITEM:
             collection_cache = None
             if root is not None:
@@ -244,18 +264,31 @@ def default(cls) -> "StacIO":
 
 class DefaultStacIO(StacIO):
     def read_text(
-        self, source: Union[str, "Link_Type"], *args: Any, **kwargs: Any
+        self, source: Union[str, "Link_Type"], *_: Any, **__: Any
     ) -> str:
+        """A concrete implementation of :meth:`StacIO.read_text <pystac.StacIO.read_text>`. Converts the
+        ``source`` argument to a string (if it is not already) and delegates to
+        :meth:`DefaultStacIO.read_text_from_href` for opening and reading the file."""
         href: Optional[str]
         if isinstance(source, str):
             href = source
         else:
             href = source.get_absolute_href()
             if href is None:
                 raise IOError(f"Could not get an absolute HREF from link {source}")
-        return self.read_text_from_href(href, *args, **kwargs)
+        return self.read_text_from_href(href)
+
+    def read_text_from_href(self, href: str) -> str:
+        """Reads file as a UTF-8 string.
+
+        If ``href`` has a "scheme" (e.g. if it starts with "https://") then this will
+        use :func:`urllib.request.urlopen` to open the file and read the contents;
+        otherwise, :func:`open` will be used to open a local file.
+
+        Args:
 
-    def read_text_from_href(self, href: str, *args: Any, **kwargs: Any) -> str:
+            href : The URI of the file to open.
+        """
         parsed = safe_urlparse(href)
         href_contents: str
         if parsed.scheme != "":
@@ -270,20 +303,33 @@ def read_text_from_href(self, href: str, *args: Any, **kwargs: Any) -> str:
         return href_contents
 
     def write_text(
-        self, dest: Union[str, "Link_Type"], txt: str, *args: Any, **kwargs: Any
+        self, dest: Union[str, "Link_Type"], txt: str, *_: Any, **__: Any
     ) -> None:
+        """A concrete implementation of :meth:`StacIO.write_text <pystac.StacIO.write_text>`. Converts the
+        ``dest`` argument to a string (if it is not already) and delegates to
+        :meth:`DefaultStacIO.write_text_from_href` for opening and reading the file."""
         href: Optional[str]
         if isinstance(dest, str):
             href = dest
         else:
             href = dest.get_absolute_href()
             if href is None:
                 raise IOError(f"Could not get an absolute HREF from link {dest}")
-        return self.write_text_to_href(href, txt, *args, **kwargs)
+        return self.write_text_to_href(href, txt)
 
     def write_text_to_href(
-        self, href: str, txt: str, *args: Any, **kwargs: Any
+        self, href: str, txt: str
     ) -> None:
+        """Writes text to file using UTF-8 encoding.
+
+        This implementation uses :func:`open` and therefore can only write to the local
+        file system.
+
+        Args:
+
+            href : The path to which the file will be written.
+            txt : The string content to write to the file.
+        """
         dirname = os.path.dirname(href)
         if dirname != "" and not os.path.isdir(dirname):
             os.makedirs(dirname)
@@ -292,16 +338,16 @@ def write_text_to_href(
 
 
 class DuplicateKeyReportingMixin(StacIO):
-    """A mixin for StacIO implementations that will report
+    """A mixin for :class:`pystac.StacIO` implementations that will report
     on duplicate keys in the JSON being read in.
 
     See https://github.com/stac-utils/pystac/issues/313
     """
 
     def json_loads(self, txt: str, *_: Any, **__: Any) -> Dict[str, Any]:
-        """Overwrites :meth:`StacIO.json_loads` as the internal method used by
-        :class:`DuplicateKeyReportingMixin` for deserializing a JSON string to a
-        dictionary while checking for duplicate object keys.
+        """Overwrites :meth:`StacIO.json_loads <pystac.StacIO.json_loads>` as the
+        internal method used by :class:`DuplicateKeyReportingMixin` for deserializing
+        a JSON string to a dictionary while checking for duplicate object keys.
 
         Raises:
 
@@ -315,8 +361,9 @@ def json_loads(self, txt: str, *_: Any, **__: Any) -> Dict[str, Any]:
     def read_json(
         self, source: Union[str, "Link_Type"], *args: Any, **kwargs: Any
     ) -> Dict[str, Any]:
-        """Overwrites :meth:`StacIO.read_json` for deserializing a JSON file to a
-        dictionary while checking for duplicate object keys.
+        """Overwrites :meth:`StacIO.read_json <pystac.StacIO.read_json>` for
+        deserializing a JSON file to a dictionary while checking for duplicate object
+        keys.
 
         Raises: