update pre-commit and use ruff for linting/formating (#105)

* update pre-commit and use ruff for linting/formating

* use python 3.11 for lint

* try pydantic 1.

---------

Co-authored-by: Jonathan Healy <jonathan.d.healy@gmail.com>

Author: vincentsarago

.github/workflows/cicd.yaml

@@ -27,7 +27,7 @@ jobs:
           cache-dependency-path: setup.py
 
       - name: Lint code
-        if: ${{ matrix.python-version == 3.8 }}
+        if: ${{ matrix.python-version == 3.11 }}
         run: |
           python -m pip install pre-commit
           pre-commit run --all-files

.pre-commit-config.yaml

@@ -1,57 +1,24 @@
 repos:
   - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
+    rev: 5.13.2
     hooks:
       - id: isort
         language_version: python
 
-  - repo: https://github.com/psf/black
-    rev: 22.12.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.3.5
     hooks:
-      - id: black
-        args: ["--safe"]
-        language_version: python
-
-  - repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
-    hooks:
-      - id: flake8
-        language_version: python
-        args: [
-            # E501 let black handle all line length decisions
-            # W503 black conflicts with "line break before operator" rule
-            # E203 black conflicts with "whitespace before ':'" rule
-            "--ignore=E501,W503,E203,C901",
-          ]
-  - repo: https://github.com/chewse/pre-commit-mirrors-pydocstyle
-    # 2.1.1
-    rev: v2.1.1
-    hooks:
-      - id: pydocstyle
-        language_version: python
-        exclude: ".*(test|scripts).*"
-        args:
-          [
-            # Check for docstring presence only
-            "--select=D1",
-          ]
-          # Don't require docstrings for tests
-          # '--match=(?!test).*\.py']
-  #    -
-  #        repo: https://github.com/pre-commit/mirrors-mypy
-  #        rev: v0.770
-  #        hooks:
-  #            -   id: mypy
-  #                language_version: python3.8
-  #                args: [--no-strict-optional, --ignore-missing-imports]
+      - id: ruff
+        args: ["--fix"]
+      - id: ruff-format
 
-  - repo: https://github.com/PyCQA/pydocstyle
-    rev: 6.3.0
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.9.0
     hooks:
-      - id: pydocstyle
+      - id: mypy
         language_version: python
-        exclude: ".*(test|scripts).*"
-        #args: [
-        # Don't require docstrings for tests
-        #'--match=(?!test|scripts).*\.py',
-        #]
+        exclude: tests/.*
+        additional_dependencies:
+        - types-requests
+        - types-attrs
+        - pydantic~=1.10

pyproject.toml

@@ -1,11 +1,28 @@
-[flake8]
-ignore = "D203"
-exclude = [".git", "__pycache__", "docs/source/conf.py", "build", "dist"]
-max-complexity = 12
-max-line-length = 90
-
 [tool.isort]
 profile = "black"
 known_first_party = "stac_fastapi.pgstac"
 known_third_party = ["rasterio", "stac-pydantic", "sqlalchemy", "geoalchemy2", "fastapi", "stac_fastapi"]
 sections = ["FUTURE", "STDLIB", "THIRDPARTY", "FIRSTPARTY", "LOCALFOLDER"]
+
+[tool.mypy]
+ignore_missing_imports = true
+namespace_packages = true
+explicit_package_bases = true
+exclude = ["tests", ".venv"]
+
+[tool.ruff]
+line-length = 90
+
+[tool.ruff.lint]
+select = [
+        "C",
+        "E",
+        "F",
+        "W",
+        "B",
+]
+ignore = [
+    "E203",  # line too long, handled by black
+    "E501",  # do not perform function calls in argument defaults
+    "B028",  # No explicit `stacklevel` keyword argument found
+]

scripts/ingest_joplin.py

@@ -1,4 +1,5 @@
 """Ingest sample data during docker-compose"""
+
 import json
 import sys
 from pathlib import Path

setup.py

@@ -64,7 +64,5 @@
     install_requires=install_requires,
     tests_require=extra_reqs["dev"],
     extras_require=extra_reqs,
-    entry_points={
-        "console_scripts": ["stac-fastapi-pgstac=stac_fastapi.pgstac.app:run"]
-    },
+    entry_points={"console_scripts": ["stac-fastapi-pgstac=stac_fastapi.pgstac.app:run"]},
 )

stac_fastapi/pgstac/app.py

@@ -46,8 +46,7 @@
 
 if enabled_extensions := os.getenv("ENABLED_EXTENSIONS"):
     extensions = [
-        extensions_map[extension_name]
-        for extension_name in enabled_extensions.split(",")
+        extensions_map[extension_name] for extension_name in enabled_extensions.split(",")
     ]
 else:
     extensions = list(extensions_map.values())
@@ -57,7 +56,7 @@
 api = StacApi(
     settings=settings,
     extensions=extensions,
-    client=CoreCrudClient(post_request_model=post_request_model),
+    client=CoreCrudClient(post_request_model=post_request_model),  # type: ignore
     response_class=ORJSONResponse,
     search_get_request_model=create_get_request_model(extensions),
     search_post_request_model=post_request_model,
@@ -90,8 +89,8 @@ def run():
             reload=settings.reload,
             root_path=os.getenv("UVICORN_ROOT_PATH", ""),
         )
-    except ImportError:
-        raise RuntimeError("Uvicorn must be installed in order to use command")
+    except ImportError as e:
+        raise RuntimeError("Uvicorn must be installed in order to use command") from e
 
 
 if __name__ == "__main__":

stac_fastapi/pgstac/core.py

@@ -1,4 +1,5 @@
 """Item crud client."""
+
 import re
 from typing import Any, Dict, List, Optional, Union
 from urllib.parse import unquote_plus, urljoin
@@ -136,7 +137,7 @@ async def _get_base_item(
 
         return item
 
-    async def _search_base(
+    async def _search_base(  # noqa: C901
         self,
         search_request: PgstacSearch,
         request: Request,
@@ -172,10 +173,10 @@ async def _search_base(
                     req=search_request_json,
                 )
                 items = await conn.fetchval(q, *p)
-        except InvalidDatetimeFormatError:
+        except InvalidDatetimeFormatError as e:
             raise InvalidQueryParameter(
                 f"Datetime parameter {search_request.datetime} is invalid."
-            )
+            ) from e
 
         next: Optional[str] = items.pop("next", None)
         prev: Optional[str] = items.pop("prev", None)
@@ -207,8 +208,8 @@ async def _add_item_links(
                 and all([collection_id, item_id])
             ):
                 feature["links"] = await ItemLinks(
-                    collection_id=collection_id,
-                    item_id=item_id,
+                    collection_id=collection_id,  # type: ignore
+                    item_id=item_id,  # type: ignore
                     request=request,
                 ).get_links(extra_links=feature.get("links"))
 
@@ -255,7 +256,7 @@ async def item_collection(
         bbox: Optional[BBox] = None,
         datetime: Optional[DateTimeType] = None,
         limit: Optional[int] = None,
-        token: str = None,
+        token: Optional[str] = None,
         **kwargs,
     ) -> ItemCollection:
         """Get all items from a specific collection.
@@ -340,7 +341,7 @@ async def post_search(
         item_collection = await self._search_base(search_request, request=request)
         return ItemCollection(**item_collection)
 
-    async def get_search(
+    async def get_search(  # noqa: C901
         self,
         request: Request,
         collections: Optional[List[str]] = None,
@@ -432,5 +433,6 @@ async def get_search(
         except ValidationError as e:
             raise HTTPException(
                 status_code=400, detail=f"Invalid parameters provided {e}"
-            )
+            ) from e
+
         return await self.post_search(search_request, request=request)

stac_fastapi/pgstac/db.py

@@ -2,7 +2,16 @@
 
 import json
 from contextlib import asynccontextmanager, contextmanager
-from typing import AsyncIterator, Callable, Dict, Generator, Literal, Union
+from typing import (
+    AsyncIterator,
+    Callable,
+    Dict,
+    Generator,
+    List,
+    Literal,
+    Optional,
+    Union,
+)
 
 import attr
 import orjson
@@ -36,14 +45,17 @@ async def con_init(conn):
 ConnectionGetter = Callable[[Request, Literal["r", "w"]], AsyncIterator[Connection]]
 
 
-async def connect_to_db(app: FastAPI, get_conn: ConnectionGetter = None) -> None:
+async def connect_to_db(
+    app: FastAPI, get_conn: Optional[ConnectionGetter] = None
+) -> None:
     """Create connection pools & connection retriever on application."""
     settings = app.state.settings
     if app.state.settings.testing:
         readpool = writepool = settings.testing_connection_string
     else:
         readpool = settings.reader_connection_string
         writepool = settings.writer_connection_string
+
     db = DB()
     app.state.readpool = await db.create_pool(readpool, settings)
     app.state.writepool = await db.create_pool(writepool, settings)
@@ -62,15 +74,13 @@ async def get_connection(
     readwrite: Literal["r", "w"] = "r",
 ) -> AsyncIterator[Connection]:
     """Retrieve connection from database conection pool."""
-    pool = (
-        request.app.state.writepool if readwrite == "w" else request.app.state.readpool
-    )
+    pool = request.app.state.writepool if readwrite == "w" else request.app.state.readpool
     with translate_pgstac_errors():
         async with pool.acquire() as conn:
             yield conn
 
 
-async def dbfunc(conn: Connection, func: str, arg: Union[str, Dict]):
+async def dbfunc(conn: Connection, func: str, arg: Union[str, Dict, List]):
     """Wrap PLPGSQL Functions.
 
     Keyword arguments:

stac_fastapi/pgstac/extensions/filter.py

@@ -1,4 +1,5 @@
 """Get Queryables."""
+
 from typing import Any, Optional
 
 from buildpg import render