Update pre-commit and ci configuration

Author: sandorkertesz

.github/workflows/ci.yml

@@ -4,20 +4,29 @@ on:
   # Trigger the workflow on push to master or develop, except tag creation
   push:
     branches:
-    - 'main'
-    - 'develop'
+    - "main"
+    - "develop"
     tags-ignore:
-    - '**'
+    - "**"
+    paths-ignore:
+    - "docs/**"
+    - "README.md"
 
   # Trigger the workflow on pull request
   pull_request:
+    paths-ignore:
+    - "docs/**"
+    - "README.md"
 
   # Trigger the workflow manually
   workflow_dispatch:
 
   # Trigger after public PR approved for CI
   pull_request_target:
     types: [labeled]
+    paths-ignore:
+    - "docs/**"
+    - "README.md"
 
 jobs:
   # Run CI including downstream packages on self-hosted runners
@@ -28,10 +37,9 @@ jobs:
     with:
       earthkit-meteo: ecmwf/earthkit-meteo@${{ github.event.pull_request.head.sha || github.sha }}
       codecov_upload: true
-      python_qa: true
+      python_qa: false
     secrets: inherit
 
-
   # Build downstream packages on HPC
   downstream-ci-hpc:
     name: downstream-ci-hpc

.github/workflows/legacy-ci.yml

@@ -24,18 +24,6 @@ defaults:
     shell: bash -l {0}
 
 jobs:
-  pre-commit:
-    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        ref: ${{ github.event.pull_request.head.sha || github.ref }}
-    - uses: actions/setup-python@v4
-      with:
-        python-version: 3.x
-    - uses: pre-commit/action@v3.0.0
-
   documentation:
     if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
     runs-on: ubuntu-latest
@@ -48,7 +36,7 @@ jobs:
       with:
         ref: ${{ github.event.pull_request.head.sha || github.ref }}
     - name: Install Conda environment with Micromamba
-      uses: mamba-org/provision-with-micromamba@v12
+      uses: mamba-org/setup-micromamba@v1
       with:
         environment-file: tests/environment-unit-tests.yml
         environment-name: DEVELOP

.github/workflows/python-pill-request.yml

@@ -0,0 +1,12 @@
+name: Code Quality checks for PRs
+
+on:
+  push:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  quality:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-precommit-run.yml@v2
+    with:
+      skip-hooks: "no-commit-to-branch"

.pre-commit-config.yaml

@@ -1,32 +1,48 @@
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.4.0
+  rev: v5.0.0
   hooks:
-  - id: trailing-whitespace
-  - id: end-of-file-fixer
+  - id: trailing-whitespace     # Trailing whitespace checker
+  - id: end-of-file-fixer     # Ensure files end in a newline
   - id: check-json
-  - id: check-yaml
+  - id: check-yaml     # Check YAML files for syntax errors only
+    args: [--unsafe, --allow-multiple-documents]
   - id: check-toml
-  # - id: check-added-large-files
-  - id: debug-statements
+      # - id: check-added-large-files
+  - id: debug-statements     # Check for debugger imports and py37+ breakpoint()
   - id: mixed-line-ending
+  - id: no-commit-to-branch     # Prevent committing to main / master
+  - id: check-merge-conflict     # Check for files that contain merge conflict
+    exclude: /README\.rst$|^docs/.*\.rst$
 - repo: https://github.com/PyCQA/isort
-  rev: 5.12.0
+  rev: 5.13.2
   hooks:
   - id: isort
+    args:
+    - -l 110
+    - --force-single-line-imports
+    - --profile black
 - repo: https://github.com/psf/black
-  rev: 23.9.1
+  rev: 24.8.0
   hooks:
   - id: black
+    args: [--line-length=110]
 - repo: https://github.com/keewis/blackdoc
   rev: v0.3.8
   hooks:
   - id: blackdoc
     additional_dependencies: [black==23.3.0]
-- repo: https://github.com/PyCQA/flake8
-  rev: 6.1.0
-  hooks:
-  - id: flake8
+    exclude: xr_engine_profile_rst\.py
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.6.9
+  hooks:
+  - id: ruff
+    exclude: '(dev/.*|.*_)\.py$'
+    args:
+    - --line-length=110
+    - --fix
+    - --exit-non-zero-on-fix
+    - --preview
 - repo: https://github.com/executablebooks/mdformat
   rev: 0.7.14
   hooks:
@@ -37,11 +53,11 @@ repos:
   hooks:
   - id: pretty-format-yaml
     args: [--autofix, --preserve-quotes]
-  - id: pretty-format-toml
-    args: [--autofix]
-- repo: https://github.com/PyCQA/pydocstyle.git
-  rev: 6.1.1
-  hooks:
-  - id: pydocstyle
-    additional_dependencies: [toml]
-    exclude: tests|docs
+- repo: https://github.com/sphinx-contrib/sphinx-lint
+  rev: v1.0.0
+  hooks:
+  - id: sphinx-lint
+- repo: https://github.com/tox-dev/pyproject-fmt
+  rev: "2.2.4"
+  hooks:
+  - id: pyproject-fmt

pyproject.toml

@@ -1,58 +1,56 @@
 [build-system]
-requires = ["setuptools>=61", "setuptools-scm>=8.0"]
+requires = [ "setuptools>=61", "setuptools-scm>=8" ]
 
 [project]
+name = "earthkit-meteo"
+description = "Meteorological computations"
+readme = "README.md"
+license = { text = "Apache License Version 2.0" }
 authors = [
-  {name = "European Centre for Medium-Range Weather Forecasts (ECMWF)", email = "software.support@ecmwf.int"}
+  { name = "European Centre for Medium-Range Weather Forecasts (ECMWF)", email = "software.support@ecmwf.int" },
 ]
+requires-python = ">=3.8"
+
 classifiers = [
   "Development Status :: 2 - Pre-Alpha",
   "Intended Audience :: Developers",
   "License :: OSI Approved :: Apache Software License",
-  "Programming Language :: Python :: 3",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3 :: Only",
   "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
   "Programming Language :: Python :: Implementation :: CPython",
   "Programming Language :: Python :: Implementation :: PyPy",
-  "Operating System :: OS Independent"
 ]
+dynamic = [ "version" ]
 dependencies = [
-  "numpy"
+  "numpy",
 ]
-description = "Meteorological computations"
-dynamic = ["version"]
-license = {text = "Apache License Version 2.0"}
-name = "earthkit-meteo"
-readme = "README.md"
-requires-python = ">= 3.8"
-
-[project.optional-dependencies]
-test = [
+optional-dependencies.test = [
   "pytest",
-  "pytest-cov"
+  "pytest-cov",
 ]
+urls.Documentation = "https://earthkit-meteo.readthedocs.io/"
+urls.Homepage = "https://github.com/ecmwf/earthkit-meteo/"
+urls.Issues = "https://github.com/ecmwf/earthkit-meteo.issues"
+urls.Repository = "https://github.com/ecmwf/earthkit-meteo/"
 
-[project.urls]
-Documentation = "https://earthkit-meteo.readthedocs.io/"
-Homepage = "https://github.com/ecmwf/earthkit-meteo/"
-Issues = "https://github.com/ecmwf/earthkit-meteo.issues"
-Repository = "https://github.com/ecmwf/earthkit-meteo/"
+[tool.setuptools.packages.find]
+include = [ "earthkit.meteo" ]
+where = [ "src/" ]
 
-[tool.coverage.run]
-branch = "true"
+[tool.setuptools_scm]
+version_file = "src/earthkit/meteo/_version.py"
 
 [tool.isort]
 profile = "black"
 
+[tool.coverage.run]
+branch = "true"
+
 [tool.pydocstyle]
-add_ignore = ["D1", "D200", "D205", "D400", "D401"]
+add_ignore = [ "D1", "D200", "D205", "D400", "D401" ]
 convention = "numpy"
-
-[tool.setuptools.packages.find]
-include = ["earthkit.meteo"]
-where = ["src/"]
-
-[tool.setuptools_scm]
-version_file = "src/earthkit/meteo/_version.py"

src/earthkit/meteo/extreme/array/cpf.py

@@ -64,10 +64,9 @@ def cpf(clim, ens, sort_clim=True, sort_ens=True):
                     idx = (qv_f < qv_c) & (qv_c_2 < qv_c)
 
                     # intersection between two lines
-                    tau_i = (
-                        tau_c * (qv_c_2[idx] - qv_f[idx])
-                        + tau_c_2 * (qv_f[idx] - qv_c[idx])
-                    ) / (qv_c_2[idx] - qv_c[idx])
+                    tau_i = (tau_c * (qv_c_2[idx] - qv_f[idx]) + tau_c_2 * (qv_f[idx] - qv_c[idx])) / (
+                        qv_c_2[idx] - qv_c[idx]
+                    )
 
                     # populate matrix, no values below 0
                     cpf[idx] = np.maximum(tau_i, 0)
@@ -85,10 +84,9 @@ def cpf(clim, ens, sort_clim=True, sort_ens=True):
 
                     idx = (qv_f > qv_c) & (qv_c_2 > qv_c) & (~mask)
 
-                    tau_i = (
-                        tau_c * (qv_c_2[idx] - qv_f[idx])
-                        + tau_c_2 * (qv_f[idx] - qv_c[idx])
-                    ) / (qv_c_2[idx] - qv_c[idx])
+                    tau_i = (tau_c * (qv_c_2[idx] - qv_f[idx]) + tau_c_2 * (qv_f[idx] - qv_c[idx])) / (
+                        qv_c_2[idx] - qv_c[idx]
+                    )
 
                     # populate matrix, no values above 1
                     cpf[idx] = np.minimum(tau_i, 1)

src/earthkit/meteo/extreme/array/efi.py

@@ -31,9 +31,7 @@ def efi(clim, ens, eps=-0.1):
         EFI values
     """
     # locate missing values
-    missing_mask = np.logical_or(
-        np.sum(np.isnan(clim), axis=0), np.sum(np.isnan(ens), axis=0)
-    )
+    missing_mask = np.logical_or(np.sum(np.isnan(clim), axis=0), np.sum(np.isnan(ens), axis=0))
 
     # Compute fraction of the forecast below climatology
     nclim, npoints = clim.shape
@@ -65,9 +63,7 @@ def efi(clim, ens, eps=-0.1):
             mask = clim[icl + 1, :] > eps
             dEFI = np.where(
                 mask,
-                (2.0 * frac[icl, :] - 1.0) * acosdiff[icl]
-                + acoef[icl] * dFdp[icl, :]
-                - proddiff[icl],
+                (2.0 * frac[icl, :] - 1.0) * acosdiff[icl] + acoef[icl] * dFdp[icl, :] - proddiff[icl],
                 0.0,
             )
             defimax = np.where(mask, -acosdiff[icl] - proddiff[icl], 0.0)
@@ -77,11 +73,7 @@ def efi(clim, ens, eps=-0.1):
         efi /= efimax
     else:
         for icl in range(nclim - 1):
-            dEFI = (
-                (2.0 * frac[icl, :] - 1.0) * acosdiff[icl]
-                + acoef[icl] * dFdp[icl, :]
-                - proddiff[icl]
-            )
+            dEFI = (2.0 * frac[icl, :] - 1.0) * acosdiff[icl] + acoef[icl] * dFdp[icl, :] - proddiff[icl]
             efi += dEFI
         efi *= 2.0 / np.pi
     ##################################

src/earthkit/meteo/extreme/array/sot.py

@@ -39,9 +39,7 @@ def sot_func(qc_tail, qc, qf, eps=-1e-4, lower_bound=-10, upper_bound=10):
     err = np.seterr(divide="ignore", invalid="ignore")
 
     min_den = np.fmax(eps, 0)
-    sot = np.where(
-        np.fabs(qc_tail - qc) > min_den, (qf - qc_tail) / (qc_tail - qc), np.nan
-    )
+    sot = np.where(np.fabs(qc_tail - qc) > min_den, (qf - qc_tail) / (qc_tail - qc), np.nan)
 
     # revert to original error state
     np.seterr(**err)
@@ -78,20 +76,12 @@ def sot(clim, ens, perc, eps=-1e4):
     numpy array (npoints)
         SOT values
     """
-    if not (isinstance(perc, int) or isinstance(perc, np.int64)) or (
-        perc < 2 or perc > 98
-    ):
-        raise Exception(
-            "Percentile value should be and Integer between 2 and 98, is {}".format(
-                perc
-            )
-        )
+    if not (isinstance(perc, int) or isinstance(perc, np.int64)) or (perc < 2 or perc > 98):
+        raise Exception("Percentile value should be and Integer between 2 and 98, is {}".format(perc))
 
     if clim.shape[0] != 101:
         raise Exception(
-            "Climatology array should contain 101 percentiles, it has {} values".format(
-                clim.shape
-            )
+            "Climatology array should contain 101 percentiles, it has {} values".format(clim.shape)
         )
 
     qc = clim[perc]
@@ -136,20 +126,12 @@ def sot_unsorted(clim, ens, perc, eps=-1e4):
     numpy array (npoints)
         SOT values
     """
-    if not (isinstance(perc, int) or isinstance(perc, np.int64)) or (
-        perc < 2 or perc > 98
-    ):
-        raise Exception(
-            "Percentile value should be and Integer between 2 and 98, is {}".format(
-                perc
-            )
-        )
+    if not (isinstance(perc, int) or isinstance(perc, np.int64)) or (perc < 2 or perc > 98):
+        raise Exception("Percentile value should be and Integer between 2 and 98, is {}".format(perc))
 
     if clim.shape[0] != 101:
         raise Exception(
-            "Climatology array should contain 101 percentiles, it has {} values".format(
-                clim.shape
-            )
+            "Climatology array should contain 101 percentiles, it has {} values".format(clim.shape)
         )
 
     if eps > 0:

src/earthkit/meteo/score/array/crps.py

@@ -48,9 +48,7 @@ def crps(x, y):
     alpha[-1] = np.fmax(-diffxy[-1], 0)  # y-x(n)
     beta[-1] = 0
     # else
-    alpha[1:-1] = np.fmin(
-        diffxx, np.fmax(-diffxy[:-1], 0)
-    )  # x(i+1)-x(i) or y-x(i) or 0
+    alpha[1:-1] = np.fmin(diffxx, np.fmax(-diffxy[:-1], 0))  # x(i+1)-x(i) or y-x(i) or 0
     beta[1:-1] = np.fmin(diffxx, np.fmax(diffxy[1:], 0))  # 0 or x(i+1)-y or x(i+1)-x(i)
 
     # compute crps