merge with dev

Author: wolfidan

.github/workflows/pyrad_tests_base.yml

@@ -1,7 +1,7 @@
 name: Test pyrad base
 env:
   MINIMAL_DEPENDENCIES: Cython numpy cartopy
-  TEST_DEPENDENCIES: pytest imageio pygrib  
+  TEST_DEPENDENCIES: pytest imageio pygrib
 
 on:
   # Triggers the workflow on push or pull request events but only for the master branch
@@ -45,7 +45,7 @@ jobs:
       - name: Clone pyart
         uses: actions/checkout@master
         with:
-          repository: ARM-DOE/pyart 
+          repository: ARM-DOE/pyart
           path: pyart
       - name: Clone pyrad-tests
         uses: actions/checkout@master
@@ -67,6 +67,10 @@ jobs:
           python -c "from pyrad.flow import main"
       - name: Run tests
         env:
-           PYRAD_TESTS_PATH: ${{github.workspace}}/pyrad-tests/
+            S3_SECRET_READ: ${{ secrets.S3_SECRET_READ }}
+            S3_KEY_READ: ${{ secrets.S3_KEY_READ }}
+            S3_SECRET_WRITE: ${{ secrets.S3_SECRET_WRITE }}
+            S3_KEY_WRITE: ${{ secrets.S3_KEY_WRITE }}
+            PYRAD_TESTS_PATH: ${{github.workspace}}/pyrad-tests/
         working-directory: ${{github.workspace}}/pyrad-tests/
         run: pytest "run_tests.py::test_base"

.github/workflows/pyrad_tests_base_dev.yml

@@ -42,7 +42,7 @@ jobs:
       - name: Clone pyart
         uses: actions/checkout@master
         with:
-          repository: ARM-DOE/pyart 
+          repository: ARM-DOE/pyart
           path: pyart
       - name: Clone pyrad-tests
         uses: actions/checkout@master
@@ -64,10 +64,13 @@ jobs:
           python -c "from pyrad.flow import main"
       - name: Run tests
         env:
-           PYRAD_TESTS_PATH: ${{github.workspace}}/pyrad-tests/
+            S3_SECRET_READ: ${{ secrets.S3_SECRET_READ }}
+            S3_KEY_READ: ${{ secrets.S3_KEY_READ }}
+            S3_SECRET_WRITE: ${{ secrets.S3_SECRET_WRITE }}
+            S3_KEY_WRITE: ${{ secrets.S3_KEY_WRITE }}
+            PYRAD_TESTS_PATH: ${{github.workspace}}/pyrad-tests/
         working-directory: ${{github.workspace}}/pyrad-tests/
         run: pytest "run_tests.py::test_base"
       - name: tmate session if tests fail
         if:  ${{ github.event_name == 'workflow_dispatch' && failure() }}
         uses: mxschmitt/action-tmate@v3
-        

.github/workflows/pyrad_tests_mch_dev.yml

@@ -6,7 +6,7 @@ on:
     branches: [ dev ]
   pull_request:
     branches: [ dev ]
-          
+
 jobs:
   unit_tests:
     name: Unit Tests (${{ matrix.python-version }}, ${{ matrix.os }})
@@ -65,12 +65,11 @@ jobs:
           # Insert tmate session here
       - name: Run tests
         env:
-           PYRAD_TESTS_PATH: ${{github.workspace}}/pyrad-tests/
-           PSRLIB_PATH: ${{github.workspace}}/src/libDX50/lib/
-           PYART_CONFIG: ${{github.workspace}}/config/pyart/mch_config.py
+            PYRAD_TESTS_PATH: ${{github.workspace}}/pyrad-tests/
+            PSRLIB_PATH: ${{github.workspace}}/src/libDX50/lib/
+            PYART_CONFIG: ${{github.workspace}}/config/pyart/mch_config.py
         working-directory: ${{github.workspace}}/pyrad-tests//
         run: pytest "run_tests.py::test_mch"
       - name: tmate session if tests fail
         if:  ${{ github.event_name == 'workflow_dispatch' && failure() }}
         uses: mxschmitt/action-tmate@v3
-        

.pre-commit-config.yaml

@@ -0,0 +1,19 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0  # Use the latest stable version
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: v0.9.3  # Use the latest stable version of ruff-pre-commit
+    hooks:
+      - id: ruff
+        args: ["--fix"]
+        files: "src/pyrad_proc/pyrad"  # Automatically fix issues when possible
+
+  - repo: https://github.com/psf/black
+    rev: 23.1.0
+    hooks:
+      - id: black

ci/environment-ci.yml

@@ -14,6 +14,7 @@ dependencies:
   - cvxopt
   - wradlib
   - xarray<2024.10.0
+  - boto3
   - xradar
   - metpy
   - pytest-cov

doc/source/overview/main.rst

@@ -16,21 +16,24 @@ locationConfigFile    STRING   File name (with full path) of the location config
 productConfigFile     STRING   File name (with full path) of the product configuration file. Described in Section 4.
 lastStateFile         STRING   File name (with full path) of the file containing the time of the last processed scan. Used in particular for real-time processing.
 imgformat             STRING/  File format(s) of the images. The following formats are supported: eps, png, and jpg. If ``saveimg`` is set to 0, this field is not used.
-                      STRARR   
+                      STRARR
 saveimgbasepath       STRING   Base directory for the images to save. The directory structure looks as follows:
                                ``<saveimgbasepath>/<name>/<YYYY-MM-DD>/<datasetname>/<prodname>/<outputname>``
                                If ``saveimg`` is set to 0, this field is not used.
-s3copypath	      STRING   OPTIONAL. Path to an S3 bucket. If provided all generated products will be written there as well using the same data structure. The format must be                                          https://bucket_name.endpoint.domain, for example https://tests.fr-par-1.linodeobjects.com/. The S3 copy procedure will only work if the 
-                               environment variables AWS_KEY and AWS_SECRET are defined in the pyrad scope. AWS_KEY contains the S3 bucket AWS key and AWS_SECRET the associated secret.
+s3EndpointRead        STRING   OPTIONAL. Url to an S3 endpoint containing input radar data. The format must be https://endpoint.domain (e.g. https://eu-central-1.linodeobjects.com/), the https:// is not mandatory.
+s3BucketRead          STRING   OPTIONAL. Name of an S3 bucket containing input radar data. It has to be used together with ``s3EndpointRead`` and ``s3PathRead`` to be able to read input radar data from a bucket. The procedure will only work by setting the environment variables S3_KEY_READ and S3_SECRET_READ.
+s3PathRead            STRING   OPTIONAL. Path where to find input radar data in an S3 bucket. The data will be retrieved at url https://s3BucketRead.s3EndpointRead/s3PathRead/*
+rm_s3_file            INT      OPTIONAL. If set input radar data files downloaded from and S3 bucket will be removed after reading.
+s3EndpointWrite       STRING   OPTIONAL. Url to an S3 endpoint where to store output data. The format must be https://endpoint.domain (e.g. https://eu-central-1.linodeobjects.com/), the https:// is not mandatory.
+s3BucketWrite         STRING   OPTIONAL. Name of an S3 bucket containing input radar data. It has to be used together with ``s3EndpointWrite`` and ``s3PathWrite`` to be able to save output data to a bucket. The procedure will only work by setting the environment variables S3_KEY_WRITE and S3_SECRET_WRITE.
+s3PathWrite           STRING   OPTIONAL. Path where to save output radar data in an S3 bucket. The data will be saved at url https://s3BucketRead.s3EndpointRead/s3PathRead/filename
 loadbasepath          STRING   OPTIONAL. Base path of saved data. By default, this field is set to ``saveimgbasepath``.
 loadname              STRING   OPTIONAL. Name of the saved data processing. Used for saved volume loading. By default, this field is set to ``name``.
-gecsxbasepath          STRING   OPTIONAL. Base path of saved visibility fields generated by the GECSX tool
-gecsxname              STRING   OPTIONAL. Name of the dataset generated by GEXSX in which the visibility fields are contained. Used for GECSX volume loading. 
+gecsxbasepath         STRING   OPTIONAL. Base path of saved visibility fields generated by the GECSX tool
+gecsxname             STRING   OPTIONAL. Name of the dataset generated by GEXSX in which the visibility fields are contained. Used for GECSX volume loading.
 dempath               STRING   OPTIONAL. Base directory of the Digital Elevation Model (DEM) files. Basically to load the radar visibility (Optional).
 smnpath               STRING   OPTIONAL. Base directory of the SwissMetNet stations data. Used in the comparison between radar data and rain gauges (Optional).
 disdropath            STRING   OPTIONAL. Base directory of the disdrometer data. Used in the comparison between radar data and disdrometers (Optional).
 solarfluxpath         STRING   OPTIONAL. Base directory of the solar flux data. Used to plot the calibration bias based on sun monitoring (Optional).
 iconpath              STRING   OPTIONAL. Base directory of the ICON (NWP model) data files.
 ====================  =======  =======================================================================================
-
-

src/pyart

@@ -1000,16 +1000,24 @@ def _generate_prod(dataset, cfg, prdname, prdfunc, dsname, voltime, runinfo=None
         filenames = prdfunc(dataset, prdcfg)
         if isinstance(filenames, str):  # convert to list if needed
             filenames = [filenames]
-        if "s3copypath" in prdcfg and filenames is not None:  # copy to S3
-            s3AccessPolicy = (
-                prdcfg["s3AccessPolicy"] if "s3AccessPolicy" in prdcfg else None
-            )
+        if (
+            "s3BucketWrite" in prdcfg
+            and "s3EndpointWrite" in prdcfg
+            and filenames is not None
+        ):  # copy to S3
+            s3AccessPolicy = prdcfg.get("s3AccessPolicy", None)
+            s3path = prdcfg.get("s3PathWrite", None)
             for fname in filenames:
                 if (
                     prdcfg["basepath"] in fname
                 ):  # only products saved to standard basepath
                     write_to_s3(
-                        fname, prdcfg["basepath"], prdcfg["s3copypath"], s3AccessPolicy
+                        fname,
+                        prdcfg["basepath"],
+                        prdcfg["s3EndpointWrite"],
+                        prdcfg["s3BucketWrite"],
+                        s3path,
+                        s3AccessPolicy,
                     )
         return False
     except Exception as inst:
@@ -1221,6 +1229,9 @@ def _create_cfg_dict(cfgfile):
     if "MFScale" not in cfg:
         cfg.update({"MFScale": 0})
 
+    if not cfg["datapath"]:  # empty datapath in case of s3 reading
+        cfg["datapath"] = ["" for rad in range(cfg["NumRadars"])]
+
     # parameters necessary to read correctly MF grid binary files
     if "BinFileParams" not in cfg:
         bin_file_params = {
@@ -1333,35 +1344,35 @@ def _create_datacfg_dict(cfg):
     datacfg.update({"DataTypeIDInFiles": cfg["DataTypeIDInFiles"]})
 
     # s3 buckets
-    if "bucket" in cfg:
+    if "s3BucketRead" in cfg:
         try:
-            datacfg["s3_key"] = os.environ["S3_IN_KEY"]
-            datacfg["s3_secret_key"] = os.environ["S3_IN_SECRET"]
+            datacfg["s3KeyRead"] = os.environ["S3_KEY_READ"]
+            datacfg["s3SecretRead"] = os.environ["S3_SECRET_READ"]
         except KeyError:
             warn(
-                "Define environment variables S3_IN_KEY and S3_IN_SECRET"
+                "Define environment variables S3_KEY_READ and S3_SECRET_READ"
                 " to get input data from S3 buckets."
             )
 
-        if "s3path" in cfg:
-            datacfg.update({"s3path": cfg["s3path"]})
+        if "s3PathRead" in cfg:
+            datacfg.update({"s3PathRead": cfg["s3PathRead"]})
         else:
-            warn("Unable to read data from s3 bucket. Define s3path")
-        if "s3_url" in cfg:
-            datacfg.update({"s3_url": cfg["s3_url"]})
+            warn("Unable to read data from s3 bucket. Define s3PathRead")
+        if "s3EndpointRead" in cfg:
+            datacfg.update({"s3EndpointRead": cfg["s3EndpointRead"]})
         else:
-            warn("Unable to read data from s3 bucket. Define s3_url")
+            warn("Unable to read data from s3 bucket. Define s3EndpointRead")
 
         if "rm_s3_file" in cfg:
             datacfg.update({"rm_s3_file": cfg["rm_s3_file"]})
 
         if (
-            "s3path" in datacfg
-            and "s3_url" in datacfg
-            and "s3_key" in datacfg
-            and "s3_secret_key" in datacfg
+            "s3PathRead" in datacfg
+            and "s3EndpointRead" in datacfg
+            and "s3KeyRead" in datacfg
+            and "s3SecretRead" in datacfg
         ):
-            datacfg.update({"bucket": cfg["bucket"]})
+            datacfg.update({"s3BucketRead": cfg["s3BucketRead"]})
 
     # Modify size of radar or radar spectra object
     datacfg.update({"elmin": cfg.get("elmin", None)})
@@ -1582,11 +1593,14 @@ def _create_prdcfg_dict(cfg, dataset, product, voltime, runinfo=None):
     prdcfg.update({"imgformat": cfg["imgformat"]})
     prdcfg.update({"RadarName": cfg["RadarName"]})
 
-    if "s3copypath" in cfg:
-        prdcfg.update({"s3copypath": cfg["s3copypath"]})
+    if "s3EndpointWrite" in cfg:
+        prdcfg.update({"s3EndpointWrite": cfg["s3EndpointWrite"]})
+    if "s3BucketWrite" in cfg:
+        prdcfg.update({"s3BucketWrite": cfg["s3BucketWrite"]})
+    if "s3PathWrite" in cfg:
+        prdcfg.update({"s3PathWrite": cfg["s3PathWrite"]})
     if "s3AccessPolicy" in cfg:
         prdcfg.update({"s3AccessPolicy": cfg["s3AccessPolicy"]})
-
     if "RadarBeamwidth" in cfg:
         prdcfg.update({"RadarBeamwidth": cfg["RadarBeamwidth"]})
     if "ppiImageConfig" in cfg:
@@ -1641,9 +1655,13 @@ def _get_datatype_list(cfg, radarnr="RADAR001"):
         if "datatype" not in cfg[dataset]:
             continue
         if isinstance(cfg[dataset]["datatype"], str):
-            (radarnr_descr, datagroup, datatype_aux, dataset_save, product_save) = (
-                get_datatype_fields(cfg[dataset]["datatype"])
-            )
+            (
+                radarnr_descr,
+                datagroup,
+                datatype_aux,
+                dataset_save,
+                product_save,
+            ) = get_datatype_fields(cfg[dataset]["datatype"])
             if datagroup != "PROC" and radarnr_descr == radarnr:
                 if (dataset_save is None) and (product_save is None):
                     datatypesdescr.add(
@@ -1667,9 +1685,13 @@ def _get_datatype_list(cfg, radarnr="RADAR001"):
                     )
         else:
             for datatype in cfg[dataset]["datatype"]:
-                (radarnr_descr, datagroup, datatype_aux, dataset_save, product_save) = (
-                    get_datatype_fields(datatype)
-                )
+                (
+                    radarnr_descr,
+                    datagroup,
+                    datatype_aux,
+                    dataset_save,
+                    product_save,
+                ) = get_datatype_fields(datatype)
                 if datagroup != "PROC" and radarnr_descr == radarnr:
                     if dataset_save is None and product_save is None:
                         datatypesdescr.add(
@@ -1799,7 +1821,7 @@ def _get_masterfile_list(datatypesdescr, starttimes, endtimes, datacfg, scan_lis
         )
         return [], None, None
 
-    if "bucket" in datacfg:
+    if "s3BucketRead" in datacfg:
         masterfilelist = get_file_list_s3(
             masterdatatypedescr, starttimes, endtimes, datacfg, scan=masterscan
         )