Merge branch 'dev'

Author: wolfidan

ci/parse_pyrad_name_mappings.py

@@ -5,43 +5,48 @@
 
 from pyrad.io import io_aux
 
-FUNCTIONS_TO_PARSE = ['get_fieldname_pyart',
-                      'get_datatype_odim',
-                      'get_datatype_metranet',
-                      'get_fieldname_icon']
+FUNCTIONS_TO_PARSE = [
+    "get_datatype_odim",
+    "get_datatype_metranet",
+    "get_fieldname_icon",
+]
 
 mainpath = Path(__file__).resolve().parent.parent
-OUT_DIRECTORY = str(Path(mainpath, 'doc', 'source', 'overview', 'mappings'))
+OUT_DIRECTORY = str(Path(mainpath, "doc", "source", "overview", "mappings"))
 
 
 for fun_name in FUNCTIONS_TO_PARSE:
-    print('Parsing function {:s}'.format(fun_name))
+    print("Parsing function {:s}".format(fun_name))
     fun = getattr(io_aux, fun_name)
-    mapping = fun_name.split('_')[-1]
+    mapping = fun_name.split("_")[-1]
     pyrad_dtypes = []
     srccode = inspect.getsource(fun)
-    srccode = srccode.split('\n')
+    srccode = srccode.split("\n")
     for line in srccode:
-        if ('datatype' in line or 'field_name' in line) and '==' in line:
-            pyrad_dtypes.append(line.split('==')[1].split(':')[
-                                0].strip().replace('"', '').replace("'",""))
-        if 'return' in line:
-            returnline = line.replace('return', '').strip()
+        if ("datatype" in line or "field_name" in line) and "==" in line:
+            pyrad_dtypes.append(
+                line.split("==")[1]
+                .split(":")[0]
+                .strip()
+                .replace('"', "")
+                .replace("'", "")
+            )
+        if "return" in line:
+            returnline = line.replace("return", "").strip()
 
     # Try to get output types from return statements of srccode
     all_values = []
     all_keys = []
-    if '{' and '}' in returnline:
-        keyname, valname = returnline.replace(
-            '{', '').replace('}', '').split(':')
+    if "{" and "}" in returnline:
+        keyname, valname = returnline.replace("{", "").replace("}", "").split(":")
         keyname = keyname.strip()
         valname = valname.strip()
         if mapping not in keyname:
-            keyname += '_' + mapping
+            keyname += "_" + mapping
     else:
         valname = returnline.strip()
     if mapping not in valname:
-        valname += '_' + mapping
+        valname += "_" + mapping
 
     for v in pyrad_dtypes:
         out = fun(v)
@@ -52,11 +57,22 @@
             all_values.append(out)
 
     dic = {}
-    dic['pyrad_name'] = pyrad_dtypes
+    dic["pyrad_name"] = pyrad_dtypes
     if len(all_keys):
         dic[keyname] = all_keys
     dic[valname] = all_values
-
     df = pd.DataFrame(dic)
-    df.to_csv(str(Path(OUT_DIRECTORY, 'pyrad_to_{:s}.txt'.format(mapping))),
-              index=False)
+    df.to_csv(
+        str(Path(OUT_DIRECTORY, "pyrad_to_{:s}.txt".format(mapping))), index=False
+    )
+
+print("Parsing function pyrad_to_pyart")
+# Treat get_datatype_pyart separately
+from pyrad.io.io_aux import pyrad_to_pyart_keys_dict
+
+dic = {
+    "pyrad_name": list(pyrad_to_pyart_keys_dict.keys()),
+    "pyart_name": list(pyrad_to_pyart_keys_dict.values()),
+}
+df = pd.DataFrame(dic)
+df.to_csv(str(Path(OUT_DIRECTORY, "pyrad_to_pyart.txt")), index=False)

config/pyart/mch_config.py

@@ -236,7 +236,8 @@
 elevation_angle = "elevation_angle"
 visibility = "visibility"
 min_vis_altitude = "min_vis_altitude"
-min_vis_altitude_above_ground = "min_vis_altitude_above_ground"
+min_vis_height_above_ground = "min_vis_height_above_ground"
+min_rad_vis_height_above_ground = "min_rad_vis_height_above_ground"
 min_vis_elevation = "min_vis_elevation"
 incident_angle = "incident_angle"
 effective_area = "effective_area"
@@ -564,7 +565,8 @@
     "elevation_angle": elevation_angle,
     "visibility": visibility,
     "min_vis_altitude": min_vis_altitude,
-    "min_vis_altitude_above_ground": min_vis_altitude_above_ground,
+    "min_vis_height_above_ground": min_vis_height_above_ground,
+    "min_rad_vis_height_above_ground": min_rad_vis_height_above_ground,
     "min_vis_elevation": min_vis_elevation,
     "incident_angle": incident_angle,
     "effective_area": effective_area,
@@ -1646,10 +1648,16 @@
         "long_name": "Minimum visible altitude",
         "coordinates": "x y",
     },
-    min_vis_altitude_above_ground: {
+    min_vis_height_above_ground: {
         "units": "meters",
-        "standard_name": "min_vis_altitude_above_ground",
-        "long_name": "Minimum visible altitude above ground",
+        "standard_name": "min_vis_height_above_ground",
+        "long_name": "Minimum visible height above ground",
+        "coordinates": "x y",
+    },
+    min_rad_vis_height_above_ground: {
+        "units": "meters",
+        "standard_name": "min_rad_vis_height_above_ground",
+        "long_name": "Minimum radar visible height above ground",
         "coordinates": "x y",
     },
     min_vis_elevation: {

doc/source/overview/main.rst

@@ -27,6 +27,7 @@ rm_s3_file            INT      OPTIONAL. If set input radar data files downloade
 s3EndpointWrite       STRING   OPTIONAL. Url to an S3 endpoint where to store output data. The format must be https://endpoint.domain (e.g. https://eu-central-1.linodeobjects.com/), the https:// is not mandatory.
 s3BucketWrite         STRING   OPTIONAL. Name of an S3 bucket containing input radar data. It has to be used together with ``s3EndpointWrite`` and ``s3PathWrite`` to be able to save output data to a bucket. The procedure will only work by setting the environment variables S3_KEY_WRITE and S3_SECRET_WRITE.
 s3PathWrite           STRING   OPTIONAL. Path where to save output radar data in an S3 bucket. The data will be saved at url https://s3BucketRead.s3EndpointRead/s3PathRead/filename
+s3SplitExtensionWrite INT      OPTIONAL. If set to 1, if files with multiple file extensions are located in the same directory, they will be separated on the s3 according to their file extension, for example myfolder/out.csv will be stored as myfolder/csv/out.csv on the S3. Default is 0 (False).
 loadbasepath          STRING   OPTIONAL. Base path of saved data. By default, this field is set to ``saveimgbasepath``.
 loadname              STRING   OPTIONAL. Name of the saved data processing. Used for saved volume loading. By default, this field is set to ``name``.
 gecsxbasepath         STRING   OPTIONAL. Base path of saved visibility fields generated by the GECSX tool

src/make_all.sh

@@ -16,4 +16,4 @@ echo 'Building PyTDA...'
 ./make_pytda.sh
 
 echo 'Building Pyrad...'
-./make_pyrad.sh
+./make_pyrad.sh

src/pyart

@@ -1007,6 +1007,15 @@ def _generate_prod(dataset, cfg, prdname, prdfunc, dsname, voltime, runinfo=None
         ):  # copy to S3
             s3AccessPolicy = prdcfg.get("s3AccessPolicy", None)
             s3path = prdcfg.get("s3PathWrite", None)
+            s3splitext = bool(prdcfg.get("s3SplitExtensionWrite", 0))
+            if s3splitext:
+                nextensions = set([os.path.splitext(f)[1] for f in filenames])
+            else:
+                nextensions = 1
+            
+            if nextensions == 1:
+                s3splitext = False
+                
             for fname in filenames:
                 if (
                     prdcfg["basepath"] in fname
@@ -1018,6 +1027,7 @@ def _generate_prod(dataset, cfg, prdname, prdfunc, dsname, voltime, runinfo=None
                         prdcfg["s3BucketWrite"],
                         s3path,
                         s3AccessPolicy,
+                        s3splitext
                     )
         return False
     except Exception as inst:
@@ -1599,6 +1609,8 @@ def _create_prdcfg_dict(cfg, dataset, product, voltime, runinfo=None):
         prdcfg.update({"s3BucketWrite": cfg["s3BucketWrite"]})
     if "s3PathWrite" in cfg:
         prdcfg.update({"s3PathWrite": cfg["s3PathWrite"]})
+    if "s3SplitExtensionWrite" in cfg:
+        prdcfg.update({"s3SplitExtensionWrite": cfg["s3SplitExtensionWrite"]})
     if "s3AccessPolicy" in cfg:
         prdcfg.update({"s3AccessPolicy": cfg["s3AccessPolicy"]})
     if "RadarBeamwidth" in cfg:

src/pyrad_proc/pyrad/flow/flow_aux.py

@@ -31,7 +31,6 @@
 import pyart
 import matplotlib.pyplot as plt
 from warnings import warn
-from copy import deepcopy
 
 import numpy as np
 
@@ -399,20 +398,16 @@ def plot_density(
         ang_step = ang[1] - ang[0]
         labelx = "ray number"
 
-    # compute percentiles of the histogram
-    az_percentile_ref = np.ma.masked_all(len(ang))
-    az_percentile_low = deepcopy(az_percentile_ref)
-    az_percentile_high = deepcopy(az_percentile_ref)
+    # Compute quantiles for each ray
+    az_percentiles = np.ma.masked_all((len(ang), len(quantiles)))
     for ray in range(len(ang)):
-        quantiles, values_ray = compute_quantiles_from_hist(
+        _, values_ray = compute_quantiles_from_hist(
             hist_obj.range["data"], field[ray, :], quantiles=quantiles
         )
+        az_percentiles[ray, :] = values_ray
 
-        az_percentile_low[ray] = values_ray[0]
-        az_percentile_ref[ray] = values_ray[1]
-        az_percentile_high[ray] = values_ray[2]
-
-    quantiles, values_sweep = compute_quantiles_from_hist(
+    # Compute overall sweep quantiles
+    _, values_sweep = compute_quantiles_from_hist(
         hist_obj.range["data"], np.ma.sum(field, axis=0), quantiles=quantiles
     )
 
@@ -469,14 +464,25 @@ def plot_density(
     # plot reference
     ax.plot(ang, np.zeros(len(ang)) + ref_value, "k--")
 
-    # plot quantiles
-    ax.plot(ang, np.zeros(len(ang)) + values_sweep[1], "r")
-    ax.plot(ang, np.zeros(len(ang)) + values_sweep[0], "r--")
-    ax.plot(ang, np.zeros(len(ang)) + values_sweep[2], "r--")
-
-    ax.plot(ang, az_percentile_ref, "k")
-    ax.plot(ang, az_percentile_low, "k--")
-    ax.plot(ang, az_percentile_high, "k--")
+    # Generate colormap for quantiles
+    quantile_colors = plt.cm.viridis_r(np.linspace(0, 1, len(quantiles)))
+
+    # Plot all quantiles
+    for i, q in enumerate(quantiles):
+        ax.plot(
+            ang,
+            np.zeros(len(ang)) + values_sweep[i],
+            color=quantile_colors[i],
+            linestyle="--",
+            label=f"{q:.1f}th quantile",
+        )
+        ax.plot(
+            ang,
+            az_percentiles[:, i],
+            color=quantile_colors[i],
+            linestyle="-",
+            label=f"{q:.1f}th quantile",
+        )
 
     # ax.autoscale(enable=True, axis='both', tight=True)
 
@@ -487,33 +493,12 @@ def plot_density(
     cb = fig.colorbar(cax)
     cb.set_label(label)
 
-    val_quant0_str = "--"
-    if values_sweep[0] is not np.ma.masked:
-        val_quant0_str = "{:.3f}".format(values_sweep[0])
-    val_quant1_str = "--"
-    if values_sweep[1] is not np.ma.masked:
-        val_quant1_str = "{:.3f}".format(values_sweep[1])
-    val_quant2_str = "--"
-    if values_sweep[2] is not np.ma.masked:
-        val_quant2_str = "{:.3f}".format(values_sweep[2])
-
-    metadata = (
-        "npoints: "
-        + str(np.ma.sum(field))
-        + "\n"
-        + str(quantiles[1])
-        + " quant: "
-        + val_quant1_str
-        + "\n"
-        + str(quantiles[0])
-        + " quant: "
-        + val_quant0_str
-        + "\n"
-        + str(quantiles[2])
-        + " quant: "
-        + val_quant2_str
-        + "\n"
-    )
+    metadata = "npoints: " + str(np.ma.sum(field)) + "\n"
+    for i, quant in enumerate(quantiles):
+        val_quant_str = "--"
+        if values_sweep[i] is not np.ma.masked:
+            val_quant_str = f"{values_sweep[i]:.3f}"
+        metadata += f"{quant} quant: {val_quant_str}\n"
 
     ax.text(
         0.05,
@@ -1641,6 +1626,7 @@ def plot_scatter_comp(
     fig=None,
     save_fig=True,
     point_format="bx",
+    write_stats=True,
 ):
     """
     plots the scatter between two time series
@@ -1674,6 +1660,8 @@ def plot_scatter_comp(
         returns the handle to the figure
     point_format : str
         format of the scatter point
+    write_stats : bool
+        If set to True will write the RMSE and bias on the plot. Default is true.
 
     Returns
     -------
@@ -1714,6 +1702,21 @@ def plot_scatter_comp(
         ax.autoscale(False)
         ax.plot(value1, value2, point_format)
 
+    if write_stats:
+        for txt in ax.texts:
+            txt.remove()
+        rmse = np.sqrt(np.nanmean((value1 - value2) ** 2))
+        bias = np.nanmean(value1 - value2)
+
+        ax.text(
+            0.01,
+            0.98,
+            f"RMSE={rmse:2.2f}\n Bias (r-g)={bias:2.2f}",
+            horizontalalignment="left",
+            verticalalignment="top",
+            transform=ax.transAxes,
+        )
+
     if save_fig:
         for fname in fname_list:
             fig.savefig(fname, dpi=dpi)