Merge branch 'dev'

Author: radarv

doc/source/overview/index.rst

@@ -1,14 +1,63 @@
 Pyrad configuration files
 ==============================
 
+.. role:: red
+
 The configuration of the data processing in pyrad is divided into three files. The main configuration file, the location configuration file describing the location of the
 weather radar and the used scans. Finally, the product configuration file describes the datasets and
 products.
 
+Two formats are accepted for configuration files. The classical pyrad format which uses the following syntax::
+
+   ppiMapImageConfig STRUCT 12
+      mapres STRING 10m
+      alpha FLOAT 0.4
+      latmin FLOAT 46.5
+      latmax FLOAT 47.1
+      lonmin FLOAT 6.5
+      lonmax FLOAT 7.5
+      lonstep FLOAT 0.1
+      latstep FLOAT 0.1
+      xsize FLOAT 18.
+      ysize FLOAT  10.
+      background_zoom INT 10
+      maps STRARR 2        # maps to overplot (cartopy)
+         OTM
+         rivers
+
+**Supported Data Types**
+
+Scalar types:
+
+* BYTE, INT, LONG, HEX: interpreted as integers (HEX supports 0x notation).
+* FLOAT, DOUBLE, EXP: interpreted as floating-point numbers.
+* BOOL: accepts true/1 (case-insensitive) → True, others → False.
+* STRING: interpreted as a string.
+
+Array types:
+
+* BYTARR, INTARR, LONARR, HEXARR: interpreted as a list of integers
+* FLTARR, DBLAR: EXPARR: interpreted as a list of floating-point numbers.
+* STRARR: interpreted as a list of strings.
+
+
+Every entry has three elements:
+
+* *fieldname*: indentifier for the field
+* *TYPE*: one of the supported scalar or array types
+* *value* (for scalar types) or *number of elements* (for STRUCT or arrays)
+
+**The number of elements must be correct or the processing will fail**
+
+
+
+:red:`Since v2.1.0 of pyrad, you can also write your config files in the YAML format.`
+A `converter script <https://github.com/MeteoSwiss/pyrad/blob/master/src/pyrad_proc/scripts/config_to_yaml.py>`_
+is also available to convert your pyrad-style config files to YAML.
 
 .. toctree::
    :maxdepth: 1
 
    main
    loc
-   prod
+   prod

doc/source/overview/main.rst

@@ -23,13 +23,13 @@ Table 2: Configuration parameters of the main configuration file
        ``<datapath>/<scanname>/<YYYY-MM-DD>/<YYYYMMDDHHMMSS00datatype>.<ext>``
    * - configpath
      - STRING
-     - Absolute base directory of the configuration files. This directory contains clutter maps, filter coefficients, antenna pattern, and the data processing configuration files. If any path within a pyrad config file is provided as a relative path, it will be converted to an absolute path, by prepending it with the path specified with "configpath".
+     - Absolute base directory of the configuration files. This directory contains clutter maps, filter coefficients, antenna pattern, and the data processing configuration files. If any path within a pyrad config file is provided as a relative path, it will be converted to an absolute path, by prepending it with the path specified with "configpath". If not provided, the directory of the main config file will be used.
    * - locationConfigFile
      - STRING
-     - File name (with full path) of the location configuration file. Described in Section 3.2.
+     - File nameof the location configuration file. If only the filename (and not directory) is provided it will be prepended with configpath. Described in Section 3.2.
    * - productConfigFile
      - STRING
-     - File name (with full path) of the product configuration file. Described in Section 4.
+     - File name of the product configuration file.  If only the filename (and not directory) is provided it will be prepended with configpath. Described in Section 4.
    * - lastStateFile
      - STRING
      - File name (with full path) of the file containing the time of the last processed scan. Used in particular for real-time processing.

src/pyart

@@ -534,6 +534,8 @@ def plot_scatter(
     rad2_name="RADAR002",
     titl="colocated radar gates",
     cmap=None,
+    vmin=None,
+    vmax=None,
 ):
     """
     2D histogram
@@ -563,6 +565,10 @@ def plot_scatter(
     cmap : str or None
         name of the colormap. If None it will be choosen the default for the
         field_name
+    vmin: float
+        Minimum value of the 2D histogram.
+    vmax: float
+        Maximum value of the 2D histogram.
 
     Returns
     -------
@@ -616,6 +622,10 @@ def plot_scatter(
     ax.set_ylabel(labely)
     ax.set_title(titl)
 
+    if vmin and vmax:
+        ax.set_xlim([vmin, vmax])
+        ax.set_ylim([vmin, vmax])
+
     cb = fig.colorbar(cax)
     cb.set_label(label)
 

src/pyrad_proc/pyrad/graph/plots.py

@@ -3879,7 +3879,9 @@ def _get_datetime(fname, datagroup, ftime_format=None):
         if ftime_format is None:
             # we assume is rad4alp format
             datetimestr = bfile[3:12]
-            fdatetime = datetime.datetime.strptime(datetimestr, "%y%j%H%M")
+            fdatetime = datetime.datetime.strptime(datetimestr, "%y%j%H%M").replace(
+                tzinfo=datetime.timezone.utc
+            )
         else:
             return find_date_in_file_name(
                 bfile, date_format=ftime_format[ftime_format.find("F") + 2 : -1]

src/pyrad_proc/pyrad/io/io_aux.py

@@ -385,8 +385,8 @@ def read_iso0_grib_data(fname):
     for grb in grbs:
         lats, lons = grb.latlons()
         values.append(grb.values)
-        date_analysis = grb.analDate
-        date_fcst.append(grb.analDate + datetime.timedelta(hours=grb["startStep"]))
+        date_analysis = grb.analDate.replace(tzinfo=datetime.timezone.utc)
+        date_fcst.append(date_analysis + datetime.timedelta(hours=grb["startStep"]))
 
     grbs.close()
 

src/pyrad_proc/pyrad/io/read_data_iso0_mf.py

@@ -1442,6 +1442,15 @@ def get_data(voltime, datatypesdescr, cfg):
             radar.radar_calibration["dBADU_to_dBm_vv"]["data"][0] = cfg["dBADUtodBmv"][
                 ind_rad
             ]
+
+    # If radar name is empty substitude the one from the config file
+    if not radar.metadata.get("instrument_name"):
+        if "RadarName" in cfg:
+            try:
+                radar.metadata["instrument_name"] = cfg["RadarName"][ind_rad]
+            except (IndexError, TypeError):
+                pass
+
     return radar
 
 
@@ -1993,7 +2002,6 @@ def merge_scans_odim(
         azmin = cfg["azmin"][ind_rad]
     if cfg["azmax"] is not None:
         azmax = cfg["azmax"][ind_rad]
-
     return pyart.util.subset_radar(
         radar,
         radar.fields.keys(),

src/pyrad_proc/pyrad/io/read_data_radar.py

@@ -2593,7 +2593,7 @@ def read_radiosounding_igra(station, dtime):
     Please see this link for a description of the columns
     https://www.ncei.noaa.gov/data/integrated-global-radiosonde-archive/doc/igra2-data-format.txt
     """
-    warn("Downloading sounding from IGRA database...")
+    print("Downloading sounding from IGRA database...")
 
     COLUMNS_SOUNDING = [
         "LVLTYPE1",
@@ -2649,7 +2649,9 @@ def read_radiosounding_igra(station, dtime):
         return
 
     # get igra code
-    code = stations_ref["CODE"][stations_ref["CODE"].str.contains(station)].iloc[0]
+    code = stations_ref["CODE"][
+        stations_ref["CODE"].str.contains(station, na=False)
+    ].iloc[0]
 
     # Check if specified time is after 2021
     if dtime > datetime.datetime(2021, 1, 1):
@@ -2702,8 +2704,8 @@ def read_radiosounding_igra(station, dtime):
     line_start = line_starts[idx_clo]
 
     if not line_start:
-        warn(
-            "Could not find a sounding in the IGRA database that corresponds to prescribed time"
+        print(
+            "Could not find a sounding in the IGRA database that corresponds to prescribed time",
         )
 
     file_content = file_content[line_start + 1 :]

src/pyrad_proc/pyrad/io/read_data_sensor.py

@@ -826,13 +826,13 @@ def process_vol_to_grid(procstatus, dscfg, radar_list=None):
             "data"
         ][0]
     else:
-        warn("beamwidth not defined. Assumed 1 deg")
+        warn("beamwidth not defined. Assumed 1 deg", use_debug=False)
         beamwidth = 1.0
 
     if radar_list_aux[0].ray_angle_res is not None:
         beam_spacing = radar_list_aux[0].ray_angle_res["data"][0]
     else:
-        warn("beam spacing not defined. Assumed 1 deg")
+        warn("beam spacing not defined. Assumed 1 deg", use_debug=False)
         beam_spacing = 1.0
 
     # cartesian mapping
@@ -1193,7 +1193,10 @@ def process_keep_roi(procstatus, dscfg, radar_list=None):
         res_cercle = dscfg.get("res_cercle", 16)
 
         if lon_centre is None or lat_centre is None:
-            warn("cercle position undefined. " "The radar position will be used")
+            warn(
+                "cercle position undefined. " "The radar position will be used",
+                use_debug=False,
+            )
             lon_centre = radar.longitude["data"][0]
             lat_centre = radar.latitude["data"][0]
 
@@ -1223,7 +1226,10 @@ def process_keep_roi(procstatus, dscfg, radar_list=None):
         sn_length = dscfg.get("sn_length", 1000.0)  # m
 
         if lon_point is None or lat_point is None:
-            warn("box position undefined. " "The radar position will be used")
+            warn(
+                "box position undefined. " "The radar position will be used",
+                use_debug=False,
+            )
             lon_point = radar.longitude["data"][0]
             lat_point = radar.latitude["data"][0]
 
@@ -1489,7 +1495,10 @@ def process_roi(procstatus, dscfg, radar_list=None):
         res_cercle = dscfg.get("res_cercle", 16)
 
         if lon_centre is None or lat_centre is None:
-            warn("cercle position undefined. " "The radar position will be used")
+            warn(
+                "cercle position undefined. " "The radar position will be used",
+                use_debug=False,
+            )
             lon_centre = radar.longitude["data"][0]
             lat_centre = radar.latitude["data"][0]
 
@@ -1519,7 +1528,10 @@ def process_roi(procstatus, dscfg, radar_list=None):
         sn_length = dscfg.get("sn_length", 1000.0)  # m
 
         if lon_point is None or lat_point is None:
-            warn("box position undefined. " "The radar position will be used")
+            warn(
+                "box position undefined. " "The radar position will be used",
+                use_debug=False,
+            )
             lon_point = radar.longitude["data"][0]
             lat_point = radar.latitude["data"][0]
 
@@ -1768,7 +1780,10 @@ def process_roi2(procstatus, dscfg, radar_list=None):
     nfields_available = 0
     for field_name in field_names_aux:
         if field_name not in radar.fields:
-            warn("Field name " + field_name + " not available in radar object")
+            warn(
+                "Field name " + field_name + " not available in radar object",
+                use_debug=False,
+            )
             continue
         field_names.append(field_name)
         nfields_available += 1
@@ -1835,7 +1850,10 @@ def process_roi2(procstatus, dscfg, radar_list=None):
         res_cercle = dscfg.get("res_cercle", 16)
 
         if lon_centre is None or lat_centre is None:
-            warn("cercle position undefined. " "The radar position will be used")
+            warn(
+                "cercle position undefined. " "The radar position will be used",
+                use_debug=False,
+            )
             lon_centre = radar.longitude["data"][0]
             lat_centre = radar.latitude["data"][0]
 
@@ -1865,7 +1883,10 @@ def process_roi2(procstatus, dscfg, radar_list=None):
         sn_length = dscfg.get("sn_length", 1000.0)  # m
 
         if lon_point is None or lat_point is None:
-            warn("box position undefined. " "The radar position will be used")
+            warn(
+                "box position undefined. " "The radar position will be used",
+                use_debug=False,
+            )
             lon_point = radar.longitude["data"][0]
             lat_point = radar.latitude["data"][0]
 
@@ -2105,7 +2126,10 @@ def process_azimuthal_average(procstatus, dscfg, radar_list=None):
     nfields_available = 0
     for field_name, datatype in zip(field_names_aux, datatypes_aux):
         if field_name not in radar.fields:
-            warn("Field name " + field_name + " not available in radar object")
+            warn(
+                "Field name " + field_name + " not available in radar object",
+                use_debug=False,
+            )
             continue
         field_names.append(field_name)
         datatypes.append(datatype)
@@ -2118,7 +2142,10 @@ def process_azimuthal_average(procstatus, dscfg, radar_list=None):
             if datatype in dscfg["lin_trans"]:
                 lin_trans[field_name] = dscfg["lin_trans"] != 0
             else:
-                warn("Averaging in linear units for {} not specified".format(datatype))
+                warn(
+                    "Averaging in linear units for {} not specified".format(datatype),
+                    use_debug=False,
+                )
 
     if nfields_available == 0:
         warn("Fields not available in radar data")
@@ -2412,7 +2439,8 @@ def process_radar_resampling(procstatus, dscfg, radar_list=None):
             radar_antenna_atsameplace = True
             warn(
                 "No target radar position specified. "
-                + "The radars are assumed co-located"
+                + "The radars are assumed co-located",
+                use_debug=False,
             )
         else:
             radar_antenna_atsameplace = False
@@ -2569,7 +2597,8 @@ def process_radar_resampling(procstatus, dscfg, radar_list=None):
                     warn(
                         "Units type for data type "
                         + datatype
-                        + " not specified. Assumed linear"
+                        + " not specified. Assumed linear",
+                        use_debug=False,
                     )
 
             use_nans.update({field_name: False})
@@ -2580,7 +2609,8 @@ def process_radar_resampling(procstatus, dscfg, radar_list=None):
                     warn(
                         "Use of nans not specified for data type "
                         + datatype
-                        + " not specified. Assumed not used"
+                        + " not specified. Assumed not used",
+                        use_debug=False,
                     )
 
             nan_value.update({field_name: 0.0})
@@ -2591,7 +2621,8 @@ def process_radar_resampling(procstatus, dscfg, radar_list=None):
                     warn(
                         "NaN value not specified for data type "
                         + datatype
-                        + " not specified. Assumed 0"
+                        + " not specified. Assumed 0",
+                        use_debug=False,
                     )
 
         # Persistent data structure

src/pyrad_proc/pyrad/proc/process_aux.py

@@ -1474,6 +1474,10 @@ def process_hydroclass(procstatus, dscfg, radar_list=None):
         weights : array of floats. Dataset keyword
             Used with HYDRO_METHOD SEMISUPERVISED. The list of weights given
             to each variable
+        frequency : float. Dataset keyword
+            the radar frequency [Hz]. If None that of the key
+            frequency in attribute instrument_parameters of the radar
+            object will be used.
         hydropath : string. Dataset keyword
             Used with HYDRO_METHOD UKMO. Directory of the UK MetOffice
             hydrometeor classification code
@@ -1576,8 +1580,10 @@ def process_hydroclass(procstatus, dscfg, radar_list=None):
             if "fzl" in dscfg:
                 freezing_level = dscfg["fzl"]
             elif "sounding" in dscfg:
-                warn("No iso0 or temperature fields were provided")
-                warn("Getting freezing level height from sounding")
+                warn(
+                    "No iso0 or temperature fields were provided, getting freezing level height from sounding",
+                    use_debug=False,
+                )
                 sounding_code = dscfg["sounding"]
                 t0 = pyart.util.datetime_utils.datetime_from_radar(radar)
                 freezing_level = read_fzl_igra(sounding_code, t0, dscfg=dscfg)
@@ -1634,6 +1640,7 @@ def process_hydroclass(procstatus, dscfg, radar_list=None):
         output_distances = dscfg.get("output_distances", False)
         vectorize = dscfg.get("vectorize", False)
         weights = dscfg.get("weights", np.array((1.0, 1.0, 1.0, 0.75, 0.5)))
+        freq = dscfg.get("frequency", None)
 
         # load centroids
         if centroids_file is not None:
@@ -1668,6 +1675,7 @@ def process_hydroclass(procstatus, dscfg, radar_list=None):
             hydro_field=None,
             entropy_field=None,
             temp_ref=temp_ref,
+            radar_freq=freq,
             compute_entropy=compute_entropy,
             output_distances=output_distances,
             vectorize=vectorize,