Merge branch 'dev'

Author: wolfidan

src/pyrad_proc/pyrad/io/read_data_sensor.py

@@ -2218,12 +2218,26 @@ def get_sensor_data(date, datatype, cfg):
 
     """
     if cfg["sensor"] == "rgage":
-        datapath = cfg["smnpath"] + date.strftime("%Y%m") + "/"
-        datafile = date.strftime("%Y%m%d") + "_" + cfg["sensorid"] + ".csv"
+        # Try several file formats
+        datapath1 = os.path.join(cfg["smnpath"], date.strftime("%Y%m"))
+        datapath2 = os.path.join(cfg["smnpath"], date.strftime("%Y-%m-%d"), 
+                                cfg["sensorid"])
+        datafile1 = os.path.join(datapath1, date.strftime("%Y%m%d")
+                                + "_" + cfg["sensorid"] + ".csv*")
+        datafile2 = os.path.join(datapath2, date.strftime("%Y%m%d")
+                                + "_" + cfg["sensorid"] + ".csv*")
+        
+        files1 = glob.glob(datafile1)
+        files2 = glob.glob(datafile2)
+        if len(files1):
+            datafile = files1[0]
+        if len(files2):
+            datafile = files2[0]
+        
         try:
-            _, sensordate, _, _, _, sensorvalue, _, _ = read_smn(datapath + datafile)
+            _, sensordate, _, _, _, sensorvalue, _, _ = read_smn(datafile)
         except Exception:
-            _, sensordate, sensorvalue = read_smn2(datapath + datafile)
+            _, sensordate, sensorvalue = read_smn2(datafile)
             if sensordate is None:
                 return None, None, None, None
         label = "RG"
@@ -2317,10 +2331,9 @@ def get_sensor_data(date, datatype, cfg):
 
     return sensordate, sensorvalue, label, period
 
-
 def read_smn(fname):
     """
-    Reads SwissMetNet data contained in a csv file
+    Reads SwissMetNet data contained in a csv or gzipped csv file.
 
     Parameters
     ----------
@@ -2329,54 +2342,45 @@ def read_smn(fname):
 
     Returns
     -------
-    smn_id, date , pressure, temp, rh, precip, wspeed, wdir : tupple
+    smn_id, date, pressure, temp, rh, precip, wspeed, wdir : tuple
         The read values
-
     """
     fill_value = 10000000.0
-    try:
-        with open(fname, "r", newline="") as csvfile:
-            # first count the lines
-            reader = csv.DictReader(csvfile)
-            nrows = sum(1 for row in reader)
-            smn_id = np.ma.empty(nrows, dtype="float32")
-            pressure = np.ma.empty(nrows, dtype="float32")
-            temp = np.ma.empty(nrows, dtype="float32")
-            rh = np.ma.empty(nrows, dtype="float32")
-            precip = np.ma.empty(nrows, dtype="float32")
-            wspeed = np.ma.empty(nrows, dtype="float32")
-            wdir = np.ma.empty(nrows, dtype="float32")
 
-            # now read the data
-            csvfile.seek(0)
-            reader = csv.DictReader(csvfile)
-            date = []
-            for i, row in enumerate(reader):
-                smn_id[i] = float(row["StationID"])
-                date.append(datetime.datetime.strptime(row["DateTime"], "%Y%m%d%H%M%S"))
-                pressure[i] = float(row["AirPressure"])
-                temp[i] = float(row["2mTemperature"])
-                rh[i] = float(row["RH"])
-                precip[i] = float(row["Precipitation"])
-                wspeed[i] = float(row["Windspeed"])
-                wdir[i] = float(row["Winddirection"])
-
-            pressure = np.ma.masked_values(pressure, fill_value)
-            temp = np.ma.masked_values(temp, fill_value)
-            rh = np.ma.masked_values(rh, fill_value)
-            precip = np.ma.masked_values(precip, fill_value)
-            wspeed = np.ma.masked_values(wspeed, fill_value)
-            wdir = np.ma.masked_values(wdir, fill_value)
-
-            # convert precip from mm/10min to mm/h
-            precip *= 6.0
-
-            csvfile.close()
-
-            return smn_id, date, pressure, temp, rh, precip, wspeed, wdir
-    except EnvironmentError as ee:
-        warn(str(ee))
-        warn("Unable to read file " + fname)
+    try:
+        # Use pandas to read the file (supports .gz files directly)
+        df = pd.read_csv(fname, compression='gzip' if fname.endswith('.gz') else None)
+
+        # Convert date strings to datetime objects
+        df['DateTime'] = pd.to_datetime(df['DateTime'], format='%Y%m%d%H%M%S')
+
+        # Identify columns by searching for keywords (handles cases like AirPressure:degC)
+        air_pressure_col = [col for col in df.columns if 'AirPressure' in col][0]
+        temp_col = [col for col in df.columns if '2mTemperature' in col][0]
+        rh_col = [col for col in df.columns if 'RH' in col][0]
+        precip_col = [col for col in df.columns if 'Precipitation' in col][0]
+        wspeed_col = [col for col in df.columns if 'Windspeed' in col][0]
+        wdir_col = [col for col in df.columns if 'Winddirection' in col][0]
+
+        # Mask invalid data (fill_value)
+        pressure = np.ma.masked_values(df[air_pressure_col].astype('float32'), fill_value)
+        temp = np.ma.masked_values(df[temp_col].astype('float32'), fill_value)
+        rh = np.ma.masked_values(df[rh_col].astype('float32'), fill_value)
+        precip = np.ma.masked_values(df[precip_col].astype('float32'), fill_value)
+        wspeed = np.ma.masked_values(df[wspeed_col].astype('float32'), fill_value)
+        wdir = np.ma.masked_values(df[wdir_col].astype('float32'), fill_value)
+
+        # Convert precip from mm/10min to mm/h
+        precip *= 6.0
+
+        # Extract smn_id and date
+        smn_id = df['StationID'].astype('float32').values
+        date = df['DateTime'].tolist()
+
+        return smn_id, date, pressure, temp, rh, precip, wspeed, wdir
+
+    except Exception as e:
+        warn(f"Unable to read file {fname}: {e}")
         return None, None, None, None, None, None, None, None
 
 

src/pyrad_proc/pyrad/proc/process_Doppler.py

@@ -65,7 +65,9 @@ def process_turbulence(procstatus, dscfg, radar_list=None):
         data set configuration. Accepted Configuration Keywords::
 
         datatype : string. Dataset keyword
-            The input data type
+            The input data type, must contain,
+            "dBuZ" or "dBZ" or "dBZc" or "dBuZv" or "dBZv" or "dBZvc" or "CNRc", and, 
+            "W" or "Wv" or "Wu" or "Wvu" or "WD" or "WDc"
         radius : float. Dataset keyword
             Search radius for calculating Eddy Dissipation Rate (EDR).
             Default 2
@@ -97,7 +99,7 @@ def process_turbulence(procstatus, dscfg, radar_list=None):
     Returns
     -------
     new_dataset : dict
-        dictionary containing the output
+        dictionary containing the output field "EDR"
     ind_rad : int
         radar index
 
@@ -235,7 +237,8 @@ def process_dealias_fourdd(procstatus, dscfg, radar_list=None):
         data set configuration. Accepted Configuration Keywords::
 
         datatype : string. Dataset keyword
-            The input data type
+            The input data type, must contain
+            "V" or "Vc"
         filt : int. Dataset keyword
             Flag controlling Bergen and Albers filter, 1 = yes, 0 = no.
         sign : int. Dataset keyword
@@ -252,7 +255,7 @@ def process_dealias_fourdd(procstatus, dscfg, radar_list=None):
     Returns
     -------
     new_dataset : dict
-        dictionary containing the output
+        dictionary containing the output field "dealV" or "dealVc" (if Vc was provided)
     ind_rad : int
         radar index
 
@@ -357,7 +360,8 @@ def process_dealias_region_based(procstatus, dscfg, radar_list=None):
         data set configuration. Accepted Configuration Keywords::
 
         datatype : string. Dataset keyword
-            The input data type
+            The input data type, must contain,
+            "V" or "Vc"
         interval_splits : int, optional
             Number of segments to split the nyquist interval into when finding
             regions of similar velocity.  More splits creates a larger number
@@ -387,7 +391,7 @@ def process_dealias_region_based(procstatus, dscfg, radar_list=None):
     Returns
     -------
     new_dataset : dict
-        dictionary containing the output
+        dictionary containing the output field "dealV" or "dealVc" (if Vc was provided)
     ind_rad : int
         radar index
 
@@ -458,7 +462,8 @@ def process_dealias_unwrap_phase(procstatus, dscfg, radar_list=None):
         data set configuration. Accepted Configuration Keywords::
 
         datatype : string. Dataset keyword
-            The input data type
+            The input data type, must contain,
+            "V" or "Vc"
         unwrap_unit : {'ray', 'sweep', 'volume'}, optional
             Unit to unwrap independently.  'ray' will unwrap each ray
             individually, 'sweep' each sweep, and 'volume' will unwrap the
@@ -473,7 +478,7 @@ def process_dealias_unwrap_phase(procstatus, dscfg, radar_list=None):
     Returns
     -------
     new_dataset : dict
-        dictionary containing the output
+        dictionary containing the output field "dealV" or "dealVc" (if Vc was provided)
     ind_rad : int
         radar index
 
@@ -534,7 +539,10 @@ def process_radial_velocity(procstatus, dscfg, radar_list=None):
         data set configuration. Accepted Configuration Keywords::
 
         datatype : string. Dataset keyword
-            The input data type
+            The input data type, must contain
+            WIND_SPEED, and,
+            WIND_DIRECTION, and,
+            wind_vel_v
         latitude, longitude : float
             arbitrary coordinates [deg] from where to compute the radial
             velocity. If any of them is None it will be the radar position
@@ -547,7 +555,7 @@ def process_radial_velocity(procstatus, dscfg, radar_list=None):
     Returns
     -------
     new_dataset : dict
-        dictionary containing the output
+        dictionary containing the output field "V"
     ind_rad : int
         radar index
 
@@ -665,7 +673,8 @@ def process_wind_vel(procstatus, dscfg, radar_list=None):
         data set configuration. Accepted Configuration Keywords::
 
         datatype : string. Dataset keyword
-            The input data type
+            The input data type, must contain
+            "V" or "Vc"
         vert_proj : Boolean
             If true the vertical projection is computed. Otherwise the
             horizontal projection is computed
@@ -675,7 +684,9 @@ def process_wind_vel(procstatus, dscfg, radar_list=None):
     Returns
     -------
     new_dataset : dict
-        dictionary containing the output
+        dictionary containing the output field
+        "wind_vel_h_az", (if vert_proj is False), or,
+        "wind_vel_v" (if vert_proj is True)
     ind_rad : int
         radar index
 
@@ -737,7 +748,7 @@ def process_windshear(procstatus, dscfg, radar_list=None):
     Returns
     -------
     new_dataset : dict
-        dictionary containing the output
+        dictionary containing the output field "windshear_v"
     ind_rad : int
         radar index
 
@@ -786,7 +797,8 @@ def process_windshear_lidar(procstatus, dscfg, radar_list=None):
         data set configuration. Accepted Configuration Keywords::
 
         datatype : string. Dataset keyword
-            The input data type
+            The input data type, must contain
+            "V" or "Vc"
         az_tol : float
             The tolerance in azimuth when looking for gates on top
             of the gate when computation is performed
@@ -797,7 +809,7 @@ def process_windshear_lidar(procstatus, dscfg, radar_list=None):
     Returns
     -------
     new_dataset : dict
-        dictionary containing the output
+        dictionary containing the output field "windshear_v"
     ind_rad : int
         radar index
 
@@ -847,14 +859,17 @@ def process_vad(procstatus, dscfg, radar_list=None):
         data set configuration. Accepted Configuration Keywords::
 
         datatype : string. Dataset keyword
-            The input data type
+            The input data type, must contain
+            "V" or "Vc"
     radar_list : list of Radar objects
         Optional. list of radar objects
 
     Returns
     -------
     new_dataset : dict
-        dictionary containing the output
+        dictionary containing the output fields
+        "wind_vel_h_u", "wind_vel_h_v", "wind_vel_v",
+        "estV", "stdV", and "diffV"
     ind_rad : int
         radar index
 
@@ -923,7 +938,9 @@ def process_dda(procstatus, dscfg, radar_list=None):
         data set configuration. Accepted Configuration Keywords::
 
         datatype : string. Dataset keyword
-            The input data type
+            The input data type, must contain
+            "V" or "Vc", and,
+            "dBuZ", "dBZ", or "dBZc"
 
         gridconfig : dictionary. Dataset keyword
             Dictionary containing some or all of this keywords:
@@ -997,7 +1014,8 @@ def process_dda(procstatus, dscfg, radar_list=None):
     Returns
     -------
     new_dataset : dict
-        dictionary containing the output
+        dictionary containing the output fields
+            "wind_vel_h_u", "wind_vel_h_v" and "wind_vel_v"
     ind_rad : int
         radar index