Specify input directory as global variable on top

Author: brandonwkerns

blob_detection/README.md

@@ -46,7 +46,8 @@ Here are the arguments:
 - Time interval in hours (optional, default = 1), e.g., `-dt 3` for every 3 h.
 - Number of CPUs to use (optional, default = 1), e.g., `-np 8` for 8 CPUs.
 
-As an __example__, to run the script for 0000 UTC 1 January 2020, run it like this:
+As an __example__, to run the script for 0000 UTC 1 January 2020,
+run it like this:
 ```
 python generate_ar_object_masks.era5.py 2020010100 2020010100
 ```
@@ -73,6 +74,7 @@ If this is successful, you should be able to run it on any time range.
 ## Input and Output
 
 ### Input 
+
 ERA-5 data were obtained using the NSF NCAR/UCAR Research Data Archive
 ([RDA](https://rda.ucar.edu/)), dataset number 633.0.
 The files needed start with `e5.oper.an.sfc.128_137_tcwv.ll025sc`
@@ -83,8 +85,25 @@ by the RDA web application to download the data.
 The input directory is to be set in the script
 `generate_ar_object_masks.era5.py`.
 
+The function `retrieve_era5_fields_nc.py` reads in the data. It takes
+a main data directory as in input. The main data directory is specified
+as the global variable DATA_DIRECTORY at the top of
+`retrieve_era5_fields_nc.py`. You need to edit this for your system.
+```
+## You need to set this for your system.
+DATA_DIRECTORY='/home/orca/data/model_anal/era5/from_rda'
+```
+Under the main data directory, it expects
+the following sub directories underneath the main data directory:
+- `tcwv/`: contains the TPW files
+- `viwve/`: eastward component of IVT
+- `viwvn/`: northward component of IVT
+- `invariant/`: Topography file
+  `e5.oper.invariant.128_129_z.ll025sc.1979010100_1979010100.nc`
+
 
 ### Output 
+
 Running this code will generate the outputs under the `mask_data/` directory.
 Subdirectories will be created by year, month, and day.
 

blob_detection/generate_ar_object_masks.era5.py

@@ -10,6 +10,8 @@
 from retrieve_era5_fields_nc import retrieve_era5_fields_nc
 from ar_objects_mask import *
 
+## You need to set this for your system.
+DATA_DIRECTORY='/home/orca/data/model_anal/era5/from_rda'
 
 # Command line args:
 # Must specify at least YYYYMMDDHH_start and YYYYMMDDHH_end
@@ -53,7 +55,8 @@ def process_a_time(time_to_process, verbose=False):
     - Save the output
     """
 
-    F = retrieve_era5_fields_nc(time_to_process, verbose=verbose)
+    F = retrieve_era5_fields_nc(time_to_process, DATA_DIRECTORY,
+                                verbose=verbose)
 
     # Initialize mask object
     mask = ar_object_mask(time_to_process)

blob_detection/retrieve_era5_fields_nc.py

@@ -6,16 +6,27 @@
 import glob
 import os
 
-def retrieve_era5_fields_nc(dt_this, verbose=True, uv200=False):
+def retrieve_era5_fields_nc(dt_this, data_directory, verbose=True, uv200=False):
     """
     retrieve_era5_fields_nc(dt_this, verbose=True, uv200=False)
     
     Read in the data fields from ERA5 for ARLiD.
+
+    Inputs:
+    - dt_this: A Python datetime-like object
+    - data_directory: The parent directory for ERA5 data.
+      The function expects the data files to be in subdirectories:
+      tcwv, viwve, and viwvn. (Also u_200mb and v_200mb if uv200=True).
+      It will also look under the invarient subdirectory for the topo file:
+      e5.oper.invariant.128_129_z.ll025sc.1979010100_1979010100.nc
     """
 
-    end_of_month_DD = calendar.monthrange(dt_this.year, dt_this.month)[1] # output is (day_of_week, day_of_month)
-    fmt_tpw = ('/home/orca/data/model_anal/era5/from_rda/tcwv/'
-        +'e5.oper.an.sfc.128_137_tcwv.ll025sc.%Y%m0100_%Y%m'+str(end_of_month_DD).zfill(2)+'23.nc')
+    # output is (day_of_week, day_of_month)
+    end_of_month_DD = calendar.monthrange(dt_this.year, dt_this.month)[1]
+
+    fmt_tpw = (f'{data_directory}/tcwv/'
+        + 'e5.oper.an.sfc.128_137_tcwv.ll025sc.%Y%m0100_%Y%m'
+        + str(end_of_month_DD).zfill(2)+'23.nc')
     fn_tpw = dt_this.strftime(fmt_tpw)
     if verbose:
         print(fn_tpw)
@@ -27,17 +38,19 @@ def retrieve_era5_fields_nc(dt_this, verbose=True, uv200=False):
         tpw = DS['TCWV'].data
 
     ## IVT
-    fmt_viwve = ('/home/orca/data/model_anal/era5/from_rda/viwve/'
-        +'e5.oper.an.vinteg.162_071_viwve.ll025sc.%Y%m0100_%Y%m'+str(end_of_month_DD).zfill(2)+'23.nc')
+    fmt_viwve = (f'{data_directory}/viwve/'
+        + 'e5.oper.an.vinteg.162_071_viwve.ll025sc.%Y%m0100_%Y%m'
+        + str(end_of_month_DD).zfill(2)+'23.nc')
     fn_viwve = dt_this.strftime(fmt_viwve)
     if verbose:
         print(fn_viwve)
     with xr.open_dataset(fn_viwve) as DS0:
         DS = DS0.sel(time=dt_this)
         viwve = DS['VIWVE'].data
 
-    fmt_viwvn = ('/home/orca/data/model_anal/era5/from_rda/viwvn/'
-        +'e5.oper.an.vinteg.162_072_viwvn.ll025sc.%Y%m0100_%Y%m'+str(end_of_month_DD).zfill(2)+'23.nc')
+    fmt_viwvn = (f'{data_directory}/viwvn/'
+        + 'e5.oper.an.vinteg.162_072_viwvn.ll025sc.%Y%m0100_%Y%m'
+        + str(end_of_month_DD).zfill(2)+'23.nc')
     fn_viwvn = dt_this.strftime(fmt_viwvn)
     if verbose:
         print(fn_viwvn)
@@ -46,8 +59,9 @@ def retrieve_era5_fields_nc(dt_this, verbose=True, uv200=False):
         viwvn = DS['VIWVN'].data
 
     if uv200:
-        fmt_v200 = ('/home/orca/data/model_anal/era5/from_rda/v_200mb/%Y/%m/'
-            +'e5.oper.an.pl.128_132_v.ll025uv.%Y%m%d00_%Y%m%d23.lev200mb.6hr.nc')
+        fmt_v200 = (f'{data_directory}/v_200mb/%Y/%m/'
+            + 'e5.oper.an.pl.128_132_v.ll025uv.%Y%m%d00_%Y%m%d23'
+            + '.lev200mb.6hr.nc')
         fn_v200 = dt_this.strftime(fmt_v200)
         if verbose:
             print(fn_v200)
@@ -59,8 +73,9 @@ def retrieve_era5_fields_nc(dt_this, verbose=True, uv200=False):
             print(f'File not found: {fn_v200}.')
             v200 = np.nan * tpw
 
-        fmt_u200 = ('/home/orca/data/model_anal/era5/from_rda/u_200mb/%Y/%m/'
-            +'e5.oper.an.pl.128_131_u.ll025uv.%Y%m%d00_%Y%m%d23.lev200mb.6hr.nc')
+        fmt_u200 = (f'{data_directory}/u_200mb/%Y/%m/'
+            + 'e5.oper.an.pl.128_131_u.ll025uv.%Y%m%d00_%Y%m%d23'
+            + '.lev200mb.6hr.nc')
         fn_u200 = dt_this.strftime(fmt_u200)
         if verbose:
             print(fn_u200)
@@ -73,7 +88,7 @@ def retrieve_era5_fields_nc(dt_this, verbose=True, uv200=False):
             u200 = np.nan * tpw
 
     ## Orography
-    fn_orog = ('/home/orca/data/model_anal/era5/from_rda/invariant/'
+    fn_orog = (f'{data_directory}/invariant/'
         +'e5.oper.invariant.128_129_z.ll025sc.1979010100_1979010100.nc')
     if verbose:
         print(fn_orog)