[1pt] PR: Add FIM capability for Guam and American Samoa

data/bridges/make_rasters_using_lidar.py

@@ -376,12 +376,19 @@ def process_bridges_lidar_data(
         # Collect results
         list_of_classification_results = [future.result() for future in executor_dict]
 
-        # Combine results into a DataFrame
-        bridges_classifications_df = pd.concat(list_of_classification_results, ignore_index=True)
+        # filter None which are for the cases when no raster is created
+        list_of_classification_results = [df for df in list_of_classification_results if df is not None]
 
-        bridges_classifications_df.to_csv(
-            os.path.join(output_dir, 'classifications_summary.csv'), index=False
-        )
+        if list_of_classification_results:
+            # Combine results into a DataFrame
+            bridges_classifications_df = pd.concat(list_of_classification_results, ignore_index=True)
+
+            bridges_classifications_df.to_csv(
+                os.path.join(output_dir, 'classifications_summary.csv'), index=False
+            )
+        else:
+            logging.info('No Raster file was created.')
+            print('No Raster file was created.')
 
         # Record run time
         end_time = datetime.now(timezone.utc)

data/bridges/pull_osm_bridges.py

@@ -28,6 +28,8 @@
 load_dotenv(f'{srcDir}/bash_variables.env')
 DEFAULT_FIM_PROJECTION_CRS = os.getenv('DEFAULT_FIM_PROJECTION_CRS')
 ALASKA_CRS = os.getenv('ALASKA_CRS')
+GUAM_CRS = os.getenv('GUAM_CRS')
+AMERICAN_SAMOA_CRS = os.getenv('AMERICAN_SAMOA_CRS')
 
 
 # Save all OSM bridge features by HUC8 to a specified folder location.
@@ -200,6 +202,10 @@ def pull_osm_features_by_huc(huc_bridge_file, huc_num, huc_geom):
 
         if str(huc_num).startswith('19'):
             gdf1 = gdf1.to_crs(ALASKA_CRS)
+        elif str(huc_num) == '22010000':
+            gdf1 = gdf1.to_crs(GUAM_CRS)
+        elif str(huc_num) == '22030001':
+            gdf1 = gdf1.to_crs(AMERICAN_SAMOA_CRS)
         else:
             gdf1 = gdf1.to_crs(DEFAULT_FIM_PROJECTION_CRS)
 
@@ -276,39 +282,29 @@ def combine_huc_features(output_dir):
     # huc8 will always have the huc8 value but huc10 might be empty
     cols_to_keep = ['osmid', 'name', 'bridge_type', 'huc8', 'huc10', 'geometry']
 
-    # Alaska
-    alaska_bridge_file_names = list(Path(output_dir).glob("huc_19*_osm_bridges.gpkg"))
-    if alaska_bridge_file_names:
-        alaska_all_bridges_gdf_raw = pd.concat(
-            [gpd.read_file(gpkg) for gpkg in alaska_bridge_file_names], ignore_index=True
-        )
-        alaska_all_bridges_gdf = alaska_all_bridges_gdf_raw[cols_to_keep]
-        alaska_all_bridges_gdf["osmid"] = alaska_all_bridges_gdf["osmid"].astype(str)
-
-        alaska_osm_bridge_file = os.path.join(output_dir, "alaska_osm_bridges.gpkg")
-
-        logging.info(f"Writing Alaska bridge lines: {alaska_osm_bridge_file}")
-
-        alaska_all_bridges_gdf.to_file(alaska_osm_bridge_file, driver="GPKG", engine='fiona')
-
-    # Rest of Conus
-    conus_bridge_file_names = list(Path(output_dir).glob("huc_*_osm_bridges.gpkg"))
-    conus_bridge_file_names = [file for file in conus_bridge_file_names if not file.name.startswith("huc_19")]
-    if conus_bridge_file_names:
-        conus_all_bridges_gdf_raw = pd.concat(
-            [gpd.read_file(gpkg) for gpkg in conus_bridge_file_names], ignore_index=True
-        )
-
-        conus_all_bridges_gdf = conus_all_bridges_gdf_raw[cols_to_keep]
-        conus_all_bridges_gdf["osmid"] = conus_all_bridges_gdf["osmid"].astype(str)
-
-        conus_osm_bridge_file = os.path.join(output_dir, "conus_osm_bridges.gpkg")
-
-        logging.info(f"Writing CONUS bridge lines: {conus_osm_bridge_file}")
-
-        conus_all_bridges_gdf.to_file(conus_osm_bridge_file, driver="GPKG", engine='fiona')
-
-    return
+    # Bucket files by region
+    buckets = {"alaska": [], "guam": [], "samoa": [], "conus": []}
+    for f in Path(output_dir).glob("huc_*_osm_bridges.gpkg"):
+        n = f.name
+        if n.startswith("huc_19"):
+            buckets["alaska"].append(f)
+        elif n.startswith("huc_22010000"):
+            buckets["guam"].append(f)
+        elif n.startswith("huc_22030001"):
+            buckets["samoa"].append(f)
+        else:
+            buckets["conus"].append(f)
+
+    # Process each bucket
+    for region, flist in buckets.items():
+        if not flist:
+            continue
+        gdf = pd.concat((gpd.read_file(f) for f in flist), ignore_index=True)[cols_to_keep]
+        if "osmid" in gdf.columns:
+            gdf["osmid"] = gdf["osmid"].astype(str)
+        out_path = os.path.join(output_dir, f"{region}_osm_bridges.gpkg")
+        logging.info(f"Writing {region.title()} bridge lines: {out_path}")
+        gdf.to_file(out_path, driver="GPKG", engine="fiona")
 
 
 def process_osm_bridges(wbd_file, output_folder, number_of_jobs, lst_hucs):

data/get_sample_data.py

@@ -226,16 +226,27 @@ def download_s3_folder(bucket_name: str, s3_folder: str, local_dir: str = None):
 
     INPUT_DEM_DOMAIN = os.environ["input_DEM_domain"]
     INPUT_DEM_DOMAIN_ALASKA = os.environ["input_DEM_domain_Alaska"]
+    INPUT_DEM_DOMAIN_GUAM = os.environ["input_DEM_domain_Guam"]
+    INPUT_DEM_DOMAIN_AMERICAN_SAMOA = os.environ["input_DEM_domain_AmericanSamoa"]
     INPUT_DEM = os.environ['input_DEM']
     INPUT_DEM_ALASKA = os.environ['input_DEM_Alaska']
+    INPUT_DEM_GUAM = os.environ['input_DEM_Guam']
+    INPUT_DEM_AMERICAN_SAMOA = os.environ['input_DEM_AmericanSamoa']
     INPUT_LANDSEA = os.environ['input_landsea']
     INPUT_LANDSEA_ALASKA = os.environ['input_landsea_Alaska']
+    INPUT_LANDSEA_GUAM = os.environ['input_landsea_Guam']
+    INPUT_LANDSEA_AMERICAN_SAMOA = os.environ['input_landsea_AmericanSamoa']
     INPUT_NLD_LEVEE_PROTECTED_AREAS = os.environ["input_nld_levee_protected_areas"]
     INPUT_NLD_LEVEE_PROTECTED_AREAS_ALASKA = os.environ["input_nld_levee_protected_areas_Alaska"]
+    INPUT_NLD_LEVEE_PROTECTED_AREAS_GUAM = os.environ["input_nld_levee_protected_areas_Guam"]
     INPUT_NWM_LAKES = os.environ['input_nwm_lakes']
     INPUT_NWM_LAKES_ALASKA = os.environ['input_nwm_lakes_Alaska']
+    INPUT_NWM_LAKES_GUAM = os.environ['input_nwm_lakes_Guam']
+    INPUT_NWM_LAKES_AMERICAN_SAMOA = os.environ['input_nwm_lakes_AmericanSamoa']
     INPUT_GL_BOUNDARIES = os.environ["input_GL_boundaries"]
     INPUT_WBD_GDB_ALASKA = os.environ["input_WBD_gdb_Alaska"]
+    INPUT_WBD_GDB_GUAM = os.environ["input_WBD_gdb_Guam"]
+    INPUT_WBD_GDB_AMERICAN_SAMOA = os.environ["input_WBD_gdb_AmericanSamoa"]
     NWM_RECUR_FILE = os.environ["nwm_recur_file"]
     INPUT_CALIB_POINTS_DIR = os.environ["input_calib_points_dir"]
 
@@ -309,6 +320,25 @@ def download_s3_folder(bucket_name: str, s3_folder: str, local_dir: str = None):
 
             __copy_file(INPUT_WBD_GDB_ALASKA, output_root_folder, input_root, bucket_path)
 
+        elif huc == '22010000':  # Guam
+            input_LANDSEA = INPUT_LANDSEA_GUAM
+            input_DEM = INPUT_DEM_GUAM
+            input_DEM_domain = INPUT_DEM_DOMAIN_GUAM
+            input_DEM_file = os.path.join(os.path.split(input_DEM_domain)[0], f'HUC8_{huc}_dem.tif')
+            input_NWM_lakes = INPUT_NWM_LAKES_GUAM
+            input_NLD_levee_protected_areas = INPUT_NLD_LEVEE_PROTECTED_AREAS_GUAM
+
+            __copy_file(INPUT_WBD_GDB_GUAM, output_root_folder, input_root, bucket_path)
+
+        elif huc == '22030001':  # American Samoa
+            input_LANDSEA = INPUT_LANDSEA_AMERICAN_SAMOA
+            input_DEM = INPUT_DEM_AMERICAN_SAMOA
+            input_DEM_domain = INPUT_DEM_DOMAIN_AMERICAN_SAMOA
+            input_DEM_file = os.path.join(os.path.split(input_DEM_domain)[0], f'HUC8_{huc}_dem.tif')
+            input_NWM_lakes = INPUT_NWM_LAKES_AMERICAN_SAMOA
+
+            __copy_file(INPUT_WBD_GDB_AMERICAN_SAMOA, output_root_folder, input_root, bucket_path)
+
         else:
             input_DEM = INPUT_DEM
             input_DEM_domain = INPUT_DEM_DOMAIN

data/nfhl/download_fema_nfhl.py

@@ -24,29 +24,41 @@ def load_wbd(huc_list):
         List of huc8
     Returns
     tuple
-        (wbd_conus, wbd_alaska)
+        (wbd_conus, wbd_alaska, wbd_guam, wbd_american_samoa)
     """
     srcDir = os.getenv('srcDir')
     load_dotenv(f'{srcDir}/bash_variables.env')
     input_WBD_gdb = os.getenv('input_WBD_gdb')
     input_WBD_gdb_Alaska = os.getenv('input_WBD_gdb_Alaska')  # alaska
+    input_WBD_gdb_Guam = os.getenv('input_WBD_gdb_Guam')  # guam
+    input_WBD_gdb_AmericanSamoa = os.getenv('input_WBD_gdb_AmericanSamoa')  # american samoa
     wbd_conus = None
     wbd_alaska = None
+    wbd_guam = None
+    wbd_american_samoa = None
     # Check if any huc8 is in AK
     has_alaska = any(huc.startswith('19') for huc in huc_list)
+    has_guam = any(huc == '22010000' for huc in huc_list)
+    has_american_samoa = any(huc == '22030001' for huc in huc_list)
 
     # Load conus wbd if needed
     if any(not huc.startswith('19') for huc in huc_list):
         if os.path.exists(input_WBD_gdb):
             wbd_conus = gpd.read_file(input_WBD_gdb)
-        else:
-            return None, None
+
     # Load AK wbd if needed
     if has_alaska and os.path.exists(input_WBD_gdb_Alaska):
         wbd_alaska = gpd.read_file(input_WBD_gdb_Alaska)
-    elif has_alaska:
-        return None, None
-    return wbd_conus, wbd_alaska
+
+    # Load Guam wbd if needed
+    if has_guam and os.path.exists(input_WBD_gdb_Guam):
+        wbd_guam = gpd.read_file(input_WBD_gdb_Guam)
+
+    # Load American Samoa wbd if needed
+    if has_american_samoa and os.path.exists(input_WBD_gdb_AmericanSamoa):
+        wbd_american_samoa = gpd.read_file(input_WBD_gdb_AmericanSamoa)
+
+    return wbd_conus, wbd_alaska, wbd_guam, wbd_american_samoa
 
 
 def process_nfhl(
@@ -100,7 +112,18 @@ def process_nfhl(
         file_logger.warning(f"No {nfhl_label} zones for HUC {huc}")
 
 
-def download_nfhl(huc, out_file, wbd_conus, wbd_alaska, geometry_type, file_logger, screen_queue, task_id):
+def download_nfhl(
+    huc,
+    out_file,
+    wbd_conus,
+    wbd_alaska,
+    wbd_guam,
+    wbd_american_samoa,
+    geometry_type,
+    file_logger,
+    screen_queue,
+    task_id,
+):
     """
     Download the NFHL flood hazard zones for a given HUC8
 
@@ -114,6 +137,8 @@ def download_nfhl(huc, out_file, wbd_conus, wbd_alaska, geometry_type, file_logg
         wbd for conus
     wbd_alaska : GeoDataFrame
         wbd for alaska
+    wbd_guam : GeoDataFrame
+        wbd for guam
     geometry_type: str
         the geometry Type for the query
     file_logger : logging.Logger
@@ -129,11 +154,26 @@ def download_nfhl(huc, out_file, wbd_conus, wbd_alaska, geometry_type, file_logg
     try:
         DEFAULT_FIM_PROJECTION_CRS = 5070
         ALASKA_CRS = 3338  # alaska
+        GUAM_CRS = 6637
+        AMERICAN_SAMOA_CRS = 32702
 
         # Select approporiate wbd and crs
         is_alaska = huc.startswith('19')
-        wbd = wbd_alaska if is_alaska else wbd_conus
-        geometryCRS = ALASKA_CRS if is_alaska else DEFAULT_FIM_PROJECTION_CRS
+        is_guam = huc == '22010000'
+        is_american_samoa = huc == '22030001'
+
+        if is_alaska:
+            wbd = wbd_alaska
+            geometryCRS = ALASKA_CRS
+        elif is_guam:
+            wbd = wbd_guam
+            geometryCRS = GUAM_CRS
+        elif is_american_samoa:
+            wbd = wbd_american_samoa
+            geometryCRS = AMERICAN_SAMOA_CRS
+        else:
+            wbd = wbd_conus
+            geometryCRS = DEFAULT_FIM_PROJECTION_CRS
 
         if wbd is None:
             file_logger.error(f'No wbd available for huc {huc}')
@@ -340,7 +380,7 @@ def download_nfhl_wrapper(huc_list, output_folder, geometryType='esriGeometryEnv
         print("")
 
         # Load wbd
-        wbd_conus, wbd_alaska = load_wbd(huc_list)
+        wbd_conus, wbd_alaska, wbd_guam, wbd_american_samoa = load_wbd(huc_list)
 
         # Tasks argument list
         huc_list = sorted(huc_list)
@@ -357,6 +397,8 @@ def download_nfhl_wrapper(huc_list, output_folder, geometryType='esriGeometryEnv
                     "out_file": os.path.join(output_folder, f"nfhl_{huc}.gpkg"),
                     "wbd_conus": wbd_conus,
                     "wbd_alaska": wbd_alaska,
+                    "wbd_guam": wbd_guam,
+                    "wbd_american_samoa": wbd_american_samoa,
                     "geometry_type": geometryType,
                 }
             )