From 9a5101059946b24bc7d570d3f964e49708317de6 Mon Sep 17 00:00:00 2001 From: RobHanna-NOAA Date: Tue, 2 Sep 2025 20:25:33 +0000 Subject: [PATCH 1/8] fix eval plots --- tools/eval_plots.py | 41 +++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/tools/eval_plots.py b/tools/eval_plots.py index 2a056beb8..716b17c6a 100644 --- a/tools/eval_plots.py +++ b/tools/eval_plots.py @@ -948,19 +948,25 @@ def convert_shapes_to_csv(workspace): if __name__ == '__main__': """ - This script has two main uses. The most common use is part of the "alpha test" system via + This script has two main uses. + + The most common use is part of the "alpha test" system generally ran after synthesize_test_cases.py. This set can be run in any environment. The second usage is called FIM Performance mode and talks to the WRDS API to get some nwm metadata. This results the script needing to be run in the OWP environments with the valid URL to WRDS. This creates FIM Performance files of fim_performance_points.csv and fim_performance_polys.csv - which are used by HV. + which are used by HV. Adding the -sp (spatial) flag creates the FIM performance output files. + + If you include the -sp flag, it needs to talk to WRDS and will neeed a config file. + That config file is defaulted as part of the -env arg which can be overridde. + If you do not include the -sp flag, the config file is not needed. Usage for FIM Performance mode. The example for output data is based on the filtered hand output normally used in OWP servers by catfim. Any valid HAND dataset can be used and the filtered "catfim" output version has all of the files needed by both CatFIM and this script. - fim_version="hand_4_6_1_4" ; python /foss_fim/tools/eval_plots.py \ - -m /outputs/${fim_version}_catfim/${fim_version}_metrics.csv \ + fim_version="hand_4_6_1_4" ; python /foss_fim/tools/eval_plots.py + -m /outputs/${fim_version}_catfim/${fim_version}_metrics.csv -w /data/fim_performance/${fim_version} -v ${fim_version} -sp -i """ @@ -1009,7 +1015,9 @@ def convert_shapes_to_csv(workspace): '-e', '--env_file', help='OPTIONAl: Docker mount path to the script environment file.' - ' Defaults to /data/config/fim_enviro_values.env.', + ' Defaults to /data/config/fim_enviro_values.env.\n' + ' The env file is only needed when you are using the -sp argument' + ' which generates spatial data generally used for FIM Performance output files.', default="/data/config/fim_enviro_values.env", required=False, ) @@ -1025,18 +1033,19 @@ def convert_shapes_to_csv(workspace): sp = args['spatial'] i = args['site_plots'] - load_dotenv(args['env_file']) + if sp: # create the spatial files generally used for FIM Performance. + load_dotenv(args['env_file']) - API_BASE_URL = os.getenv('API_BASE_URL') - if API_BASE_URL is None: - raise ValueError( - 'API base url not found. ' - 'Ensure inundation_mapping/tools/ has an .env file with the following info: ' - 'API_BASE_URL, WBD_LAYER, NWM_FLOWS_MS, ' - 'USGS_METADATA_URL, USGS_DOWNLOAD_URL' - ) - WBD_LAYER = os.getenv("WBD_LAYER") - API_BASE_URL = os.getenv("API_BASE_URL") + API_BASE_URL = os.getenv('API_BASE_URL') + if API_BASE_URL is None: + raise ValueError( + 'API base url not found. ' + 'Ensure inundation_mapping/tools/ has an .env file with the following info: ' + 'API_BASE_URL, WBD_LAYER, NWM_FLOWS_MS, ' + 'USGS_METADATA_URL, USGS_DOWNLOAD_URL' + ) + WBD_LAYER = os.getenv("WBD_LAYER") + API_BASE_URL = os.getenv("API_BASE_URL") # Run eval_plots function print('The following AHPS sites are considered "BAD_SITES": ' + ', '.join(BAD_SITES)) From 5494b6f33638cd81dbaf278d8acdd8de06843617 Mon Sep 17 00:00:00 2001 From: Rob Hanna Date: Tue, 2 Sep 2025 22:27:16 +0000 Subject: [PATCH 2/8] minor adjustments and added shp to gpkg fix --- tools/eval_plots.py | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/tools/eval_plots.py b/tools/eval_plots.py index 716b17c6a..96a1d4333 100644 --- a/tools/eval_plots.py +++ b/tools/eval_plots.py @@ -865,9 +865,11 @@ def eval_plots( # Join spatial data to metric data gdf['nws_lid'] = gdf['nws_lid'].str.lower() joined = gdf.merge(all_ahps_datasets, on='nws_lid') - # Project to VIZ projection and write to file + # Project to VIZ projection and write to file (the csv is for HV, the gpkg is for debugging) joined = joined.to_crs(VIZ_PROJECTION) - joined.to_file(Path(workspace) / 'fim_performance_points.shp', engine='fiona') + points_file_name = os.path.join(workspace, 'fim_performance_points.gpkg') + joined.to_file(points_file_name, driver="GPKG", engine='fiona') + joined.to_csv(points_file_name.replace(".gpkg", ".csv")) else: print( 'NWS/USGS MS datasets not analyzed, no spatial data created.\n' @@ -924,8 +926,10 @@ def eval_plots( wbd_with_metrics.rename(columns={'benchmark_source': 'source'}, inplace=True) # Project to VIZ projection wbd_with_metrics = wbd_with_metrics.to_crs(VIZ_PROJECTION) - # Write out to file - wbd_with_metrics.to_file(Path(workspace) / 'fim_performance_polys.shp', engine='fiona') + # Write out to file (the csv is for HV, the gpkg is for debugging) + wbd_with_metrics_file_name = os.path.join(workspace, 'fim_performance_polys.gpkg') + wbd_with_metrics.to_file(wbd_with_metrics_file_name, driver="GPKG", engine='fiona') + wbd_with_metrics.to_csv(wbd_with_metrics_file_name.replace(".gpkg", ".csv")) else: print( 'BLE/IFC/RAS2FIM FR datasets not analyzed, no spatial data created.\n' @@ -933,17 +937,6 @@ def eval_plots( ) -def convert_shapes_to_csv(workspace): - # Convert any geopackage in the root level of output_mapping_dir to CSV and rename. - shape_list = glob.glob(os.path.join(workspace, '*.shp')) - for shape in shape_list: - gdf = gpd.read_file(shape) - parent_directory = os.path.split(shape)[0] - file_name = shape.replace('.shp', '.csv') - csv_output_path = os.path.join(parent_directory, file_name) - gdf.to_csv(csv_output_path) - - ####################################################################### if __name__ == '__main__': @@ -1051,7 +1044,3 @@ def convert_shapes_to_csv(workspace): print('The following AHPS sites are considered "BAD_SITES": ' + ', '.join(BAD_SITES)) print('The following query is used to filter AHPS: ' + DISCARD_AHPS_QUERY) eval_plots(metrics_csv=m, workspace=w, versions=v, stats=s, spatial=sp, site_barplots=i) - - # Convert output shapefiles to CSV - print("Converting to CSVs...") - convert_shapes_to_csv(w) From af1b4fe6e69f467983e28fbaf4d3d901f722a984 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CRobHanna-NOAA=E2=80=9D?= <“Robert.Hanna@NOAA.gov”> Date: Tue, 2 Sep 2025 22:37:15 +0000 Subject: [PATCH 3/8] updated the changelog --- docs/CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index bd3d3722c..254b87435 100755 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,6 +1,19 @@ All notable changes to this project will be documented in this file. We follow the [Semantic Versioning 2.0.0](http://semver.org/) format. +## v4.8.__._ - 2025-09-02 - [PR#1642](https://github.com/NOAA-OWP/inundation-mapping/pull/1642) + +Adjust eval_plot.py so it only looks for the config file when in -sp (spatial) mode. Spatial mode is used when running the tool to create FIM_performance files. In that mode, it can only run in OWP servers as it needs to talk to internal servers. + +Also udpated the tools to create gpkg files instead of shape file. It does continue to create csv as files as before. + +This update does not affect use of the tool as part of regular alpha testing. + +### Changes +- `tools\eval_plots.py`: as described above. +
+ + ## v4.8.10.3 - 2025-08-29 - [PR#1627](https://github.com/NOAA-OWP/inundation-mapping/pull/1627) Adds gcsfs dependency to allow retrieval of NWM output from the Google Cloud Service. From 04fbeda75fb760af69f3284372d28af13f6b28ea Mon Sep 17 00:00:00 2001 From: RyanSpies-NOAA Date: Fri, 5 Sep 2025 16:48:29 +0000 Subject: [PATCH 4/8] Added zfill padding to include leading zero to huc8 csv read --- tools/eval_plots.py | 3 +++ tools/eval_plots_stackedbar.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/tools/eval_plots.py b/tools/eval_plots.py index 96a1d4333..db27ea062 100644 --- a/tools/eval_plots.py +++ b/tools/eval_plots.py @@ -574,6 +574,9 @@ def eval_plots( # Import metrics csv as DataFrame and initialize all_datasets dictionary csv_df = pd.read_csv(metrics_csv, dtype={'huc': str}) + # Ensure all HUC values are 8 characters long with leading zeros if needed + csv_df['huc'] = csv_df['huc'].str.zfill(8) + # If versions are supplied then filter out if versions: # Filter out versions based on supplied version list diff --git a/tools/eval_plots_stackedbar.py b/tools/eval_plots_stackedbar.py index 1a07c3921..6ecd0a0c6 100644 --- a/tools/eval_plots_stackedbar.py +++ b/tools/eval_plots_stackedbar.py @@ -33,6 +33,8 @@ def eval_plot_stack_data_prep(metric_csv, versions=[]): # Load in FIM 4 CSV and select the proper version metrics metrics = pd.read_csv(metric_csv, dtype={"huc": str}) + # Ensure all HUC values are 8 characters long with leading zeros if needed + metrics['huc'] = metrics['huc'].str.zfill(8) if versions: versions = list(versions) # Check to make sure requested versions are in the metrics file From dc83f764ed2a658f79e485b48c5fdd099e76172b Mon Sep 17 00:00:00 2001 From: Rob Hanna - NOAA <90854818+RobHanna-NOAA@users.noreply.github.com> Date: Mon, 8 Sep 2025 09:09:39 -0600 Subject: [PATCH 5/8] Update CHANGELOG.md --- docs/CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 254b87435..f989b5b7c 100755 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -5,15 +5,16 @@ We follow the [Semantic Versioning 2.0.0](http://semver.org/) format. Adjust eval_plot.py so it only looks for the config file when in -sp (spatial) mode. Spatial mode is used when running the tool to create FIM_performance files. In that mode, it can only run in OWP servers as it needs to talk to internal servers. -Also udpated the tools to create gpkg files instead of shape file. It does continue to create csv as files as before. +Also updated the tools to create gpkg files instead of shape file. It does continue to create csv as files as before. Adjustments for zero padding HUC numbers when applicable was also added. This update does not affect use of the tool as part of regular alpha testing. ### Changes -- `tools\eval_plots.py`: as described above. +- `tools\` + - `eval_plots.py`: as described above including the zero padding HUC fix. + - `eval_plots_stackedbar.py`: zero padding HUC fix.
- ## v4.8.10.3 - 2025-08-29 - [PR#1627](https://github.com/NOAA-OWP/inundation-mapping/pull/1627) Adds gcsfs dependency to allow retrieval of NWM output from the Google Cloud Service. From 04dea0b7084f5dfafe7d60414603a37400c86db7 Mon Sep 17 00:00:00 2001 From: Rob Hanna Date: Tue, 9 Sep 2025 14:55:59 +0000 Subject: [PATCH 6/8] Minor last min adjustments --- tools/eval_plots.py | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/tools/eval_plots.py b/tools/eval_plots.py index db27ea062..64bc87fa4 100644 --- a/tools/eval_plots.py +++ b/tools/eval_plots.py @@ -571,6 +571,22 @@ def eval_plots( ''' + if sp: # create the spatial files generally used for FIM Performance. + load_dotenv(args['env_file']) + + API_BASE_URL = os.getenv('API_BASE_URL') + if API_BASE_URL is None: + raise ValueError( + 'API base url not found. ' + 'Check your env config path or file and ensure it has a correct value for API_BASE_URL' + ) + WBD_LAYER = os.getenv("WBD_LAYER") + + # Run eval_plots function + print('The following AHPS sites are considered "BAD_SITES": ' + ', '.join(BAD_SITES)) + print('The following query is used to filter AHPS: ' + DISCARD_AHPS_QUERY) + + # Import metrics csv as DataFrame and initialize all_datasets dictionary csv_df = pd.read_csv(metrics_csv, dtype={'huc': str}) @@ -1010,7 +1026,7 @@ def eval_plots( parser.add_argument( '-e', '--env_file', - help='OPTIONAl: Docker mount path to the script environment file.' + help='OPTIONAL: Docker mount path to the script environment file.' ' Defaults to /data/config/fim_enviro_values.env.\n' ' The env file is only needed when you are using the -sp argument' ' which generates spatial data generally used for FIM Performance output files.', @@ -1029,21 +1045,4 @@ def eval_plots( sp = args['spatial'] i = args['site_plots'] - if sp: # create the spatial files generally used for FIM Performance. - load_dotenv(args['env_file']) - - API_BASE_URL = os.getenv('API_BASE_URL') - if API_BASE_URL is None: - raise ValueError( - 'API base url not found. ' - 'Ensure inundation_mapping/tools/ has an .env file with the following info: ' - 'API_BASE_URL, WBD_LAYER, NWM_FLOWS_MS, ' - 'USGS_METADATA_URL, USGS_DOWNLOAD_URL' - ) - WBD_LAYER = os.getenv("WBD_LAYER") - API_BASE_URL = os.getenv("API_BASE_URL") - - # Run eval_plots function - print('The following AHPS sites are considered "BAD_SITES": ' + ', '.join(BAD_SITES)) - print('The following query is used to filter AHPS: ' + DISCARD_AHPS_QUERY) eval_plots(metrics_csv=m, workspace=w, versions=v, stats=s, spatial=sp, site_barplots=i) From bfaa35f9b4a171e7b4f38cdcfd639757d8caba99 Mon Sep 17 00:00:00 2001 From: Rob Hanna Date: Tue, 9 Sep 2025 15:00:45 +0000 Subject: [PATCH 7/8] linting fix --- tools/eval_plots.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/eval_plots.py b/tools/eval_plots.py index 64bc87fa4..09a96a355 100644 --- a/tools/eval_plots.py +++ b/tools/eval_plots.py @@ -586,7 +586,6 @@ def eval_plots( print('The following AHPS sites are considered "BAD_SITES": ' + ', '.join(BAD_SITES)) print('The following query is used to filter AHPS: ' + DISCARD_AHPS_QUERY) - # Import metrics csv as DataFrame and initialize all_datasets dictionary csv_df = pd.read_csv(metrics_csv, dtype={'huc': str}) From dddbbc378a4035c2e54f9778768b15337a8cad52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CRobHanna-NOAA=E2=80=9D?= <“Robert.Hanna@NOAA.gov”> Date: Fri, 19 Sep 2025 17:28:48 +0000 Subject: [PATCH 8/8] merge dev and update changelog --- docs/CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index a2ed8774f..2729772b8 100755 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,7 +1,7 @@ All notable changes to this project will be documented in this file. We follow the [Semantic Versioning 2.0.0](http://semver.org/) format. -## v4.8.__._ - 2025-09-02 - [PR#1642](https://github.com/NOAA-OWP/inundation-mapping/pull/1642) +## v4.8.10.6 - 2025-09-19 - [PR#1642](https://github.com/NOAA-OWP/inundation-mapping/pull/1642) Adjust eval_plot.py so it only looks for the config file when in -sp (spatial) mode. Spatial mode is used when running the tool to create FIM_performance files. In that mode, it can only run in OWP servers as it needs to talk to internal servers. @@ -13,6 +13,8 @@ This update does not affect use of the tool as part of regular alpha testing. - `tools\` - `eval_plots.py`: as described above including the zero padding HUC fix. - `eval_plots_stackedbar.py`: zero padding HUC fix. +
+ ## v4.8.10.5 - 2025-09-19 - [PR#1606](https://github.com/NOAA-OWP/inundation-mapping/pull/1606) Added new optional input argument to inundate scripts and `synthesize_test_cases.py` to use the `precalb_discharge_cms` values in the hydrotable instead of the default `discharge_cms`.