[1pt] PR: Stop Whitebox from downloading during runtime

Dockerfile

@@ -10,7 +10,7 @@ ENV taudemDir=$depDir/taudem/bin
 ENV taudemDir2=$depDir/taudem_accelerated_flowDirections/taudem/build/bin
 
 # remove reference to missing repo
-RUN rm /etc/apt/sources.list.d/apache-arrow.sources
+# RUN rm /etc/apt/sources.list.d/apache-arrow.sources
 
 RUN apt-get update && apt-get install -y git  && rm -rf /var/lib/apt/lists/*
 
@@ -70,20 +70,12 @@ RUN mkdir -p $depDir
 COPY --from=builder $depDir $depDir
 
 # remove reference to missing repo
-RUN rm /etc/apt/sources.list.d/apache-arrow.sources
-
-RUN apt-get update --fix-missing && apt-get install -y openjdk-21-jdk && rm -rf /var/lib/apt/lists/*
-RUN apt update --fix-missing
-
-# An older version of openjdk still exists on the file system but was never cleaned up
-# After research, we realized, it just needs file cleanup. Leaving it there is triggering security warnings
-# RUN apt-get remove -y openjdk-17-jdk  (not installed, just residue left)
-RUN rm -rf ./usr/lib/jvm/*java-1.17* && \
-    rm -rf ./usr/lib/jvm/.java-1.17* && \
-    rm -rdf ./usr/lib/jvm/java-17*
+# RUN rm /etc/apt/sources.list.d/apache-arrow.sources
 
+# RUN apt-get update --fix-missing && apt-get install -y openjdk-19-jdk && rm -rf /var/lib/apt/lists/*
 
-RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt install -y p7zip-full python3-pip time mpich parallel libgeos-dev expect tmux rsync tzdata
+RUN apt update --fix-missing
+RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt install -y p7zip-full python3-pip time mpich parallel libgeos-dev expect tmux rsync tzdata wget
 
 RUN apt auto-remove
 
@@ -99,9 +91,7 @@ ENV PYTHONUNBUFFERED=TRUE
 
 ## ADD TO PATHS ##
 ENV PATH="$projectDir:${PATH}"
-#ENV PATH=${PATH}:$projectDir:$projectDir/$srcDir:$projectDir/tools
-# Jul 17, 2024: Even though PYTHONPATH isn't used, it still seems to want it.
-ENV PYTHONPATH=${PATH}:$srcDir:$projectDir/tools
+ENV PYTHONPATH=${PYTHONPATH}:$srcDir:$projectDir/unit_tests:$projectDir/tools
 
 ## install python 3 modules ##
 
@@ -120,12 +110,10 @@ RUN pip3 install pipenv==2024.0.1 && PIP_NO_CACHE_DIR=off pipenv install --syste
 # We download and unzip it to the same file folder that pip deployed the whitebox library.
 # Whitebox also attempts to always download a folder called testdata regardless of use.
 # We added an empty folder to fake out whitebox_tools.py so it doesn't try to download the folder
-
-# RUN wbox_path=/usr/local/lib/python3.10/dist-packages/whitebox/WBT && \
-#     wget -P $wbox_path https://www.whiteboxgeo.com/WBT_Linux/WhiteboxTools_linux_musl.zip && \
-#     unzip -o $wbox_path/WhiteboxTools_linux_musl.zip -d $wbox_path && \
-#     cp $wbox_path/whitebox_tools $wbox_path && \
-#     mkdir $wbox_path/testdata
+ENV WBT_PATH=/usr/local/lib/python3.10/dist-packages/whitebox/WBT
+RUN wget -P $WBT_PATH https://www.whiteboxgeo.com/WBT_Linux/WhiteboxTools_linux_musl.zip && \
+    unzip -o $WBT_PATH/WhiteboxTools_linux_musl.zip -d $WBT_PATH && \
+    cp $WBT_PATH/WhiteboxTools_linux_amd64/WBT/whitebox_tools $WBT_PATH
 # ----------------------------------
 
 ## RUN UMASK TO CHANGE DEFAULT PERMISSIONS ##

Pipfile

@@ -50,7 +50,7 @@ seaborn = "==0.12.2"
 tables = "==3.8.0"
 tqdm = "==4.66.3"
 urllib3 = "==1.26.19"
-whitebox = "==2.3.4"
+whitebox = "==2.3.5"
 xarray = "==2024.6.0"
 zarr = "==2.18.0"
 

docs/CHANGELOG.md

@@ -1,6 +1,20 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.5.x.x - 2024-09-17 - [PR#1293](https://github.com/NOAA-OWP/inundation-mapping/pull/1293)
+
+Stops Whitebox from downloading during runtime. Not only does this slow down the `fim_pipeline` runtime, but it leaves a vulnerability to changes in the file being downloaded from Whitebox and/or errors if the Whitebox server is down.
+
+### Changes
+
+- `Dockerfile`: sets `WBT_PATH` environment variable which stops the Whitebox download (Whitebox is manually downloaded during `docker build`).
+- `Pipfile` and `Pipfile.lock`: Upgrades Whitebox to v2.3.5.
+- `fim_pre_processing.sh`: Removes unnecessary `WBT_PATH` assignment.
+- `src/`
+    `agreedem.py` and `unique_pixel_and_allocation.py`: Sets `whitebox_dir` to `WBT_PATH`.
+
+<br/><br/>
+
 ## v4.5.8.0 - 2024-09-13 - [PR#1165](https://github.com/NOAA-OWP/inundation-mapping/pull/1165)
 
 This PR was originally intended to get Alaska HUCs incorporated into CatFIM, but there were a very, very large array of problems and the tool was unable to run. We have made some major modifications and many more will come in the near future. There are partial hooks and commented code for Alaska integration, but temporarily disabled are included and will be handled by a separate branch / PR.

fim_pre_processing.sh

@@ -122,7 +122,7 @@ fi
 # outputsDir & workDir come from the Dockerfile
 outputDestDir=$outputsDir/$runName
 tempRunDir=$workDir/$runName
-export WBT_PATH=${tempRunDir}/whitebox_temp
+# export WBT_PATH=${tempRunDir}/whitebox_temp
 
 # default values
 if [ "$envFile" = "" ]; then envFile=/$projectDir/config/params_template.env; fi

src/agreedem.py

@@ -50,6 +50,7 @@ def agreedem(
     # Set wbt envs
     wbt = whitebox.WhiteboxTools()
     wbt.set_verbose_mode(False)
+    wbt.set_whitebox_dir(os.environ.get("WBT_PATH"))
 
     # ------------------------------------------------------------------
     # 1. From Hellweger documentation: Compute the vector grid

src/unique_pixel_and_allocation.py

@@ -34,6 +34,7 @@ def stream_pixel_zones(stream_pixels, unique_stream_pixels):
     # Set wbt envs
     wbt = whitebox.WhiteboxTools()
     wbt.set_verbose_mode(False)
+    wbt.set_whitebox_dir(os.environ.get("WBT_PATH"))
 
     workspace = os.path.dirname(unique_stream_pixels)
     base = os.path.basename(unique_stream_pixels)