Merge pull request #10 from open-AIMS/cache-warmup

Add cache warmup

Author: ConnectedSystems

Manifest.toml

@@ -2,7 +2,7 @@
 
 julia_version = "1.10.5"
 manifest_format = "2.0"
-project_hash = "d0ffae0ac62d29f3001ef8e43624c0daaa93de19"
+project_hash = "23aa2c97b270f292dd6c68842031110f00d039d9"
 
 [[deps.AbstractFFTs]]
 deps = ["LinearAlgebra"]

Project.toml

@@ -35,6 +35,7 @@ Oxygen = "df9a0d86-3283-4920-82dc-4555fc0d1d8b"
 Proj = "c94c279d-25a6-4763-9509-64d165bea63e"
 ProtoBuf = "3349acd9-ac6a-5e09-bcdb-63829b23a429"
 Rasters = "a3a2b9e3-a471-40c9-b274-f788e487c689"
+Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"

README.md

@@ -18,7 +18,8 @@ These are currently the files/data created in Step/Script 1a in https://github.c
 PREPPED_DATA_DIR = "C:/some_path_to_data/MPA/"
 
 [server_config]
-CACHE_DIR = "<some location to cache geotiffs>"
+TIFF_CACHE_DIR = "<some location to cache geotiffs>"
+REGIONAL_CACHE_DIR = "<some location to cache regional datasets>"
 DEBUG_MODE = "false"  # Optional, disables file caching and displays debug logs
 COG_THREADS = "2"  # Optional, Number of threads to use when creating COGs (defaults to 1)
 TILE_SIZE = "256"  # Optional, tile block size to use (defaults to 256)

src/ReefGuideAPI.jl

@@ -5,6 +5,8 @@ using
     Glob,
     TOML
 
+using Serialization
+
 using DataFrames
 using OrderedCollections
 using Memoization
@@ -37,18 +39,38 @@ function get_regions()
     return regions
 end
 
+"""
+    setup_regional_data(config::Dict)
+
+Load regional data to act as an in-memory cache.
+
+# Arguments
+- `config` : Configuration settings, typically loaded from a TOML file.
+- `reef_data_path` : Path to pre-prepared reef data
+
+# Returns
+OrderedDict of `RegionalCriteria` for each region.
+"""
 function setup_regional_data(config::Dict)
-    return setup_regional_data(config["prepped_data"]["PREPPED_DATA_DIR"])
-end
-function setup_regional_data(reef_data_path::String)
     if @isdefined(REGIONAL_DATA)
         @debug "Using previously generated regional data store."
         sleep(1)  # Pause so message is noticeably visible
         return REGIONAL_DATA
     end
 
+    # Check disk-based store
+    reg_cache_dir = config["server_config"]["REGIONAL_CACHE_DIR"]
+    reg_cache_fn = joinpath(reg_cache_dir, "regional_cache.dat")
+    if isfile(reg_cache_fn)
+        @debug "Loading regional data cache from disk"
+        @eval const REGIONAL_DATA = deserialize($(reg_cache_fn))
+        return REGIONAL_DATA
+    end
+
     @debug "Setting up regional data store..."
 
+    reef_data_path = config["prepped_data"]["PREPPED_DATA_DIR"]
+
     regional_assessment_data = OrderedDict{String, RegionalCriteria}()
     for reg in get_regions()
         data_paths = String[]
@@ -74,7 +96,8 @@ function setup_regional_data(reef_data_path::String)
                 elseif occursin("flat", string(dp))
                     flat_table = GeoParquet.read(parq_file)
                 else
-                    error("Unknown lookup found: $(parq_file)")
+                    msg = "Unknown lookup found: $(parq_file). Must be 'slope' or 'flat'"
+                    throw(ArgumentError(msg))
                 end
             end
         end
@@ -96,19 +119,28 @@ function setup_regional_data(reef_data_path::String)
         )
     end
 
+    # Store cache on disk to avoid excessive cold startup times
+    @debug "Saving regional data cache to disk"
+    serialize(reg_cache_fn, regional_assessment_data)
+
     # Remember, `@eval` runs in global scope.
     @eval const REGIONAL_DATA = $(regional_assessment_data)
 
     return REGIONAL_DATA
 end
 
-function _cache_location(config)
+"""
+    _cache_location(config::Dict)::String
+
+Retrieve cache location for geotiffs.
+"""
+function _cache_location(config::Dict)::String
     cache_loc = try
         in_debug = "DEBUG_MODE" in config["server_config"]
         if in_debug && lowercase(config["server_config"]["DEBUG_MODE"]) == "true"
             mktempdir()
         else
-            config["server_config"]["CACHE_DIR"]
+            config["server_config"]["TIFF_CACHE_DIR"]
         end
     catch
         mktempdir()
@@ -117,7 +149,12 @@ function _cache_location(config)
     return cache_loc
 end
 
-function n_gdal_threads(config)::String
+"""
+    n_gdal_threads(config::Dict)::String
+
+Retrieve the configured number of threads to use when writing COGs with GDAL.
+"""
+function n_gdal_threads(config::Dict)::String
     n_cog_threads = try
         config["server_config"]["COG_THREADS"]
     catch
@@ -127,7 +164,12 @@ function n_gdal_threads(config)::String
     return n_cog_threads
 end
 
-function tile_size(config)::Tuple
+"""
+    tile_size(config::Dict)::Tuple
+
+Retrieve the configured size of map tiles in pixels (width and height / lon and lat).
+"""
+function tile_size(config::Dict)::Tuple
     tile_dims = try
         res = parse(Int, config["server_config"]["TILE_SIZE"])
         (res, res)
@@ -138,23 +180,33 @@ function tile_size(config)::Tuple
     return tile_dims
 end
 
+"""
+    warmup_cache(config_path::String)
+
+Invokes warm up of regional data cache to reduce later spin up times.
+"""
+function warmup_cache(config_path::String)
+    config = TOML.parsefile(config_path)
+    setup_regional_data(config)
+end
+
 function start_server(config_path)
-    println("Launching server...please wait") 
+    @info "Launching server... please wait"
 
-    println("Parsing configuration from $(config_path)...") 
+    @info "Parsing configuration from $(config_path)..."
     config = TOML.parsefile(config_path)
-    println("Successfully parsed configuration.") 
+    @info "Successfully parsed configuration."
 
-    println("Setting up region routes...")
+    @info "Setting up region routes..."
     setup_region_routes(config)
-    println("Completed region routes setup.")
+    @info "Completed region routes setup."
 
-    println("Setting up tile routes...")
+    @info "Setting up tile routes..."
     setup_tile_routes()
-    println("Completed tile routes setup.")
+    @info "Completed tile routes setup."
 
-    println("Initialisation complete, starting server on port 8000.") 
-    println("Starting with $(Threads.nthreads()) threads...") 
+    @info "Initialisation complete, starting server on port 8000."
+    @info "Starting with $(Threads.nthreads()) threads..."
     if Threads.nthreads() > 1
         serveparallel(host="0.0.0.0", port=8000)
     else