Merge branch 'main' into jwt-auth

Author: ConnectedSystems

Project.toml

@@ -37,6 +37,7 @@ Oxygen = "df9a0d86-3283-4920-82dc-4555fc0d1d8b"
 Proj = "c94c279d-25a6-4763-9509-64d165bea63e"
 ProtoBuf = "3349acd9-ac6a-5e09-bcdb-63829b23a429"
 Rasters = "a3a2b9e3-a471-40c9-b274-f788e487c689"
+Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"

README.md

@@ -18,7 +18,8 @@ These are currently the files/data created in Step/Script 1a in https://github.c
 PREPPED_DATA_DIR = "C:/some_path_to_data/MPA/"
 
 [server_config]
-CACHE_DIR = "<some location to cache geotiffs>"
+TIFF_CACHE_DIR = "<some location to cache geotiffs>"
+REGIONAL_CACHE_DIR = "<some location to cache regional datasets>"
 DEBUG_MODE = "false"  # Optional, disables file caching and displays debug logs
 COG_THREADS = "2"  # Optional, Number of threads to use when creating COGs (defaults to 1)
 TILE_SIZE = "256"  # Optional, tile block size to use (defaults to 256)

src/ReefGuideAPI.jl

@@ -5,6 +5,8 @@ using
     Glob,
     TOML
 
+using Serialization
+
 using DataFrames
 using OrderedCollections
 using Memoization
@@ -38,18 +40,38 @@ function get_regions()
     return regions
 end
 
+"""
+    setup_regional_data(config::Dict)
+
+Load regional data to act as an in-memory cache.
+
+# Arguments
+- `config` : Configuration settings, typically loaded from a TOML file.
+- `reef_data_path` : Path to pre-prepared reef data
+
+# Returns
+OrderedDict of `RegionalCriteria` for each region.
+"""
 function setup_regional_data(config::Dict)
-    return setup_regional_data(config["prepped_data"]["PREPPED_DATA_DIR"])
-end
-function setup_regional_data(reef_data_path::String)
     if @isdefined(REGIONAL_DATA)
         @debug "Using previously generated regional data store."
         sleep(1)  # Pause so message is noticeably visible
         return REGIONAL_DATA
     end
 
+    # Check disk-based store
+    reg_cache_dir = config["server_config"]["REGIONAL_CACHE_DIR"]
+    reg_cache_fn = joinpath(reg_cache_dir, "regional_cache.dat")
+    if isfile(reg_cache_fn)
+        @debug "Loading regional data cache from disk"
+        @eval const REGIONAL_DATA = deserialize($(reg_cache_fn))
+        return REGIONAL_DATA
+    end
+
     @debug "Setting up regional data store..."
 
+    reef_data_path = config["prepped_data"]["PREPPED_DATA_DIR"]
+
     regional_assessment_data = OrderedDict{String, RegionalCriteria}()
     for reg in get_regions()
         data_paths = String[]
@@ -75,7 +97,8 @@ function setup_regional_data(reef_data_path::String)
                 elseif occursin("flat", string(dp))
                     flat_table = GeoParquet.read(parq_file)
                 else
-                    error("Unknown lookup found: $(parq_file)")
+                    msg = "Unknown lookup found: $(parq_file). Must be 'slope' or 'flat'"
+                    throw(ArgumentError(msg))
                 end
             end
         end
@@ -97,19 +120,28 @@ function setup_regional_data(reef_data_path::String)
         )
     end
 
+    # Store cache on disk to avoid excessive cold startup times
+    @debug "Saving regional data cache to disk"
+    serialize(reg_cache_fn, regional_assessment_data)
+
     # Remember, `@eval` runs in global scope.
     @eval const REGIONAL_DATA = $(regional_assessment_data)
 
     return REGIONAL_DATA
 end
 
-function _cache_location(config)
+"""
+    _cache_location(config::Dict)::String
+
+Retrieve cache location for geotiffs.
+"""
+function _cache_location(config::Dict)::String
     cache_loc = try
         in_debug = "DEBUG_MODE" in config["server_config"]
         if in_debug && lowercase(config["server_config"]["DEBUG_MODE"]) == "true"
             mktempdir()
         else
-            config["server_config"]["CACHE_DIR"]
+            config["server_config"]["TIFF_CACHE_DIR"]
         end
     catch
         mktempdir()
@@ -118,7 +150,12 @@ function _cache_location(config)
     return cache_loc
 end
 
-function n_gdal_threads(config)::String
+"""
+    n_gdal_threads(config::Dict)::String
+
+Retrieve the configured number of threads to use when writing COGs with GDAL.
+"""
+function n_gdal_threads(config::Dict)::String
     n_cog_threads = try
         config["server_config"]["COG_THREADS"]
     catch
@@ -128,7 +165,12 @@ function n_gdal_threads(config)::String
     return n_cog_threads
 end
 
-function tile_size(config)::Tuple
+"""
+    tile_size(config::Dict)::Tuple
+
+Retrieve the configured size of map tiles in pixels (width and height / lon and lat).
+"""
+function tile_size(config::Dict)::Tuple
     tile_dims = try
         res = parse(Int, config["server_config"]["TILE_SIZE"])
         (res, res)
@@ -145,8 +187,19 @@ function get_auth_middleware(config :: Dict)
     return [auth]
 end
 
+"""
+    warmup_cache(config_path::String)
+
+Invokes warm up of regional data cache to reduce later spin up times.
+"""
+function warmup_cache(config_path::String)
+    config = TOML.parsefile(config_path)
+    setup_regional_data(config)
+end
+
+
 function start_server(config_path)
-    @info "Launching server...please wait"
+    @info "Launching server... please wait"
 
     @info "Parsing configuration from $(config_path)..."
     config = TOML.parsefile(config_path)
@@ -163,6 +216,13 @@ function start_server(config_path)
 
     @info "Setting up tile routes..."
     setup_tile_routes(auth)
+
+    @info "Setting up region routes..."
+    setup_region_routes(config)
+    @info "Completed region routes setup."
+
+    @info "Setting up tile routes..."
+    setup_tile_routes()
     @info "Completed tile routes setup."
 
     @info "Initialisation complete, starting server on port 8000."