From 90b2055415a45e4a0807b48fe8811ea1bde78b00 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Wed, 21 May 2025 12:12:58 +1000 Subject: [PATCH 01/26] WIP on improving criteria parsing/defaults Signed-off-by: Peter Baker --- src/job_worker/Helpers.jl | 198 ++++++++++++++++++++++++++++++++++++++ src/job_worker/Jobs.jl | 58 +++++------ src/job_worker/Worker.jl | 1 + 3 files changed, 225 insertions(+), 32 deletions(-) create mode 100644 src/job_worker/Helpers.jl diff --git a/src/job_worker/Helpers.jl b/src/job_worker/Helpers.jl new file mode 100644 index 0000000..7fa837c --- /dev/null +++ b/src/job_worker/Helpers.jl @@ -0,0 +1,198 @@ +""" +Helper methods for criteria parsing etc. +""" + +# Default threshold when not provided in inputs +const DEFAULT_SUITABILITY_THRESHOLD = 50 + +""" +Min/max storage for environmental criteria +""" +mutable struct Range + min::Float32 + max::Float32 + label::String +end + +""" +Serialises the value of the range to min:max format +""" +function serialise_range(range::Range)::String + return "$(range.min):$(range.max)" +end + +""" +Builds a dictionary key-value pair from a Range +""" +function range_entry_to_kvp(range::Range)::Tuple{String,String} + return (range.label, serialise_range(range)) +end + +""" +Typed/structured ranges for all notable criteria +""" +mutable struct RelevantRanges + depth::Range + slope::Range + turbidity::Range + waves_height::Range + waves_period::Range + rugosity::Range + + function RelevantRanges(; + depth::Range, + slope::Range, + turbidity::Range, + waves_height::Range, + waves_period::Range, + rugosity::Range + ) + return new(depth, slope, turbidity, waves_height, waves_period, rugosity) + end +end + +""" +Dumps the relevant ranges into a dictionary following the appropriate style for +other methods +""" +function relevant_ranges_to_dict(ranges::RelevantRanges)::Dict{String,String} + return Dict{String,String}( + range_entry_to_kvp.([ + ranges.depth, + ranges.slope, + ranges.turbidity, + ranges.waves_height, + ranges.waves_period, + ranges.rugosity + ]) + ) +end + +""" +Converts DataFrame criteria ranges to a structured RelevantRanges object +""" +function structured_ranges_from_criteria_ranges(criteria_ranges::DataFrame)::RelevantRanges + # Extract min/max values from the criteria_ranges DataFrame + depth_range = Range( + criteria_ranges[1, "Depth"], + criteria_ranges[2, "Depth"], + "Depth" + ) + + slope_range = Range( + criteria_ranges[1, "Slope"], + criteria_ranges[2, "Slope"], + "Slope" + ) + + turbidity_range = Range( + criteria_ranges[1, "Turbidity"], + criteria_ranges[2, "Turbidity"], + "Turbidity" + ) + + waves_height_range = Range( + criteria_ranges[1, "WavesHs"], + criteria_ranges[2, "WavesHs"], + "WavesHs" + ) + + waves_period_range = Range( + criteria_ranges[1, "WavesTp"], + criteria_ranges[2, "WavesTp"], + "WavesTp" + ) + + rugosity_range = Range( + criteria_ranges[1, "Rugosity"], + criteria_ranges[2, "Rugosity"], + "Rugosity" + ) + + return RelevantRanges( + depth=depth_range, + slope=slope_range, + turbidity=turbidity_range, + waves_height=waves_height_range, + waves_period=waves_period_range, + rugosity=rugosity_range + ) +end + +""" +Applies optional min/max overrides to a Range object +""" +function apply_optional_overrides!(range::Range, min_value::OptionalValue{Float64}, max_value::OptionalValue{Float64}) + if !isnothing(min_value) + range.min = min_value + end + if !isnothing(max_value) + range.max = max_value + end +end + +""" +Applies all criteria overrides from input to a RelevantRanges object +""" +function apply_criteria_overrides!(ranges::RelevantRanges, criteria::Union{RegionalAssessmentInput, SuitabilityAssessmentInput}) + # Apply overrides for each range + apply_optional_overrides!(ranges.depth, criteria.depth_min, criteria.depth_max) + apply_optional_overrides!(ranges.slope, criteria.slope_min, criteria.slope_max) + apply_optional_overrides!(ranges.rugosity, criteria.rugosity_min, criteria.rugosity_max) + apply_optional_overrides!(ranges.waves_period, criteria.waves_period_min, criteria.waves_period_max) + apply_optional_overrides!(ranges.waves_height, criteria.waves_height_min, criteria.waves_height_max) +end + +""" +Builds a parameters dictionary from RegionalAssessmentInput, applying overrides as needed +""" +function build_params_dictionary_from_regional_input( + # The regional criteria job input + criteria::RegionalAssessmentInput, + # Criteria ranges as [[min,max], name] i.e. [2, name] = max of name + criteria_ranges::DataFrame +)::Dict{String,String} + # Get the structured ranges + default_ranges = structured_ranges_from_criteria_ranges(criteria_ranges) + + # Apply all overrides + apply_criteria_overrides!(default_ranges, criteria) + + # Base dictionary of ranges + ranges_dict = relevant_ranges_to_dict(default_ranges) + + # Add in suitability threshold + threshold_value = isnothing(criteria.threshold) ? DEFAULT_SUITABILITY_THRESHOLD : criteria.threshold + ranges_dict["SuitabilityThreshold"] = "$(threshold_value)" + + return ranges_dict +end + +""" +Builds a parameters dictionary from SuitabilityAssessmentInput, applying overrides and adding suitability-specific parameters +""" +function build_params_dictionary_from_suitability_input( + # The suitability criteria job input + criteria::SuitabilityAssessmentInput, + # Criteria ranges as [[min,max], name] i.e. [2, name] = max of name + criteria_ranges::DataFrame +)::Dict{String,String} + # Get the structured ranges + default_ranges = structured_ranges_from_criteria_ranges(criteria_ranges) + + # Apply all overrides + apply_criteria_overrides!(default_ranges, criteria) + + # Base dictionary of ranges + ranges_dict = relevant_ranges_to_dict(default_ranges) + + # Add in suitability threshold + threshold_value = isnothing(criteria.threshold) ? DEFAULT_SUITABILITY_THRESHOLD : criteria.threshold + ranges_dict["SuitabilityThreshold"] = "$(threshold_value)" + + # Suitability specific entries + ranges_dict["xdist"] = "$(criteria.x_dist)" + ranges_dict["ydist"] = "$(criteria.y_dist)" + + return ranges_dict +end diff --git a/src/job_worker/Jobs.jl b/src/job_worker/Jobs.jl index abd252e..0c67b96 100644 --- a/src/job_worker/Jobs.jl +++ b/src/job_worker/Jobs.jl @@ -7,6 +7,8 @@ using JSON3 using Logging using Dates +const OptionalValue{T} = Union{T,Nothing}; + """ create_job_id(query_params::Dict)::String @@ -276,22 +278,18 @@ struct RegionalAssessmentInput <: AbstractJobInput region::String "The type of reef, slopes or flats" reef_type::String - - # Criteria - "The depth range (min)" - depth_min::Float64 - "The depth range (max)" - depth_max::Float64 - "The slope range (min)" - slope_min::Float64 - "The slope range (max)" - slope_max::Float64 - "The rugosity range (min)" - rugosity_min::Float64 - "The rugosity range (max)" - rugosity_max::Float64 - "Suitability threshold (min)" - threshold::Int64 + # Criteria (all optional - defaulting to min/max of criteria) + depth_min::OptionalValue{Float64} + depth_max::OptionalValue{Float64} + slope_min::OptionalValue{Float64} + slope_max::OptionalValue{Float64} + rugosity_min::OptionalValue{Float64} + rugosity_max::OptionalValue{Float64} + waves_period_min::OptionalValue{Float64} + waves_period_max::OptionalValue{Float64} + waves_height_min::OptionalValue{Float64} + waves_height_max::OptionalValue{Float64} + threshold::OptionalValue{Int64} end """ @@ -385,27 +383,23 @@ Input payload for SUITABILITY_ASSESSMENT job """ struct SuitabilityAssessmentInput <: AbstractJobInput # High level config - "Region for assessment" region::String "The type of reef, slopes or flats" reef_type::String - # Criteria - "The depth range (min)" - depth_min::Float64 - "The depth range (max)" - depth_max::Float64 - "The slope range (min)" - slope_min::Float64 - "The slope range (max)" - slope_max::Float64 - "The rugosity range (min)" - rugosity_min::Float64 - "The rugosity range (max)" - rugosity_max::Float64 - "Suitability threshold (min)" - threshold::Int64 + depth_min::OptionalValue{Float64} + depth_max::OptionalValue{Float64} + slope_min::OptionalValue{Float64} + slope_max::OptionalValue{Float64} + rugosity_min::OptionalValue{Float64} + rugosity_max::OptionalValue{Float64} + waves_period_min::OptionalValue{Float64} + waves_period_max::OptionalValue{Float64} + waves_height_min::OptionalValue{Float64} + waves_height_max::OptionalValue{Float64} + threshold::OptionalValue{Int64} + # Unique to suitability assessment - required "Length dimension of target polygon" x_dist::Int64 "Width dimension of target polygon" diff --git a/src/job_worker/Worker.jl b/src/job_worker/Worker.jl index 34b5118..f0cf25d 100644 --- a/src/job_worker/Worker.jl +++ b/src/job_worker/Worker.jl @@ -14,6 +14,7 @@ include("ECS.jl") include("HttpClient.jl") include("Jobs.jl") include("Storage.jl") +include("Helpers.jl") """ Represents a job that needs to be processed From f239fd155029b6ae6e7c742188727a81929a9414 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Wed, 21 May 2025 12:18:46 +1000 Subject: [PATCH 02/26] Integrating helpers with job routes Signed-off-by: Peter Baker --- src/job_worker/Helpers.jl | 53 +++++++++++++++++++++++---------------- src/job_worker/Jobs.jl | 42 +++++++++++++------------------ 2 files changed, 50 insertions(+), 45 deletions(-) diff --git a/src/job_worker/Helpers.jl b/src/job_worker/Helpers.jl index 7fa837c..19b6b5a 100644 --- a/src/job_worker/Helpers.jl +++ b/src/job_worker/Helpers.jl @@ -3,10 +3,10 @@ Helper methods for criteria parsing etc. """ # Default threshold when not provided in inputs -const DEFAULT_SUITABILITY_THRESHOLD = 50 +const DEFAULT_SUITABILITY_THRESHOLD = 80 """ -Min/max storage for environmental criteria +Min/max storage for criteria """ mutable struct Range min::Float32 @@ -29,7 +29,7 @@ function range_entry_to_kvp(range::Range)::Tuple{String,String} end """ -Typed/structured ranges for all notable criteria +Typed/structured ranges for all common ranged criteria """ mutable struct RelevantRanges depth::Range @@ -109,7 +109,7 @@ function structured_ranges_from_criteria_ranges(criteria_ranges::DataFrame)::Rel "Rugosity" ) - return RelevantRanges( + return RelevantRanges(; depth=depth_range, slope=slope_range, turbidity=turbidity_range, @@ -122,7 +122,9 @@ end """ Applies optional min/max overrides to a Range object """ -function apply_optional_overrides!(range::Range, min_value::OptionalValue{Float64}, max_value::OptionalValue{Float64}) +function apply_optional_overrides!( + range::Range, min_value::OptionalValue{Float64}, max_value::OptionalValue{Float64} +) if !isnothing(min_value) range.min = min_value end @@ -134,19 +136,26 @@ end """ Applies all criteria overrides from input to a RelevantRanges object """ -function apply_criteria_overrides!(ranges::RelevantRanges, criteria::Union{RegionalAssessmentInput, SuitabilityAssessmentInput}) +function apply_criteria_overrides!( + ranges::RelevantRanges, + criteria::Union{RegionalAssessmentInput,SuitabilityAssessmentInput} +) # Apply overrides for each range apply_optional_overrides!(ranges.depth, criteria.depth_min, criteria.depth_max) apply_optional_overrides!(ranges.slope, criteria.slope_min, criteria.slope_max) apply_optional_overrides!(ranges.rugosity, criteria.rugosity_min, criteria.rugosity_max) - apply_optional_overrides!(ranges.waves_period, criteria.waves_period_min, criteria.waves_period_max) - apply_optional_overrides!(ranges.waves_height, criteria.waves_height_min, criteria.waves_height_max) + apply_optional_overrides!( + ranges.waves_period, criteria.waves_period_min, criteria.waves_period_max + ) + return apply_optional_overrides!( + ranges.waves_height, criteria.waves_height_min, criteria.waves_height_max + ) end """ Builds a parameters dictionary from RegionalAssessmentInput, applying overrides as needed """ -function build_params_dictionary_from_regional_input( +function build_params_dictionary_from_regional_input(; # The regional criteria job input criteria::RegionalAssessmentInput, # Criteria ranges as [[min,max], name] i.e. [2, name] = max of name @@ -154,24 +163,25 @@ function build_params_dictionary_from_regional_input( )::Dict{String,String} # Get the structured ranges default_ranges = structured_ranges_from_criteria_ranges(criteria_ranges) - + # Apply all overrides apply_criteria_overrides!(default_ranges, criteria) - + # Base dictionary of ranges ranges_dict = relevant_ranges_to_dict(default_ranges) - + # Add in suitability threshold - threshold_value = isnothing(criteria.threshold) ? DEFAULT_SUITABILITY_THRESHOLD : criteria.threshold + threshold_value = + isnothing(criteria.threshold) ? DEFAULT_SUITABILITY_THRESHOLD : criteria.threshold ranges_dict["SuitabilityThreshold"] = "$(threshold_value)" - + return ranges_dict end """ Builds a parameters dictionary from SuitabilityAssessmentInput, applying overrides and adding suitability-specific parameters """ -function build_params_dictionary_from_suitability_input( +function build_params_dictionary_from_suitability_input(; # The suitability criteria job input criteria::SuitabilityAssessmentInput, # Criteria ranges as [[min,max], name] i.e. [2, name] = max of name @@ -179,20 +189,21 @@ function build_params_dictionary_from_suitability_input( )::Dict{String,String} # Get the structured ranges default_ranges = structured_ranges_from_criteria_ranges(criteria_ranges) - + # Apply all overrides apply_criteria_overrides!(default_ranges, criteria) - + # Base dictionary of ranges ranges_dict = relevant_ranges_to_dict(default_ranges) - + # Add in suitability threshold - threshold_value = isnothing(criteria.threshold) ? DEFAULT_SUITABILITY_THRESHOLD : criteria.threshold + threshold_value = + isnothing(criteria.threshold) ? DEFAULT_SUITABILITY_THRESHOLD : criteria.threshold ranges_dict["SuitabilityThreshold"] = "$(threshold_value)" - + # Suitability specific entries ranges_dict["xdist"] = "$(criteria.x_dist)" ranges_dict["ydist"] = "$(criteria.y_dist)" - + return ranges_dict end diff --git a/src/job_worker/Jobs.jl b/src/job_worker/Jobs.jl index 0c67b96..ef025cb 100644 --- a/src/job_worker/Jobs.jl +++ b/src/job_worker/Jobs.jl @@ -327,25 +327,23 @@ function handle_job( reg = input.region rtype = input.reef_type - # This is the format expected by prior methods TODO improve the separation - # of concerns between query string parameters and function inputs - the API - # routing concerns should not be exposed to the application layer - qp::Dict{String,String} = Dict{String,String}( - "Depth" => "$(input.depth_min):$(input.depth_max)", - "Slope" => "$(input.slope_min):$(input.slope_max)", - "Rugosity" => "$(input.rugosity_min):$(input.rugosity_max)", - "SuitabilityThreshold" => "$(input.threshold)" + # Build the fully populated query params - noting that this merges defaults + # computed as part of the regional data setup with the user provided values + # (if present) + criteria_dictionary = build_params_dictionary_from_regional_input(; + criteria=input, + criteria_ranges=reg_assess_data["criteria_ranges"] ) assessed_fn = build_regional_assessment_file_path(; - query_params=qp, region=reg, reef_type=rtype, ext="tiff", config + query_params=criteria_dictionary, region=reg, reef_type=rtype, ext="tiff", config ) @debug "COG File name: $(assessed_fn)" if !isfile(assessed_fn) @debug "File system cache was not hit for this task" @debug "Assessing region $(reg)" - assessed = assess_region(reg_assess_data, reg, qp, rtype) + assessed = assess_region(reg_assess_data, reg, criteria_dictionary, rtype) @debug now() "Writing COG of regional assessment to $(assessed_fn)" _write_cog(assessed_fn, assessed, config) @@ -441,27 +439,23 @@ function handle_job( reg = input.region rtype = input.reef_type - # This is the format expected by prior methods TODO improve the separation - # of concerns between query string parameters and function inputs - the API - # routing concerns should not be exposed to the application layer - qp::Dict{String,String} = Dict{String,String}( - "Depth" => "$(input.depth_min):$(input.depth_max)", - "Slope" => "$(input.slope_min):$(input.slope_max)", - "Rugosity" => "$(input.rugosity_min):$(input.rugosity_max)", - "SuitabilityThreshold" => "$(input.threshold)", - "xdist" => "$(input.x_dist)", - "ydist" => "$(input.y_dist)" + # Build the fully populated query params - noting that this merges defaults + # computed as part of the regional data setup with the user provided values + # (if present) + criteria_dictionary = build_params_dictionary_from_regional_input(; + criteria=input, + criteria_ranges=reg_assess_data["criteria_ranges"] ) assessed_fn = build_regional_assessment_file_path(; - query_params=qp, region=reg, reef_type=rtype, ext="tiff", config + query_params=criteria_dictionary, region=reg, reef_type=rtype, ext="tiff", config ) @debug "COG File name: $(assessed_fn)" if !isfile(assessed_fn) @debug "File system cache was not hit for this task" @debug "Assessing region $(reg)" - assessed = assess_region(reg_assess_data, reg, qp, rtype) + assessed = assess_region(reg_assess_data, reg, criteria_dictionary, rtype) @debug "Writing COG to $(assessed_fn)" _write_cog(assessed_fn, assessed, config) @@ -472,8 +466,8 @@ function handle_job( end # Extract criteria and assessment - pixel_criteria = extract_criteria(qp, search_criteria()) - deploy_site_criteria = extract_criteria(qp, site_criteria()) + pixel_criteria = extract_criteria(criteria_dictionary, search_criteria()) + deploy_site_criteria = extract_criteria(criteria_dictionary, site_criteria()) @debug "Performing site assessment" best_sites = filter_sites( From 25f2a2428441d47cd1d39842b38084846d7121cc Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Wed, 21 May 2025 12:45:21 +1000 Subject: [PATCH 03/26] Debug messages and adding out of bounds warnings Signed-off-by: Peter Baker --- src/job_worker/Helpers.jl | 8 ++++++++ src/job_worker/Jobs.jl | 3 +++ 2 files changed, 11 insertions(+) diff --git a/src/job_worker/Helpers.jl b/src/job_worker/Helpers.jl index 19b6b5a..423cf1b 100644 --- a/src/job_worker/Helpers.jl +++ b/src/job_worker/Helpers.jl @@ -121,14 +121,22 @@ end """ Applies optional min/max overrides to a Range object + +Warns if suggested overrides are beyond the dataset bounds. """ function apply_optional_overrides!( range::Range, min_value::OptionalValue{Float64}, max_value::OptionalValue{Float64} ) if !isnothing(min_value) + if min_value < range.min + @warn "Minimum range value ($(min_value)) for $(range.label) was outside of the dataset bounds ($(range.min)). Proceeding regardless." + end range.min = min_value end if !isnothing(max_value) + if max_value < range.max + @warn "Maximum range value ($(max_value)) for $(range.label) was outside of the dataset bounds ($(range.max)). Proceeding regardless." + end range.max = max_value end end diff --git a/src/job_worker/Jobs.jl b/src/job_worker/Jobs.jl index ef025cb..77d476e 100644 --- a/src/job_worker/Jobs.jl +++ b/src/job_worker/Jobs.jl @@ -335,6 +335,8 @@ function handle_job( criteria_ranges=reg_assess_data["criteria_ranges"] ) + @debug "Criteria after merging default and provided ranges" criteria_dictionary + assessed_fn = build_regional_assessment_file_path(; query_params=criteria_dictionary, region=reg, reef_type=rtype, ext="tiff", config ) @@ -446,6 +448,7 @@ function handle_job( criteria=input, criteria_ranges=reg_assess_data["criteria_ranges"] ) + @debug "Criteria after merging default and provided ranges" criteria_dictionary assessed_fn = build_regional_assessment_file_path(; query_params=criteria_dictionary, region=reg, reef_type=rtype, ext="tiff", config From 77b740f651f0068b8357384416188d5965a199ad Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Wed, 21 May 2025 17:09:34 +1000 Subject: [PATCH 04/26] WIP Signed-off-by: Peter Baker --- src/ReefGuideAPI.jl | 1 + src/server_cache.jl | 1 + src/setup.jl | 421 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 423 insertions(+) create mode 100644 src/setup.jl diff --git a/src/ReefGuideAPI.jl b/src/ReefGuideAPI.jl index 9d09d9a..92b2a70 100644 --- a/src/ReefGuideAPI.jl +++ b/src/ReefGuideAPI.jl @@ -26,6 +26,7 @@ using include("job_worker/Worker.jl") +include("setup.jl") include("Middleware.jl") include("admin.jl") include("file_io.jl") diff --git a/src/server_cache.jl b/src/server_cache.jl index 82097a3..1f4d3c3 100644 --- a/src/server_cache.jl +++ b/src/server_cache.jl @@ -102,6 +102,7 @@ function initialize_regional_data_cache(reef_data_path::String, reg_cache_fn::St # Pre-extract long/lat coordinates coords = GI.coordinates.(slope_table.geometry) + # For each entry, extract slope_table[!, :lons] .= first.(coords) slope_table[!, :lats] .= last.(coords) diff --git a/src/setup.jl b/src/setup.jl new file mode 100644 index 0000000..ae9ed8a --- /dev/null +++ b/src/setup.jl @@ -0,0 +1,421 @@ +using Rasters + +const REGIONAL_DATA_CACHE_FILENAME = "regional_cache_v2.dat" + +@enum Region begin + TOWNSVILLE_WHITSUNDAY + CAIRNS_COOKTOWN + MACKAY_CAPRICORN + FAR_NORTHERN +end + +struct RegionDetails + region::Region + display_name::String + id::String + + function RegionDetails(; + region::Region, + display_name::String, + id::String + ) + return new(region, display_name, id) + end +end + +const REGIONS::Vector{RegionDetails} = [ + RegionDetails(; + region=Region.TOWNSVILLE_WHITSUNDAY, + display_name="Townsville/Whitsunday Management Area", + id="Townsville-Whitsunday" + ), + RegionDetails(; + region=Region.CAIRNS_COOKTOWN, + display_name="Cairns/Cooktown Management Area", + id="Cairns-Cooktown" + ), + RegionDetails(; + region=Region.MACKAY_CAPRICORN, + display_name="Mackay/Capricorn Management Area", + id="Mackay-Capricorn" + ), + RegionDetails(; + region=Region.FAR_NORTHERN, + display_name="Far Northern Management Area", + id="Far-Northern" + ) +] + +""" +Simple struct to contain min/max values +""" +mutable struct Bounds + min::Float32 + max::Float32 + + function Bounds(; min::Number, max::Number) + return new(parse(Float32, min), parse(Float32, max)) + end +end + +struct CriteriaMetadata + "System ID used to uniquely identify this criteria" + id::String + "What is the suffix of this criteria in the data?" + file_suffix::String + "Pretty display name, can be changed with a guarantee of no runtime consequences/errors" + display_label::String + + function CriteriaMetadata(; + id::String, + file_suffix::String, + display_label::String + ) + return new( + id, + file_suffix, + display_label + ) + end +end + +struct AssessmentCriteria + depth::CriteriaMetadata + slope::CriteriaMetadata + turbidity::CriteriaMetadata + waves_height::CriteriaMetadata + waves_period::CriteriaMetadata + rugosity::CriteriaMetadata + + function AssessmentCriteria(; + depth::CriteriaMetadata, + slope::CriteriaMetadata, + turbidity::CriteriaMetadata, + waves_height::CriteriaMetadata, + waves_period::CriteriaMetadata, + rugosity::CriteriaMetadata + ) + return new( + depth, slope, turbidity, waves_height, waves_period, rugosity + ) + end +end + +const ASSESSMENT_CRITERIA::AssessmentCriteria = AssessmentCriteria(; + depth=CriteriaMetadata(; + id="Depth", + file_suffix="_bathy", + display_label="Depth" + ), + slope=CriteriaMetadata(; + id="Slope", + file_suffix="_slope", + display_label="Slope" + ), + turbidity=CriteriaMetadata(; + id="Turbidity", + file_suffix="_turbid", + display_label="Turbidity" + ), + waves_height=CriteriaMetadata(; + id="WavesHs", + file_suffix="_waves_Hs", + display_label="Wave Height (m)" + ), + waves_period=CriteriaMetadata(; + id="WavesTp", + file_suffix="_waves_Tp", + display_label="Wave Period (s)" + ), + rugosity=CriteriaMetadata(; + id="Rugosity", + file_suffix="_rugosity", + display_label="Rugosity" + ) +) + +struct RegionalCriteriaEntry + "Other details about the criteria" + metadata::CriteriaMetadata + "Bounds (min/max)" + bounds::CriteriaBounds + + function RegionCriteriaEntry(; + metadata::CriteriaMetadata, + bounds::Bounds + ) + return new( + metadata, + bounds + ) + end +end + +struct RegionalCriteria + depth::RegionalCriteriaEntry + slope::RegionalCriteriaEntry + turbidity::RegionalCriteriaEntry + waves_height::RegionalCriteriaEntry + waves_period::RegionalCriteriaEntry + rugosity::RegionalCriteriaEntry + + function AssessmentCriteria(; + depth_bounds::Bounds, + slope_bounds::Bounds, + turbidity_bounds::Bounds, + waves_height_bounds::Bounds, + waves_period_bounds::Bounds, + rugosity_bounds::Bounds + ) + return new( + RegionalCriteriaEntry(; + metadata=ASSESSMENT_CRITERIA.depth, bounds=depth_bounds + ), + RegionalCriteriaEntry(; + metadata=ASSESSMENT_CRITERIA.slope, bounds=slope_bounds + ), + RegionalCriteriaEntry(; + metadata=ASSESSMENT_CRITERIA.turbidity, bounds=turbidity_bounds + ), + RegionalCriteriaEntry(; + metadata=ASSESSMENT_CRITERIA.waves_height, bounds=waves_height_bounds + ), + RegionalCriteriaEntry(; + metadata=ASSESSMENT_CRITERIA.waves_period, bounds=waves_period_bounds + ), + RegionalCriteriaEntry(; + metadata=ASSESSMENT_CRITERIA.rugosity, bounds=rugosity_bounds + ) + ) + end +end + +# Helps to find the slopes lookup file - note this is a bit fragile +const SLOPES_LOOKUP_SUFFIX = "_lookup.parq" +function get_slope_parquet_filepath(region::RegionDetails)::String + return "$(region.id)$(ASSESSMENT_CRITERIA.slope.file_suffix)$(SLOPES_LOOKUP_SUFFIX)" +end + +function populate_data_driven_criteria(region::RegionDetails)::RegionalCriteria + regions = get_regions() + + # Collate available criteria across all regions + criteria_names = [] + for r in regions + criteria_names = append!(criteria_names, names(reg_assess_data[regions[1]].stack)) + end + + unique!(criteria_names) + + # Ignore the valid data layer (not a "true" criteria layer) + criteria_names = [c for c in criteria_names if c != :ValidSlopes] + + # Create entries for the min and max + criteria_ranges = DataFrame([c => Number[NaN, NaN] for c in criteria_names]...) + + # Populate dataframe with data from first region + for cn in criteria_names + try + c_values = reg_assess_data[regions[1]].valid_slopes[:, cn] + criteria_ranges[:, cn] .= extrema(c_values) + catch err + # If error is an ArgumentError, then the criteria name was not found + # for this specific region and can be safely skipped. + if !(err isa ArgumentError) + rethrow(err) + end + end + end + + # Check for the extrema in other regions + for r in regions[2:end], cn in criteria_names + reg_criteria = names(reg_assess_data[r].valid_slopes) + if cn ∉ reg_criteria + # If the criteria column does not exist in `valid_slopes` (data not + # available/applicable for region) then it can be safely skipped. + continue + end + + c_values = reg_assess_data[r].valid_slopes[:, cn] + min_max = extrema(c_values) + + # Populate with the min/max + criteria_ranges[1, cn] .= min(criteria_ranges[1, cn], min_max[1]) + criteria_ranges[2, cn] .= max(criteria_ranges[2, cn], min_max[2]) + end + + reg_assess_data["criteria_ranges"] = criteria_ranges + + return nothing +end + +mutable struct RegionalDataEntry + "The unique ID of the region" + region_id::String + "Other metadata about the region" + region_metadata::RegionDetails + "The rasters associated with this region" + raster_stack::Rasters.RasterStack + "A DataFrame containing coordinates of valid slope reefs" + slope_coordinates::DataFrame + "Information about the criteria for this region" + criteria_ranges::AssessmentCriteria + + function RegionalDataEntry(; + region_id::String, + region_metadata::RegionDetails, + raster_stack::Rasters.RasterStack, + slope_coordinates::DataFrame, + criteria_ranges::Vector{RegionalCriteria} + ) + return new( + region_id, region_metadata, raster_stack, slope_coordinates, criteria_ranges + ) + end +end + +const RegionalDataType = Dict{String,RegionalDataEntry} +REGIONAL_DATA::OptionalValue{RegionalDataType} = nothing + +function check_existing_regional_data_from_memory()::OptionalValue{RegionalDataType} + if !isnothing(REGIONAL_DATA) + @debug "Using previously generated regional data store." + return REGIONAL_DATA + end + return nothing +end + +function check_existing_regional_data_from_disk( + cache_directory::String +)::OptionalValue{RegionalDataType} + # check if we have an existing cache file + reg_cache_filename = joinpath(cache_directory, REGIONAL_DATA_CACHE_FILENAME) + if isfile(reg_cache_filename) + @debug "Loading regional data cache from disk" + try + return deserialize(reg_cache_filename) + catch err + @warn "Failed to deserialize $(reg_cache_filename) with error:" err + # Invalidating cache + rm(reg_cache_filename) + end + end + # no success + return nothing +end + +function find_data_source_for_criteria(; + data_source_directory::String, region::RegionDetails, criteria::CriteriaMetadata +)::String + # ascertain the file pattern + matched_files = glob("$(region.id)*$(criteria.file_suffix).tif", data_source_directory) + + # ensure there is exactly one matching file + if length(matched_files) == 0 + throw(ErrorException("Did not find data for the criteria: $(criteria.id).")) + end + if length(matched_files) > 1 + throw( + ErrorException( + "Found more than one data source match for criteria: $(criteria.id). This is ambiguous, unsure how to proceed." + ) + ) + end + + return first(matched_files) +end + +""" +For each row of the GeoDataFrame, adds a lons/lats entry which is a vector of +coordinates +""" +function add_lat_long_columns_to_dataframe(df::DataFrame)::Nothing + # This returns a Vector{Tuple{number, number}} i.e. list of lat/lons + coords = GI.coordinates.(df.geometry) + # This adds a new column where each value is the first entry from the tuple (i.e. lon) + df[!, :lons] .= first.(coords) + # This adds a new column where each value is the second entry from the tuple (i.e. lat) + df[!, :lats] .= last.(coords) + return nothing +end + +function initialise_data(config::Dict)::RegionalDataType + regional_data::RegionalDataType = Dict() + data_source_directory = config["prepped_data"]["PREPPED_DATA_DIR"] + # iterate through regions + for region_details::RegionDetails in REGIONS + @debug "$(now()) : Initializing cache for $(region_details)" + + # Setup data paths and names arrays + data_paths = String[] + data_names = String[] + + slope_table = nothing + + # TODO others + assessable_criteria = [ + ASSESSMENT_CRITERIA.depth + ] + + for criteria::CriteriaMetadata in assessable_criteria + # Get the file name of the matching data layer (.tif) and add it to + # the raster stack + push!( + data_paths, + find_data_source_for_criteria(; + data_source_directory, + region, + criteria + ) + ) + # Add name to - this is helpful so you can reference the raster in + # the raster stack + push!(data_names, criteria.id) + end + + # Extract the slope table for this region + slope_table::DataFrame = GeoParquet.read( + get_slope_parquet_filepath(region_details) + ) + + # Extract the lat longs + add_lat_long_columns_to_dataframe(slope_table) + + # Build the raster stack + raster_stack = RasterStack(data_paths; name=data_names, lazy=true) + + regional_data[region_details.id] = RegionalDataEntry(; + region_id=region_details.id, + region_metadata=region_details, + raster_stack=raster_stack, + slope_coordinates=slope_table, + criteria_ranges=nothing) + end +end + +""" +""" +function initialise_data_with_cache(config::Dict; force_cache_invalidation::Bool=false) + # Use module scope variable here + global REGIONAL_DATA + + # Where is cache located? + regional_cache_directory = config["server_config"]["REGIONAL_CACHE_DIR"] + + # do we have an in-memory cache available? + local_data = check_existing_regional_data_from_memory() + if !isnothing(local_data) + REGIONAL_DATA = local_data + return nothing + end + + # do we have disk cache available? + disk_data = check_existing_regional_data_from_disk(regional_cache_directory) + if !isnothing(disk_data) + REGIONAL_DATA = disk_data + return nothing + end + + # No cache exists - recompute + +end From c8450433b183fe916475bb82b05cf6fb26f14772 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Thu, 22 May 2025 11:49:11 +1000 Subject: [PATCH 05/26] Equivalent functionality done I think Signed-off-by: Peter Baker --- src/ReefGuideAPI.jl | 2 +- src/criteria_assessment/criteria.jl | 12 +- src/criteria_assessment/query_thresholds.jl | 14 +- src/server_cache.jl | 6 +- src/setup.jl | 240 ++++++++++++-------- 5 files changed, 160 insertions(+), 114 deletions(-) diff --git a/src/ReefGuideAPI.jl b/src/ReefGuideAPI.jl index 92b2a70..7a7ff0e 100644 --- a/src/ReefGuideAPI.jl +++ b/src/ReefGuideAPI.jl @@ -117,7 +117,7 @@ function start_worker() end export - RegionalCriteria, + OldRegionalCriteria, criteria_data_map # Methods to assess/identify deployment "plots" of reef. diff --git a/src/criteria_assessment/criteria.jl b/src/criteria_assessment/criteria.jl index 064d9a9..591574f 100644 --- a/src/criteria_assessment/criteria.jl +++ b/src/criteria_assessment/criteria.jl @@ -55,26 +55,26 @@ function extract_criteria(qp::T, criteria::Vector{String})::T where {T<:Dict{Str ) end -struct RegionalCriteria{T} +struct OldRegionalCriteria{T} stack::RasterStack valid_slopes::T valid_flats::T end -function valid_slope_lon_inds(reg::RegionalCriteria) +function valid_slope_lon_inds(reg::OldRegionalCriteria) return reg.valid_slopes.lon_idx end -function valid_slope_lat_inds(reg::RegionalCriteria) +function valid_slope_lat_inds(reg::OldRegionalCriteria) return reg.valid_slopes.lat_idx end -function valid_flat_lon_inds(reg::RegionalCriteria) +function valid_flat_lon_inds(reg::OldRegionalCriteria) return reg.valid_flats.lon_idx end -function valid_flat_lat_inds(reg::RegionalCriteria) +function valid_flat_lat_inds(reg::OldRegionalCriteria) return reg.valid_flats.lat_idx end -function Base.show(io::IO, ::MIME"text/plain", z::RegionalCriteria) +function Base.show(io::IO, ::MIME"text/plain", z::OldRegionalCriteria) # TODO: Include the extent println(""" Criteria: $(names(z.stack)) diff --git a/src/criteria_assessment/query_thresholds.jl b/src/criteria_assessment/query_thresholds.jl index 4885979..eaec606 100644 --- a/src/criteria_assessment/query_thresholds.jl +++ b/src/criteria_assessment/query_thresholds.jl @@ -196,7 +196,7 @@ end """ apply_criteria_lookup( - reg_criteria::RegionalCriteria, + reg_criteria::OldRegionalCriteria, rtype::Symbol, ruleset::Vector{CriteriaBounds{Function}} ) @@ -204,7 +204,7 @@ end Filter lookup table by applying user defined `ruleset` criteria. # Arguments -- `reg_criteria` : RegionalCriteria containing valid_rtype lookup table for filtering. +- `reg_criteria` : OldRegionalCriteria containing valid_rtype lookup table for filtering. - `rtype` : Flats or slope category for assessment. - `ruleset` : User defined ruleset for upper and lower bounds. @@ -212,7 +212,7 @@ Filter lookup table by applying user defined `ruleset` criteria. Filtered lookup table containing points that meet all criteria in `ruleset`. """ function apply_criteria_lookup( - reg_criteria::RegionalCriteria, + reg_criteria::OldRegionalCriteria, rtype::Symbol, ruleset )::DataFrame @@ -240,7 +240,7 @@ applied to a set of criteria. - Ones indicate locations to **keep**. # Arguments -- `reg_criteria` : RegionalCriteria to assess +- `reg_criteria` : OldRegionalCriteria to assess - `rtype` : reef type to assess (`:slopes` or `:flats`) - `crit_map` : List of criteria thresholds to apply (see `apply_criteria_thresholds()`) - `lons` : Longitudinal extent (min and max, required when generating masks for tiles) @@ -250,7 +250,7 @@ applied to a set of criteria. True/false mask indicating locations within desired thresholds. """ function threshold_mask( - reg_criteria::RegionalCriteria, + reg_criteria::OldRegionalCriteria, rtype::Symbol, crit_map::Vector{CriteriaBounds{Function}} )::Raster @@ -264,7 +264,7 @@ function threshold_mask( return mask_layer end function threshold_mask( - reg_criteria::RegionalCriteria, + reg_criteria::OldRegionalCriteria, rtype::Symbol, crit_map::Vector{CriteriaBounds{Function}}, lons::Tuple, @@ -310,7 +310,7 @@ applied to a set of criteria. # Arguments - `fn` : File to write geotiff to -- `reg_criteria` : RegionalCriteria to assess +- `reg_criteria` : OldRegionalCriteria to assess - `rtype` : reef type to assess (`:slopes` or `:flats`) - `crit_map` : List of criteria thresholds to apply (see `apply_criteria_thresholds()`) diff --git a/src/server_cache.jl b/src/server_cache.jl index 1f4d3c3..7a7800b 100644 --- a/src/server_cache.jl +++ b/src/server_cache.jl @@ -66,7 +66,7 @@ data and save to `reg_cache_fn` path. """ function initialize_regional_data_cache(reef_data_path::String, reg_cache_fn::String) regional_assessment_data = OrderedDict{ - String,Union{RegionalCriteria,DataFrame,Dict} + String,Union{OldRegionalCriteria,DataFrame,Dict} }() for reg in get_regions() @debug "$(now()) : Initializing cache for $reg" @@ -111,7 +111,7 @@ function initialize_regional_data_cache(reef_data_path::String, reg_cache_fn::St # flat_table[!, :lats] .= last.(coords) rst_stack = RasterStack(data_paths; name=data_names, lazy=true) - regional_assessment_data[reg] = RegionalCriteria( + regional_assessment_data[reg] = OldRegionalCriteria( rst_stack, slope_table, slope_table[[1], :] # Dummy entry for flat_table @@ -146,7 +146,7 @@ Load regional data to act as an in-memory cache. - `reef_data_path` : Path to pre-prepared reef data # Returns -OrderedDict of `RegionalCriteria` for each region. +OrderedDict of `OldRegionalCriteria` for each region. """ function setup_regional_data(config::Dict) reef_data_path = config["prepped_data"]["PREPPED_DATA_DIR"] diff --git a/src/setup.jl b/src/setup.jl index ae9ed8a..d5b2dad 100644 --- a/src/setup.jl +++ b/src/setup.jl @@ -58,6 +58,10 @@ mutable struct Bounds end end +function bounds_from_tuple(min_max::Tuple{Number,Number})::Bounds + return Bounds(; min=min_max[1], max=min_max[2]) +end + struct CriteriaMetadata "System ID used to uniquely identify this criteria" id::String @@ -159,7 +163,7 @@ struct RegionalCriteria waves_period::RegionalCriteriaEntry rugosity::RegionalCriteriaEntry - function AssessmentCriteria(; + function RegionalCriteria(; depth_bounds::Bounds, slope_bounds::Bounds, turbidity_bounds::Bounds, @@ -190,65 +194,42 @@ struct RegionalCriteria end end +""" +Takes a slope_table data frame and builds out the region specific bounds for +each criteria defined as the extrema of present values over the region +""" +function build_assessment_criteria_from_slope_table(table::DataFrame)::RegionalCriteria + const depth_bounds = bounds_from_tuple(extrema(table[:, ASSESSMENT_CRITERIA.depth.id])) + const slope_bounds = bounds_from_tuple(extrema(table[:, ASSESSMENT_CRITERIA.slope.id])) + const turbidity_bounds = bounds_from_tuple( + extrema(table[:, ASSESSMENT_CRITERIA.turbidity.id]) + ) + const waves_height_bounds = bounds_from_tuple( + extrema(table[:, ASSESSMENT_CRITERIA.waves_height.id]) + ) + const waves_period_bounds = bounds_from_tuple( + extrema(table[:, ASSESSMENT_CRITERIA.waves_period.id]) + ) + const rugosity_bounds = bounds_from_tuple( + extrema(table[:, ASSESSMENT_CRITERIA.rugosity.id]) + ) + + return RegionalCriteria(; + depth_bounds, + slope_bounds, + turbidity_bounds, + waves_height_bounds, + waves_period_bounds, + rugosity_bounds + ) +end + # Helps to find the slopes lookup file - note this is a bit fragile const SLOPES_LOOKUP_SUFFIX = "_lookup.parq" function get_slope_parquet_filepath(region::RegionDetails)::String return "$(region.id)$(ASSESSMENT_CRITERIA.slope.file_suffix)$(SLOPES_LOOKUP_SUFFIX)" end -function populate_data_driven_criteria(region::RegionDetails)::RegionalCriteria - regions = get_regions() - - # Collate available criteria across all regions - criteria_names = [] - for r in regions - criteria_names = append!(criteria_names, names(reg_assess_data[regions[1]].stack)) - end - - unique!(criteria_names) - - # Ignore the valid data layer (not a "true" criteria layer) - criteria_names = [c for c in criteria_names if c != :ValidSlopes] - - # Create entries for the min and max - criteria_ranges = DataFrame([c => Number[NaN, NaN] for c in criteria_names]...) - - # Populate dataframe with data from first region - for cn in criteria_names - try - c_values = reg_assess_data[regions[1]].valid_slopes[:, cn] - criteria_ranges[:, cn] .= extrema(c_values) - catch err - # If error is an ArgumentError, then the criteria name was not found - # for this specific region and can be safely skipped. - if !(err isa ArgumentError) - rethrow(err) - end - end - end - - # Check for the extrema in other regions - for r in regions[2:end], cn in criteria_names - reg_criteria = names(reg_assess_data[r].valid_slopes) - if cn ∉ reg_criteria - # If the criteria column does not exist in `valid_slopes` (data not - # available/applicable for region) then it can be safely skipped. - continue - end - - c_values = reg_assess_data[r].valid_slopes[:, cn] - min_max = extrema(c_values) - - # Populate with the min/max - criteria_ranges[1, cn] .= min(criteria_ranges[1, cn], min_max[1]) - criteria_ranges[2, cn] .= max(criteria_ranges[2, cn], min_max[2]) - end - - reg_assess_data["criteria_ranges"] = criteria_ranges - - return nothing -end - mutable struct RegionalDataEntry "The unique ID of the region" region_id::String @@ -257,27 +238,44 @@ mutable struct RegionalDataEntry "The rasters associated with this region" raster_stack::Rasters.RasterStack "A DataFrame containing coordinates of valid slope reefs" - slope_coordinates::DataFrame + slope_table::DataFrame "Information about the criteria for this region" - criteria_ranges::AssessmentCriteria + criteria::RegionalCriteria + "The canonical reef outlines geo data frame" + reef_outlines::DataFrame function RegionalDataEntry(; region_id::String, region_metadata::RegionDetails, raster_stack::Rasters.RasterStack, - slope_coordinates::DataFrame, - criteria_ranges::Vector{RegionalCriteria} + slope_table::DataFrame, + criteria::RegionalCriteria, + reef_outlines::DataFrame ) return new( - region_id, region_metadata, raster_stack, slope_coordinates, criteria_ranges + region_id, region_metadata, raster_stack, slope_table, criteria, reef_outlines ) end end -const RegionalDataType = Dict{String,RegionalDataEntry} -REGIONAL_DATA::OptionalValue{RegionalDataType} = nothing +const RegionalDataMapType = Dict{String,RegionalDataEntry} +struct RegionalData + "Dictionary mapping the region ID (see region details) to the data entry" + regions::RegionalDataMapType + "Canonical reef outlines" + reef_outlines::DataFrame + + function RegionalData(; + regions::Dict{String,RegionalDataEntry}, + reef_outlines::DataFrame + ) + return new(regions, reef_outlines) + end +end + +REGIONAL_DATA::OptionalValue{RegionalData} = nothing -function check_existing_regional_data_from_memory()::OptionalValue{RegionalDataType} +function check_existing_regional_data_from_memory()::OptionalValue{RegionalData} if !isnothing(REGIONAL_DATA) @debug "Using previously generated regional data store." return REGIONAL_DATA @@ -287,7 +285,7 @@ end function check_existing_regional_data_from_disk( cache_directory::String -)::OptionalValue{RegionalDataType} +)::OptionalValue{RegionalData} # check if we have an existing cache file reg_cache_filename = joinpath(cache_directory, REGIONAL_DATA_CACHE_FILENAME) if isfile(reg_cache_filename) @@ -326,8 +324,8 @@ function find_data_source_for_criteria(; end """ -For each row of the GeoDataFrame, adds a lons/lats entry which is a vector of -coordinates +For each row of the GeoDataFrame, adds a lons/lats entry which is the coordinate +of the centroid of the geometry of that reef """ function add_lat_long_columns_to_dataframe(df::DataFrame)::Nothing # This returns a Vector{Tuple{number, number}} i.e. list of lat/lons @@ -339,12 +337,22 @@ function add_lat_long_columns_to_dataframe(df::DataFrame)::Nothing return nothing end -function initialise_data(config::Dict)::RegionalDataType - regional_data::RegionalDataType = Dict() +const DEFAULT_CANONICAL_REEFS_FILE_NAME = "rrap_canonical_outlines.gpkg" +function load_canonical_reefs( + source_dir::String; file_name::String=DEFAULT_CANONICAL_REEFS_FILE_NAME +)::DataFrame + # Load in the reef canonical outlines geodataframe + return GDF.read( + joinpath(source_dir, file_name) + ) +end + +function initialise_data(config::Dict)::RegionalData + regional_data::RegionalDataMapType = Dict() data_source_directory = config["prepped_data"]["PREPPED_DATA_DIR"] # iterate through regions - for region_details::RegionDetails in REGIONS - @debug "$(now()) : Initializing cache for $(region_details)" + for region_metadata::RegionDetails in REGIONS + @debug "$(now()) : Initializing cache for $(region_metadata)" # Setup data paths and names arrays data_paths = String[] @@ -352,11 +360,25 @@ function initialise_data(config::Dict)::RegionalDataType slope_table = nothing - # TODO others + # list out the criteria we are interested in pulling out raster and + # criteria ranges for assessable_criteria = [ - ASSESSMENT_CRITERIA.depth + ASSESSMENT_CRITERIA.depth, + ASSESSMENT_CRITERIA.rugosity, + ASSESSMENT_CRITERIA.slope, + ASSESSMENT_CRITERIA.turbidity, + ASSESSMENT_CRITERIA.waves_height, + ASSESSMENT_CRITERIA.waves_period ] + # Extract the slope table for this region + slope_table::DataFrame = GeoParquet.read( + get_slope_parquet_filepath(region_metadata) + ) + + # Extract the lat longs + add_lat_long_columns_to_dataframe(slope_table) + for criteria::CriteriaMetadata in assessable_criteria # Get the file name of the matching data layer (.tif) and add it to # the raster stack @@ -373,28 +395,42 @@ function initialise_data(config::Dict)::RegionalDataType push!(data_names, criteria.id) end - # Extract the slope table for this region - slope_table::DataFrame = GeoParquet.read( - get_slope_parquet_filepath(region_details) + # Determine the criteria and extrema + criteria::AssessmentCriteria = build_assessment_criteria_from_slope_table( + slope_table ) - # Extract the lat longs - add_lat_long_columns_to_dataframe(slope_table) - # Build the raster stack raster_stack = RasterStack(data_paths; name=data_names, lazy=true) - regional_data[region_details.id] = RegionalDataEntry(; - region_id=region_details.id, - region_metadata=region_details, - raster_stack=raster_stack, - slope_coordinates=slope_table, - criteria_ranges=nothing) + # Add entry to the regional data dictionary + regional_data[region_metadata.id] = RegionalDataEntry(; + region_id=region_metadata.id, + region_metadata, + raster_stack, + slope_table, + criteria) + end + + # Load canonical reefs + canonical_reefs = load_canonical_reefs(data_source_directory) + + # All done - return populated data + return RegionalData(; regions=regional_data, reef_outlines=canonical_reefs) +end + +function get_empty_tile_path(config::Dict)::String + cache_location = _cache_location(config) + return joinpath(cache_location, "no_data_tile.png") +end + +function setup_empty_tile_cache(config::Dict)::String + file_path = get_empty_tile_path(config) + if !isfile(file_path) + save(file_path, zeros(RGBA, tile_size(config))) end end -""" -""" function initialise_data_with_cache(config::Dict; force_cache_invalidation::Bool=false) # Use module scope variable here global REGIONAL_DATA @@ -402,20 +438,30 @@ function initialise_data_with_cache(config::Dict; force_cache_invalidation::Bool # Where is cache located? regional_cache_directory = config["server_config"]["REGIONAL_CACHE_DIR"] - # do we have an in-memory cache available? - local_data = check_existing_regional_data_from_memory() - if !isnothing(local_data) - REGIONAL_DATA = local_data - return nothing - end + if !force_cache_invalidation + # do we have an in-memory cache available? + local_data = check_existing_regional_data_from_memory() + if !isnothing(local_data) && !force_cache_invalidation + REGIONAL_DATA = local_data + return nothing + end - # do we have disk cache available? - disk_data = check_existing_regional_data_from_disk(regional_cache_directory) - if !isnothing(disk_data) - REGIONAL_DATA = disk_data - return nothing + # do we have disk cache available? + disk_data = check_existing_regional_data_from_disk(regional_cache_directory) + if !isnothing(disk_data) && !force_cache_invalidation + REGIONAL_DATA = disk_data + return nothing + end end - # No cache exists - recompute + # No cache exists OR we are forcing invalidation + regional_data = initialise_data(config) + + # Update the global variable + REGIONAL_DATA = regional_data + # Also ensure our empty tile cache is ready + setup_empty_tile_cache(config) + + return nothing end From 844a688365e07d7f777d9323657fb521fd884dd9 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Thu, 22 May 2025 12:39:36 +1000 Subject: [PATCH 06/26] Starting to deprecate, restructuring, documentation Signed-off-by: Peter Baker --- src/criteria_assessment/_generators_dev.jl | 50 -- src/criteria_assessment/criteria.jl | 144 ---- .../regional_assessment.jl | 9 +- src/criteria_assessment/tiles.jl | 2 +- src/job_worker/Jobs.jl | 4 +- src/server_cache.jl | 208 ----- src/setup.jl | 751 ++++++++++++------ 7 files changed, 540 insertions(+), 628 deletions(-) delete mode 100644 src/criteria_assessment/_generators_dev.jl delete mode 100644 src/criteria_assessment/criteria.jl delete mode 100644 src/server_cache.jl diff --git a/src/criteria_assessment/_generators_dev.jl b/src/criteria_assessment/_generators_dev.jl deleted file mode 100644 index 894d4c3..0000000 --- a/src/criteria_assessment/_generators_dev.jl +++ /dev/null @@ -1,50 +0,0 @@ - -abstract type DeploymentCriteria end - -""" - generate_criteria(name::Symbol, field_spec::Dict) - generate_criteria(name::Symbol, fields::Vector{Symbol}) - generate_criteria(name::Symbol) - -Generate structs for a given criteria -""" -function generate_criteria(name::Symbol, field_spec::Dict) - fields = [:($(Symbol(k))::$(typeof(v)) = $(v)) for (k, v) in field_spec] - @eval @kwdef struct $(name) - $(fields...) - end -end -function generate_criteria(name::Symbol, fields::Vector{Symbol}) - # fields = [:($(Symbol(field))) for field in field_spec] - @eval struct $(name) - $(fields...) - end -end -function generate_criteria(name::Symbol) - pascal_case_name = Symbol(name, :Criteria) - - @eval struct $(pascal_case_name){T<:Union{Int64,Float64}} <: DeploymentCriteria - data::Raster - lower_bound::T - upper_bound::T - end - - # Data-only constructor - @eval function $(pascal_case_name)(data) - return $(pascal_case_name)(data, minimum(data), maximum(data)) - end - - @eval export $(pascal_case_name) -end - -function generate_criteria_structs()::Nothing - patterns = criteria_data_map() - - # Create structs for each criteria - for k in keys(patterns) - generate_criteria(k) - end -end - -# function generate_criteria_state()::Nothing -# end diff --git a/src/criteria_assessment/criteria.jl b/src/criteria_assessment/criteria.jl deleted file mode 100644 index 591574f..0000000 --- a/src/criteria_assessment/criteria.jl +++ /dev/null @@ -1,144 +0,0 @@ -using Base.Threads - -using Dates -using StructTypes - -import Rasters: Between -using Oxygen: json, Request - -include("query_parser.jl") -include("tiles.jl") -include("site_identification.jl") - -const REEF_TYPE = [:slopes, :flats] - -# HTTP response headers for COG files -const COG_HEADERS = [ - "Cache-Control" => "max-age=86400, no-transform" -] - -function criteria_data_map() - # TODO: Load from config? - return OrderedDict( - :Depth => "_bathy", - :Benthic => "_benthic", - :Geomorphic => "_geomorphic", - :Slope => "_slope", - :Turbidity => "_turbid", - :WavesHs => "_waves_Hs", - :WavesTp => "_waves_Tp", - :Rugosity => "_rugosity", - :ValidSlopes => "_valid_slopes", - :ValidFlats => "_valid_flats" - - # Unused datasets - # :PortDistSlopes => "_PortDistSlopes", - # :PortDistFlats => "_PortDistFlats" - ) -end - -function search_criteria()::Vector{String} - return string.(keys(criteria_data_map())) -end - -function site_criteria()::Vector{String} - return ["SuitabilityThreshold", "xdist", "ydist"] -end - -function suitability_criteria()::Vector{String} - return vcat(search_criteria(), ["SuitabilityThreshold"]) -end - -function extract_criteria(qp::T, criteria::Vector{String})::T where {T<:Dict{String,String}} - return filter( - k -> string(k.first) ∈ criteria, qp - ) -end - -struct OldRegionalCriteria{T} - stack::RasterStack - valid_slopes::T - valid_flats::T -end - -function valid_slope_lon_inds(reg::OldRegionalCriteria) - return reg.valid_slopes.lon_idx -end -function valid_slope_lat_inds(reg::OldRegionalCriteria) - return reg.valid_slopes.lat_idx -end -function valid_flat_lon_inds(reg::OldRegionalCriteria) - return reg.valid_flats.lon_idx -end -function valid_flat_lat_inds(reg::OldRegionalCriteria) - return reg.valid_flats.lat_idx -end - -function Base.show(io::IO, ::MIME"text/plain", z::OldRegionalCriteria) - # TODO: Include the extent - println(""" - Criteria: $(names(z.stack)) - Number of valid slope locations: $(nrow(z.valid_slopes)) - Number of valid flat locations: $(nrow(z.valid_flats)) - """) - return nothing -end - -struct CriteriaBounds{F<:Function} - name::Symbol - lower_bound::Float32 - upper_bound::Float32 - rule::F - - function CriteriaBounds(name::String, lb::S, ub::S)::CriteriaBounds where {S<:String} - lower_bound::Float32 = parse(Float32, lb) - upper_bound::Float32 = parse(Float32, ub) - func = (x) -> lower_bound .<= x .<= upper_bound - - return new{Function}(Symbol(name), lower_bound, upper_bound, func) - end -end - -# Define struct type definition to auto-serialize/deserialize to JSON -StructTypes.StructType(::Type{CriteriaBounds}) = StructTypes.Struct() - -""" - criteria_middleware(handle) - -Creates middleware that parses a criteria query before reaching an endpoint - -# Example -`https://somewhere:8000/suitability/assess/region-name/reeftype?criteria_names=Depth,Slope&lb=-9.0,0.0&ub=-2.0,40` -""" -function criteria_middleware(handle) - function (req) - fd = queryparams(req) - - criteria_names = string.(split(fd["criteria_names"], ",")) - lbs = string.(split(fd["lb"], ",")) - ubs = string.(split(fd["ub"], ",")) - - return handle(CriteriaBounds.(criteria_names, lbs, ubs)) - end -end - -function setup_criteria_routes(config, auth) - reg_assess_data = setup_regional_data(config) - - @get auth("/criteria/ranges") function (req::Request) - # Transform cached criteria ranges to a dictionary for return as json. - criteria_ranges = reg_assess_data["criteria_ranges"] - criteria_names = names(criteria_ranges) - - @debug "Transforming criteria dataframe to JSON" - ret = OrderedDict() - for cn in criteria_names - ret[cn] = OrderedDict( - :min_val => criteria_ranges[1, cn], - :max_val => criteria_ranges[2, cn] - ) - end - - return json(ret) - end -end diff --git a/src/criteria_assessment/regional_assessment.jl b/src/criteria_assessment/regional_assessment.jl index 52ea504..bf942db 100644 --- a/src/criteria_assessment/regional_assessment.jl +++ b/src/criteria_assessment/regional_assessment.jl @@ -2,8 +2,13 @@ DEPRECATED: To remove once jobs are migrated to new system fully """ +# HTTP response headers for COG files +const COG_HEADERS = [ + "Cache-Control" => "max-age=86400, no-transform" +] + function setup_job_routes(config, auth) - reg_assess_data = setup_regional_data(config) + reg_assess_data = get_regional_data(config) @get auth("/job/details/{job_id}") function (req::Request, job_id::String) srv = DiskService(_cache_location(config)) @@ -121,7 +126,7 @@ end Set up endpoints for regional assessment. """ function setup_region_routes(config, auth) - reg_assess_data = setup_regional_data(config) + reg_assess_data = get_regional_data(config) @get auth("/assess/{reg}/{rtype}") function (req::Request, reg::String, rtype::String) qp = queryparams(req) diff --git a/src/criteria_assessment/tiles.jl b/src/criteria_assessment/tiles.jl index 12abdb8..664f9a3 100644 --- a/src/criteria_assessment/tiles.jl +++ b/src/criteria_assessment/tiles.jl @@ -176,7 +176,7 @@ function setup_tile_routes(config, auth) ) end - reg_assess_data = setup_regional_data(config) + reg_assess_data = get_regional_data(config) @get auth("/tile/{z}/{x}/{y}") function (req::Request, z::Int64, x::Int64, y::Int64) # http://127.0.0.1:8000/tile/{z}/{x}/{y}?region=Cairns-Cooktown&rtype=slopes&Depth=-9.0:0.0&Slope=0.0:40.0&Rugosity=0.0:3.0 # http://127.0.0.1:8000/tile/8/231/139?region=Cairns-Cooktown&rtype=slopes&Depth=-9.0:0.0&Slope=0.0:40.0&Rugosity=0.0:3.0 diff --git a/src/job_worker/Jobs.jl b/src/job_worker/Jobs.jl index 77d476e..a2149d0 100644 --- a/src/job_worker/Jobs.jl +++ b/src/job_worker/Jobs.jl @@ -318,7 +318,7 @@ function handle_job( @info "Configuration parsing complete." @info "Setting up regional assessment data" - reg_assess_data = setup_regional_data(config) + reg_assess_data = get_regional_data(config) @info "Done setting up regional assessment data" @info "Performing regional assessment" @@ -432,7 +432,7 @@ function handle_job( @info "Configuration parsing complete." @info "Setting up regional assessment data" - reg_assess_data = setup_regional_data(config) + reg_assess_data = get_regional_data(config) @info "Done setting up regional assessment data" @info "Performing regional assessment (dependency of site assessment)" diff --git a/src/server_cache.jl b/src/server_cache.jl deleted file mode 100644 index 7a7800b..0000000 --- a/src/server_cache.jl +++ /dev/null @@ -1,208 +0,0 @@ -"""Methods to prep and cache server data""" - -""" - _prep_criteria_ranges(reg_assess_data::OrderedDict)::Nothing - -Determine min/max values for each criteria layer and store as entry in data store. -""" -function _prep_criteria_ranges(reg_assess_data::OrderedDict)::Nothing - regions = get_regions() - - # Collate available criteria across all regions - criteria_names = [] - for r in regions - criteria_names = append!(criteria_names, names(reg_assess_data[regions[1]].stack)) - end - - unique!(criteria_names) - - # Ignore the valid data layer (not a "true" criteria layer) - criteria_names = [c for c in criteria_names if c != :ValidSlopes] - - # Create entries for the min and max - criteria_ranges = DataFrame([c => Number[NaN, NaN] for c in criteria_names]...) - - # Populate dataframe with data from first region - for cn in criteria_names - try - c_values = reg_assess_data[regions[1]].valid_slopes[:, cn] - criteria_ranges[:, cn] .= extrema(c_values) - catch err - # If error is an ArgumentError, then the criteria name was not found - # for this specific region and can be safely skipped. - if !(err isa ArgumentError) - rethrow(err) - end - end - end - - # Check for the extrema in other regions - for r in regions[2:end], cn in criteria_names - reg_criteria = names(reg_assess_data[r].valid_slopes) - if cn ∉ reg_criteria - # If the criteria column does not exist in `valid_slopes` (data not - # available/applicable for region) then it can be safely skipped. - continue - end - - c_values = reg_assess_data[r].valid_slopes[:, cn] - min_max = extrema(c_values) - - # Populate with the min/max - criteria_ranges[1, cn] .= min(criteria_ranges[1, cn], min_max[1]) - criteria_ranges[2, cn] .= max(criteria_ranges[2, cn], min_max[2]) - end - - reg_assess_data["criteria_ranges"] = criteria_ranges - - return nothing -end - -""" - initialize_regional_data_cache(reef_data_path::String, reg_cache_fn::String) - -Create initial regional data store with data from `reef_data_path`, excluding geospatial -data and save to `reg_cache_fn` path. -""" -function initialize_regional_data_cache(reef_data_path::String, reg_cache_fn::String) - regional_assessment_data = OrderedDict{ - String,Union{OldRegionalCriteria,DataFrame,Dict} - }() - for reg in get_regions() - @debug "$(now()) : Initializing cache for $reg" - data_paths = String[] - data_names = String[] - - slope_table = nothing - flat_table = nothing - - for (k, dp) in criteria_data_map() - g = glob("$reg*$dp.tif", reef_data_path) - if length(g) == 0 - continue - end - - push!(data_paths, first(g)) - push!(data_names, string(k)) - if occursin("valid", string(dp)) - # Load up Parquet files - parq_file = replace(first(g), ".tif" => "_lookup.parq") - - if occursin("slope", string(dp)) - slope_table = GeoParquet.read(parq_file) - elseif occursin("flat", string(dp)) - @warn "Skipping data for reef flats as it is currently unused" - # flat_table = GeoParquet.read(parq_file) - else - msg = "Unknown lookup found: $(parq_file). Must be 'slope' or 'flat'" - throw(ArgumentError(msg)) - end - end - end - - # Pre-extract long/lat coordinates - coords = GI.coordinates.(slope_table.geometry) - # For each entry, extract - slope_table[!, :lons] .= first.(coords) - slope_table[!, :lats] .= last.(coords) - - # coords = GI.coordinates.(flat_table.geometry) - # flat_table[!, :lons] .= first.(coords) - # flat_table[!, :lats] .= last.(coords) - - rst_stack = RasterStack(data_paths; name=data_names, lazy=true) - regional_assessment_data[reg] = OldRegionalCriteria( - rst_stack, - slope_table, - slope_table[[1], :] # Dummy entry for flat_table - ) - - @debug "$(now()) : Finished initialization for $reg" - end - - regional_assessment_data["region_long_names"] = Dict( - "FarNorthern" => "Far Northern Management Area", - "Cairns-Cooktown" => "Cairns/Cooktown Management Area", - "Townsville-Whitsunday" => "Townsville/Whitsunday Management Area", - "Mackay-Capricorn" => "Mackay/Capricorn Management Area" - ) - - _prep_criteria_ranges(regional_assessment_data) - - # Store cache on disk to avoid excessive cold startup times - @debug "Saving regional data cache to disk" - serialize(reg_cache_fn, regional_assessment_data) - - return regional_assessment_data -end - -""" - setup_regional_data(config::Dict) - -Load regional data to act as an in-memory cache. - -# Arguments -- `config` : Configuration settings, typically loaded from a TOML file. -- `reef_data_path` : Path to pre-prepared reef data - -# Returns -OrderedDict of `OldRegionalCriteria` for each region. -""" -function setup_regional_data(config::Dict) - reef_data_path = config["prepped_data"]["PREPPED_DATA_DIR"] - reg_cache_dir = config["server_config"]["REGIONAL_CACHE_DIR"] - reg_cache_fn = joinpath(reg_cache_dir, "regional_cache.dat") - - if @isdefined(REGIONAL_DATA) - @debug "Using previously generated regional data store." - elseif isfile(reg_cache_fn) - @debug "Loading regional data cache from disk" - # Updates to packages like DiskArrays can break deserialization - try - @eval const REGIONAL_DATA = deserialize($(reg_cache_fn)) - catch err - @warn "Failed to deserialize $(reg_cache_fn) with error:" err - rm(reg_cache_fn) - end - end - - if !@isdefined(REGIONAL_DATA) - @debug "Setting up regional data store..." - regional_assessment_data = initialize_regional_data_cache( - reef_data_path, - reg_cache_fn - ) - # Remember, `@eval` runs in global scope. - @eval const REGIONAL_DATA = $(regional_assessment_data) - end - - # If REGIONAL_DATA is defined, but failed to load supporting data that cannot be - # cached to disk, such as the reef outlines, (e.g., due to incorrect config), then it - # will cause errors later on. - # Then there's no way to address this, even between web server sessions, as `const` - # values cannot be modified. - # Here, we check for existence and try to load again if needed. - if !haskey(REGIONAL_DATA, "reef_outlines") - reef_outline_path = joinpath(reef_data_path, "rrap_canonical_outlines.gpkg") - REGIONAL_DATA["reef_outlines"] = GDF.read(reef_outline_path) - end - - return REGIONAL_DATA -end - -""" - warmup_cache(config_path::String) - -Invokes warm up of regional data cache to reduce later spin up times. -""" -function warmup_cache(config_path::String) - config = TOML.parsefile(config_path) - - # Create re-usable empty tile - no_data_path = cache_filename(Dict("no_data" => "none"), config, "no_data", "png") - if !isfile(no_data_path) - save(no_data_path, zeros(RGBA, tile_size(config))) - end - - return setup_regional_data(config) -end diff --git a/src/setup.jl b/src/setup.jl index d5b2dad..1bc8041 100644 --- a/src/setup.jl +++ b/src/setup.jl @@ -1,73 +1,71 @@ using Rasters +using Glob +using GeoParquet +using Serialization +using Oxygen: json, Request +using FileIO + +# ============================================================================= +# Constants and Configuration +# ============================================================================= const REGIONAL_DATA_CACHE_FILENAME = "regional_cache_v2.dat" +const SLOPES_LOOKUP_SUFFIX = "_lookup.parq" +const DEFAULT_CANONICAL_REEFS_FILE_NAME = "rrap_canonical_outlines.gpkg" -@enum Region begin - TOWNSVILLE_WHITSUNDAY - CAIRNS_COOKTOWN - MACKAY_CAPRICORN - FAR_NORTHERN -end +# Global variable to store regional data cache +REGIONAL_DATA::OptionalValue{RegionalData} = nothing + +# ============================================================================= +# Core Data Structures +# ============================================================================= -struct RegionDetails - region::Region +""" +Metadata container for regional information. + +# Fields +- `display_name::String` : Human-readable name for UI display +- `id::String` : Unique system identifier (changing this affects data loading) +""" +struct RegionMetadata display_name::String id::String - function RegionDetails(; - region::Region, + function RegionMetadata(; display_name::String, id::String ) - return new(region, display_name, id) + return new(display_name, id) end end -const REGIONS::Vector{RegionDetails} = [ - RegionDetails(; - region=Region.TOWNSVILLE_WHITSUNDAY, - display_name="Townsville/Whitsunday Management Area", - id="Townsville-Whitsunday" - ), - RegionDetails(; - region=Region.CAIRNS_COOKTOWN, - display_name="Cairns/Cooktown Management Area", - id="Cairns-Cooktown" - ), - RegionDetails(; - region=Region.MACKAY_CAPRICORN, - display_name="Mackay/Capricorn Management Area", - id="Mackay-Capricorn" - ), - RegionDetails(; - region=Region.FAR_NORTHERN, - display_name="Far Northern Management Area", - id="Far-Northern" - ) -] - """ -Simple struct to contain min/max values +Simple container for minimum and maximum boundary values. + +# Fields +- `min::Float32` : Minimum value +- `max::Float32` : Maximum value """ mutable struct Bounds min::Float32 max::Float32 function Bounds(; min::Number, max::Number) - return new(parse(Float32, min), parse(Float32, max)) + return new(Float32(min), Float32(max)) end end -function bounds_from_tuple(min_max::Tuple{Number,Number})::Bounds - return Bounds(; min=min_max[1], max=min_max[2]) -end +""" +Metadata for assessment criteria including file naming conventions. +# Fields +- `id::String` : Unique system identifier for the criteria +- `file_suffix::String` : File suffix pattern for data files +- `display_label::String` : Human-readable label for UI display +""" struct CriteriaMetadata - "System ID used to uniquely identify this criteria" id::String - "What is the suffix of this criteria in the data?" file_suffix::String - "Pretty display name, can be changed with a guarantee of no runtime consequences/errors" display_label::String function CriteriaMetadata(; @@ -75,14 +73,21 @@ struct CriteriaMetadata file_suffix::String, display_label::String ) - return new( - id, - file_suffix, - display_label - ) + return new(id, file_suffix, display_label) end end +""" +Container for all assessment criteria metadata. + +# Fields +- `depth::CriteriaMetadata` : Bathymetry/depth criteria +- `slope::CriteriaMetadata` : Slope gradient criteria +- `turbidity::CriteriaMetadata` : Water turbidity criteria +- `waves_height::CriteriaMetadata` : Wave height criteria +- `waves_period::CriteriaMetadata` : Wave period criteria +- `rugosity::CriteriaMetadata` : Seafloor rugosity criteria +""" struct AssessmentCriteria depth::CriteriaMetadata slope::CriteriaMetadata @@ -105,56 +110,36 @@ struct AssessmentCriteria end end -const ASSESSMENT_CRITERIA::AssessmentCriteria = AssessmentCriteria(; - depth=CriteriaMetadata(; - id="Depth", - file_suffix="_bathy", - display_label="Depth" - ), - slope=CriteriaMetadata(; - id="Slope", - file_suffix="_slope", - display_label="Slope" - ), - turbidity=CriteriaMetadata(; - id="Turbidity", - file_suffix="_turbid", - display_label="Turbidity" - ), - waves_height=CriteriaMetadata(; - id="WavesHs", - file_suffix="_waves_Hs", - display_label="Wave Height (m)" - ), - waves_period=CriteriaMetadata(; - id="WavesTp", - file_suffix="_waves_Tp", - display_label="Wave Period (s)" - ), - rugosity=CriteriaMetadata(; - id="Rugosity", - file_suffix="_rugosity", - display_label="Rugosity" - ) -) +""" +Combines criteria metadata with regional boundary values. +# Fields +- `metadata::CriteriaMetadata` : Criteria definition and metadata +- `bounds::Bounds` : Min/max values for this criteria in the region +""" struct RegionalCriteriaEntry - "Other details about the criteria" metadata::CriteriaMetadata - "Bounds (min/max)" - bounds::CriteriaBounds + bounds::Bounds - function RegionCriteriaEntry(; + function RegionalCriteriaEntry(; metadata::CriteriaMetadata, bounds::Bounds ) - return new( - metadata, - bounds - ) + return new(metadata, bounds) end end +""" +Complete set of regional criteria with computed bounds for each parameter. + +# Fields +- `depth::RegionalCriteriaEntry` : Depth criteria and bounds +- `slope::RegionalCriteriaEntry` : Slope criteria and bounds +- `turbidity::RegionalCriteriaEntry` : Turbidity criteria and bounds +- `waves_height::RegionalCriteriaEntry` : Wave height criteria and bounds +- `waves_period::RegionalCriteriaEntry` : Wave period criteria and bounds +- `rugosity::RegionalCriteriaEntry` : Rugosity criteria and bounds +""" struct RegionalCriteria depth::RegionalCriteriaEntry slope::RegionalCriteriaEntry @@ -195,10 +180,212 @@ struct RegionalCriteria end """ -Takes a slope_table data frame and builds out the region specific bounds for -each criteria defined as the extrema of present values over the region +Complete data package for a single region including rasters and metadata. + +# Fields +- `region_id::String` : Unique identifier for the region +- `region_metadata::RegionMetadata` : Display metadata for the region +- `raster_stack::Rasters.RasterStack` : Geospatial raster data layers +- `slope_table::DataFrame` : Coordinates and values for valid slope reef locations +- `criteria::RegionalCriteria` : Computed criteria bounds for this region +""" +mutable struct RegionalDataEntry + region_id::String + region_metadata::RegionMetadata + raster_stack::Rasters.RasterStack + slope_table::DataFrame + criteria::RegionalCriteria + + function RegionalDataEntry(; + region_id::String, + region_metadata::RegionMetadata, + raster_stack::Rasters.RasterStack, + slope_table::DataFrame, + criteria::RegionalCriteria + ) + return new(region_id, region_metadata, raster_stack, slope_table, criteria) + end +end + +""" +Top-level container for all regional data and reef outlines. + +# Fields +- `regions::Dict{String,RegionalDataEntry}` : Regional data indexed by region ID +- `reef_outlines::DataFrame` : Canonical reef outline geometries +""" +struct RegionalData + regions::Dict{String,RegionalDataEntry} + reef_outlines::DataFrame + + function RegionalData(; + regions::Dict{String,RegionalDataEntry}, + reef_outlines::DataFrame + ) + return new(regions, reef_outlines) + end +end + +# ============================================================================= +# Configuration Constants +# ============================================================================= + +# Define all available regions for the assessment system +const REGIONS::Vector{RegionMetadata} = [ + RegionMetadata(; + display_name="Townsville/Whitsunday Management Area", + id="Townsville-Whitsunday" + ), + RegionMetadata(; + display_name="Cairns/Cooktown Management Area", + id="Cairns-Cooktown" + ), + RegionMetadata(; + display_name="Mackay/Capricorn Management Area", + id="Mackay-Capricorn" + ), + RegionMetadata(; + display_name="Far Northern Management Area", + id="Far-Northern" + ) +] + +# Define all assessment criteria with file naming conventions +const ASSESSMENT_CRITERIA::AssessmentCriteria = AssessmentCriteria(; + depth=CriteriaMetadata(; + id="Depth", + file_suffix="_bathy", + display_label="Depth" + ), + slope=CriteriaMetadata(; + id="Slope", + file_suffix="_slope", + display_label="Slope" + ), + turbidity=CriteriaMetadata(; + id="Turbidity", + file_suffix="_turbid", + display_label="Turbidity" + ), + waves_height=CriteriaMetadata(; + id="WavesHs", + file_suffix="_waves_Hs", + display_label="Wave Height (m)" + ), + waves_period=CriteriaMetadata(; + id="WavesTp", + file_suffix="_waves_Tp", + display_label="Wave Period (s)" + ), + rugosity=CriteriaMetadata(; + id="Rugosity", + file_suffix="_rugosity", + display_label="Rugosity" + ) +) + +# Convenience list for iteration over all criteria +const ASSESSMENT_CRITERIA_LIST::Vector{CriteriaMetadata} = [ + ASSESSMENT_CRITERIA.depth, + ASSESSMENT_CRITERIA.slope, + ASSESSMENT_CRITERIA.turbidity, + ASSESSMENT_CRITERIA.waves_height, + ASSESSMENT_CRITERIA.waves_period, + ASSESSMENT_CRITERIA.rugosity +] + +# Type alias for cleaner code +const RegionalDataMapType = Dict{String,RegionalDataEntry} + +# ============================================================================= +# Utility Functions +# ============================================================================= + +""" +Convert a min/max tuple to a Bounds struct. + +# Arguments +- `min_max::Tuple{Number,Number}` : Tuple containing (minimum, maximum) values + +# Returns +`Bounds` struct with converted float values. +""" +function bounds_from_tuple(min_max::Tuple{Number,Number})::Bounds + return Bounds(; min=min_max[1], max=min_max[2]) +end + +""" +Generate the filename for slope lookup data for a given region. + +# Arguments +- `region::RegionMetadata` : Region metadata containing ID + +# Returns +String filename in format "{region_id}_slope_lookup.parq" +""" +function get_slope_parquet_filename(region::RegionMetadata)::String + return "$(region.id)$(ASSESSMENT_CRITERIA.slope.file_suffix)$(SLOPES_LOOKUP_SUFFIX)" +end + +""" +Create a dictionary mapping criteria IDs to regional criteria entries. + +# Arguments +- `region_data::RegionalDataEntry` : Regional data containing criteria information + +# Returns +Dictionary with criteria ID strings as keys and RegionalCriteriaEntry as values. +""" +function build_regional_criteria_dictionary( + region_data::RegionalDataEntry +)::Dict{String,RegionalCriteriaEntry} + return Dict( + ASSESSMENT_CRITERIA.depth.id => region_data.criteria.depth, + ASSESSMENT_CRITERIA.slope.id => region_data.criteria.slope, + ASSESSMENT_CRITERIA.turbidity.id => region_data.criteria.turbidity, + ASSESSMENT_CRITERIA.waves_height.id => region_data.criteria.waves_height, + ASSESSMENT_CRITERIA.waves_period.id => region_data.criteria.waves_period, + ASSESSMENT_CRITERIA.rugosity.id => region_data.criteria.rugosity + ) +end + +""" +Add longitude and latitude columns to a DataFrame based on geometry centroids. + +Modifies the input DataFrame by adding 'lons' and 'lats' columns extracted +from the centroid coordinates of each geometry feature. + +# Arguments +- `df::DataFrame` : DataFrame with geometry column containing spatial features +""" +function add_lat_long_columns_to_dataframe(df::DataFrame)::Nothing + # Extract coordinate tuples from geometry centroids + coords = GI.coordinates.(df.geometry) + # Add longitude column (first coordinate) + df[!, :lons] .= first.(coords) + # Add latitude column (second coordinate) + df[!, :lats] .= last.(coords) + return nothing +end + +# ============================================================================= +# Data Loading and Processing Functions +# ============================================================================= + +""" +Build regional criteria bounds from slope table data. + +Computes min/max bounds for each assessment criteria by finding extrema +in the slope table data columns. + +# Arguments +- `table::DataFrame` : Slope table containing criteria data columns + +# Returns +`RegionalCriteria` struct with computed bounds for all criteria. """ function build_assessment_criteria_from_slope_table(table::DataFrame)::RegionalCriteria + # Compute bounds for each criteria using column extrema const depth_bounds = bounds_from_tuple(extrema(table[:, ASSESSMENT_CRITERIA.depth.id])) const slope_bounds = bounds_from_tuple(extrema(table[:, ASSESSMENT_CRITERIA.slope.id])) const turbidity_bounds = bounds_from_tuple( @@ -224,57 +411,74 @@ function build_assessment_criteria_from_slope_table(table::DataFrame)::RegionalC ) end -# Helps to find the slopes lookup file - note this is a bit fragile -const SLOPES_LOOKUP_SUFFIX = "_lookup.parq" -function get_slope_parquet_filepath(region::RegionDetails)::String - return "$(region.id)$(ASSESSMENT_CRITERIA.slope.file_suffix)$(SLOPES_LOOKUP_SUFFIX)" -end +""" +Find the data source file for a specific criteria and region. -mutable struct RegionalDataEntry - "The unique ID of the region" - region_id::String - "Other metadata about the region" - region_metadata::RegionDetails - "The rasters associated with this region" - raster_stack::Rasters.RasterStack - "A DataFrame containing coordinates of valid slope reefs" - slope_table::DataFrame - "Information about the criteria for this region" - criteria::RegionalCriteria - "The canonical reef outlines geo data frame" - reef_outlines::DataFrame +Uses glob pattern matching to locate the appropriate .tif file based on +region ID and criteria file suffix. - function RegionalDataEntry(; - region_id::String, - region_metadata::RegionDetails, - raster_stack::Rasters.RasterStack, - slope_table::DataFrame, - criteria::RegionalCriteria, - reef_outlines::DataFrame - ) - return new( - region_id, region_metadata, raster_stack, slope_table, criteria, reef_outlines +# Arguments +- `data_source_directory::String` : Directory containing data files +- `region::RegionMetadata` : Region metadata for ID matching +- `criteria::CriteriaMetadata` : Criteria metadata for suffix matching + +# Returns +String path to the matching data file. + +# Throws +- `ErrorException` : If zero or multiple matching files are found +""" +function find_data_source_for_criteria(; + data_source_directory::String, + region::RegionMetadata, + criteria::CriteriaMetadata +)::String + # Search for files matching the pattern: {region_id}*{criteria_suffix}.tif + matched_files = glob("$(region.id)*$(criteria.file_suffix).tif", data_source_directory) + + # Validate exactly one match exists + if length(matched_files) == 0 + throw(ErrorException("Did not find data for the criteria: $(criteria.id).")) + end + if length(matched_files) > 1 + throw( + ErrorException( + "Found more than one data source match for criteria: $(criteria.id). This is ambiguous, unsure how to proceed." + ) ) end + + return first(matched_files) end -const RegionalDataMapType = Dict{String,RegionalDataEntry} -struct RegionalData - "Dictionary mapping the region ID (see region details) to the data entry" - regions::RegionalDataMapType - "Canonical reef outlines" - reef_outlines::DataFrame +""" +Load canonical reef outline geometries from geopackage file. - function RegionalData(; - regions::Dict{String,RegionalDataEntry}, - reef_outlines::DataFrame - ) - return new(regions, reef_outlines) - end +# Arguments +- `source_dir::String` : Directory containing the reef outlines file +- `file_name::String` : Name of the geopackage file (defaults to canonical name) + +# Returns +DataFrame containing reef geometry data. +""" +function load_canonical_reefs( + source_dir::String; + file_name::String=DEFAULT_CANONICAL_REEFS_FILE_NAME +)::DataFrame + # Load reef outlines from geopackage format + return GDF.read(joinpath(source_dir, file_name)) end -REGIONAL_DATA::OptionalValue{RegionalData} = nothing +# ============================================================================= +# Cache Management Functions +# ============================================================================= +""" +Check if regional data is available in memory cache. + +# Returns +`RegionalData` if available in memory, `nothing` otherwise. +""" function check_existing_regional_data_from_memory()::OptionalValue{RegionalData} if !isnothing(REGIONAL_DATA) @debug "Using previously generated regional data store." @@ -283,185 +487,290 @@ function check_existing_regional_data_from_memory()::OptionalValue{RegionalData} return nothing end +""" +Check if regional data cache exists on disk and attempt to load it. + +# Arguments +- `cache_directory::String` : Directory where cache files are stored + +# Returns +`RegionalData` if successfully loaded from disk, `nothing` otherwise. +""" function check_existing_regional_data_from_disk( cache_directory::String )::OptionalValue{RegionalData} - # check if we have an existing cache file + # Construct cache file path reg_cache_filename = joinpath(cache_directory, REGIONAL_DATA_CACHE_FILENAME) + if isfile(reg_cache_filename) @debug "Loading regional data cache from disk" try return deserialize(reg_cache_filename) catch err @warn "Failed to deserialize $(reg_cache_filename) with error:" err - # Invalidating cache + # Remove corrupted cache file rm(reg_cache_filename) end end - # no success + # No cache available or load failed return nothing end -function find_data_source_for_criteria(; - data_source_directory::String, region::RegionDetails, criteria::CriteriaMetadata -)::String - # ascertain the file pattern - matched_files = glob("$(region.id)*$(criteria.file_suffix).tif", data_source_directory) +""" +Get the file path for the empty tile cache. - # ensure there is exactly one matching file - if length(matched_files) == 0 - throw(ErrorException("Did not find data for the criteria: $(criteria.id).")) - end - if length(matched_files) > 1 - throw( - ErrorException( - "Found more than one data source match for criteria: $(criteria.id). This is ambiguous, unsure how to proceed." - ) - ) - end +# Arguments +- `config::Dict` : Configuration dictionary containing cache settings - return first(matched_files) +# Returns +String path to the empty tile cache file. +""" +function get_empty_tile_path(config::Dict)::String + cache_location = _cache_location(config) + return joinpath(cache_location, "no_data_tile.png") end """ -For each row of the GeoDataFrame, adds a lons/lats entry which is the coordinate -of the centroid of the geometry of that reef +Initialize empty tile cache if it doesn't exist. + +Creates a blank PNG tile used for areas with no data coverage. + +# Arguments +- `config::Dict` : Configuration dictionary containing tile and cache settings """ -function add_lat_long_columns_to_dataframe(df::DataFrame)::Nothing - # This returns a Vector{Tuple{number, number}} i.e. list of lat/lons - coords = GI.coordinates.(df.geometry) - # This adds a new column where each value is the first entry from the tuple (i.e. lon) - df[!, :lons] .= first.(coords) - # This adds a new column where each value is the second entry from the tuple (i.e. lat) - df[!, :lats] .= last.(coords) +function setup_empty_tile_cache(config::Dict)::Nothing + file_path = get_empty_tile_path(config) + if !isfile(file_path) + # Create empty RGBA tile with configured dimensions + save(file_path, zeros(RGBA, tile_size(config))) + end return nothing end -const DEFAULT_CANONICAL_REEFS_FILE_NAME = "rrap_canonical_outlines.gpkg" -function load_canonical_reefs( - source_dir::String; file_name::String=DEFAULT_CANONICAL_REEFS_FILE_NAME -)::DataFrame - # Load in the reef canonical outlines geodataframe - return GDF.read( - joinpath(source_dir, file_name) - ) -end +# ============================================================================= +# Main Data Initialization Functions +# ============================================================================= + +""" +Initialize all regional data from source files. +Loads raster data, slope tables, and computes criteria bounds for all regions. +This is the main data loading function that builds the complete data structure. + +# Arguments +- `config::Dict` : Configuration dictionary containing data directory paths + +# Returns +`RegionalData` struct containing all loaded and processed regional information. +""" function initialise_data(config::Dict)::RegionalData regional_data::RegionalDataMapType = Dict() data_source_directory = config["prepped_data"]["PREPPED_DATA_DIR"] - # iterate through regions - for region_metadata::RegionDetails in REGIONS + + # Process each region sequentially + for region_metadata::RegionMetadata in REGIONS @debug "$(now()) : Initializing cache for $(region_metadata)" - # Setup data paths and names arrays + # Initialize data collection arrays data_paths = String[] data_names = String[] - slope_table = nothing - - # list out the criteria we are interested in pulling out raster and - # criteria ranges for - assessable_criteria = [ - ASSESSMENT_CRITERIA.depth, - ASSESSMENT_CRITERIA.rugosity, - ASSESSMENT_CRITERIA.slope, - ASSESSMENT_CRITERIA.turbidity, - ASSESSMENT_CRITERIA.waves_height, - ASSESSMENT_CRITERIA.waves_period - ] - - # Extract the slope table for this region + # Load slope table containing valid reef coordinates and criteria values slope_table::DataFrame = GeoParquet.read( - get_slope_parquet_filepath(region_metadata) + joinpath(data_source_directory, get_slope_parquet_filename(region_metadata)) ) - # Extract the lat longs + # Add coordinate columns for spatial referencing add_lat_long_columns_to_dataframe(slope_table) - for criteria::CriteriaMetadata in assessable_criteria - # Get the file name of the matching data layer (.tif) and add it to - # the raster stack + # Collect raster file paths for all criteria + for criteria::CriteriaMetadata in ASSESSMENT_CRITERIA_LIST + # Find the corresponding .tif file for this criteria push!( data_paths, find_data_source_for_criteria(; data_source_directory, - region, + region=region_metadata, criteria ) ) - # Add name to - this is helpful so you can reference the raster in - # the raster stack + # Use criteria ID as the raster layer name push!(data_names, criteria.id) end - # Determine the criteria and extrema - criteria::AssessmentCriteria = build_assessment_criteria_from_slope_table( - slope_table - ) + # Compute regional criteria bounds from slope table data + criteria::RegionalCriteria = build_assessment_criteria_from_slope_table(slope_table) - # Build the raster stack + # Create lazy-loaded raster stack from all criteria files raster_stack = RasterStack(data_paths; name=data_names, lazy=true) - # Add entry to the regional data dictionary + # Store complete regional data entry regional_data[region_metadata.id] = RegionalDataEntry(; region_id=region_metadata.id, region_metadata, raster_stack, slope_table, - criteria) + criteria + ) end - # Load canonical reefs + # Load canonical reef outlines that apply to all regions canonical_reefs = load_canonical_reefs(data_source_directory) - # All done - return populated data + # Return complete regional data structure return RegionalData(; regions=regional_data, reef_outlines=canonical_reefs) end -function get_empty_tile_path(config::Dict)::String - cache_location = _cache_location(config) - return joinpath(cache_location, "no_data_tile.png") -end +""" +Initialize regional data with caching support. -function setup_empty_tile_cache(config::Dict)::String - file_path = get_empty_tile_path(config) - if !isfile(file_path) - save(file_path, zeros(RGBA, tile_size(config))) - end -end +Attempts to load from memory cache, then disk cache, before falling back +to full data initialization. Handles cache invalidation and saves new data to disk. +# Arguments +- `config::Dict` : Configuration dictionary containing cache and data settings +- `force_cache_invalidation::Bool` : If true, bypass all caches and reload data +""" function initialise_data_with_cache(config::Dict; force_cache_invalidation::Bool=false) - # Use module scope variable here + # Access global cache variable global REGIONAL_DATA - # Where is cache located? + # Determine cache directory location regional_cache_directory = config["server_config"]["REGIONAL_CACHE_DIR"] if !force_cache_invalidation - # do we have an in-memory cache available? + # Try memory cache first (fastest) local_data = check_existing_regional_data_from_memory() - if !isnothing(local_data) && !force_cache_invalidation + if !isnothing(local_data) REGIONAL_DATA = local_data return nothing end - # do we have disk cache available? + # Try disk cache second (faster than full reload) disk_data = check_existing_regional_data_from_disk(regional_cache_directory) - if !isnothing(disk_data) && !force_cache_invalidation + if !isnothing(disk_data) REGIONAL_DATA = disk_data return nothing end end - # No cache exists OR we are forcing invalidation + # No cache available or forced invalidation - load from source regional_data = initialise_data(config) - # Update the global variable + # Update global cache REGIONAL_DATA = regional_data - # Also ensure our empty tile cache is ready + # Initialize empty tile cache for map rendering setup_empty_tile_cache(config) + # Save to disk for future use + @debug "Saving regional data cache to disk" + serialize( + joinpath(regional_cache_directory, REGIONAL_DATA_CACHE_FILENAME), + regional_data + ) + return nothing end + +""" +Get regional data with automatic cache management. + +Primary interface for accessing regional data. Handles initialization +and caching automatically. + +# Arguments +- `config::Dict` : Configuration dictionary + +# Returns +`RegionalData` struct containing all regional information. +""" +function get_regional_data(config::Dict)::RegionalData + # Ensure data is loaded (with caching) + initialise_data_with_cache(config) + # Return cached data + return REGIONAL_DATA +end + +# ============================================================================= +# Display and Routing Functions +# ============================================================================= + +""" +Custom display format for RegionalDataEntry showing key statistics. +""" +function Base.show(io::IO, ::MIME"text/plain", z::RegionalDataEntry) + println(""" + Criteria: $(names(z.raster_stack)) + Number of valid slope locations: $(nrow(z.slope_table)) + """) + return nothing +end + +""" +Setup HTTP routes for criteria information endpoints. + +Creates REST endpoints for accessing regional criteria bounds and metadata. + +# Arguments +- `config` : Configuration object +- `auth` : Authentication/authorization handler +""" +function setup_criteria_routes(config, auth) + regional_data::RegionalData = get_regional_data(config) + + # Endpoint: GET /criteria/{region}/ranges + # Returns JSON with min/max values for all criteria in specified region + @get auth("/criteria/{region}/ranges") function (_::Request, region::String) + @debug "Transforming criteria information to JSON for region $(region)" + output_dict = OrderedDict() + + # Build lookup dictionary for regional criteria + regional_criteria_lookup = build_regional_criteria_dictionary( + regional_data.regions[region] + ) + + # Format each criteria with min/max bounds + for (id::String, criteria::RegionalCriteriaEntry) in regional_criteria_lookup + output_dict[id] = OrderedDict( + :min_val => criteria.bounds.min, + :max_val => criteria.bounds.max + ) + end + + return json(output_dict) + end +end + +""" +========== +DEPRECATED +========== +""" +function criteria_data_map() + # TODO: Load from config? + return OrderedDict( + :Depth => "_bathy", + :Benthic => "_benthic", + :Geomorphic => "_geomorphic", + :Slope => "_slope", + :Turbidity => "_turbid", + :WavesHs => "_waves_Hs", + :WavesTp => "_waves_Tp", + :Rugosity => "_rugosity", + :ValidSlopes => "_valid_slopes", + :ValidFlats => "_valid_flats" + ) +end + +function search_criteria()::Vector{String} + return string.(keys(criteria_data_map())) +end + +function site_criteria()::Vector{String} + return ["SuitabilityThreshold", "xdist", "ydist"] +end + +function suitability_criteria()::Vector{String} + return vcat(search_criteria(), ["SuitabilityThreshold"]) +end From cf8813717000201cde368ad17a39ea9f383e9c0b Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Thu, 22 May 2025 14:03:33 +1000 Subject: [PATCH 07/26] Working systemtically through optionality of criteria on a per region basis Signed-off-by: Peter Baker --- src/ReefGuideAPI.jl | 6 +- src/criteria_assessment/query_thresholds.jl | 40 +- src/setup.jl | 704 ++++++++++++++++---- 3 files changed, 604 insertions(+), 146 deletions(-) diff --git a/src/ReefGuideAPI.jl b/src/ReefGuideAPI.jl index 7a7ff0e..fb8664a 100644 --- a/src/ReefGuideAPI.jl +++ b/src/ReefGuideAPI.jl @@ -30,13 +30,11 @@ include("setup.jl") include("Middleware.jl") include("admin.jl") include("file_io.jl") -include("server_cache.jl") # TODO Remove these due to deprecation include("job_management/JobInterface.jl") include("job_management/DiskService.jl") -include("criteria_assessment/criteria.jl") include("criteria_assessment/query_thresholds.jl") include("criteria_assessment/regional_assessment.jl") @@ -111,6 +109,10 @@ This is a blocking operation until the worker times out. function start_worker() @info "Initializing worker from environment variables..." worker = create_worker_from_env() + @info "Parsing TOML config" + config = TOML.parsefile(worker.config.config_path) + @info "Loading regional data" + initialise_data_with_cache(config) @info "Starting worker loop from ReefGuideAPI.jl" start(worker) @info "Worker closed itself..." diff --git a/src/criteria_assessment/query_thresholds.jl b/src/criteria_assessment/query_thresholds.jl index eaec606..582ffc7 100644 --- a/src/criteria_assessment/query_thresholds.jl +++ b/src/criteria_assessment/query_thresholds.jl @@ -173,26 +173,28 @@ function apply_criteria_thresholds( res = Raster(criteria_stack.Depth; data=sparse(data), missingval=0) return res end -function apply_criteria_thresholds( - criteria_stack::T, - lookup::DataFrame, - ruleset::Vector{CriteriaBounds{Function}} -)::Raster where {T} - # Result store - data = falses(size(criteria_stack)) - res_lookup = trues(nrow(lookup)) - for threshold in ruleset - res_lookup .= res_lookup .& threshold.rule(lookup[!, threshold.name]) - end - - tmp = lookup[res_lookup, [:lon_idx, :lat_idx]] - data[CartesianIndex.(tmp.lon_idx, tmp.lat_idx)] .= true - - res = Raster(criteria_stack.Depth; data=sparse(data), missingval=0) - - return res -end +# TODO need this? +#function apply_criteria_thresholds( +# criteria_stack::T, +# lookup::DataFrame, +# ruleset::Vector{CriteriaBounds{Function}} +#)::Raster where {T} +# # Result store +# data = falses(size(criteria_stack)) +# +# res_lookup = trues(nrow(lookup)) +# for threshold in ruleset +# res_lookup .= res_lookup .& threshold.rule(lookup[!, threshold.name]) +# end +# +# tmp = lookup[res_lookup, [:lon_idx, :lat_idx]] +# data[CartesianIndex.(tmp.lon_idx, tmp.lat_idx)] .= true +# +# res = Raster(criteria_stack.Depth; data=sparse(data), missingval=0) +# +# return res +#end """ apply_criteria_lookup( diff --git a/src/setup.jl b/src/setup.jl index 1bc8041..d90bb11 100644 --- a/src/setup.jl +++ b/src/setup.jl @@ -3,19 +3,17 @@ using Glob using GeoParquet using Serialization using Oxygen: json, Request -using FileIO +using Logging +using Images # ============================================================================= # Constants and Configuration # ============================================================================= const REGIONAL_DATA_CACHE_FILENAME = "regional_cache_v2.dat" -const SLOPES_LOOKUP_SUFFIX = "_lookup.parq" +const SLOPES_LOOKUP_SUFFIX = "_valid_slopes_lookup.parq" const DEFAULT_CANONICAL_REEFS_FILE_NAME = "rrap_canonical_outlines.gpkg" -# Global variable to store regional data cache -REGIONAL_DATA::OptionalValue{RegionalData} = nothing - # ============================================================================= # Core Data Structures # ============================================================================= @@ -26,16 +24,20 @@ Metadata container for regional information. # Fields - `display_name::String` : Human-readable name for UI display - `id::String` : Unique system identifier (changing this affects data loading) +- `available_criteria::Vector{String}` : the criteria IDs that are relevant to this region """ struct RegionMetadata display_name::String id::String + available_criteria::Vector{String} function RegionMetadata(; display_name::String, - id::String + id::String, + available_criteria::Vector{String} ) - return new(display_name, id) + @debug "Creating RegionMetadata" display_name id + return new(display_name, id, available_criteria) end end @@ -104,6 +106,7 @@ struct AssessmentCriteria waves_period::CriteriaMetadata, rugosity::CriteriaMetadata ) + @debug "Initializing AssessmentCriteria with $(length(fieldnames(AssessmentCriteria))) criteria types" return new( depth, slope, turbidity, waves_height, waves_period, rugosity ) @@ -132,46 +135,63 @@ end """ Complete set of regional criteria with computed bounds for each parameter. +Note: Not all regions have all criteria available. Each field is optional to +accommodate varying data availability across different regions. + # Fields -- `depth::RegionalCriteriaEntry` : Depth criteria and bounds -- `slope::RegionalCriteriaEntry` : Slope criteria and bounds -- `turbidity::RegionalCriteriaEntry` : Turbidity criteria and bounds -- `waves_height::RegionalCriteriaEntry` : Wave height criteria and bounds -- `waves_period::RegionalCriteriaEntry` : Wave period criteria and bounds -- `rugosity::RegionalCriteriaEntry` : Rugosity criteria and bounds +- `depth::Union{RegionalCriteriaEntry,Nothing}` : Depth criteria and bounds + (optional) +- `slope::Union{RegionalCriteriaEntry,Nothing}` : Slope criteria and bounds + (optional) +- `turbidity::Union{RegionalCriteriaEntry,Nothing}` : Turbidity criteria and + bounds (optional) +- `waves_height::Union{RegionalCriteriaEntry,Nothing}` : Wave height criteria + and bounds (optional) +- `waves_period::Union{RegionalCriteriaEntry,Nothing}` : Wave period criteria + and bounds (optional) +- `rugosity::Union{RegionalCriteriaEntry,Nothing}` : Rugosity criteria and + bounds (optional) """ struct RegionalCriteria - depth::RegionalCriteriaEntry - slope::RegionalCriteriaEntry - turbidity::RegionalCriteriaEntry - waves_height::RegionalCriteriaEntry - waves_period::RegionalCriteriaEntry - rugosity::RegionalCriteriaEntry + depth::OptionalValue{RegionalCriteriaEntry} + slope::OptionalValue{RegionalCriteriaEntry} + turbidity::OptionalValue{RegionalCriteriaEntry} + waves_height::OptionalValue{RegionalCriteriaEntry} + waves_period::OptionalValue{RegionalCriteriaEntry} + rugosity::OptionalValue{RegionalCriteriaEntry} function RegionalCriteria(; - depth_bounds::Bounds, - slope_bounds::Bounds, - turbidity_bounds::Bounds, - waves_height_bounds::Bounds, - waves_period_bounds::Bounds, - rugosity_bounds::Bounds + depth_bounds::OptionalValue{Bounds}=nothing, + slope_bounds::OptionalValue{Bounds}=nothing, + turbidity_bounds::OptionalValue{Bounds}=nothing, + waves_height_bounds::OptionalValue{Bounds}=nothing, + waves_period_bounds::OptionalValue{Bounds}=nothing, + rugosity_bounds::OptionalValue{Bounds}=nothing ) + @debug "Creating RegionalCriteria with computed bounds for available criteria" return new( + # All criteria are now optional + isnothing(depth_bounds) ? nothing : RegionalCriteriaEntry(; metadata=ASSESSMENT_CRITERIA.depth, bounds=depth_bounds ), + isnothing(slope_bounds) ? nothing : RegionalCriteriaEntry(; metadata=ASSESSMENT_CRITERIA.slope, bounds=slope_bounds ), + isnothing(turbidity_bounds) ? nothing : RegionalCriteriaEntry(; metadata=ASSESSMENT_CRITERIA.turbidity, bounds=turbidity_bounds ), + isnothing(waves_height_bounds) ? nothing : RegionalCriteriaEntry(; metadata=ASSESSMENT_CRITERIA.waves_height, bounds=waves_height_bounds ), + isnothing(waves_period_bounds) ? nothing : RegionalCriteriaEntry(; metadata=ASSESSMENT_CRITERIA.waves_period, bounds=waves_period_bounds ), + isnothing(rugosity_bounds) ? nothing : RegionalCriteriaEntry(; metadata=ASSESSMENT_CRITERIA.rugosity, bounds=rugosity_bounds ) @@ -182,14 +202,18 @@ end """ Complete data package for a single region including rasters and metadata. +Validates during construction that all criteria instantiated have a +corresponding layer in the RasterStack + # Fields - `region_id::String` : Unique identifier for the region - `region_metadata::RegionMetadata` : Display metadata for the region - `raster_stack::Rasters.RasterStack` : Geospatial raster data layers -- `slope_table::DataFrame` : Coordinates and values for valid slope reef locations +- `slope_table::DataFrame` : Coordinates and values for valid slope reef + locations - `criteria::RegionalCriteria` : Computed criteria bounds for this region """ -mutable struct RegionalDataEntry +struct RegionalDataEntry region_id::String region_metadata::RegionMetadata raster_stack::Rasters.RasterStack @@ -203,10 +227,90 @@ mutable struct RegionalDataEntry slope_table::DataFrame, criteria::RegionalCriteria ) + # Get available layers and expected criteria from metadata + raster_layer_names = Set(string.(names(raster_stack))) + expected_criteria_set = Set(region_metadata.available_criteria) + + # Collect criteria that are actually instantiated (non-nothing) + instantiated_criteria = String[] + missing_from_criteria = String[] + missing_from_rasters = String[] + + # Check each criteria field for instantiation + for field_name in fieldnames(RegionalCriteria) + criteria_entry = getfield(criteria, field_name) + if !isnothing(criteria_entry) + layer_id = criteria_entry.metadata.id + push!(instantiated_criteria, layer_id) + + # Validate this instantiated criteria has a corresponding raster layer + if layer_id ∉ raster_layer_names + push!(missing_from_rasters, layer_id) + end + end + end + + # Cross-validate: ensure all expected criteria from metadata are instantiated + for expected_criteria_id in expected_criteria_set + if expected_criteria_id ∉ instantiated_criteria + push!(missing_from_criteria, expected_criteria_id) + end + end + + # Cross-validate: ensure all raster layers correspond to expected criteria + unexpected_layers = String[] + for layer_name in raster_layer_names + if layer_name ∉ expected_criteria_set + push!(unexpected_layers, layer_name) + end + end + + # Report validation errors + validation_errors = String[] + + if !isempty(missing_from_rasters) + push!( + validation_errors, + "RasterStack missing layers for instantiated criteria: $(join(missing_from_rasters, ", "))" + ) + end + + if !isempty(missing_from_criteria) + push!( + validation_errors, + "RegionalCriteria missing expected criteria from metadata: $(join(missing_from_criteria, ", "))" + ) + end + + if !isempty(unexpected_layers) + push!( + validation_errors, + "RasterStack contains unexpected layers not in metadata: $(join(unexpected_layers, ", "))" + ) + end + + # If any validation errors, log and throw + if !isempty(validation_errors) + @error "Validation failed for region $(region_metadata.display_name)" region_id validation_errors available_in_metadata = collect( + expected_criteria_set + ) instantiated_criteria available_raster_layers = collect(raster_layer_names) + + error("RegionalDataEntry validation failed: $(join(validation_errors, "; "))") + end + + @info "Created RegionalDataEntry for $(region_metadata.display_name)" region_id slope_locations = nrow( + slope_table + ) raster_layers = length(names(raster_stack)) validated_criteria_layers = length( + instantiated_criteria + ) expected_criteria = length(expected_criteria_set) + return new(region_id, region_metadata, raster_stack, slope_table, criteria) end end +# Type alias +const RegionalDataMapType = Dict{String,RegionalDataEntry} + """ Top-level container for all regional data and reef outlines. @@ -215,13 +319,16 @@ Top-level container for all regional data and reef outlines. - `reef_outlines::DataFrame` : Canonical reef outline geometries """ struct RegionalData - regions::Dict{String,RegionalDataEntry} + regions::RegionalDataMapType reef_outlines::DataFrame function RegionalData(; - regions::Dict{String,RegionalDataEntry}, + regions::RegionalDataMapType, reef_outlines::DataFrame ) + total_locations = sum(nrow(entry.slope_table) for entry in values(regions)) + @info "RegionalData initialized" num_regions = length(regions) total_valid_locations = + total_locations reef_outlines = nrow(reef_outlines) return new(regions, reef_outlines) end end @@ -230,26 +337,6 @@ end # Configuration Constants # ============================================================================= -# Define all available regions for the assessment system -const REGIONS::Vector{RegionMetadata} = [ - RegionMetadata(; - display_name="Townsville/Whitsunday Management Area", - id="Townsville-Whitsunday" - ), - RegionMetadata(; - display_name="Cairns/Cooktown Management Area", - id="Cairns-Cooktown" - ), - RegionMetadata(; - display_name="Mackay/Capricorn Management Area", - id="Mackay-Capricorn" - ), - RegionMetadata(; - display_name="Far Northern Management Area", - id="Far-Northern" - ) -] - # Define all assessment criteria with file naming conventions const ASSESSMENT_CRITERIA::AssessmentCriteria = AssessmentCriteria(; depth=CriteriaMetadata(; @@ -294,8 +381,43 @@ const ASSESSMENT_CRITERIA_LIST::Vector{CriteriaMetadata} = [ ASSESSMENT_CRITERIA.rugosity ] -# Type alias for cleaner code -const RegionalDataMapType = Dict{String,RegionalDataEntry} +# Normal list - only Townsville has rugosity +const BASE_CRITERIA_IDS::Vector{String} = [ + criteria.id for + criteria in ASSESSMENT_CRITERIA_LIST if criteria.id != ASSESSMENT_CRITERIA.rugosity.id +] +# All +const ALL_CRITERIA_IDS::Vector{String} = [ + criteria.id for + criteria in ASSESSMENT_CRITERIA_LIST +] + +# Define all available regions for the assessment system +const REGIONS::Vector{RegionMetadata} = [ + RegionMetadata(; + display_name="Townsville/Whitsunday Management Area", + id="Townsville-Whitsunday", + available_criteria=ALL_CRITERIA_IDS + ), + RegionMetadata(; + display_name="Cairns/Cooktown Management Area", + id="Cairns-Cooktown", + available_criteria=BASE_CRITERIA_IDS + ), + RegionMetadata(; + display_name="Mackay/Capricorn Management Area", + id="Mackay-Capricorn", + available_criteria=BASE_CRITERIA_IDS + ), + RegionMetadata(; + display_name="Far Northern Management Area", + id="FarNorthern", + available_criteria=BASE_CRITERIA_IDS + ) +] + +# GLOBAL variable to store regional data cache +REGIONAL_DATA::OptionalValue{RegionalData} = nothing # ============================================================================= # Utility Functions @@ -324,29 +446,78 @@ Generate the filename for slope lookup data for a given region. String filename in format "{region_id}_slope_lookup.parq" """ function get_slope_parquet_filename(region::RegionMetadata)::String - return "$(region.id)$(ASSESSMENT_CRITERIA.slope.file_suffix)$(SLOPES_LOOKUP_SUFFIX)" + filename = "$(region.id)$(SLOPES_LOOKUP_SUFFIX)" + @debug "Generated slope parquet filename" region_id = region.id filename + return filename end """ Create a dictionary mapping criteria IDs to regional criteria entries. +NOTE: Only includes criteria that are available for the region, as specified in the +region metadata and actually instantiated in the RegionalCriteria struct. + # Arguments - `region_data::RegionalDataEntry` : Regional data containing criteria information # Returns Dictionary with criteria ID strings as keys and RegionalCriteriaEntry as values. +Only includes criteria that are both listed in region metadata and available as non-nothing. """ function build_regional_criteria_dictionary( region_data::RegionalDataEntry )::Dict{String,RegionalCriteriaEntry} - return Dict( - ASSESSMENT_CRITERIA.depth.id => region_data.criteria.depth, - ASSESSMENT_CRITERIA.slope.id => region_data.criteria.slope, - ASSESSMENT_CRITERIA.turbidity.id => region_data.criteria.turbidity, - ASSESSMENT_CRITERIA.waves_height.id => region_data.criteria.waves_height, - ASSESSMENT_CRITERIA.waves_period.id => region_data.criteria.waves_period, - ASSESSMENT_CRITERIA.rugosity.id => region_data.criteria.rugosity - ) + @debug "Building criteria dictionary for region" region_id = region_data.region_id available_in_metadata = + region_data.region_metadata.available_criteria + + criteria_dict = Dict{String,RegionalCriteriaEntry}() + + # Only process criteria that are listed as available in the region metadata + available_criteria_set = Set(region_data.region_metadata.available_criteria) + + # Check depth + if ASSESSMENT_CRITERIA.depth.id ∈ available_criteria_set && + !isnothing(region_data.criteria.depth) + criteria_dict[ASSESSMENT_CRITERIA.depth.id] = region_data.criteria.depth + end + + # Check slope + if ASSESSMENT_CRITERIA.slope.id ∈ available_criteria_set && + !isnothing(region_data.criteria.slope) + criteria_dict[ASSESSMENT_CRITERIA.slope.id] = region_data.criteria.slope + end + + # Check turbidity + if ASSESSMENT_CRITERIA.turbidity.id ∈ available_criteria_set && + !isnothing(region_data.criteria.turbidity) + criteria_dict[ASSESSMENT_CRITERIA.turbidity.id] = region_data.criteria.turbidity + end + + # Check waves_height + if ASSESSMENT_CRITERIA.waves_height.id ∈ available_criteria_set && + !isnothing(region_data.criteria.waves_height) + criteria_dict[ASSESSMENT_CRITERIA.waves_height.id] = + region_data.criteria.waves_height + end + + # Check waves_period + if ASSESSMENT_CRITERIA.waves_period.id ∈ available_criteria_set && + !isnothing(region_data.criteria.waves_period) + criteria_dict[ASSESSMENT_CRITERIA.waves_period.id] = + region_data.criteria.waves_period + end + + # Check rugosity + if ASSESSMENT_CRITERIA.rugosity.id ∈ available_criteria_set && + !isnothing(region_data.criteria.rugosity) + criteria_dict[ASSESSMENT_CRITERIA.rugosity.id] = region_data.criteria.rugosity + end + + @debug "Built criteria dictionary" region_id = region_data.region_id available_in_metadata = length( + available_criteria_set + ) actually_available = length(criteria_dict) criteria_ids = collect(keys(criteria_dict)) + + return criteria_dict end """ @@ -359,12 +530,14 @@ from the centroid coordinates of each geometry feature. - `df::DataFrame` : DataFrame with geometry column containing spatial features """ function add_lat_long_columns_to_dataframe(df::DataFrame)::Nothing + @debug "Adding lat/long columns to DataFrame" num_rows = nrow(df) # Extract coordinate tuples from geometry centroids coords = GI.coordinates.(df.geometry) # Add longitude column (first coordinate) df[!, :lons] .= first.(coords) # Add latitude column (second coordinate) df[!, :lats] .= last.(coords) + @debug "Successfully added coordinate columns" return nothing end @@ -376,30 +549,81 @@ end Build regional criteria bounds from slope table data. Computes min/max bounds for each assessment criteria by finding extrema -in the slope table data columns. +in the slope table data columns. Only processes criteria that are available +for the specific region as defined in the region metadata. # Arguments - `table::DataFrame` : Slope table containing criteria data columns +- `region_metadata::RegionMetadata` : Region metadata specifying available criteria # Returns -`RegionalCriteria` struct with computed bounds for all criteria. -""" -function build_assessment_criteria_from_slope_table(table::DataFrame)::RegionalCriteria - # Compute bounds for each criteria using column extrema - const depth_bounds = bounds_from_tuple(extrema(table[:, ASSESSMENT_CRITERIA.depth.id])) - const slope_bounds = bounds_from_tuple(extrema(table[:, ASSESSMENT_CRITERIA.slope.id])) - const turbidity_bounds = bounds_from_tuple( - extrema(table[:, ASSESSMENT_CRITERIA.turbidity.id]) - ) - const waves_height_bounds = bounds_from_tuple( - extrema(table[:, ASSESSMENT_CRITERIA.waves_height.id]) +`RegionalCriteria` struct with computed bounds for available criteria only. +""" +function build_assessment_criteria_from_slope_table( + table::DataFrame, + region_metadata::RegionMetadata +)::RegionalCriteria + @debug "Computing assessment criteria bounds from slope table" table_rows = nrow(table) region_id = + region_metadata.id available_criteria = region_metadata.available_criteria + + # Create set for efficient lookup + available_criteria_set = Set(region_metadata.available_criteria) + + # Initialize all bounds as nothing + depth_bounds = nothing + slope_bounds = nothing + turbidity_bounds = nothing + waves_height_bounds = nothing + waves_period_bounds = nothing + rugosity_bounds = nothing + + # Helper function to compute bounds for a specific criteria + function compute_criteria_bounds( + criteria_metadata::CriteriaMetadata, criteria_name::String ) - const waves_period_bounds = bounds_from_tuple( - extrema(table[:, ASSESSMENT_CRITERIA.waves_period.id]) + if criteria_metadata.id ∈ available_criteria_set + if hasproperty(table, Symbol(criteria_metadata.id)) + bounds = bounds_from_tuple(extrema(table[:, criteria_metadata.id])) + @debug "Computed $(criteria_name) bounds" range = "$(bounds.min):$(bounds.max)" + return bounds + else + @error "Region metadata lists $(criteria_name) as available but column missing from slope table" region_id = + region_metadata.id column = criteria_metadata.id + throw( + ErrorException( + "Missing required column '$(criteria_metadata.id)' in slope table for region $(region_metadata.id)" + ) + ) + end + end + return nothing + end + + # Compute bounds only for available criteria + depth_bounds = compute_criteria_bounds(ASSESSMENT_CRITERIA.depth, "depth") + slope_bounds = compute_criteria_bounds(ASSESSMENT_CRITERIA.slope, "slope") + turbidity_bounds = compute_criteria_bounds(ASSESSMENT_CRITERIA.turbidity, "turbidity") + waves_height_bounds = compute_criteria_bounds( + ASSESSMENT_CRITERIA.waves_height, "waves_height" ) - const rugosity_bounds = bounds_from_tuple( - extrema(table[:, ASSESSMENT_CRITERIA.rugosity.id]) + waves_period_bounds = compute_criteria_bounds( + ASSESSMENT_CRITERIA.waves_period, "waves_period" ) + rugosity_bounds = compute_criteria_bounds(ASSESSMENT_CRITERIA.rugosity, "rugosity") + + computed_criteria = length([ + b for b in [ + depth_bounds, + slope_bounds, + turbidity_bounds, + waves_height_bounds, + waves_period_bounds, + rugosity_bounds + ] if !isnothing(b) + ]) + @debug "Computed criteria bounds" region_id = region_metadata.id total_available = length( + available_criteria_set + ) total_computed = computed_criteria return RegionalCriteria(; depth_bounds, @@ -433,14 +657,21 @@ function find_data_source_for_criteria(; region::RegionMetadata, criteria::CriteriaMetadata )::String + pattern = "$(region.id)*$(criteria.file_suffix).tif" + @debug "Searching for data file" pattern directory = data_source_directory + # Search for files matching the pattern: {region_id}*{criteria_suffix}.tif - matched_files = glob("$(region.id)*$(criteria.file_suffix).tif", data_source_directory) + matched_files = glob(pattern, data_source_directory) # Validate exactly one match exists if length(matched_files) == 0 + @error "No data file found for criteria" criteria_id = criteria.id region_id = + region.id pattern throw(ErrorException("Did not find data for the criteria: $(criteria.id).")) end if length(matched_files) > 1 + @error "Multiple data files found for criteria - ambiguous match" criteria_id = + criteria.id region_id = region.id matched_files throw( ErrorException( "Found more than one data source match for criteria: $(criteria.id). This is ambiguous, unsure how to proceed." @@ -448,7 +679,9 @@ function find_data_source_for_criteria(; ) end - return first(matched_files) + file_path = first(matched_files) + @debug "Found data file" criteria_id = criteria.id file_path + return file_path end """ @@ -465,8 +698,17 @@ function load_canonical_reefs( source_dir::String; file_name::String=DEFAULT_CANONICAL_REEFS_FILE_NAME )::DataFrame - # Load reef outlines from geopackage format - return GDF.read(joinpath(source_dir, file_name)) + file_path = joinpath(source_dir, file_name) + @info "Loading canonical reef outlines" file_path + + try + reef_data = GDF.read(file_path) + @info "Successfully loaded reef outlines" num_reefs = nrow(reef_data) + return reef_data + catch e + @error "Failed to load canonical reef outlines" file_path error = e + rethrow(e) + end end # ============================================================================= @@ -481,9 +723,10 @@ Check if regional data is available in memory cache. """ function check_existing_regional_data_from_memory()::OptionalValue{RegionalData} if !isnothing(REGIONAL_DATA) - @debug "Using previously generated regional data store." + @info "Using cached regional data from memory" return REGIONAL_DATA end + @debug "No regional data found in memory cache" return nothing end @@ -503,14 +746,19 @@ function check_existing_regional_data_from_disk( reg_cache_filename = joinpath(cache_directory, REGIONAL_DATA_CACHE_FILENAME) if isfile(reg_cache_filename) - @debug "Loading regional data cache from disk" + @info "Loading regional data from disk cache" cache_file = reg_cache_filename try - return deserialize(reg_cache_filename) + data = deserialize(reg_cache_filename) + @info "Successfully loaded regional data from disk cache" + return data catch err - @warn "Failed to deserialize $(reg_cache_filename) with error:" err + @warn "Failed to deserialize regional data cache - removing corrupted file" cache_file = + reg_cache_filename error = err # Remove corrupted cache file rm(reg_cache_filename) end + else + @debug "No disk cache file found" expected_path = reg_cache_filename end # No cache available or load failed return nothing @@ -541,8 +789,11 @@ Creates a blank PNG tile used for areas with no data coverage. function setup_empty_tile_cache(config::Dict)::Nothing file_path = get_empty_tile_path(config) if !isfile(file_path) + @info "Creating empty tile cache" file_path # Create empty RGBA tile with configured dimensions save(file_path, zeros(RGBA, tile_size(config))) + else + @debug "Empty tile cache already exists" file_path end return nothing end @@ -564,60 +815,102 @@ This is the main data loading function that builds the complete data structure. `RegionalData` struct containing all loaded and processed regional information. """ function initialise_data(config::Dict)::RegionalData + @info "Starting regional data initialization from source files" + regional_data::RegionalDataMapType = Dict() data_source_directory = config["prepped_data"]["PREPPED_DATA_DIR"] + @info "Using data source directory" directory = data_source_directory # Process each region sequentially for region_metadata::RegionMetadata in REGIONS - @debug "$(now()) : Initializing cache for $(region_metadata)" + @info "Processing region" region = region_metadata.display_name region_id = + region_metadata.id # Initialize data collection arrays data_paths = String[] data_names = String[] # Load slope table containing valid reef coordinates and criteria values - slope_table::DataFrame = GeoParquet.read( - joinpath(data_source_directory, get_slope_parquet_filename(region_metadata)) - ) + slope_filename = get_slope_parquet_filename(region_metadata) + slope_file_path = joinpath(data_source_directory, slope_filename) + @debug "Loading slope table" file_path = slope_file_path + + try + slope_table::DataFrame = GeoParquet.read(slope_file_path) + @info "Loaded slope table" region_id = region_metadata.id num_locations = nrow( + slope_table + ) - # Add coordinate columns for spatial referencing - add_lat_long_columns_to_dataframe(slope_table) + # Add coordinate columns for spatial referencing + add_lat_long_columns_to_dataframe(slope_table) - # Collect raster file paths for all criteria - for criteria::CriteriaMetadata in ASSESSMENT_CRITERIA_LIST - # Find the corresponding .tif file for this criteria - push!( - data_paths, - find_data_source_for_criteria(; + # Filter criteria list to only those available for this region + available_criteria_set = Set(region_metadata.available_criteria) + region_criteria_list = filter( + criteria -> criteria.id ∈ available_criteria_set, + ASSESSMENT_CRITERIA_LIST + ) + + @debug "Filtered criteria for region" region_id = region_metadata.id total_criteria = length( + ASSESSMENT_CRITERIA_LIST + ) available_criteria = length(region_criteria_list) criteria_ids = [ + c.id for c in region_criteria_list + ] + + # Collect raster file paths for available criteria only + for criteria::CriteriaMetadata in region_criteria_list + @debug "Processing criteria" criteria_id = criteria.id region_id = + region_metadata.id + + # Find the corresponding .tif file for this criteria + data_file_path = find_data_source_for_criteria(; data_source_directory, region=region_metadata, criteria ) - ) - # Use criteria ID as the raster layer name - push!(data_names, criteria.id) - end - # Compute regional criteria bounds from slope table data - criteria::RegionalCriteria = build_assessment_criteria_from_slope_table(slope_table) + push!(data_paths, data_file_path) + # Use criteria ID as the raster layer name + push!(data_names, criteria.id) + end - # Create lazy-loaded raster stack from all criteria files - raster_stack = RasterStack(data_paths; name=data_names, lazy=true) + @info "Found all criteria data files for region" region_id = region_metadata.id num_criteria = length( + data_paths + ) available_criteria = join([c.id for c in region_criteria_list], ", ") - # Store complete regional data entry - regional_data[region_metadata.id] = RegionalDataEntry(; - region_id=region_metadata.id, - region_metadata, - raster_stack, - slope_table, - criteria - ) + # Compute regional criteria bounds from slope table data + criteria::RegionalCriteria = build_assessment_criteria_from_slope_table( + slope_table, region_metadata + ) + + # Create lazy-loaded raster stack from all criteria files + @debug "Creating raster stack" region_id = region_metadata.id num_layers = length( + data_paths + ) + raster_stack = RasterStack(data_paths; name=data_names, lazy=true) + + # Store complete regional data entry + regional_data[region_metadata.id] = RegionalDataEntry(; + region_id=region_metadata.id, + region_metadata, + raster_stack, + slope_table, + criteria + ) + + catch e + @error "Failed to process region data" region_id = region_metadata.id error = e + rethrow(e) + end end + @info "Completed processing all regions" num_regions = length(regional_data) + # Load canonical reef outlines that apply to all regions canonical_reefs = load_canonical_reefs(data_source_directory) # Return complete regional data structure + @info "Regional data initialization completed successfully" return RegionalData(; regions=regional_data, reef_outlines=canonical_reefs) end @@ -632,11 +925,14 @@ to full data initialization. Handles cache invalidation and saves new data to di - `force_cache_invalidation::Bool` : If true, bypass all caches and reload data """ function initialise_data_with_cache(config::Dict; force_cache_invalidation::Bool=false) + @info "Initializing regional data with caching" force_cache_invalidation + # Access global cache variable global REGIONAL_DATA # Determine cache directory location regional_cache_directory = config["server_config"]["REGIONAL_CACHE_DIR"] + @debug "Using cache directory" directory = regional_cache_directory if !force_cache_invalidation # Try memory cache first (fastest) @@ -652,9 +948,12 @@ function initialise_data_with_cache(config::Dict; force_cache_invalidation::Bool REGIONAL_DATA = disk_data return nothing end + else + @info "Cache invalidation forced - reloading from source files" end # No cache available or forced invalidation - load from source + @info "Loading regional data from source files (no cache available)" regional_data = initialise_data(config) # Update global cache @@ -664,11 +963,16 @@ function initialise_data_with_cache(config::Dict; force_cache_invalidation::Bool setup_empty_tile_cache(config) # Save to disk for future use - @debug "Saving regional data cache to disk" - serialize( - joinpath(regional_cache_directory, REGIONAL_DATA_CACHE_FILENAME), - regional_data - ) + @info "Saving regional data to disk cache" cache_directory = regional_cache_directory + try + serialize( + joinpath(regional_cache_directory, REGIONAL_DATA_CACHE_FILENAME), + regional_data + ) + @info "Successfully saved regional data cache to disk" + catch e + @warn "Failed to save regional data cache to disk" error = e + end return nothing end @@ -686,6 +990,7 @@ and caching automatically. `RegionalData` struct containing all regional information. """ function get_regional_data(config::Dict)::RegionalData + @debug "Getting regional data with automatic cache management" # Ensure data is loaded (with caching) initialise_data_with_cache(config) # Return cached data @@ -693,20 +998,84 @@ function get_regional_data(config::Dict)::RegionalData end # ============================================================================= -# Display and Routing Functions +# Display Methods # ============================================================================= """ -Custom display format for RegionalDataEntry showing key statistics. +Enhanced display format for RegionalDataEntry showing key statistics. """ -function Base.show(io::IO, ::MIME"text/plain", z::RegionalDataEntry) - println(""" - Criteria: $(names(z.raster_stack)) - Number of valid slope locations: $(nrow(z.slope_table)) - """) - return nothing +function Base.show(io::IO, ::MIME"text/plain", entry::RegionalDataEntry) + println(io, "RegionalDataEntry: $(entry.region_metadata.display_name)") + println(io, " Region ID: $(entry.region_id)") + println(io, " Raster layers: $(join(names(entry.raster_stack), ", "))") + println(io, " Valid slope locations: $(nrow(entry.slope_table))") + println( + io, " Available criteria: $(join(entry.region_metadata.available_criteria, ", "))" + ) + println(io, " Criteria bounds:") + + # Show each criteria with its bounds, only for non-nothing entries + for field_name in fieldnames(RegionalCriteria) + criteria_entry = getfield(entry.criteria, field_name) + if !isnothing(criteria_entry) + min_val = round(criteria_entry.bounds.min; digits=2) + max_val = round(criteria_entry.bounds.max; digits=2) + println( + io, " $(criteria_entry.metadata.display_label): $(min_val) - $(max_val)" + ) + else + # Get the criteria name for display + criteria_name = if field_name == :depth + ASSESSMENT_CRITERIA.depth.display_label + elseif field_name == :slope + ASSESSMENT_CRITERIA.slope.display_label + elseif field_name == :turbidity + ASSESSMENT_CRITERIA.turbidity.display_label + elseif field_name == :waves_height + ASSESSMENT_CRITERIA.waves_height.display_label + elseif field_name == :waves_period + ASSESSMENT_CRITERIA.waves_period.display_label + elseif field_name == :rugosity + ASSESSMENT_CRITERIA.rugosity.display_label + else + string(field_name) + end + println(io, " $(criteria_name): Not available") + end + end end +""" +Display format for RegionalData showing system overview. +""" +function Base.show(io::IO, ::MIME"text/plain", data::RegionalData) + total_locations = sum(nrow(entry.slope_table) for entry in values(data.regions)) + + println(io, "RegionalData:") + println(io, " Regions: $(length(data.regions))") + println(io, " Total valid locations: $(total_locations)") + println(io, " Reef outlines: $(nrow(data.reef_outlines))") + println(io, "") + + println(io, "Regional breakdown:") + for (_, region_entry) in data.regions + locations = nrow(region_entry.slope_table) + println( + io, " $(region_entry.region_metadata.display_name): $(locations) locations" + ) + end + println(io, "") + + return println( + io, + "Assessment criteria: $(join([c.display_label for c in ASSESSMENT_CRITERIA_LIST], ", "))" + ) +end + +# ============================================================================= +# Routes +# ============================================================================= + """ Setup HTTP routes for criteria information endpoints. @@ -717,11 +1086,21 @@ Creates REST endpoints for accessing regional criteria bounds and metadata. - `auth` : Authentication/authorization handler """ function setup_criteria_routes(config, auth) + @info "Setting up criteria routes" regional_data::RegionalData = get_regional_data(config) # Endpoint: GET /criteria/{region}/ranges # Returns JSON with min/max values for all criteria in specified region @get auth("/criteria/{region}/ranges") function (_::Request, region::String) + @info "Processing criteria ranges request" region + + if !haskey(regional_data.regions, region) + @warn "Request for unknown region" region available_regions = keys( + regional_data.regions + ) + return json(Dict("error" => "Region not found")) + end + @debug "Transforming criteria information to JSON for region $(region)" output_dict = OrderedDict() @@ -738,8 +1117,11 @@ function setup_criteria_routes(config, auth) ) end + @debug "Returning criteria ranges" region num_criteria = length(output_dict) return json(output_dict) end + + @info "Criteria routes setup completed" end """ @@ -774,3 +1156,75 @@ end function suitability_criteria()::Vector{String} return vcat(search_criteria(), ["SuitabilityThreshold"]) end + +function criteria_data_map() + # TODO: Load from config? + return OrderedDict( + :Depth => "_bathy", + :Benthic => "_benthic", + :Geomorphic => "_geomorphic", + :Slope => "_slope", + :Turbidity => "_turbid", + :WavesHs => "_waves_Hs", + :WavesTp => "_waves_Tp", + :Rugosity => "_rugosity", + :ValidSlopes => "_valid_slopes", + :ValidFlats => "_valid_flats" + + # Unused datasets + # :PortDistSlopes => "_PortDistSlopes", + # :PortDistFlats => "_PortDistFlats" + ) +end + +function search_criteria()::Vector{String} + return string.(keys(criteria_data_map())) +end + +function site_criteria()::Vector{String} + return ["SuitabilityThreshold", "xdist", "ydist"] +end + +function suitability_criteria()::Vector{String} + return vcat(search_criteria(), ["SuitabilityThreshold"]) +end + +function extract_criteria(qp::T, criteria::Vector{String})::T where {T<:Dict{String,String}} + return filter( + k -> string(k.first) ∈ criteria, qp + ) +end + +struct OldRegionalCriteria{T} + stack::RasterStack + valid_slopes::T + valid_flats::T +end + +function valid_slope_lon_inds(reg::OldRegionalCriteria) + return reg.valid_slopes.lon_idx +end +function valid_slope_lat_inds(reg::OldRegionalCriteria) + return reg.valid_slopes.lat_idx +end +function valid_flat_lon_inds(reg::OldRegionalCriteria) + return reg.valid_flats.lon_idx +end +function valid_flat_lat_inds(reg::OldRegionalCriteria) + return reg.valid_flats.lat_idx +end + +struct CriteriaBounds{F<:Function} + name::Symbol + lower_bound::Float32 + upper_bound::Float32 + rule::F + + function CriteriaBounds(name::String, lb::S, ub::S)::CriteriaBounds where {S<:String} + lower_bound::Float32 = parse(Float32, lb) + upper_bound::Float32 = parse(Float32, ub) + func = (x) -> lower_bound .<= x .<= upper_bound + + return new{Function}(Symbol(name), lower_bound, upper_bound, func) + end +end From b4a38f63dfcfa81586be1730df8f4f6dec042250 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Thu, 22 May 2025 16:16:09 +1000 Subject: [PATCH 08/26] Working through regional assessment candidate implementation Signed-off-by: Peter Baker --- src/ReefGuideAPI.jl | 20 +- src/RegionalDataHelpers.jl | 505 ++++++++++++++++++ src/criteria_assessment/query_thresholds.jl | 24 + .../site_identification.jl | 41 ++ src/job_worker/Helpers.jl | 217 -------- src/job_worker/Jobs.jl | 53 +- src/job_worker/Worker.jl | 1 - src/setup.jl | 19 + 8 files changed, 606 insertions(+), 274 deletions(-) create mode 100644 src/RegionalDataHelpers.jl delete mode 100644 src/job_worker/Helpers.jl diff --git a/src/ReefGuideAPI.jl b/src/ReefGuideAPI.jl index fb8664a..c5a507a 100644 --- a/src/ReefGuideAPI.jl +++ b/src/ReefGuideAPI.jl @@ -26,7 +26,10 @@ using include("job_worker/Worker.jl") +# New work including setup logic and helper functions include("setup.jl") +include("RegionalDataHelpers.jl") + include("Middleware.jl") include("admin.jl") include("file_io.jl") @@ -37,21 +40,11 @@ include("job_management/DiskService.jl") include("criteria_assessment/query_thresholds.jl") include("criteria_assessment/regional_assessment.jl") +include("criteria_assessment/site_identification.jl") include("site_assessment/common_functions.jl") include("site_assessment/best_fit_polygons.jl") -function get_regions() - # TODO: Comes from config? - regions = String[ - "Townsville-Whitsunday", - "Cairns-Cooktown", - "Mackay-Capricorn", - "FarNorthern" - ] - - return regions -end function get_auth_router(config::Dict) # Setup auth middleware - depends on config.toml - can return identity func @@ -62,11 +55,12 @@ end function start_server(config_path) @info "Launching server... please wait" - warmup_cache(config_path) - @info "Parsing configuration from $(config_path)..." config = TOML.parsefile(config_path) + @info "Initialising regional data and setting up tile cache" + initialise_data_with_cache(config) + @info "Setting up auth middleware and router." auth = get_auth_router(config) diff --git a/src/RegionalDataHelpers.jl b/src/RegionalDataHelpers.jl new file mode 100644 index 0000000..16517f1 --- /dev/null +++ b/src/RegionalDataHelpers.jl @@ -0,0 +1,505 @@ +# ============================================================================= +# Assessment Parameters Constants +# ============================================================================= + +const DEFAULT_SUITABILITY_THRESHOLD::Int32 = 80 + +# ============================================================================= +# Assessment Parameters Data Structures +# ============================================================================= + +""" +Payload for regional assessment actions - this includes all merged bounds and +regional data. + +# Fields +- `region::String` : The region that is being assessed +- `regional_criteria::RegionalCriteria` : The criteria to assess, including user provided bounds +- `region_data::RegionalDataEntry` : The data to consider for this region +- `suitability_threshold::Int32` : The cutoff to consider a site suitable +""" +struct RegionalAssessmentParameters + region::String + regional_criteria::RegionalCriteria + region_data::RegionalDataEntry + suitability_threshold::Int32 + + function RegionalAssessmentParameters(; + region::String, + regional_criteria::RegionalCriteria, + region_data::RegionalDataEntry, + suitability_threshold::Int32 + ) + @debug "Created RegionalAssessmentParameters" region suitability_threshold + return new(region, regional_criteria, region_data, suitability_threshold) + end +end + +""" +Payload for suitability assessment actions - this includes all merged bounds and +regional data plus spatial dimensions. + +# Fields +- `region::String` : The region that is being assessed +- `regional_criteria::RegionalCriteria` : The criteria to assess, including user provided bounds +- `region_data::RegionalDataEntry` : The data to consider for this region +- `suitability_threshold::Int32` : The cutoff to consider a site suitable +- `x_dist::Int32` : X dimension of polygon (metres) +- `y_dist::Int32` : Y dimension of polygon (metres) +""" +struct SuitabilityAssessmentParameters + region::String + regional_criteria::RegionalCriteria + region_data::RegionalDataEntry + suitability_threshold::Int32 + x_dist::Int32 + y_dist::Int32 + + function SuitabilityAssessmentParameters(; + region::String, + regional_criteria::RegionalCriteria, + region_data::RegionalDataEntry, + suitability_threshold::Int32, + x_dist::Int32, + y_dist::Int32 + ) + @debug "Created SuitabilityAssessmentParameters" region suitability_threshold x_dist y_dist + return new( + region, regional_criteria, region_data, suitability_threshold, x_dist, y_dist + ) + end +end + +# ============================================================================= +# Assessment Parameters Builder Functions +# ============================================================================= + +""" +Merge user-specified bounds with regional defaults. + +Creates bounds using user values where provided, falling back to regional +bounds for unspecified values. Returns nothing if regional criteria is not available. + +# Arguments +- `user_min::OptionalValue{Float64}` : User-specified minimum value (optional) +- `user_max::OptionalValue{Float64}` : User-specified maximum value (optional) +- `regional_criteria::OptionalValue{RegionalCriteriaEntry}` : Regional criteria with default bounds (optional) + +# Returns +`Bounds` struct with merged values, or `nothing` if regional criteria unavailable. +""" +function merge_bounds( + user_min::OptionalValue{Float64}, + user_max::OptionalValue{Float64}, + regional_criteria::OptionalValue{RegionalCriteriaEntry} +)::OptionalValue{Bounds} + if isnothing(regional_criteria) + return nothing + end + + bounds = Bounds(; + min=!isnothing(user_min) ? user_min : regional_criteria.bounds.min, + max=!isnothing(user_max) ? user_max : regional_criteria.bounds.max + ) + + @debug "Merged bounds" min_val = bounds.min max_val = bounds.max user_specified_min = + !isnothing(user_min) user_specified_max = !isnothing(user_max) + + return bounds +end + +""" +Build regional assessment parameters from user input and regional data. + +Creates a complete parameter set for regional assessment by merging user-specified +criteria bounds with regional defaults. Validates that the specified region exists. + +# Arguments +- `input::RegionalAssessmentInput` : User input containing assessment parameters +- `regional_data::RegionalData` : Complete regional data for validation and defaults + +# Returns +`RegionalAssessmentParameters` struct ready for assessment execution. + +# Throws +- `ErrorException` : If specified region is not found in regional data +""" +function build_regional_assessment_parameters( + input::RegionalAssessmentInput, + regional_data::RegionalData +)::RegionalAssessmentParameters + @info "Building regional assessment parameters" region = input.region + + # Validate region exists + if !haskey(regional_data.regions, input.region) + available_regions = collect(keys(regional_data.regions)) + @error "Region not found in regional data" region = input.region available_regions + throw( + ErrorException( + "Regional data did not have data for region $(input.region). Available regions: $(join(available_regions, ", "))" + ) + ) + end + + region_data = regional_data.regions[input.region] + + # Extract threshold with default fallback + threshold = + !isnothing(input.threshold) ? input.threshold : DEFAULT_SUITABILITY_THRESHOLD + + # Build merged criteria + regional_criteria = RegionalCriteria(; + depth_bounds=merge_bounds( + input.depth_min, input.depth_max, region_data.criteria.depth + ), + slope_bounds=merge_bounds( + input.slope_min, input.slope_max, region_data.criteria.slope + ), + waves_height_bounds=merge_bounds( + input.waves_height_min, + input.waves_height_max, + region_data.criteria.waves_height + ), + waves_period_bounds=merge_bounds( + input.waves_period_min, + input.waves_period_max, + region_data.criteria.waves_period + ), + rugosity_bounds=merge_bounds( + input.rugosity_min, input.rugosity_max, region_data.criteria.rugosity + ), + # Turbidity is not user-configurable, always use regional bounds + turbidity_bounds=merge_bounds(nothing, nothing, region_data.criteria.turbidity) + ) + + # Count active criteria for logging + active_criteria = length([ + b for b in [ + regional_criteria.depth, regional_criteria.slope, regional_criteria.turbidity, + regional_criteria.waves_height, regional_criteria.waves_period, + regional_criteria.rugosity + ] if !isnothing(b) + ]) + + @info "Built regional assessment parameters" region = input.region threshold active_criteria user_specified_threshold = + !isnothing(input.threshold) + + return RegionalAssessmentParameters(; + region=input.region, + regional_criteria, + region_data, + suitability_threshold=Int32(threshold) + ) +end + +""" +Build suitability assessment parameters from user input and regional data. + +Creates a complete parameter set for suitability assessment by merging user-specified +criteria bounds with regional defaults. Includes spatial dimensions for polygon analysis. + +# Arguments +- `input::SuitabilityAssessmentInput` : User input containing assessment parameters and spatial dimensions +- `regional_data::RegionalData` : Complete regional data for validation and defaults + +# Returns +`SuitabilityAssessmentParameters` struct ready for assessment execution. + +# Throws +- `ErrorException` : If specified region is not found in regional data +""" +function build_suitability_assessment_parameters( + input::SuitabilityAssessmentInput, + regional_data::RegionalData +)::SuitabilityAssessmentParameters + @info "Building suitability assessment parameters" region = input.region x_dist = + input.x_dist y_dist = input.y_dist + + # Validate region exists + if !haskey(regional_data.regions, input.region) + available_regions = collect(keys(regional_data.regions)) + @error "Region not found in regional data" region = input.region available_regions + throw( + ErrorException( + "Regional data did not have data for region $(input.region). Available regions: $(join(available_regions, ", "))" + ) + ) + end + + region_data = regional_data.regions[input.region] + + # Extract threshold with default fallback + threshold = + !isnothing(input.threshold) ? input.threshold : DEFAULT_SUITABILITY_THRESHOLD + + # Build merged criteria + regional_criteria = RegionalCriteria(; + depth_bounds=merge_bounds( + input.depth_min, input.depth_max, region_data.criteria.depth + ), + slope_bounds=merge_bounds( + input.slope_min, input.slope_max, region_data.criteria.slope + ), + waves_height_bounds=merge_bounds( + input.waves_height_min, + input.waves_height_max, + region_data.criteria.waves_height + ), + waves_period_bounds=merge_bounds( + input.waves_period_min, + input.waves_period_max, + region_data.criteria.waves_period + ), + rugosity_bounds=merge_bounds( + input.rugosity_min, input.rugosity_max, region_data.criteria.rugosity + ), + # Turbidity is not user-configurable, always use regional bounds + turbidity_bounds=merge_bounds(nothing, nothing, region_data.criteria.turbidity) + ) + + # Count active criteria for logging + active_criteria = length([ + b for b in [ + regional_criteria.depth, regional_criteria.slope, regional_criteria.turbidity, + regional_criteria.waves_height, regional_criteria.waves_period, + regional_criteria.rugosity + ] if !isnothing(b) + ]) + + @info "Built suitability assessment parameters" region = input.region threshold active_criteria x_dist = + input.x_dist y_dist = input.y_dist user_specified_threshold = + !isnothing(input.threshold) + + return SuitabilityAssessmentParameters(; + region=input.region, + regional_criteria, + region_data, + suitability_threshold=Int32(threshold), + x_dist=Int32(input.x_dist), + y_dist=Int32(input.y_dist) + ) +end + +# ============================================================================= +# Cache File Name Generation +# ============================================================================= + +""" +Builds a hash by combining strings and hashing result +""" +function build_hash_from_components(components::Vector{String})::String + return string(hash(join(components, "|"))) +end + +""" +Combines present regional criteria including bounds into hash components +""" +function get_hash_components_from_regional_criteria( + criteria::RegionalCriteria +)::Vector{String} + hash_components::Vector{String} = [] + for field in REGIONAL_CRITERIA_SYMBOLS + criteria_entry::OptionalValue{RegionalCriteriaEntry} = getfield( + criteria, field + ) + if !isnothing(criteria_entry) + push!( + hash_components, + "$(field)_$(criteria_entry.bounds.min)_$(criteria_entry.bounds.max)" + ) + else + push!(hash_components, "$(field)_null") + end + end + return hash_components +end + +""" +Generate a deterministic hash string for RegionalAssessmentParameters. + +Creates a consistent hash based on assessment parameters that can be used +for cache file naming. Same parameters will always produce the same hash. + +# Arguments +- `params::RegionalAssessmentParameters` : Assessment parameters to hash + +# Returns +String hash suitable for use in cache file names. +""" +function regional_assessment_params_hash(params::RegionalAssessmentParameters)::String + @debug "Generating hash for regional assessment parameters" region = params.region threshold = + params.suitability_threshold + + # Create hash input from key parameters + hash_components = [ + params.region, + string(params.suitability_threshold) + ] + + # Add criteria bounds to hash (only non-nothing criteria) + hash_components::Vector{String} = [ + hash_components; + get_hash_components_from_regional_criteria(params.regional_criteria) + ] + + # Create deterministic hash + hash_string = build_hash_from_components(hash_components) + + @debug "Generated assessment parameters hash" hash = hash_string components_count = length( + hash_components + ) + + return hash_string +end + +""" +Generate a deterministic hash string for SuitabilityAssessmentParameters. + +Creates a consistent hash based on assessment parameters that can be used +for cache file naming. Same parameters will always produce the same hash. + +# Arguments +- `params::SuitabilityAssessmentParameters` : Assessment parameters to hash + +# Returns +String hash suitable for use in cache file names. +""" +function suitability_assessment_params_hash(params::SuitabilityAssessmentParameters)::String + @debug "Generating hash for suitability assessment parameters" region = params.region threshold = + params.suitability_threshold x_dist = params.x_dist y_dist = params.y_dist + + # Create hash input from key parameters including spatial dimensions + hash_components = [ + params.region, + string(params.suitability_threshold), + string(params.x_dist), + string(params.y_dist) + ] + + # Add criteria bounds to hash (only non-nothing criteria) + hash_components::Vector{String} = [ + hash_components; + get_hash_components_from_regional_criteria(params.regional_criteria) + ] + + # Create deterministic hash + hash_string = build_hash_from_components(hash_components) + + @debug "Generated suitability parameters hash" hash = hash_string components_count = length( + hash_components + ) + + return hash_string +end + +""" +Build predictable file path for regional assessment results in configured cache +location. + +Creates a complete file path for caching regional assessment results using the +configured cache directory and deterministic parameter-based naming. + +# Arguments +- `params::RegionalAssessmentParameters` : Regional assessment parameters +- `ext::String` : File extension for the cache file +- `config::Dict` : Configuration dictionary containing cache settings + +# Returns +String path to cache file location. +""" +function build_regional_assessment_file_path( + params::RegionalAssessmentParameters; + ext::String, + config::Dict +)::String + @debug "Building file path for regional assessment cache" region = params.region ext + + cache_path = _cache_location(config) + param_hash = regional_assessment_params_hash(params) + filename = "$(param_hash)_$(params.region)_regional_assessment.$(ext)" + file_path = joinpath(cache_path, filename) + + @debug "Built regional assessment file path" file_path region = params.region hash = + param_hash + + return file_path +end + +""" +Build predictable file path for suitability assessment results in configured cache +location. + +Creates a complete file path for caching suitability assessment results using the +configured cache directory and deterministic parameter-based naming. + +# Arguments +- `params::SuitabilityAssessmentParameters` : Suitability assessment parameters +- `ext::String` : File extension for the cache file +- `config::Dict` : Configuration dictionary containing cache settings + +# Returns +String path to cache file location. +""" +function build_suitability_assessment_file_path( + params::SuitabilityAssessmentParameters; + ext::String, + config::Dict +)::String + @debug "Building file path for suitability assessment cache" region = params.region ext + + cache_path = _cache_location(config) + param_hash = suitability_assessment_params_hash(params) + filename = "$(param_hash)_$(params.region)_suitability_assessment.$(ext)" + file_path = joinpath(cache_path, filename) + + @debug "Built suitability assessment file path" file_path region = params.region hash = + param_hash + + return file_path +end + +""" +Convert RegionalCriteria to a vector of CriteriaBounds for assessment processing. + +Transforms the RegionalCriteria struct into CriteriaBounds objects that include +evaluation functions. Only includes criteria that are available (non-nothing). + +# Arguments +- `regional_criteria::RegionalCriteria` : Regional criteria with bounds to convert + +# Returns +Vector of `CriteriaBounds` objects for available criteria. +""" +function build_criteria_bounds_from_regional_criteria( + regional_criteria::RegionalCriteria +)::Vector{CriteriaBounds} + @debug "Converting RegionalCriteria to CriteriaBounds vector" + + criteria_bounds = CriteriaBounds[] + + # Process each criteria field + criteria_mapping = [ + (Symbol(criteria.id), criteria) for criteria in ASSESSMENT_CRITERIA_LIST + ] + + for (field_symbol, criteria_metadata) in criteria_mapping + criteria_entry = getfield(regional_criteria, field_symbol) + + if !isnothing(criteria_entry) + bounds = CriteriaBounds( + criteria_metadata.id, + criteria_entry.bounds.min, + criteria_entry.bounds.max + ) + push!(criteria_bounds, bounds) + else + @debug "Skipped criteria - not available" criteria_id = criteria_metadata.id + end + end + + @debug "Built CriteriaBounds vector" total_criteria = length(criteria_bounds) criteria_ids = [ + String(cb.name) for cb in criteria_bounds + ] + + return criteria_bounds +end diff --git a/src/criteria_assessment/query_thresholds.jl b/src/criteria_assessment/query_thresholds.jl index 582ffc7..42f26b5 100644 --- a/src/criteria_assessment/query_thresholds.jl +++ b/src/criteria_assessment/query_thresholds.jl @@ -231,6 +231,7 @@ function apply_criteria_lookup( end """ + threshold_mask(params :: RegionalAssessmentParameters)::Raster threshold_mask(reg_criteria, rtype::Symbol, crit_map)::Raster threshold_mask(reg_criteria, rtype::Symbol, crit_map, lons::Tuple, lats::Tuple)::Raster @@ -300,6 +301,29 @@ function threshold_mask( return rebuild(view_of_data, sparse(convert.(UInt8, view_of_data))) end +""" +Handles threshold masking using the integrated assessment parameter struct +""" +function threshold_mask( + params::RegionalAssessmentParameters +)::Raster + # build out a set of criteria filters using the regional criteria + # NOTE this will only filter over available criteria + filters = build_criteria_bounds_from_regional_criteria(params.regional_criteria) + + # map our regional criteria + mask_layer = apply_criteria_thresholds( + # This is the raster stack + params.region_data.raster_stack, + # The slope table dataframe + params.region_data.slope_table, + # The list of criteria bounds + filters + ) + + return mask_layer +end + """ generate_criteria_mask!(fn::String, rst_stack::RasterStack, lookup::DataFrame, ruleset::Vector{CriteriaBounds{Function}}) diff --git a/src/criteria_assessment/site_identification.jl b/src/criteria_assessment/site_identification.jl index 2142c9b..9123db0 100644 --- a/src/criteria_assessment/site_identification.jl +++ b/src/criteria_assessment/site_identification.jl @@ -129,6 +129,22 @@ function mask_region(reg_assess_data, reg, qp, rtype) return mask_data end +""" +# Arguments +- params::RegionalAssessmentParameters - parameters needed to perform assessment + +# Returns +Raster of region with locations that meet criteria masked. +""" +function mask_region(params::RegionalAssessmentParameters) + @debug "$(now()) : Masking area based on criteria" + mask_data = threshold_mask( + params + ) + + return mask_data +end + """ lookup_assess_region(reg_assess_data, reg, qp, rtype; x_dist=100.0, y_dist=100.0) @@ -209,6 +225,31 @@ end Perform raster suitability assessment based on user-defined criteria. +# Arguments +- params :: RegionalAssessmentParameters + +# Returns +GeoTiff file of surrounding hectare suitability (1-100%) based on the criteria bounds input +by a user. +""" +function assess_region(params::RegionalAssessmentParameters)::Raster + # Make mask of suitable locations + @debug "$(now()) : Creating mask for region" + mask_data = mask_region(params::RegionalAssessmentParameters) + + # Assess remaining pixels for their suitability + @debug "$(now()) : Calculating proportional suitability score" + suitability_scores = proportion_suitable(mask_data.data) + + @debug "$(now()) : Rebuilding raster and returning results" + return rebuild(mask_data, suitability_scores) +end + +""" + assess_region(reg_assess_data, reg, qp, rtype) + +Perform raster suitability assessment based on user-defined criteria. + # Arguments - `reg_assess_data` : Dictionary containing the regional data paths, reef outlines and \ full region names. diff --git a/src/job_worker/Helpers.jl b/src/job_worker/Helpers.jl deleted file mode 100644 index 423cf1b..0000000 --- a/src/job_worker/Helpers.jl +++ /dev/null @@ -1,217 +0,0 @@ -""" -Helper methods for criteria parsing etc. -""" - -# Default threshold when not provided in inputs -const DEFAULT_SUITABILITY_THRESHOLD = 80 - -""" -Min/max storage for criteria -""" -mutable struct Range - min::Float32 - max::Float32 - label::String -end - -""" -Serialises the value of the range to min:max format -""" -function serialise_range(range::Range)::String - return "$(range.min):$(range.max)" -end - -""" -Builds a dictionary key-value pair from a Range -""" -function range_entry_to_kvp(range::Range)::Tuple{String,String} - return (range.label, serialise_range(range)) -end - -""" -Typed/structured ranges for all common ranged criteria -""" -mutable struct RelevantRanges - depth::Range - slope::Range - turbidity::Range - waves_height::Range - waves_period::Range - rugosity::Range - - function RelevantRanges(; - depth::Range, - slope::Range, - turbidity::Range, - waves_height::Range, - waves_period::Range, - rugosity::Range - ) - return new(depth, slope, turbidity, waves_height, waves_period, rugosity) - end -end - -""" -Dumps the relevant ranges into a dictionary following the appropriate style for -other methods -""" -function relevant_ranges_to_dict(ranges::RelevantRanges)::Dict{String,String} - return Dict{String,String}( - range_entry_to_kvp.([ - ranges.depth, - ranges.slope, - ranges.turbidity, - ranges.waves_height, - ranges.waves_period, - ranges.rugosity - ]) - ) -end - -""" -Converts DataFrame criteria ranges to a structured RelevantRanges object -""" -function structured_ranges_from_criteria_ranges(criteria_ranges::DataFrame)::RelevantRanges - # Extract min/max values from the criteria_ranges DataFrame - depth_range = Range( - criteria_ranges[1, "Depth"], - criteria_ranges[2, "Depth"], - "Depth" - ) - - slope_range = Range( - criteria_ranges[1, "Slope"], - criteria_ranges[2, "Slope"], - "Slope" - ) - - turbidity_range = Range( - criteria_ranges[1, "Turbidity"], - criteria_ranges[2, "Turbidity"], - "Turbidity" - ) - - waves_height_range = Range( - criteria_ranges[1, "WavesHs"], - criteria_ranges[2, "WavesHs"], - "WavesHs" - ) - - waves_period_range = Range( - criteria_ranges[1, "WavesTp"], - criteria_ranges[2, "WavesTp"], - "WavesTp" - ) - - rugosity_range = Range( - criteria_ranges[1, "Rugosity"], - criteria_ranges[2, "Rugosity"], - "Rugosity" - ) - - return RelevantRanges(; - depth=depth_range, - slope=slope_range, - turbidity=turbidity_range, - waves_height=waves_height_range, - waves_period=waves_period_range, - rugosity=rugosity_range - ) -end - -""" -Applies optional min/max overrides to a Range object - -Warns if suggested overrides are beyond the dataset bounds. -""" -function apply_optional_overrides!( - range::Range, min_value::OptionalValue{Float64}, max_value::OptionalValue{Float64} -) - if !isnothing(min_value) - if min_value < range.min - @warn "Minimum range value ($(min_value)) for $(range.label) was outside of the dataset bounds ($(range.min)). Proceeding regardless." - end - range.min = min_value - end - if !isnothing(max_value) - if max_value < range.max - @warn "Maximum range value ($(max_value)) for $(range.label) was outside of the dataset bounds ($(range.max)). Proceeding regardless." - end - range.max = max_value - end -end - -""" -Applies all criteria overrides from input to a RelevantRanges object -""" -function apply_criteria_overrides!( - ranges::RelevantRanges, - criteria::Union{RegionalAssessmentInput,SuitabilityAssessmentInput} -) - # Apply overrides for each range - apply_optional_overrides!(ranges.depth, criteria.depth_min, criteria.depth_max) - apply_optional_overrides!(ranges.slope, criteria.slope_min, criteria.slope_max) - apply_optional_overrides!(ranges.rugosity, criteria.rugosity_min, criteria.rugosity_max) - apply_optional_overrides!( - ranges.waves_period, criteria.waves_period_min, criteria.waves_period_max - ) - return apply_optional_overrides!( - ranges.waves_height, criteria.waves_height_min, criteria.waves_height_max - ) -end - -""" -Builds a parameters dictionary from RegionalAssessmentInput, applying overrides as needed -""" -function build_params_dictionary_from_regional_input(; - # The regional criteria job input - criteria::RegionalAssessmentInput, - # Criteria ranges as [[min,max], name] i.e. [2, name] = max of name - criteria_ranges::DataFrame -)::Dict{String,String} - # Get the structured ranges - default_ranges = structured_ranges_from_criteria_ranges(criteria_ranges) - - # Apply all overrides - apply_criteria_overrides!(default_ranges, criteria) - - # Base dictionary of ranges - ranges_dict = relevant_ranges_to_dict(default_ranges) - - # Add in suitability threshold - threshold_value = - isnothing(criteria.threshold) ? DEFAULT_SUITABILITY_THRESHOLD : criteria.threshold - ranges_dict["SuitabilityThreshold"] = "$(threshold_value)" - - return ranges_dict -end - -""" -Builds a parameters dictionary from SuitabilityAssessmentInput, applying overrides and adding suitability-specific parameters -""" -function build_params_dictionary_from_suitability_input(; - # The suitability criteria job input - criteria::SuitabilityAssessmentInput, - # Criteria ranges as [[min,max], name] i.e. [2, name] = max of name - criteria_ranges::DataFrame -)::Dict{String,String} - # Get the structured ranges - default_ranges = structured_ranges_from_criteria_ranges(criteria_ranges) - - # Apply all overrides - apply_criteria_overrides!(default_ranges, criteria) - - # Base dictionary of ranges - ranges_dict = relevant_ranges_to_dict(default_ranges) - - # Add in suitability threshold - threshold_value = - isnothing(criteria.threshold) ? DEFAULT_SUITABILITY_THRESHOLD : criteria.threshold - ranges_dict["SuitabilityThreshold"] = "$(threshold_value)" - - # Suitability specific entries - ranges_dict["xdist"] = "$(criteria.x_dist)" - ranges_dict["ydist"] = "$(criteria.y_dist)" - - return ranges_dict -end diff --git a/src/job_worker/Jobs.jl b/src/job_worker/Jobs.jl index a2149d0..7ae1627 100644 --- a/src/job_worker/Jobs.jl +++ b/src/job_worker/Jobs.jl @@ -18,29 +18,6 @@ function create_job_id(query_params::Dict)::String return string(hash(query_params)) end -""" -Builds a predictable file name based on extracted regional assessment criteria -in the configured cache location. -""" -function build_regional_assessment_file_path(; - query_params::Dict, region::String, reef_type::String, ext::String, - config::Dict -)::String - @debug "Ascertaining file name for regional assessment" - cache_path = _cache_location(config) - - # Use only the criteria relevant to regional assessment - regional_assess_criteria = extract_criteria(query_params, suitability_criteria()) - - # NOTE: This is where we could add additional hash params to invalidate - # cache - job_id = create_job_id(regional_assess_criteria) - - return joinpath( - cache_path, "$(job_id)_$(region)_$(reef_type)_regional_assessment.$(ext)" - ) -end - # ================ # Type Definitions # ================ @@ -318,34 +295,24 @@ function handle_job( @info "Configuration parsing complete." @info "Setting up regional assessment data" - reg_assess_data = get_regional_data(config) + reg_assess_data::RegionalData = get_regional_data(config) @info "Done setting up regional assessment data" - @info "Performing regional assessment" - - # Pull out these parameters in the format previously expected - reg = input.region - rtype = input.reef_type - - # Build the fully populated query params - noting that this merges defaults - # computed as part of the regional data setup with the user provided values - # (if present) - criteria_dictionary = build_params_dictionary_from_regional_input(; - criteria=input, - criteria_ranges=reg_assess_data["criteria_ranges"] + @info "Compiling regional assessment parameters from regional data and input data" + params = build_regional_assessment_parameters( + input, + reg_assess_data ) + @info "Done compiling parameters" - @debug "Criteria after merging default and provided ranges" criteria_dictionary - - assessed_fn = build_regional_assessment_file_path(; - query_params=criteria_dictionary, region=reg, reef_type=rtype, ext="tiff", config - ) + @info "Performing regional assessment" + assessed_fn = build_regional_assessment_file_path(params; ext="tiff", config) @debug "COG File name: $(assessed_fn)" if !isfile(assessed_fn) @debug "File system cache was not hit for this task" - @debug "Assessing region $(reg)" - assessed = assess_region(reg_assess_data, reg, criteria_dictionary, rtype) + @debug "Assessing region $(params.region)" + assessed = assess_region(params) @debug now() "Writing COG of regional assessment to $(assessed_fn)" _write_cog(assessed_fn, assessed, config) diff --git a/src/job_worker/Worker.jl b/src/job_worker/Worker.jl index f0cf25d..34b5118 100644 --- a/src/job_worker/Worker.jl +++ b/src/job_worker/Worker.jl @@ -14,7 +14,6 @@ include("ECS.jl") include("HttpClient.jl") include("Jobs.jl") include("Storage.jl") -include("Helpers.jl") """ Represents a job that needs to be processed diff --git a/src/setup.jl b/src/setup.jl index d90bb11..dddfa9b 100644 --- a/src/setup.jl +++ b/src/setup.jl @@ -5,6 +5,7 @@ using Serialization using Oxygen: json, Request using Logging using Images +include("criteria_assessment/tiles.jl") # ============================================================================= # Constants and Configuration @@ -199,6 +200,17 @@ struct RegionalCriteria end end +# A lookup list of all symbols/criteria available on the regional criteria +# object, helpful when iterating through it's values +const REGIONAL_CRITERIA_SYMBOLS::Vector{Symbol} = [ + :depth, + :slope, + :turbidity, + :waves_height, + :waves_period, + :rugosity +] + """ Complete data package for a single region including rasters and metadata. @@ -1227,4 +1239,11 @@ struct CriteriaBounds{F<:Function} return new{Function}(Symbol(name), lower_bound, upper_bound, func) end + + function CriteriaBounds( + name::String, lb::Float32, ub::Float32 + )::CriteriaBounds + func = (x) -> lb .<= x .<= ub + return new{Function}(Symbol(name), lb, ub, func) + end end From 4e4e0a06dc717557dec52bf2ce3f81aeb1912dff Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Thu, 22 May 2025 17:01:25 +1000 Subject: [PATCH 09/26] WIP Signed-off-by: Peter Baker --- src/RegionalDataHelpers.jl | 13 ++- src/criteria_assessment/query_thresholds.jl | 85 ++++++++++++++----- .../site_identification.jl | 49 +++++++++++ src/job_worker/Jobs.jl | 48 +++-------- 4 files changed, 131 insertions(+), 64 deletions(-) diff --git a/src/RegionalDataHelpers.jl b/src/RegionalDataHelpers.jl index 16517f1..777017f 100644 --- a/src/RegionalDataHelpers.jl +++ b/src/RegionalDataHelpers.jl @@ -477,23 +477,20 @@ function build_criteria_bounds_from_regional_criteria( criteria_bounds = CriteriaBounds[] - # Process each criteria field - criteria_mapping = [ - (Symbol(criteria.id), criteria) for criteria in ASSESSMENT_CRITERIA_LIST - ] - - for (field_symbol, criteria_metadata) in criteria_mapping + for field_symbol in REGIONAL_CRITERIA_SYMBOLS criteria_entry = getfield(regional_criteria, field_symbol) if !isnothing(criteria_entry) bounds = CriteriaBounds( - criteria_metadata.id, + # Field to get in the data + criteria_entry.metadata.id, + # Min/max bounds criteria_entry.bounds.min, criteria_entry.bounds.max ) push!(criteria_bounds, bounds) else - @debug "Skipped criteria - not available" criteria_id = criteria_metadata.id + @debug "Skipped criteria - not available" criteria_id = String(field_symbol) end end diff --git a/src/criteria_assessment/query_thresholds.jl b/src/criteria_assessment/query_thresholds.jl index 42f26b5..c5d07be 100644 --- a/src/criteria_assessment/query_thresholds.jl +++ b/src/criteria_assessment/query_thresholds.jl @@ -154,6 +154,17 @@ function apply_criteria_thresholds( return apply_criteria_thresholds(criteria_stack, lookup, ruleset) end +function apply_criteria_thresholds( + criteria_stack::RasterStack, + lookup::DataFrame, + ruleset::Vector{CriteriaBounds} +)::Raster + ruleset = NamedTuple{([c.name for c in ruleset]...,)}( + Tuple([c.rule for c in ruleset]) + ) + return apply_criteria_thresholds(criteria_stack, lookup, ruleset) +end + function apply_criteria_thresholds( criteria_stack::RasterStack, lookup::DataFrame, @@ -174,28 +185,6 @@ function apply_criteria_thresholds( return res end -# TODO need this? -#function apply_criteria_thresholds( -# criteria_stack::T, -# lookup::DataFrame, -# ruleset::Vector{CriteriaBounds{Function}} -#)::Raster where {T} -# # Result store -# data = falses(size(criteria_stack)) -# -# res_lookup = trues(nrow(lookup)) -# for threshold in ruleset -# res_lookup .= res_lookup .& threshold.rule(lookup[!, threshold.name]) -# end -# -# tmp = lookup[res_lookup, [:lon_idx, :lat_idx]] -# data[CartesianIndex.(tmp.lon_idx, tmp.lat_idx)] .= true -# -# res = Raster(criteria_stack.Depth; data=sparse(data), missingval=0) -# -# return res -#end - """ apply_criteria_lookup( reg_criteria::OldRegionalCriteria, @@ -230,6 +219,57 @@ function apply_criteria_lookup( return lookup end +""" + apply_criteria_lookup( + raster_stack::RasterStack, + rtype::Symbol, + ruleset::Vector{CriteriaBounds{Function}} + ) + +Filter lookup table by applying user defined `ruleset` criteria. + +# Arguments +- `reg_criteria` : OldRegionalCriteria containing valid_rtype lookup table for filtering. +- `rtype` : Flats or slope category for assessment. +- `ruleset` : User defined ruleset for upper and lower bounds. + +# Returns +Filtered lookup table containing points that meet all criteria in `ruleset`. +""" +function apply_criteria_lookup( + reg_criteria::OldRegionalCriteria, + rtype::Symbol, + ruleset +)::DataFrame + lookup = getfield(reg_criteria, Symbol(:valid_, rtype)) + lookup.all_crit .= 1 + + for threshold in ruleset + lookup.all_crit = lookup.all_crit .& threshold.rule(lookup[!, threshold.name]) + end + + lookup = lookup[BitVector(lookup.all_crit), :] + + return lookup +end + +function apply_criteria_lookup( + slope_table::DataFrame, + ruleset::Vector{CriteriaBounds} +)::DataFrame + # TODO FINISH + slope_table.all_crit .= 1 + + for threshold in ruleset + slope_table.all_crit = + slope_table.all_crit .& threshold.rule(lookup[!, threshold.name]) + end + + lookup = lookup[BitVector(lookup.all_crit), :] + + return lookup +end + """ threshold_mask(params :: RegionalAssessmentParameters)::Raster threshold_mask(reg_criteria, rtype::Symbol, crit_map)::Raster @@ -312,6 +352,7 @@ function threshold_mask( filters = build_criteria_bounds_from_regional_criteria(params.regional_criteria) # map our regional criteria + @debug "Applying criteria thresholds to generate mask layer" mask_layer = apply_criteria_thresholds( # This is the raster stack params.region_data.raster_stack, diff --git a/src/criteria_assessment/site_identification.jl b/src/criteria_assessment/site_identification.jl index 9123db0..507f41e 100644 --- a/src/criteria_assessment/site_identification.jl +++ b/src/criteria_assessment/site_identification.jl @@ -407,3 +407,52 @@ function assess_sites( return initial_polygons end + +function assess_sites(; + params::SuitabilityAssessmentParameters, + regional_raster::Raster +) + target_crs = convert(EPSG, crs(regional_raster)) + suitability_threshold = params.suitability_threshold + region = params.region + + @debug "$(now()) : Identifying search pixels for $(region)" + target_locs = search_lookup(regional_raster, suitability_threshold) + + if size(target_locs, 1) == 0 + # No viable set of locations, return empty dataframe + return DataFrame(; + score=[], + orientation=[], + qc_flag=[], + geometry=[] + ) + end + + # Otherwise, create the file + @debug "$(now()) : Assessing criteria table for $(region)" + # Get criteria bounds list from criteria + filters = build_criteria_bounds_from_regional_criteria(params.regional_criteria) + + crit_pixels::DataFrame = apply_criteria_lookup( + params.region_data.raster_stack, + Symbol(rtype), + filters + ) + + res = abs(step(dims(regional_raster, X))) + x_dist = parse(Int64, site_criteria["xdist"]) + y_dist = parse(Int64, site_criteria["ydist"]) + @debug "$(now()) : Assessing $(size(target_locs, 1)) candidate locations in $(region)." + @debug "Finding optimal site alignment" + initial_polygons = find_optimal_site_alignment( + crit_pixels, + target_locs, + res, + x_dist, + y_dist, + target_crs + ) + + return initial_polygons +end diff --git a/src/job_worker/Jobs.jl b/src/job_worker/Jobs.jl index 7ae1627..a212a57 100644 --- a/src/job_worker/Jobs.jl +++ b/src/job_worker/Jobs.jl @@ -9,15 +9,6 @@ using Dates const OptionalValue{T} = Union{T,Nothing}; -""" - create_job_id(query_params::Dict)::String - -Generate a job id based on query parameters. -""" -function create_job_id(query_params::Dict)::String - return string(hash(query_params)) -end - # ================ # Type Definitions # ================ @@ -390,8 +381,7 @@ Handler for the suitability assessment job. """ function handle_job( ::SuitabilityAssessmentHandler, input::SuitabilityAssessmentInput, - context::HandlerContext -)::SuitabilityAssessmentOutput + context::HandlerContext)::SuitabilityAssessmentOutput @info "Initiating site assessment task" @info "Parsing configuration from $(context.config_path)..." @@ -399,40 +389,30 @@ function handle_job( @info "Configuration parsing complete." @info "Setting up regional assessment data" - reg_assess_data = get_regional_data(config) + reg_assess_data::RegionalData = get_regional_data(config) @info "Done setting up regional assessment data" - @info "Performing regional assessment (dependency of site assessment)" - - # Pull out these parameters in the format previously expected - reg = input.region - rtype = input.reef_type - - # Build the fully populated query params - noting that this merges defaults - # computed as part of the regional data setup with the user provided values - # (if present) - criteria_dictionary = build_params_dictionary_from_regional_input(; - criteria=input, - criteria_ranges=reg_assess_data["criteria_ranges"] + @info "Compiling regional assessment parameters from regional data and input data" + params = build_suitability_assessment_parameters( + input, + reg_assess_data ) - @debug "Criteria after merging default and provided ranges" criteria_dictionary + @info "Done compiling parameters" - assessed_fn = build_regional_assessment_file_path(; - query_params=criteria_dictionary, region=reg, reef_type=rtype, ext="tiff", config - ) + @info "Performing regional assessment" + assessed_fn = build_regional_assessment_file_path(params; ext="tiff", config) @debug "COG File name: $(assessed_fn)" if !isfile(assessed_fn) @debug "File system cache was not hit for this task" - @debug "Assessing region $(reg)" - assessed = assess_region(reg_assess_data, reg, criteria_dictionary, rtype) + @debug "Assessing region $(params.region)" + assessed = assess_region(params) - @debug "Writing COG to $(assessed_fn)" + @debug now() "Writing COG of regional assessment to $(assessed_fn)" _write_cog(assessed_fn, assessed, config) + @debug now() "Finished writing cog " else - @info "Cache hit - skipping regional assessment process!" - @debug "Pulling out raster from cache" - assessed = Raster(assessed_fn; missingval=0, lazy=true) + @info "Cache hit - skipping regional assessment process..." end # Extract criteria and assessment From 8baea1316fb052a910f798f081b46c30c317bc9a Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Fri, 23 May 2025 09:22:23 +1000 Subject: [PATCH 10/26] Site assessment using new parameters Signed-off-by: Peter Baker --- src/criteria_assessment/query_thresholds.jl | 11 ++++++----- src/criteria_assessment/site_identification.jl | 10 ++++------ 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/criteria_assessment/query_thresholds.jl b/src/criteria_assessment/query_thresholds.jl index c5d07be..9f1f3c2 100644 --- a/src/criteria_assessment/query_thresholds.jl +++ b/src/criteria_assessment/query_thresholds.jl @@ -253,21 +253,22 @@ function apply_criteria_lookup( return lookup end +""" +Filters the slope table (which contains raster param values too) by building a +bit mask AND'd for all thresholds +""" function apply_criteria_lookup( slope_table::DataFrame, ruleset::Vector{CriteriaBounds} )::DataFrame - # TODO FINISH slope_table.all_crit .= 1 for threshold in ruleset slope_table.all_crit = - slope_table.all_crit .& threshold.rule(lookup[!, threshold.name]) + slope_table.all_crit .& threshold.rule(slope_table[!, threshold.name]) end - lookup = lookup[BitVector(lookup.all_crit), :] - - return lookup + return slope_table[BitVector(slope_table.all_crit), :] end """ diff --git a/src/criteria_assessment/site_identification.jl b/src/criteria_assessment/site_identification.jl index 507f41e..3733871 100644 --- a/src/criteria_assessment/site_identification.jl +++ b/src/criteria_assessment/site_identification.jl @@ -435,22 +435,20 @@ function assess_sites(; filters = build_criteria_bounds_from_regional_criteria(params.regional_criteria) crit_pixels::DataFrame = apply_criteria_lookup( - params.region_data.raster_stack, - Symbol(rtype), + # Slope table + params.region_data.slope_table, filters ) res = abs(step(dims(regional_raster, X))) - x_dist = parse(Int64, site_criteria["xdist"]) - y_dist = parse(Int64, site_criteria["ydist"]) @debug "$(now()) : Assessing $(size(target_locs, 1)) candidate locations in $(region)." @debug "Finding optimal site alignment" initial_polygons = find_optimal_site_alignment( crit_pixels, target_locs, res, - x_dist, - y_dist, + params.x_dist, + params.y_dist, target_crs ) From 766230841bef2868751531c923984b84cc21b49e Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Fri, 23 May 2025 13:20:11 +1000 Subject: [PATCH 11/26] Pre-refactor, working Signed-off-by: Peter Baker --- src/ReefGuideAPI.jl | 10 +- .../site_identification.jl | 2 +- src/job_worker/{Config.jl => config.jl} | 0 src/job_worker/{ECS.jl => ecs.jl} | 0 src/job_worker/{Jobs.jl => handlers.jl} | 41 +- .../{HttpClient.jl => http_client.jl} | 0 src/job_worker/index.jl | 6 + .../{Storage.jl => storage_client.jl} | 0 src/job_worker/{Worker.jl => worker.jl} | 5 - .../assessment_interfaces.jl} | 238 +++------- src/utility/deprecated.jl | 111 +++++ src/utility/helpers.jl | 192 ++++++++ src/utility/index.jl | 5 + .../regions_criteria_setup.jl} | 440 +++--------------- src/utility/routes.jl | 51 ++ 15 files changed, 532 insertions(+), 569 deletions(-) rename src/job_worker/{Config.jl => config.jl} (100%) rename src/job_worker/{ECS.jl => ecs.jl} (100%) rename src/job_worker/{Jobs.jl => handlers.jl} (91%) rename src/job_worker/{HttpClient.jl => http_client.jl} (100%) create mode 100644 src/job_worker/index.jl rename src/job_worker/{Storage.jl => storage_client.jl} (100%) rename src/job_worker/{Worker.jl => worker.jl} (99%) rename src/{RegionalDataHelpers.jl => utility/assessment_interfaces.jl} (63%) create mode 100644 src/utility/deprecated.jl create mode 100644 src/utility/helpers.jl create mode 100644 src/utility/index.jl rename src/{setup.jl => utility/regions_criteria_setup.jl} (71%) create mode 100644 src/utility/routes.jl diff --git a/src/ReefGuideAPI.jl b/src/ReefGuideAPI.jl index c5a507a..17bd7ea 100644 --- a/src/ReefGuideAPI.jl +++ b/src/ReefGuideAPI.jl @@ -24,11 +24,11 @@ using HTTP, Oxygen -include("job_worker/Worker.jl") +# Worker system +include("job_worker/index.jl") -# New work including setup logic and helper functions -include("setup.jl") -include("RegionalDataHelpers.jl") +# Utilities and helpers for assessments +include("utility/index.jl") include("Middleware.jl") include("admin.jl") @@ -44,7 +44,7 @@ include("criteria_assessment/site_identification.jl") include("site_assessment/common_functions.jl") include("site_assessment/best_fit_polygons.jl") - +include("criteria_assessment/tiles.jl") function get_auth_router(config::Dict) # Setup auth middleware - depends on config.toml - can return identity func diff --git a/src/criteria_assessment/site_identification.jl b/src/criteria_assessment/site_identification.jl index 3733871..edac031 100644 --- a/src/criteria_assessment/site_identification.jl +++ b/src/criteria_assessment/site_identification.jl @@ -408,7 +408,7 @@ function assess_sites( return initial_polygons end -function assess_sites(; +function assess_sites( params::SuitabilityAssessmentParameters, regional_raster::Raster ) diff --git a/src/job_worker/Config.jl b/src/job_worker/config.jl similarity index 100% rename from src/job_worker/Config.jl rename to src/job_worker/config.jl diff --git a/src/job_worker/ECS.jl b/src/job_worker/ecs.jl similarity index 100% rename from src/job_worker/ECS.jl rename to src/job_worker/ecs.jl diff --git a/src/job_worker/Jobs.jl b/src/job_worker/handlers.jl similarity index 91% rename from src/job_worker/Jobs.jl rename to src/job_worker/handlers.jl index a212a57..855d4cc 100644 --- a/src/job_worker/Jobs.jl +++ b/src/job_worker/handlers.jl @@ -338,6 +338,9 @@ end """ Input payload for SUITABILITY_ASSESSMENT job + +NOTE this is a RegionalAssessmentInput (and more) and therefore also an +AbstractJobInput """ struct SuitabilityAssessmentInput <: AbstractJobInput # High level config @@ -389,46 +392,50 @@ function handle_job( @info "Configuration parsing complete." @info "Setting up regional assessment data" - reg_assess_data::RegionalData = get_regional_data(config) + regional_data::RegionalData = get_regional_data(config) @info "Done setting up regional assessment data" - @info "Compiling regional assessment parameters from regional data and input data" - params = build_suitability_assessment_parameters( + @info "Compiling suitability assessment parameters from regional data and job inputs" + params::SuitabilityAssessmentParameters = build_suitability_assessment_parameters( input, - reg_assess_data + regional_data ) @info "Done compiling parameters" + @debug "Converting suitability job into regional job for regional assessment" + regional_params = regional_params_from_suitability(params) + @debug "Conversion complete" + @info "Performing regional assessment" - assessed_fn = build_regional_assessment_file_path(params; ext="tiff", config) - @debug "COG File name: $(assessed_fn)" + regional_assessment_fn = build_regional_assessment_file_path( + regional_params; ext="tiff", config=config + ) + @debug "COG File name: $(regional_assessment_fn)" - if !isfile(assessed_fn) + if !isfile(regional_assessment_fn) @debug "File system cache was not hit for this task" @debug "Assessing region $(params.region)" - assessed = assess_region(params) + regional_raster = assess_region(regional_params) - @debug now() "Writing COG of regional assessment to $(assessed_fn)" - _write_cog(assessed_fn, assessed, config) + @debug now() "Writing COG of regional assessment to $(regional_assessment_fn)" + _write_cog(regional_assessment_fn, regional_raster, config) @debug now() "Finished writing cog " else @info "Cache hit - skipping regional assessment process..." + @debug "Pulling out raster from cache" + regional_raster = Raster(regional_assessment_fn; missingval=0, lazy=true) end - # Extract criteria and assessment - pixel_criteria = extract_criteria(criteria_dictionary, search_criteria()) - deploy_site_criteria = extract_criteria(criteria_dictionary, site_criteria()) - @debug "Performing site assessment" best_sites = filter_sites( assess_sites( - reg_assess_data, reg, rtype, pixel_criteria, deploy_site_criteria, - assessed + params, + regional_raster ) ) # Specifically clear from memory to invoke garbage collector - assessed = nothing + regional_raster = nothing @debug "Writing to temporary file" geojson_name = "$(tempname()).geojson" diff --git a/src/job_worker/HttpClient.jl b/src/job_worker/http_client.jl similarity index 100% rename from src/job_worker/HttpClient.jl rename to src/job_worker/http_client.jl diff --git a/src/job_worker/index.jl b/src/job_worker/index.jl new file mode 100644 index 0000000..60ab689 --- /dev/null +++ b/src/job_worker/index.jl @@ -0,0 +1,6 @@ +include("config.jl") +include("ecs.jl") +include("http_client.jl") +include("handlers.jl") +include("storage_client.jl") +include("worker.jl") diff --git a/src/job_worker/Storage.jl b/src/job_worker/storage_client.jl similarity index 100% rename from src/job_worker/Storage.jl rename to src/job_worker/storage_client.jl diff --git a/src/job_worker/Worker.jl b/src/job_worker/worker.jl similarity index 99% rename from src/job_worker/Worker.jl rename to src/job_worker/worker.jl index 34b5118..99054b6 100644 --- a/src/job_worker/Worker.jl +++ b/src/job_worker/worker.jl @@ -9,11 +9,6 @@ using Random using Logging using JSON3 -include("Config.jl") -include("ECS.jl") -include("HttpClient.jl") -include("Jobs.jl") -include("Storage.jl") """ Represents a job that needs to be processed diff --git a/src/RegionalDataHelpers.jl b/src/utility/assessment_interfaces.jl similarity index 63% rename from src/RegionalDataHelpers.jl rename to src/utility/assessment_interfaces.jl index 777017f..0b6ba2c 100644 --- a/src/RegionalDataHelpers.jl +++ b/src/utility/assessment_interfaces.jl @@ -2,7 +2,7 @@ # Assessment Parameters Constants # ============================================================================= -const DEFAULT_SUITABILITY_THRESHOLD::Int32 = 80 +const DEFAULT_SUITABILITY_THRESHOLD::Int64 = 80 # ============================================================================= # Assessment Parameters Data Structures @@ -16,19 +16,19 @@ regional data. - `region::String` : The region that is being assessed - `regional_criteria::RegionalCriteria` : The criteria to assess, including user provided bounds - `region_data::RegionalDataEntry` : The data to consider for this region -- `suitability_threshold::Int32` : The cutoff to consider a site suitable +- `suitability_threshold::Int64` : The cutoff to consider a site suitable """ struct RegionalAssessmentParameters region::String regional_criteria::RegionalCriteria region_data::RegionalDataEntry - suitability_threshold::Int32 + suitability_threshold::Int64 function RegionalAssessmentParameters(; region::String, regional_criteria::RegionalCriteria, region_data::RegionalDataEntry, - suitability_threshold::Int32 + suitability_threshold::Int64 ) @debug "Created RegionalAssessmentParameters" region suitability_threshold return new(region, regional_criteria, region_data, suitability_threshold) @@ -43,25 +43,28 @@ regional data plus spatial dimensions. - `region::String` : The region that is being assessed - `regional_criteria::RegionalCriteria` : The criteria to assess, including user provided bounds - `region_data::RegionalDataEntry` : The data to consider for this region -- `suitability_threshold::Int32` : The cutoff to consider a site suitable -- `x_dist::Int32` : X dimension of polygon (metres) -- `y_dist::Int32` : Y dimension of polygon (metres) +- `suitability_threshold::Int64` : The cutoff to consider a site suitable +- `x_dist::Int64` : X dimension of polygon (metres) +- `y_dist::Int64` : Y dimension of polygon (metres) """ struct SuitabilityAssessmentParameters + # Regional criteria region::String regional_criteria::RegionalCriteria region_data::RegionalDataEntry - suitability_threshold::Int32 - x_dist::Int32 - y_dist::Int32 + suitability_threshold::Int64 + + # Additional criteria + x_dist::Int64 + y_dist::Int64 function SuitabilityAssessmentParameters(; region::String, regional_criteria::RegionalCriteria, region_data::RegionalDataEntry, - suitability_threshold::Int32, - x_dist::Int32, - y_dist::Int32 + suitability_threshold::Int64, + x_dist::Int64, + y_dist::Int64 ) @debug "Created SuitabilityAssessmentParameters" region suitability_threshold x_dist y_dist return new( @@ -71,7 +74,7 @@ struct SuitabilityAssessmentParameters end # ============================================================================= -# Assessment Parameters Builder Functions +# Assessment Parameters Helper Functions # ============================================================================= """ @@ -188,7 +191,7 @@ function build_regional_assessment_parameters( region=input.region, regional_criteria, region_data, - suitability_threshold=Int32(threshold) + suitability_threshold=Int64(threshold) ) end @@ -215,105 +218,24 @@ function build_suitability_assessment_parameters( @info "Building suitability assessment parameters" region = input.region x_dist = input.x_dist y_dist = input.y_dist - # Validate region exists - if !haskey(regional_data.regions, input.region) - available_regions = collect(keys(regional_data.regions)) - @error "Region not found in regional data" region = input.region available_regions - throw( - ErrorException( - "Regional data did not have data for region $(input.region). Available regions: $(join(available_regions, ", "))" - ) - ) - end - - region_data = regional_data.regions[input.region] - - # Extract threshold with default fallback - threshold = - !isnothing(input.threshold) ? input.threshold : DEFAULT_SUITABILITY_THRESHOLD - - # Build merged criteria - regional_criteria = RegionalCriteria(; - depth_bounds=merge_bounds( - input.depth_min, input.depth_max, region_data.criteria.depth - ), - slope_bounds=merge_bounds( - input.slope_min, input.slope_max, region_data.criteria.slope - ), - waves_height_bounds=merge_bounds( - input.waves_height_min, - input.waves_height_max, - region_data.criteria.waves_height - ), - waves_period_bounds=merge_bounds( - input.waves_period_min, - input.waves_period_max, - region_data.criteria.waves_period - ), - rugosity_bounds=merge_bounds( - input.rugosity_min, input.rugosity_max, region_data.criteria.rugosity - ), - # Turbidity is not user-configurable, always use regional bounds - turbidity_bounds=merge_bounds(nothing, nothing, region_data.criteria.turbidity) + @debug "Building regional parameters first" + regional_input = regional_job_from_suitability_job(input) + regional_parameters = build_regional_assessment_parameters( + regional_input, + regional_data ) - - # Count active criteria for logging - active_criteria = length([ - b for b in [ - regional_criteria.depth, regional_criteria.slope, regional_criteria.turbidity, - regional_criteria.waves_height, regional_criteria.waves_period, - regional_criteria.rugosity - ] if !isnothing(b) - ]) - - @info "Built suitability assessment parameters" region = input.region threshold active_criteria x_dist = - input.x_dist y_dist = input.y_dist user_specified_threshold = - !isnothing(input.threshold) - + @debug "Extending regional parameters with suitability inputs x_dist and ydist" x = + input.x_dist y = input.y_dist return SuitabilityAssessmentParameters(; - region=input.region, - regional_criteria, - region_data, - suitability_threshold=Int32(threshold), - x_dist=Int32(input.x_dist), - y_dist=Int32(input.y_dist) + region=regional_parameters.region, + regional_criteria=regional_parameters.regional_criteria, + region_data=regional_parameters.region_data, + suitability_threshold=regional_parameters.suitability_threshold, + x_dist=input.x_dist, + y_dist=input.y_dist ) end -# ============================================================================= -# Cache File Name Generation -# ============================================================================= - -""" -Builds a hash by combining strings and hashing result -""" -function build_hash_from_components(components::Vector{String})::String - return string(hash(join(components, "|"))) -end - -""" -Combines present regional criteria including bounds into hash components -""" -function get_hash_components_from_regional_criteria( - criteria::RegionalCriteria -)::Vector{String} - hash_components::Vector{String} = [] - for field in REGIONAL_CRITERIA_SYMBOLS - criteria_entry::OptionalValue{RegionalCriteriaEntry} = getfield( - criteria, field - ) - if !isnothing(criteria_entry) - push!( - hash_components, - "$(field)_$(criteria_entry.bounds.min)_$(criteria_entry.bounds.max)" - ) - else - push!(hash_components, "$(field)_null") - end - end - return hash_components -end - """ Generate a deterministic hash string for RegionalAssessmentParameters. @@ -425,78 +347,40 @@ function build_regional_assessment_file_path( return file_path end -""" -Build predictable file path for suitability assessment results in configured cache -location. -Creates a complete file path for caching suitability assessment results using the -configured cache directory and deterministic parameter-based naming. - -# Arguments -- `params::SuitabilityAssessmentParameters` : Suitability assessment parameters -- `ext::String` : File extension for the cache file -- `config::Dict` : Configuration dictionary containing cache settings - -# Returns -String path to cache file location. """ -function build_suitability_assessment_file_path( - params::SuitabilityAssessmentParameters; - ext::String, - config::Dict -)::String - @debug "Building file path for suitability assessment cache" region = params.region ext - - cache_path = _cache_location(config) - param_hash = suitability_assessment_params_hash(params) - filename = "$(param_hash)_$(params.region)_suitability_assessment.$(ext)" - file_path = joinpath(cache_path, filename) - - @debug "Built suitability assessment file path" file_path region = params.region hash = - param_hash - - return file_path +Converts parameters from a suitability job into a regional job +""" +function regional_job_from_suitability_job( + suitability_job::SuitabilityAssessmentInput +)::RegionalAssessmentInput + return RegionalAssessmentInput( + suitability_job.region, + suitability_job.reef_type, + suitability_job.depth_min, + suitability_job.depth_max, + suitability_job.slope_min, + suitability_job.slope_max, + suitability_job.rugosity_min, + suitability_job.rugosity_max, + suitability_job.waves_period_min, + suitability_job.waves_period_max, + suitability_job.waves_height_min, + suitability_job.waves_height_max, + suitability_job.threshold + ) end """ -Convert RegionalCriteria to a vector of CriteriaBounds for assessment processing. - -Transforms the RegionalCriteria struct into CriteriaBounds objects that include -evaluation functions. Only includes criteria that are available (non-nothing). - -# Arguments -- `regional_criteria::RegionalCriteria` : Regional criteria with bounds to convert - -# Returns -Vector of `CriteriaBounds` objects for available criteria. +Converts parameters from a suitability assessment into a regional assessment """ -function build_criteria_bounds_from_regional_criteria( - regional_criteria::RegionalCriteria -)::Vector{CriteriaBounds} - @debug "Converting RegionalCriteria to CriteriaBounds vector" - - criteria_bounds = CriteriaBounds[] - - for field_symbol in REGIONAL_CRITERIA_SYMBOLS - criteria_entry = getfield(regional_criteria, field_symbol) - - if !isnothing(criteria_entry) - bounds = CriteriaBounds( - # Field to get in the data - criteria_entry.metadata.id, - # Min/max bounds - criteria_entry.bounds.min, - criteria_entry.bounds.max - ) - push!(criteria_bounds, bounds) - else - @debug "Skipped criteria - not available" criteria_id = String(field_symbol) - end - end - - @debug "Built CriteriaBounds vector" total_criteria = length(criteria_bounds) criteria_ids = [ - String(cb.name) for cb in criteria_bounds - ] - - return criteria_bounds +function regional_params_from_suitability( + suitability_params::SuitabilityAssessmentParameters +)::RegionalAssessmentParameters + return RegionalAssessmentParameters(; + region=suitability_params.region, + regional_criteria=suitability_params.regional_criteria, + region_data=suitability_params.region_data, + suitability_threshold=suitability_params.suitability_threshold + ) end diff --git a/src/utility/deprecated.jl b/src/utility/deprecated.jl new file mode 100644 index 0000000..a0151aa --- /dev/null +++ b/src/utility/deprecated.jl @@ -0,0 +1,111 @@ +""" +========== +DEPRECATED +========== +""" +function criteria_data_map() + # TODO: Load from config? + return OrderedDict( + :Depth => "_bathy", + :Benthic => "_benthic", + :Geomorphic => "_geomorphic", + :Slope => "_slope", + :Turbidity => "_turbid", + :WavesHs => "_waves_Hs", + :WavesTp => "_waves_Tp", + :Rugosity => "_rugosity", + :ValidSlopes => "_valid_slopes", + :ValidFlats => "_valid_flats" + ) +end + +function search_criteria()::Vector{String} + return string.(keys(criteria_data_map())) +end + +function site_criteria()::Vector{String} + return ["SuitabilityThreshold", "xdist", "ydist"] +end + +function suitability_criteria()::Vector{String} + return vcat(search_criteria(), ["SuitabilityThreshold"]) +end + +function criteria_data_map() + # TODO: Load from config? + return OrderedDict( + :Depth => "_bathy", + :Benthic => "_benthic", + :Geomorphic => "_geomorphic", + :Slope => "_slope", + :Turbidity => "_turbid", + :WavesHs => "_waves_Hs", + :WavesTp => "_waves_Tp", + :Rugosity => "_rugosity", + :ValidSlopes => "_valid_slopes", + :ValidFlats => "_valid_flats" + + # Unused datasets + # :PortDistSlopes => "_PortDistSlopes", + # :PortDistFlats => "_PortDistFlats" + ) +end + +function search_criteria()::Vector{String} + return string.(keys(criteria_data_map())) +end + +function site_criteria()::Vector{String} + return ["SuitabilityThreshold", "xdist", "ydist"] +end + +function suitability_criteria()::Vector{String} + return vcat(search_criteria(), ["SuitabilityThreshold"]) +end + +function extract_criteria(qp::T, criteria::Vector{String})::T where {T<:Dict{String,String}} + return filter( + k -> string(k.first) ∈ criteria, qp + ) +end + +struct OldRegionalCriteria{T} + stack::RasterStack + valid_slopes::T + valid_flats::T +end + +function valid_slope_lon_inds(reg::OldRegionalCriteria) + return reg.valid_slopes.lon_idx +end +function valid_slope_lat_inds(reg::OldRegionalCriteria) + return reg.valid_slopes.lat_idx +end +function valid_flat_lon_inds(reg::OldRegionalCriteria) + return reg.valid_flats.lon_idx +end +function valid_flat_lat_inds(reg::OldRegionalCriteria) + return reg.valid_flats.lat_idx +end + +struct CriteriaBounds{F<:Function} + name::Symbol + lower_bound::Float32 + upper_bound::Float32 + rule::F + + function CriteriaBounds(name::String, lb::S, ub::S)::CriteriaBounds where {S<:String} + lower_bound::Float32 = parse(Float32, lb) + upper_bound::Float32 = parse(Float32, ub) + func = (x) -> lower_bound .<= x .<= upper_bound + + return new{Function}(Symbol(name), lower_bound, upper_bound, func) + end + + function CriteriaBounds( + name::String, lb::Float32, ub::Float32 + )::CriteriaBounds + func = (x) -> lb .<= x .<= ub + return new{Function}(Symbol(name), lb, ub, func) + end +end diff --git a/src/utility/helpers.jl b/src/utility/helpers.jl new file mode 100644 index 0000000..8542d8f --- /dev/null +++ b/src/utility/helpers.jl @@ -0,0 +1,192 @@ +""" +Builds a hash by combining strings and hashing result +""" +function build_hash_from_components(components::Vector{String})::String + return string(hash(join(components, "|"))) +end + +""" +Combines present regional criteria including bounds into hash components +""" +function get_hash_components_from_regional_criteria( + criteria::RegionalCriteria +)::Vector{String} + hash_components::Vector{String} = [] + for field in REGIONAL_CRITERIA_SYMBOLS + criteria_entry::OptionalValue{RegionalCriteriaEntry} = getfield( + criteria, field + ) + if !isnothing(criteria_entry) + push!( + hash_components, + "$(field)_$(criteria_entry.bounds.min)_$(criteria_entry.bounds.max)" + ) + else + push!(hash_components, "$(field)_null") + end + end + return hash_components +end + +""" +Convert RegionalCriteria to a vector of CriteriaBounds for assessment processing. + +Transforms the RegionalCriteria struct into CriteriaBounds objects that include +evaluation functions. Only includes criteria that are available (non-nothing). + +# Arguments +- `regional_criteria::RegionalCriteria` : Regional criteria with bounds to convert + +# Returns +Vector of `CriteriaBounds` objects for available criteria. +""" +function build_criteria_bounds_from_regional_criteria( + regional_criteria::RegionalCriteria +)::Vector{CriteriaBounds} + @debug "Converting RegionalCriteria to CriteriaBounds vector" + + criteria_bounds = CriteriaBounds[] + + for field_symbol in REGIONAL_CRITERIA_SYMBOLS + criteria_entry = getfield(regional_criteria, field_symbol) + + if !isnothing(criteria_entry) + bounds = CriteriaBounds( + # Field to get in the data + criteria_entry.metadata.id, + # Min/max bounds + criteria_entry.bounds.min, + criteria_entry.bounds.max + ) + push!(criteria_bounds, bounds) + else + @debug "Skipped criteria - not available" criteria_id = String(field_symbol) + end + end + + @debug "Built CriteriaBounds vector" total_criteria = length(criteria_bounds) criteria_ids = [ + String(cb.name) for cb in criteria_bounds + ] + + return criteria_bounds +end + +""" +Convert a min/max tuple to a Bounds struct. + +# Arguments +- `min_max::Tuple{Number,Number}` : Tuple containing (minimum, maximum) values + +# Returns +`Bounds` struct with converted float values. +""" +function bounds_from_tuple(min_max::Tuple{Number,Number})::Bounds + return Bounds(; min=min_max[1], max=min_max[2]) +end + +""" +Generate the filename for slope lookup data for a given region. + +# Arguments +- `region::RegionMetadata` : Region metadata containing ID + +# Returns +String filename in format "{region_id}_slope_lookup.parq" +""" +function get_slope_parquet_filename(region::RegionMetadata)::String + filename = "$(region.id)$(SLOPES_LOOKUP_SUFFIX)" + @debug "Generated slope parquet filename" region_id = region.id filename + return filename +end + +""" +Create a dictionary mapping criteria IDs to regional criteria entries. + +NOTE: Only includes criteria that are available for the region, as specified in the +region metadata and actually instantiated in the RegionalCriteria struct. + +Uses the defined set of symbols on the regional criteria struct to iterate through + +# Arguments +- `region_data::RegionalDataEntry` : Regional data containing criteria information + +# Returns +Dictionary with criteria ID strings as keys and RegionalCriteriaEntry as values. +Only includes criteria that are both listed in region metadata and available as non-nothing. +""" +function build_regional_criteria_dictionary( + region_data::RegionalDataEntry +)::Dict{String,RegionalCriteriaEntry} + @debug "Building criteria dictionary for region" region_id = region_data.region_id available_in_metadata = + region_data.region_metadata.available_criteria + + regional_criteria = region_data.criteria + criteria_dict = Dict{String,RegionalCriteriaEntry}() + + # Only process criteria that are listed as available in the region metadata + available_criteria_set = Set(region_data.region_metadata.available_criteria) + + for symbol in REGIONAL_CRITERIA_SYMBOLS + possible_value::OptionalValue{RegionalCriteriaEntry} = getfield( + regional_criteria, symbol + ) + if ( + !isnothing(possible_value) && + possible_value.metadata.id ∈ available_criteria_set + ) + criteria_dict[possible_value.metadata.id] = possible_value + end + end + + @debug "Built criteria dictionary" region_id = region_data.region_id available_in_metadata = length( + available_criteria_set + ) actually_available = length(criteria_dict) criteria_ids = collect(keys(criteria_dict)) + + return criteria_dict +end + +""" +Given a dictionary mapping criteria ID -> optional bounds, builds out a +RegionalCriteria object. +""" +function build_regional_criteria_from_criteria_dictionary( + criteria::Dict{String,OptionalValue{Bounds}} +) + function check_criteria(metadata::CriteriaMetadata)::OptionalValue{Bounds} + if haskey(criteria, metadata.id) && !isnothing(criteria[metadata.id]) + return criteria[metadata.id] + end + return nothing + end + + @debug "Creating RegionalCriteria by assessing each entry of criteria dictionary" + return RegionalCriteria(; + depth_bounds=check_criteria(ASSESSMENT_CRITERIA.depth), + slope_bounds=check_criteria(ASSESSMENT_CRITERIA.slope), + turbidity_bounds=check_criteria(ASSESSMENT_CRITERIA.turbidity), + waves_height_bounds=check_criteria(ASSESSMENT_CRITERIA.waves_height), + waves_period_bounds=check_criteria(ASSESSMENT_CRITERIA.waves_period), + rugosity_bounds=check_criteria(ASSESSMENT_CRITERIA.rugosity) + ) +end + +""" +Add longitude and latitude columns to a DataFrame based on geometry centroids. + +Modifies the input DataFrame by adding 'lons' and 'lats' columns extracted +from the centroid coordinates of each geometry feature. + +# Arguments +- `df::DataFrame` : DataFrame with geometry column containing spatial features +""" +function add_lat_long_columns_to_dataframe(df::DataFrame)::Nothing + @debug "Adding lat/long columns to DataFrame" num_rows = nrow(df) + # Extract coordinate tuples from geometry centroids + coords = GI.coordinates.(df.geometry) + # Add longitude column (first coordinate) + df[!, :lons] .= first.(coords) + # Add latitude column (second coordinate) + df[!, :lats] .= last.(coords) + @debug "Successfully added coordinate columns" + return nothing +end diff --git a/src/utility/index.jl b/src/utility/index.jl new file mode 100644 index 0000000..3f24c0f --- /dev/null +++ b/src/utility/index.jl @@ -0,0 +1,5 @@ +include("regions_criteria_setup.jl") +include("helpers.jl") +include("deprecated.jl") +include("routes.jl") +include("assessment_interfaces.jl") diff --git a/src/setup.jl b/src/utility/regions_criteria_setup.jl similarity index 71% rename from src/setup.jl rename to src/utility/regions_criteria_setup.jl index dddfa9b..9dc11e3 100644 --- a/src/setup.jl +++ b/src/utility/regions_criteria_setup.jl @@ -4,8 +4,7 @@ using GeoParquet using Serialization using Oxygen: json, Request using Logging -using Images -include("criteria_assessment/tiles.jl") +using Images, ImageIO, Interpolations # ============================================================================= # Constants and Configuration @@ -65,21 +64,69 @@ Metadata for assessment criteria including file naming conventions. - `id::String` : Unique system identifier for the criteria - `file_suffix::String` : File suffix pattern for data files - `display_label::String` : Human-readable label for UI display +- `description::String` : Human-readable info about this criteria +- `units::String` : Human-readable info about relevant units """ struct CriteriaMetadata id::String file_suffix::String display_label::String + description::String + units::String function CriteriaMetadata(; id::String, file_suffix::String, - display_label::String + display_label::String, + description::String, + units::String ) - return new(id, file_suffix, display_label) + return new(id, file_suffix, display_label, description, units) end end +# NOTE: This is where you add to list of all possible criteria +const AVAILABLE_CRITERIA_METADATA::Vector{CriteriaMetadata} = [ + CriteriaMetadata(; + id="Depth", + file_suffix="_bathy", + display_label="Depth", + description="TODO", + units="TODO" + ), + CriteriaMetadata(; + id="Slope", + file_suffix="_slope", + display_label="Slope", + description="TODO", + units="TODO" + ), + CriteriaMetadata(; + id="Turbidity", + file_suffix="_turbid", + display_label="Turbidity", + description="TODO", + units="TODO"), + CriteriaMetadata(; + id="WavesHs", + file_suffix="_waves_Hs", + display_label="Wave Height (m)", + description="TODO", + units="TODO"), + CriteriaMetadata(; + id="WavesTp", + file_suffix="_waves_Tp", + display_label="Wave Period (s)", + description="TODO", + units="TODO"), + CriteriaMetadata(; + id="Rugosity", + file_suffix="_rugosity", + display_label="Rugosity", + description="TODO", + units="TODO") +] + """ Container for all assessment criteria metadata. @@ -202,14 +249,7 @@ end # A lookup list of all symbols/criteria available on the regional criteria # object, helpful when iterating through it's values -const REGIONAL_CRITERIA_SYMBOLS::Vector{Symbol} = [ - :depth, - :slope, - :turbidity, - :waves_height, - :waves_period, - :rugosity -] +const REGIONAL_CRITERIA_SYMBOLS::Vector{Symbol} = collect(fieldnames(RegionalCriteria)) """ Complete data package for a single region including rasters and metadata. @@ -249,7 +289,7 @@ struct RegionalDataEntry missing_from_rasters = String[] # Check each criteria field for instantiation - for field_name in fieldnames(RegionalCriteria) + for field_name in REGIONAL_CRITERIA_SYMBOLS criteria_entry = getfield(criteria, field_name) if !isnothing(criteria_entry) layer_id = criteria_entry.metadata.id @@ -383,14 +423,9 @@ const ASSESSMENT_CRITERIA::AssessmentCriteria = AssessmentCriteria(; ) ) -# Convenience list for iteration over all criteria +# Convenience list for iteration over all criteria metadata const ASSESSMENT_CRITERIA_LIST::Vector{CriteriaMetadata} = [ - ASSESSMENT_CRITERIA.depth, - ASSESSMENT_CRITERIA.slope, - ASSESSMENT_CRITERIA.turbidity, - ASSESSMENT_CRITERIA.waves_height, - ASSESSMENT_CRITERIA.waves_period, - ASSESSMENT_CRITERIA.rugosity + getfield(ASSESSMENT_CRITERIA, name) for name in fieldnames(AssessmentCriteria) ] # Normal list - only Townsville has rugosity @@ -398,13 +433,14 @@ const BASE_CRITERIA_IDS::Vector{String} = [ criteria.id for criteria in ASSESSMENT_CRITERIA_LIST if criteria.id != ASSESSMENT_CRITERIA.rugosity.id ] -# All +# All criteria const ALL_CRITERIA_IDS::Vector{String} = [ criteria.id for criteria in ASSESSMENT_CRITERIA_LIST ] -# Define all available regions for the assessment system +# Define all available regions for the assessment system NOTE: Here is where you +# configure which criteria are available per region const REGIONS::Vector{RegionMetadata} = [ RegionMetadata(; display_name="Townsville/Whitsunday Management Area", @@ -431,128 +467,6 @@ const REGIONS::Vector{RegionMetadata} = [ # GLOBAL variable to store regional data cache REGIONAL_DATA::OptionalValue{RegionalData} = nothing -# ============================================================================= -# Utility Functions -# ============================================================================= - -""" -Convert a min/max tuple to a Bounds struct. - -# Arguments -- `min_max::Tuple{Number,Number}` : Tuple containing (minimum, maximum) values - -# Returns -`Bounds` struct with converted float values. -""" -function bounds_from_tuple(min_max::Tuple{Number,Number})::Bounds - return Bounds(; min=min_max[1], max=min_max[2]) -end - -""" -Generate the filename for slope lookup data for a given region. - -# Arguments -- `region::RegionMetadata` : Region metadata containing ID - -# Returns -String filename in format "{region_id}_slope_lookup.parq" -""" -function get_slope_parquet_filename(region::RegionMetadata)::String - filename = "$(region.id)$(SLOPES_LOOKUP_SUFFIX)" - @debug "Generated slope parquet filename" region_id = region.id filename - return filename -end - -""" -Create a dictionary mapping criteria IDs to regional criteria entries. - -NOTE: Only includes criteria that are available for the region, as specified in the -region metadata and actually instantiated in the RegionalCriteria struct. - -# Arguments -- `region_data::RegionalDataEntry` : Regional data containing criteria information - -# Returns -Dictionary with criteria ID strings as keys and RegionalCriteriaEntry as values. -Only includes criteria that are both listed in region metadata and available as non-nothing. -""" -function build_regional_criteria_dictionary( - region_data::RegionalDataEntry -)::Dict{String,RegionalCriteriaEntry} - @debug "Building criteria dictionary for region" region_id = region_data.region_id available_in_metadata = - region_data.region_metadata.available_criteria - - criteria_dict = Dict{String,RegionalCriteriaEntry}() - - # Only process criteria that are listed as available in the region metadata - available_criteria_set = Set(region_data.region_metadata.available_criteria) - - # Check depth - if ASSESSMENT_CRITERIA.depth.id ∈ available_criteria_set && - !isnothing(region_data.criteria.depth) - criteria_dict[ASSESSMENT_CRITERIA.depth.id] = region_data.criteria.depth - end - - # Check slope - if ASSESSMENT_CRITERIA.slope.id ∈ available_criteria_set && - !isnothing(region_data.criteria.slope) - criteria_dict[ASSESSMENT_CRITERIA.slope.id] = region_data.criteria.slope - end - - # Check turbidity - if ASSESSMENT_CRITERIA.turbidity.id ∈ available_criteria_set && - !isnothing(region_data.criteria.turbidity) - criteria_dict[ASSESSMENT_CRITERIA.turbidity.id] = region_data.criteria.turbidity - end - - # Check waves_height - if ASSESSMENT_CRITERIA.waves_height.id ∈ available_criteria_set && - !isnothing(region_data.criteria.waves_height) - criteria_dict[ASSESSMENT_CRITERIA.waves_height.id] = - region_data.criteria.waves_height - end - - # Check waves_period - if ASSESSMENT_CRITERIA.waves_period.id ∈ available_criteria_set && - !isnothing(region_data.criteria.waves_period) - criteria_dict[ASSESSMENT_CRITERIA.waves_period.id] = - region_data.criteria.waves_period - end - - # Check rugosity - if ASSESSMENT_CRITERIA.rugosity.id ∈ available_criteria_set && - !isnothing(region_data.criteria.rugosity) - criteria_dict[ASSESSMENT_CRITERIA.rugosity.id] = region_data.criteria.rugosity - end - - @debug "Built criteria dictionary" region_id = region_data.region_id available_in_metadata = length( - available_criteria_set - ) actually_available = length(criteria_dict) criteria_ids = collect(keys(criteria_dict)) - - return criteria_dict -end - -""" -Add longitude and latitude columns to a DataFrame based on geometry centroids. - -Modifies the input DataFrame by adding 'lons' and 'lats' columns extracted -from the centroid coordinates of each geometry feature. - -# Arguments -- `df::DataFrame` : DataFrame with geometry column containing spatial features -""" -function add_lat_long_columns_to_dataframe(df::DataFrame)::Nothing - @debug "Adding lat/long columns to DataFrame" num_rows = nrow(df) - # Extract coordinate tuples from geometry centroids - coords = GI.coordinates.(df.geometry) - # Add longitude column (first coordinate) - df[!, :lons] .= first.(coords) - # Add latitude column (second coordinate) - df[!, :lats] .= last.(coords) - @debug "Successfully added coordinate columns" - return nothing -end - # ============================================================================= # Data Loading and Processing Functions # ============================================================================= @@ -571,39 +485,28 @@ for the specific region as defined in the region metadata. # Returns `RegionalCriteria` struct with computed bounds for available criteria only. """ -function build_assessment_criteria_from_slope_table( +function derive_criteria_bounds_from_slope_table( table::DataFrame, region_metadata::RegionMetadata )::RegionalCriteria @debug "Computing assessment criteria bounds from slope table" table_rows = nrow(table) region_id = region_metadata.id available_criteria = region_metadata.available_criteria - # Create set for efficient lookup - available_criteria_set = Set(region_metadata.available_criteria) - - # Initialize all bounds as nothing - depth_bounds = nothing - slope_bounds = nothing - turbidity_bounds = nothing - waves_height_bounds = nothing - waves_period_bounds = nothing - rugosity_bounds = nothing - # Helper function to compute bounds for a specific criteria function compute_criteria_bounds( - criteria_metadata::CriteriaMetadata, criteria_name::String + criteria::CriteriaMetadata ) - if criteria_metadata.id ∈ available_criteria_set - if hasproperty(table, Symbol(criteria_metadata.id)) - bounds = bounds_from_tuple(extrema(table[:, criteria_metadata.id])) - @debug "Computed $(criteria_name) bounds" range = "$(bounds.min):$(bounds.max)" + if criteria.id ∈ region_metadata.available_criteria + if hasproperty(table, Symbol(criteria.id)) + bounds = bounds_from_tuple(extrema(table[:, criteria.id])) + @debug "Computed $(criteria.display_label) bounds" range = "$(bounds.min):$(bounds.max)" return bounds else - @error "Region metadata lists $(criteria_name) as available but column missing from slope table" region_id = - region_metadata.id column = criteria_metadata.id + @error "Region metadata lists $(criteria.display_label) with id $(criteria.id) as available but column missing from slope table" region_id = + region_metadata.id column = criteria.id throw( ErrorException( - "Missing required column '$(criteria_metadata.id)' in slope table for region $(region_metadata.id)" + "Missing required column '$(criteria.id)' in slope table for region $(region_metadata.id)" ) ) end @@ -611,40 +514,14 @@ function build_assessment_criteria_from_slope_table( return nothing end - # Compute bounds only for available criteria - depth_bounds = compute_criteria_bounds(ASSESSMENT_CRITERIA.depth, "depth") - slope_bounds = compute_criteria_bounds(ASSESSMENT_CRITERIA.slope, "slope") - turbidity_bounds = compute_criteria_bounds(ASSESSMENT_CRITERIA.turbidity, "turbidity") - waves_height_bounds = compute_criteria_bounds( - ASSESSMENT_CRITERIA.waves_height, "waves_height" - ) - waves_period_bounds = compute_criteria_bounds( - ASSESSMENT_CRITERIA.waves_period, "waves_period" - ) - rugosity_bounds = compute_criteria_bounds(ASSESSMENT_CRITERIA.rugosity, "rugosity") - - computed_criteria = length([ - b for b in [ - depth_bounds, - slope_bounds, - turbidity_bounds, - waves_height_bounds, - waves_period_bounds, - rugosity_bounds - ] if !isnothing(b) - ]) - @debug "Computed criteria bounds" region_id = region_metadata.id total_available = length( - available_criteria_set - ) total_computed = computed_criteria - - return RegionalCriteria(; - depth_bounds, - slope_bounds, - turbidity_bounds, - waves_height_bounds, - waves_period_bounds, - rugosity_bounds - ) + # For each criteria, if available in region, try to find bounds + criteria_dict::Dict{String,OptionalValue{Bounds}} = Dict() + for criteria in ASSESSMENT_CRITERIA_LIST + criteria_dict[criteria.id] = compute_criteria_bounds(criteria) + end + + @debug "Completed computation of criteria bounds, returning populated RegionalCriteria" + return build_regional_criteria_from_criteria_dictionary(criteria_dict) end """ @@ -712,7 +589,6 @@ function load_canonical_reefs( )::DataFrame file_path = joinpath(source_dir, file_name) @info "Loading canonical reef outlines" file_path - try reef_data = GDF.read(file_path) @info "Successfully loaded reef outlines" num_reefs = nrow(reef_data) @@ -891,7 +767,7 @@ function initialise_data(config::Dict)::RegionalData ) available_criteria = join([c.id for c in region_criteria_list], ", ") # Compute regional criteria bounds from slope table data - criteria::RegionalCriteria = build_assessment_criteria_from_slope_table( + criteria::RegionalCriteria = derive_criteria_bounds_from_slope_table( slope_table, region_metadata ) @@ -1083,167 +959,3 @@ function Base.show(io::IO, ::MIME"text/plain", data::RegionalData) "Assessment criteria: $(join([c.display_label for c in ASSESSMENT_CRITERIA_LIST], ", "))" ) end - -# ============================================================================= -# Routes -# ============================================================================= - -""" -Setup HTTP routes for criteria information endpoints. - -Creates REST endpoints for accessing regional criteria bounds and metadata. - -# Arguments -- `config` : Configuration object -- `auth` : Authentication/authorization handler -""" -function setup_criteria_routes(config, auth) - @info "Setting up criteria routes" - regional_data::RegionalData = get_regional_data(config) - - # Endpoint: GET /criteria/{region}/ranges - # Returns JSON with min/max values for all criteria in specified region - @get auth("/criteria/{region}/ranges") function (_::Request, region::String) - @info "Processing criteria ranges request" region - - if !haskey(regional_data.regions, region) - @warn "Request for unknown region" region available_regions = keys( - regional_data.regions - ) - return json(Dict("error" => "Region not found")) - end - - @debug "Transforming criteria information to JSON for region $(region)" - output_dict = OrderedDict() - - # Build lookup dictionary for regional criteria - regional_criteria_lookup = build_regional_criteria_dictionary( - regional_data.regions[region] - ) - - # Format each criteria with min/max bounds - for (id::String, criteria::RegionalCriteriaEntry) in regional_criteria_lookup - output_dict[id] = OrderedDict( - :min_val => criteria.bounds.min, - :max_val => criteria.bounds.max - ) - end - - @debug "Returning criteria ranges" region num_criteria = length(output_dict) - return json(output_dict) - end - - @info "Criteria routes setup completed" -end - -""" -========== -DEPRECATED -========== -""" -function criteria_data_map() - # TODO: Load from config? - return OrderedDict( - :Depth => "_bathy", - :Benthic => "_benthic", - :Geomorphic => "_geomorphic", - :Slope => "_slope", - :Turbidity => "_turbid", - :WavesHs => "_waves_Hs", - :WavesTp => "_waves_Tp", - :Rugosity => "_rugosity", - :ValidSlopes => "_valid_slopes", - :ValidFlats => "_valid_flats" - ) -end - -function search_criteria()::Vector{String} - return string.(keys(criteria_data_map())) -end - -function site_criteria()::Vector{String} - return ["SuitabilityThreshold", "xdist", "ydist"] -end - -function suitability_criteria()::Vector{String} - return vcat(search_criteria(), ["SuitabilityThreshold"]) -end - -function criteria_data_map() - # TODO: Load from config? - return OrderedDict( - :Depth => "_bathy", - :Benthic => "_benthic", - :Geomorphic => "_geomorphic", - :Slope => "_slope", - :Turbidity => "_turbid", - :WavesHs => "_waves_Hs", - :WavesTp => "_waves_Tp", - :Rugosity => "_rugosity", - :ValidSlopes => "_valid_slopes", - :ValidFlats => "_valid_flats" - - # Unused datasets - # :PortDistSlopes => "_PortDistSlopes", - # :PortDistFlats => "_PortDistFlats" - ) -end - -function search_criteria()::Vector{String} - return string.(keys(criteria_data_map())) -end - -function site_criteria()::Vector{String} - return ["SuitabilityThreshold", "xdist", "ydist"] -end - -function suitability_criteria()::Vector{String} - return vcat(search_criteria(), ["SuitabilityThreshold"]) -end - -function extract_criteria(qp::T, criteria::Vector{String})::T where {T<:Dict{String,String}} - return filter( - k -> string(k.first) ∈ criteria, qp - ) -end - -struct OldRegionalCriteria{T} - stack::RasterStack - valid_slopes::T - valid_flats::T -end - -function valid_slope_lon_inds(reg::OldRegionalCriteria) - return reg.valid_slopes.lon_idx -end -function valid_slope_lat_inds(reg::OldRegionalCriteria) - return reg.valid_slopes.lat_idx -end -function valid_flat_lon_inds(reg::OldRegionalCriteria) - return reg.valid_flats.lon_idx -end -function valid_flat_lat_inds(reg::OldRegionalCriteria) - return reg.valid_flats.lat_idx -end - -struct CriteriaBounds{F<:Function} - name::Symbol - lower_bound::Float32 - upper_bound::Float32 - rule::F - - function CriteriaBounds(name::String, lb::S, ub::S)::CriteriaBounds where {S<:String} - lower_bound::Float32 = parse(Float32, lb) - upper_bound::Float32 = parse(Float32, ub) - func = (x) -> lower_bound .<= x .<= upper_bound - - return new{Function}(Symbol(name), lower_bound, upper_bound, func) - end - - function CriteriaBounds( - name::String, lb::Float32, ub::Float32 - )::CriteriaBounds - func = (x) -> lb .<= x .<= ub - return new{Function}(Symbol(name), lb, ub, func) - end -end diff --git a/src/utility/routes.jl b/src/utility/routes.jl new file mode 100644 index 0000000..ae50f92 --- /dev/null +++ b/src/utility/routes.jl @@ -0,0 +1,51 @@ +# ============================================================================= +# Routes - utility/misc routes not related to jobs or key tasks +# ============================================================================= + +""" +Setup HTTP routes for criteria information endpoints. + +Creates REST endpoints for accessing regional criteria bounds and metadata. + +# Arguments +- `config` : Configuration object +- `auth` : Authentication/authorization handler +""" +function setup_criteria_routes(config, auth) + @info "Setting up criteria routes" + regional_data::RegionalData = get_regional_data(config) + + # Endpoint: GET /criteria/{region}/ranges + # Returns JSON with min/max values for all criteria in specified region + @get auth("/criteria/{region}/ranges") function (_::Request, region::String) + @info "Processing criteria ranges request" region + + if !haskey(regional_data.regions, region) + @warn "Request for unknown region" region available_regions = keys( + regional_data.regions + ) + return json(Dict("error" => "Region not found")) + end + + @debug "Transforming criteria information to JSON for region $(region)" + output_dict = OrderedDict() + + # Build lookup dictionary for regional criteria + regional_criteria_lookup = build_regional_criteria_dictionary( + regional_data.regions[region] + ) + + # Format each criteria with min/max bounds + for (id::String, criteria::RegionalCriteriaEntry) in regional_criteria_lookup + output_dict[id] = OrderedDict( + :min_val => criteria.bounds.min, + :max_val => criteria.bounds.max + ) + end + + @debug "Returning criteria ranges" region num_criteria = length(output_dict) + return json(output_dict) + end + + @info "Criteria routes setup completed" +end From 907ebfeb676b177eb87b661262286b83c4cb2f05 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Fri, 23 May 2025 14:08:48 +1000 Subject: [PATCH 12/26] Refactoring to simplify - removing the struct hardcodes instead using a dictionary mapping criteria ID to metadata and bounds - updating helpers etc Signed-off-by: Peter Baker --- src/job_worker/handlers.jl | 2 +- src/utility/assessment_interfaces.jl | 78 +++---- src/utility/helpers.jl | 136 ++---------- src/utility/regions_criteria_setup.jl | 296 +++++++------------------- src/utility/routes.jl | 6 +- 5 files changed, 136 insertions(+), 382 deletions(-) diff --git a/src/job_worker/handlers.jl b/src/job_worker/handlers.jl index 855d4cc..db15747 100644 --- a/src/job_worker/handlers.jl +++ b/src/job_worker/handlers.jl @@ -403,7 +403,7 @@ function handle_job( @info "Done compiling parameters" @debug "Converting suitability job into regional job for regional assessment" - regional_params = regional_params_from_suitability(params) + regional_params = regional_params_from_suitability_params(params) @debug "Conversion complete" @info "Performing regional assessment" diff --git a/src/utility/assessment_interfaces.jl b/src/utility/assessment_interfaces.jl index 0b6ba2c..8c138c2 100644 --- a/src/utility/assessment_interfaces.jl +++ b/src/utility/assessment_interfaces.jl @@ -14,19 +14,19 @@ regional data. # Fields - `region::String` : The region that is being assessed -- `regional_criteria::RegionalCriteria` : The criteria to assess, including user provided bounds +- `regional_criteria::BoundedCriteriaDict` : The criteria to assess, including user provided bounds - `region_data::RegionalDataEntry` : The data to consider for this region - `suitability_threshold::Int64` : The cutoff to consider a site suitable """ struct RegionalAssessmentParameters region::String - regional_criteria::RegionalCriteria + regional_criteria::BoundedCriteriaDict region_data::RegionalDataEntry suitability_threshold::Int64 function RegionalAssessmentParameters(; region::String, - regional_criteria::RegionalCriteria, + regional_criteria::BoundedCriteriaDict, region_data::RegionalDataEntry, suitability_threshold::Int64 ) @@ -41,7 +41,7 @@ regional data plus spatial dimensions. # Fields - `region::String` : The region that is being assessed -- `regional_criteria::RegionalCriteria` : The criteria to assess, including user provided bounds +- `regional_criteria::BoundedCriteriaDict` : The criteria to assess, including user provided bounds - `region_data::RegionalDataEntry` : The data to consider for this region - `suitability_threshold::Int64` : The cutoff to consider a site suitable - `x_dist::Int64` : X dimension of polygon (metres) @@ -50,7 +50,7 @@ regional data plus spatial dimensions. struct SuitabilityAssessmentParameters # Regional criteria region::String - regional_criteria::RegionalCriteria + regional_criteria::BoundedCriteriaDict region_data::RegionalDataEntry suitability_threshold::Int64 @@ -60,7 +60,7 @@ struct SuitabilityAssessmentParameters function SuitabilityAssessmentParameters(; region::String, - regional_criteria::RegionalCriteria, + regional_criteria::BoundedCriteriaDict, region_data::RegionalDataEntry, suitability_threshold::Int64, x_dist::Int64, @@ -94,15 +94,15 @@ bounds for unspecified values. Returns nothing if regional criteria is not avail function merge_bounds( user_min::OptionalValue{Float64}, user_max::OptionalValue{Float64}, - regional_criteria::OptionalValue{RegionalCriteriaEntry} + criteria::OptionalValue{BoundedCriteria} )::OptionalValue{Bounds} - if isnothing(regional_criteria) + if isnothing(criteria) return nothing end bounds = Bounds(; - min=!isnothing(user_min) ? user_min : regional_criteria.bounds.min, - max=!isnothing(user_max) ? user_max : regional_criteria.bounds.max + min=!isnothing(user_min) ? user_min : criteria.bounds.min, + max=!isnothing(user_max) ? user_max : criteria.bounds.max ) @debug "Merged bounds" min_val = bounds.min max_val = bounds.max user_specified_min = @@ -111,6 +111,16 @@ function merge_bounds( return bounds end +# Parameter mapping: criteria_id => (min_field, max_field) or nothing +const PARAM_MAP::Dict{String,OptionalValue{Tuple{Symbol,Symbol}}} = Dict( + "Depth" => (:depth_min, :depth_max), + "Slope" => (:slope_min, :slope_max), + "Turbidity" => nothing, # Not user-configurable + "WavesHs" => (:waves_height_min, :waves_height_max), + "WavesTp" => (:waves_period_min, :waves_period_max), + "Rugosity" => (:rugosity_min, :rugosity_max) +) + """ Build regional assessment parameters from user input and regional data. @@ -150,42 +160,17 @@ function build_regional_assessment_parameters( threshold = !isnothing(input.threshold) ? input.threshold : DEFAULT_SUITABILITY_THRESHOLD - # Build merged criteria - regional_criteria = RegionalCriteria(; - depth_bounds=merge_bounds( - input.depth_min, input.depth_max, region_data.criteria.depth - ), - slope_bounds=merge_bounds( - input.slope_min, input.slope_max, region_data.criteria.slope - ), - waves_height_bounds=merge_bounds( - input.waves_height_min, - input.waves_height_max, - region_data.criteria.waves_height - ), - waves_period_bounds=merge_bounds( - input.waves_period_min, - input.waves_period_max, - region_data.criteria.waves_period - ), - rugosity_bounds=merge_bounds( - input.rugosity_min, input.rugosity_max, region_data.criteria.rugosity - ), - # Turbidity is not user-configurable, always use regional bounds - turbidity_bounds=merge_bounds(nothing, nothing, region_data.criteria.turbidity) - ) - - # Count active criteria for logging - active_criteria = length([ - b for b in [ - regional_criteria.depth, regional_criteria.slope, regional_criteria.turbidity, - regional_criteria.waves_height, regional_criteria.waves_period, - regional_criteria.rugosity - ] if !isnothing(b) - ]) + regional_criteria::BoundedCriteriaDict = Dict() - @info "Built regional assessment parameters" region = input.region threshold active_criteria user_specified_threshold = - !isnothing(input.threshold) + for (criteria_id, possible_symbols) in PARAM_MAP + regional_bounds = get(region_data.criteria, criteria_id, nothing) + user_bounds = + isnothing(possible_symbols) ? nothing : + (get(input, first(possible_symbols)), get(input, first(possible_symbols))) + regional_criteria[criteria_id] = merge_bounds( + user_bounds[1], user_bounds[2], regional_bounds + ) + end return RegionalAssessmentParameters(; region=input.region, @@ -347,7 +332,6 @@ function build_regional_assessment_file_path( return file_path end - """ Converts parameters from a suitability job into a regional job """ @@ -374,7 +358,7 @@ end """ Converts parameters from a suitability assessment into a regional assessment """ -function regional_params_from_suitability( +function regional_params_from_suitability_params( suitability_params::SuitabilityAssessmentParameters )::RegionalAssessmentParameters return RegionalAssessmentParameters(; diff --git a/src/utility/helpers.jl b/src/utility/helpers.jl index 8542d8f..e4085e1 100644 --- a/src/utility/helpers.jl +++ b/src/utility/helpers.jl @@ -6,68 +6,47 @@ function build_hash_from_components(components::Vector{String})::String end """ -Combines present regional criteria including bounds into hash components +Returns a hash component for bounded criteria """ function get_hash_components_from_regional_criteria( - criteria::RegionalCriteria + criteria::BoundedCriteriaDict )::Vector{String} - hash_components::Vector{String} = [] - for field in REGIONAL_CRITERIA_SYMBOLS - criteria_entry::OptionalValue{RegionalCriteriaEntry} = getfield( - criteria, field - ) - if !isnothing(criteria_entry) - push!( - hash_components, - "$(field)_$(criteria_entry.bounds.min)_$(criteria_entry.bounds.max)" - ) - else - push!(hash_components, "$(field)_null") - end - end - return hash_components + return [hash(criteria)] end """ -Convert RegionalCriteria to a vector of CriteriaBounds for assessment processing. +Convert BoundedCriteriaDict to a vector of CriteriaBounds for assessment processing. +Transforms the BoundedCriteriaDict into CriteriaBounds objects that include +evaluation functions. Only includes criteria that are available in the dictionary. -Transforms the RegionalCriteria struct into CriteriaBounds objects that include -evaluation functions. Only includes criteria that are available (non-nothing). - -# Arguments -- `regional_criteria::RegionalCriteria` : Regional criteria with bounds to convert +# Arguments +- `bounded_criteria_dict::BoundedCriteriaDict` : Dictionary of bounded criteria to convert # Returns Vector of `CriteriaBounds` objects for available criteria. """ function build_criteria_bounds_from_regional_criteria( - regional_criteria::RegionalCriteria + bounded_criteria_dict::BoundedCriteriaDict )::Vector{CriteriaBounds} - @debug "Converting RegionalCriteria to CriteriaBounds vector" - + @debug "Converting BoundedCriteriaDict to CriteriaBounds vector" criteria_bounds = CriteriaBounds[] - for field_symbol in REGIONAL_CRITERIA_SYMBOLS - criteria_entry = getfield(regional_criteria, field_symbol) - - if !isnothing(criteria_entry) - bounds = CriteriaBounds( - # Field to get in the data - criteria_entry.metadata.id, - # Min/max bounds - criteria_entry.bounds.min, - criteria_entry.bounds.max - ) - push!(criteria_bounds, bounds) - else - @debug "Skipped criteria - not available" criteria_id = String(field_symbol) - end + for (criteria_id, bounded_criteria) in bounded_criteria_dict + bounds = CriteriaBounds( + # Field to get in the data + bounded_criteria.metadata.id, + # Min/max bounds + bounded_criteria.bounds.min, + bounded_criteria.bounds.max + ) + push!(criteria_bounds, bounds) + @debug "Added criteria bounds" criteria_id = criteria_id min_val = + bounded_criteria.bounds.min max_val = bounded_criteria.bounds.max end @debug "Built CriteriaBounds vector" total_criteria = length(criteria_bounds) criteria_ids = [ - String(cb.name) for cb in criteria_bounds + cb.name for cb in criteria_bounds ] - return criteria_bounds end @@ -99,77 +78,6 @@ function get_slope_parquet_filename(region::RegionMetadata)::String return filename end -""" -Create a dictionary mapping criteria IDs to regional criteria entries. - -NOTE: Only includes criteria that are available for the region, as specified in the -region metadata and actually instantiated in the RegionalCriteria struct. - -Uses the defined set of symbols on the regional criteria struct to iterate through - -# Arguments -- `region_data::RegionalDataEntry` : Regional data containing criteria information - -# Returns -Dictionary with criteria ID strings as keys and RegionalCriteriaEntry as values. -Only includes criteria that are both listed in region metadata and available as non-nothing. -""" -function build_regional_criteria_dictionary( - region_data::RegionalDataEntry -)::Dict{String,RegionalCriteriaEntry} - @debug "Building criteria dictionary for region" region_id = region_data.region_id available_in_metadata = - region_data.region_metadata.available_criteria - - regional_criteria = region_data.criteria - criteria_dict = Dict{String,RegionalCriteriaEntry}() - - # Only process criteria that are listed as available in the region metadata - available_criteria_set = Set(region_data.region_metadata.available_criteria) - - for symbol in REGIONAL_CRITERIA_SYMBOLS - possible_value::OptionalValue{RegionalCriteriaEntry} = getfield( - regional_criteria, symbol - ) - if ( - !isnothing(possible_value) && - possible_value.metadata.id ∈ available_criteria_set - ) - criteria_dict[possible_value.metadata.id] = possible_value - end - end - - @debug "Built criteria dictionary" region_id = region_data.region_id available_in_metadata = length( - available_criteria_set - ) actually_available = length(criteria_dict) criteria_ids = collect(keys(criteria_dict)) - - return criteria_dict -end - -""" -Given a dictionary mapping criteria ID -> optional bounds, builds out a -RegionalCriteria object. -""" -function build_regional_criteria_from_criteria_dictionary( - criteria::Dict{String,OptionalValue{Bounds}} -) - function check_criteria(metadata::CriteriaMetadata)::OptionalValue{Bounds} - if haskey(criteria, metadata.id) && !isnothing(criteria[metadata.id]) - return criteria[metadata.id] - end - return nothing - end - - @debug "Creating RegionalCriteria by assessing each entry of criteria dictionary" - return RegionalCriteria(; - depth_bounds=check_criteria(ASSESSMENT_CRITERIA.depth), - slope_bounds=check_criteria(ASSESSMENT_CRITERIA.slope), - turbidity_bounds=check_criteria(ASSESSMENT_CRITERIA.turbidity), - waves_height_bounds=check_criteria(ASSESSMENT_CRITERIA.waves_height), - waves_period_bounds=check_criteria(ASSESSMENT_CRITERIA.waves_period), - rugosity_bounds=check_criteria(ASSESSMENT_CRITERIA.rugosity) - ) -end - """ Add longitude and latitude columns to a DataFrame based on geometry centroids. diff --git a/src/utility/regions_criteria_setup.jl b/src/utility/regions_criteria_setup.jl index 9dc11e3..7eaf855 100644 --- a/src/utility/regions_criteria_setup.jl +++ b/src/utility/regions_criteria_setup.jl @@ -57,6 +57,13 @@ mutable struct Bounds end end +""" +Helper function to create Bounds from a tuple. +""" +function bounds_from_tuple(extrema_tuple::Tuple{T,T}) where T<:Number + return Bounds(; min=extrema_tuple[1], max=extrema_tuple[2]) +end + """ Metadata for assessment criteria including file naming conventions. @@ -86,80 +93,53 @@ struct CriteriaMetadata end # NOTE: This is where you add to list of all possible criteria -const AVAILABLE_CRITERIA_METADATA::Vector{CriteriaMetadata} = [ - CriteriaMetadata(; +const ASSESSMENT_CRITERIA::Dict{String,CriteriaMetadata} = Dict( + "Depth" => CriteriaMetadata(; id="Depth", file_suffix="_bathy", display_label="Depth", description="TODO", units="TODO" ), - CriteriaMetadata(; + "Slope" => CriteriaMetadata(; id="Slope", file_suffix="_slope", display_label="Slope", description="TODO", units="TODO" ), - CriteriaMetadata(; + "Turbidity" => CriteriaMetadata(; id="Turbidity", file_suffix="_turbid", display_label="Turbidity", description="TODO", - units="TODO"), - CriteriaMetadata(; + units="TODO" + ), + "WavesHs" => CriteriaMetadata(; id="WavesHs", file_suffix="_waves_Hs", display_label="Wave Height (m)", description="TODO", - units="TODO"), - CriteriaMetadata(; + units="TODO" + ), + "WavesTp" => CriteriaMetadata(; id="WavesTp", file_suffix="_waves_Tp", display_label="Wave Period (s)", description="TODO", - units="TODO"), - CriteriaMetadata(; + units="TODO" + ), + "Rugosity" => CriteriaMetadata(; id="Rugosity", file_suffix="_rugosity", display_label="Rugosity", description="TODO", - units="TODO") -] - -""" -Container for all assessment criteria metadata. - -# Fields -- `depth::CriteriaMetadata` : Bathymetry/depth criteria -- `slope::CriteriaMetadata` : Slope gradient criteria -- `turbidity::CriteriaMetadata` : Water turbidity criteria -- `waves_height::CriteriaMetadata` : Wave height criteria -- `waves_period::CriteriaMetadata` : Wave period criteria -- `rugosity::CriteriaMetadata` : Seafloor rugosity criteria -""" -struct AssessmentCriteria - depth::CriteriaMetadata - slope::CriteriaMetadata - turbidity::CriteriaMetadata - waves_height::CriteriaMetadata - waves_period::CriteriaMetadata - rugosity::CriteriaMetadata - - function AssessmentCriteria(; - depth::CriteriaMetadata, - slope::CriteriaMetadata, - turbidity::CriteriaMetadata, - waves_height::CriteriaMetadata, - waves_period::CriteriaMetadata, - rugosity::CriteriaMetadata + units="TODO" ) - @debug "Initializing AssessmentCriteria with $(length(fieldnames(AssessmentCriteria))) criteria types" - return new( - depth, slope, turbidity, waves_height, waves_period, rugosity - ) - end -end +) + +# Create list from the dictionary values +const ASSESSMENT_CRITERIA_LIST = collect(values(ASSESSMENT_CRITERIA)) """ Combines criteria metadata with regional boundary values. @@ -168,11 +148,11 @@ Combines criteria metadata with regional boundary values. - `metadata::CriteriaMetadata` : Criteria definition and metadata - `bounds::Bounds` : Min/max values for this criteria in the region """ -struct RegionalCriteriaEntry +struct BoundedCriteria metadata::CriteriaMetadata bounds::Bounds - function RegionalCriteriaEntry(; + function BoundedCriteria(; metadata::CriteriaMetadata, bounds::Bounds ) @@ -180,76 +160,8 @@ struct RegionalCriteriaEntry end end -""" -Complete set of regional criteria with computed bounds for each parameter. - -Note: Not all regions have all criteria available. Each field is optional to -accommodate varying data availability across different regions. - -# Fields -- `depth::Union{RegionalCriteriaEntry,Nothing}` : Depth criteria and bounds - (optional) -- `slope::Union{RegionalCriteriaEntry,Nothing}` : Slope criteria and bounds - (optional) -- `turbidity::Union{RegionalCriteriaEntry,Nothing}` : Turbidity criteria and - bounds (optional) -- `waves_height::Union{RegionalCriteriaEntry,Nothing}` : Wave height criteria - and bounds (optional) -- `waves_period::Union{RegionalCriteriaEntry,Nothing}` : Wave period criteria - and bounds (optional) -- `rugosity::Union{RegionalCriteriaEntry,Nothing}` : Rugosity criteria and - bounds (optional) -""" -struct RegionalCriteria - depth::OptionalValue{RegionalCriteriaEntry} - slope::OptionalValue{RegionalCriteriaEntry} - turbidity::OptionalValue{RegionalCriteriaEntry} - waves_height::OptionalValue{RegionalCriteriaEntry} - waves_period::OptionalValue{RegionalCriteriaEntry} - rugosity::OptionalValue{RegionalCriteriaEntry} - - function RegionalCriteria(; - depth_bounds::OptionalValue{Bounds}=nothing, - slope_bounds::OptionalValue{Bounds}=nothing, - turbidity_bounds::OptionalValue{Bounds}=nothing, - waves_height_bounds::OptionalValue{Bounds}=nothing, - waves_period_bounds::OptionalValue{Bounds}=nothing, - rugosity_bounds::OptionalValue{Bounds}=nothing - ) - @debug "Creating RegionalCriteria with computed bounds for available criteria" - return new( - # All criteria are now optional - isnothing(depth_bounds) ? nothing : - RegionalCriteriaEntry(; - metadata=ASSESSMENT_CRITERIA.depth, bounds=depth_bounds - ), - isnothing(slope_bounds) ? nothing : - RegionalCriteriaEntry(; - metadata=ASSESSMENT_CRITERIA.slope, bounds=slope_bounds - ), - isnothing(turbidity_bounds) ? nothing : - RegionalCriteriaEntry(; - metadata=ASSESSMENT_CRITERIA.turbidity, bounds=turbidity_bounds - ), - isnothing(waves_height_bounds) ? nothing : - RegionalCriteriaEntry(; - metadata=ASSESSMENT_CRITERIA.waves_height, bounds=waves_height_bounds - ), - isnothing(waves_period_bounds) ? nothing : - RegionalCriteriaEntry(; - metadata=ASSESSMENT_CRITERIA.waves_period, bounds=waves_period_bounds - ), - isnothing(rugosity_bounds) ? nothing : - RegionalCriteriaEntry(; - metadata=ASSESSMENT_CRITERIA.rugosity, bounds=rugosity_bounds - ) - ) - end -end - -# A lookup list of all symbols/criteria available on the regional criteria -# object, helpful when iterating through it's values -const REGIONAL_CRITERIA_SYMBOLS::Vector{Symbol} = collect(fieldnames(RegionalCriteria)) +# Maps criteria ID to bounded criteria +const BoundedCriteriaDict = Dict{String,BoundedCriteria} """ Complete data package for a single region including rasters and metadata. @@ -263,21 +175,21 @@ corresponding layer in the RasterStack - `raster_stack::Rasters.RasterStack` : Geospatial raster data layers - `slope_table::DataFrame` : Coordinates and values for valid slope reef locations -- `criteria::RegionalCriteria` : Computed criteria bounds for this region +- `criteria::Dict{String, BoundedCriteria}` : Computed criteria bounds for this region """ struct RegionalDataEntry region_id::String region_metadata::RegionMetadata raster_stack::Rasters.RasterStack slope_table::DataFrame - criteria::RegionalCriteria + criteria::BoundedCriteriaDict function RegionalDataEntry(; region_id::String, region_metadata::RegionMetadata, raster_stack::Rasters.RasterStack, slope_table::DataFrame, - criteria::RegionalCriteria + criteria::BoundedCriteriaDict ) # Get available layers and expected criteria from metadata raster_layer_names = Set(string.(names(raster_stack))) @@ -289,8 +201,8 @@ struct RegionalDataEntry missing_from_rasters = String[] # Check each criteria field for instantiation - for field_name in REGIONAL_CRITERIA_SYMBOLS - criteria_entry = getfield(criteria, field_name) + for desired_criteria in expected_criteria_set + criteria_entry = get(criteria, desired_criteria, nothing) if !isnothing(criteria_entry) layer_id = criteria_entry.metadata.id push!(instantiated_criteria, layer_id) @@ -361,21 +273,21 @@ struct RegionalDataEntry end # Type alias -const RegionalDataMapType = Dict{String,RegionalDataEntry} +const RegionalDataDict = Dict{String,RegionalDataEntry} """ Top-level container for all regional data and reef outlines. # Fields -- `regions::Dict{String,RegionalDataEntry}` : Regional data indexed by region ID +- `regions::RegionalDataDict` : Regional data indexed by region ID - `reef_outlines::DataFrame` : Canonical reef outline geometries """ struct RegionalData - regions::RegionalDataMapType + regions::RegionalDataDict reef_outlines::DataFrame function RegionalData(; - regions::RegionalDataMapType, + regions::RegionalDataDict, reef_outlines::DataFrame ) total_locations = sum(nrow(entry.slope_table) for entry in values(regions)) @@ -389,54 +301,13 @@ end # Configuration Constants # ============================================================================= -# Define all assessment criteria with file naming conventions -const ASSESSMENT_CRITERIA::AssessmentCriteria = AssessmentCriteria(; - depth=CriteriaMetadata(; - id="Depth", - file_suffix="_bathy", - display_label="Depth" - ), - slope=CriteriaMetadata(; - id="Slope", - file_suffix="_slope", - display_label="Slope" - ), - turbidity=CriteriaMetadata(; - id="Turbidity", - file_suffix="_turbid", - display_label="Turbidity" - ), - waves_height=CriteriaMetadata(; - id="WavesHs", - file_suffix="_waves_Hs", - display_label="Wave Height (m)" - ), - waves_period=CriteriaMetadata(; - id="WavesTp", - file_suffix="_waves_Tp", - display_label="Wave Period (s)" - ), - rugosity=CriteriaMetadata(; - id="Rugosity", - file_suffix="_rugosity", - display_label="Rugosity" - ) -) - -# Convenience list for iteration over all criteria metadata -const ASSESSMENT_CRITERIA_LIST::Vector{CriteriaMetadata} = [ - getfield(ASSESSMENT_CRITERIA, name) for name in fieldnames(AssessmentCriteria) -] - -# Normal list - only Townsville has rugosity +# Normal list - only Townsville has rugosity! const BASE_CRITERIA_IDS::Vector{String} = [ - criteria.id for - criteria in ASSESSMENT_CRITERIA_LIST if criteria.id != ASSESSMENT_CRITERIA.rugosity.id + criteria.id for criteria in values(ASSESSMENT_CRITERIA) if criteria.id != "Rugosity" ] # All criteria const ALL_CRITERIA_IDS::Vector{String} = [ - criteria.id for - criteria in ASSESSMENT_CRITERIA_LIST + criteria.id for criteria in values(ASSESSMENT_CRITERIA) ] # Define all available regions for the assessment system NOTE: Here is where you @@ -483,24 +354,24 @@ for the specific region as defined in the region metadata. - `region_metadata::RegionMetadata` : Region metadata specifying available criteria # Returns -`RegionalCriteria` struct with computed bounds for available criteria only. +`BoundedCriteriaDict` with computed bounds for relevant criteria """ function derive_criteria_bounds_from_slope_table( table::DataFrame, region_metadata::RegionMetadata -)::RegionalCriteria +)::BoundedCriteriaDict @debug "Computing assessment criteria bounds from slope table" table_rows = nrow(table) region_id = region_metadata.id available_criteria = region_metadata.available_criteria # Helper function to compute bounds for a specific criteria function compute_criteria_bounds( criteria::CriteriaMetadata - ) + )::Union{BoundedCriteria,Nothing} if criteria.id ∈ region_metadata.available_criteria if hasproperty(table, Symbol(criteria.id)) bounds = bounds_from_tuple(extrema(table[:, criteria.id])) @debug "Computed $(criteria.display_label) bounds" range = "$(bounds.min):$(bounds.max)" - return bounds + return BoundedCriteria(; metadata=criteria, bounds=bounds) else @error "Region metadata lists $(criteria.display_label) with id $(criteria.id) as available but column missing from slope table" region_id = region_metadata.id column = criteria.id @@ -515,13 +386,17 @@ function derive_criteria_bounds_from_slope_table( end # For each criteria, if available in region, try to find bounds - criteria_dict::Dict{String,OptionalValue{Bounds}} = Dict() - for criteria in ASSESSMENT_CRITERIA_LIST - criteria_dict[criteria.id] = compute_criteria_bounds(criteria) + criteria_dict::BoundedCriteriaDict = Dict() + for criteria in values(ASSESSMENT_CRITERIA) + bounded_criteria = compute_criteria_bounds(criteria) + # Only include criteria relevant to the region + if !isnothing(bounded_criteria) + criteria_dict[criteria.id] = bounded_criteria + end end - @debug "Completed computation of criteria bounds, returning populated RegionalCriteria" - return build_regional_criteria_from_criteria_dictionary(criteria_dict) + @debug "Completed computation of criteria bounds, returning populated BoundedCriteriaDict" + return criteria_dict end """ @@ -705,7 +580,7 @@ This is the main data loading function that builds the complete data structure. function initialise_data(config::Dict)::RegionalData @info "Starting regional data initialization from source files" - regional_data::RegionalDataMapType = Dict() + regional_data::RegionalDataDict = Dict() data_source_directory = config["prepped_data"]["PREPPED_DATA_DIR"] @info "Using data source directory" directory = data_source_directory @@ -733,14 +608,13 @@ function initialise_data(config::Dict)::RegionalData add_lat_long_columns_to_dataframe(slope_table) # Filter criteria list to only those available for this region - available_criteria_set = Set(region_metadata.available_criteria) - region_criteria_list = filter( - criteria -> criteria.id ∈ available_criteria_set, - ASSESSMENT_CRITERIA_LIST - ) + available_criteria::Vector{String} = region_metadata.available_criteria + region_criteria_list::Vector{CriteriaMetadata} = [ + ASSESSMENT_CRITERIA[id] for id in available_criteria + ] @debug "Filtered criteria for region" region_id = region_metadata.id total_criteria = length( - ASSESSMENT_CRITERIA_LIST + ASSESSMENT_CRITERIA ) available_criteria = length(region_criteria_list) criteria_ids = [ c.id for c in region_criteria_list ] @@ -767,7 +641,7 @@ function initialise_data(config::Dict)::RegionalData ) available_criteria = join([c.id for c in region_criteria_list], ", ") # Compute regional criteria bounds from slope table data - criteria::RegionalCriteria = derive_criteria_bounds_from_slope_table( + bounds::BoundedCriteriaDict = derive_criteria_bounds_from_slope_table( slope_table, region_metadata ) @@ -783,7 +657,7 @@ function initialise_data(config::Dict)::RegionalData region_metadata, raster_stack, slope_table, - criteria + criteria=bounds ) catch e @@ -902,33 +776,23 @@ function Base.show(io::IO, ::MIME"text/plain", entry::RegionalDataEntry) ) println(io, " Criteria bounds:") - # Show each criteria with its bounds, only for non-nothing entries - for field_name in fieldnames(RegionalCriteria) - criteria_entry = getfield(entry.criteria, field_name) - if !isnothing(criteria_entry) - min_val = round(criteria_entry.bounds.min; digits=2) - max_val = round(criteria_entry.bounds.max; digits=2) - println( - io, " $(criteria_entry.metadata.display_label): $(min_val) - $(max_val)" - ) - else - # Get the criteria name for display - criteria_name = if field_name == :depth - ASSESSMENT_CRITERIA.depth.display_label - elseif field_name == :slope - ASSESSMENT_CRITERIA.slope.display_label - elseif field_name == :turbidity - ASSESSMENT_CRITERIA.turbidity.display_label - elseif field_name == :waves_height - ASSESSMENT_CRITERIA.waves_height.display_label - elseif field_name == :waves_period - ASSESSMENT_CRITERIA.waves_period.display_label - elseif field_name == :rugosity - ASSESSMENT_CRITERIA.rugosity.display_label - else - string(field_name) - end - println(io, " $(criteria_name): Not available") + # Show each criteria with its bounds from the dictionary + for (_, bounded_criteria) in entry.criteria + min_val = round(bounded_criteria.bounds.min; digits=2) + max_val = round(bounded_criteria.bounds.max; digits=2) + display_label = bounded_criteria.metadata.display_label + println(io, " $(display_label): $(min_val) - $(max_val)") + end + + # Show criteria that are expected but not available + expected_criteria = Set(entry.region_metadata.available_criteria) + available_criteria = Set(keys(entry.criteria)) + missing_criteria = setdiff(expected_criteria, available_criteria) + + for criteria_id in missing_criteria + if haskey(ASSESSMENT_CRITERIA, criteria_id) + display_label = ASSESSMENT_CRITERIA[criteria_id].display_label + println(io, " $(display_label): Not available") end end end @@ -956,6 +820,6 @@ function Base.show(io::IO, ::MIME"text/plain", data::RegionalData) return println( io, - "Assessment criteria: $(join([c.display_label for c in ASSESSMENT_CRITERIA_LIST], ", "))" + "Assessment criteria: $(join([c.display_label for c in values(ASSESSMENT_CRITERIA)], ", "))" ) end diff --git a/src/utility/routes.jl b/src/utility/routes.jl index ae50f92..007e953 100644 --- a/src/utility/routes.jl +++ b/src/utility/routes.jl @@ -31,12 +31,10 @@ function setup_criteria_routes(config, auth) output_dict = OrderedDict() # Build lookup dictionary for regional criteria - regional_criteria_lookup = build_regional_criteria_dictionary( - regional_data.regions[region] - ) + regional_criteria_lookup = regional_data.regions[region].criteria # Format each criteria with min/max bounds - for (id::String, criteria::RegionalCriteriaEntry) in regional_criteria_lookup + for (id::String, criteria::BoundedCriteria) in regional_criteria_lookup output_dict[id] = OrderedDict( :min_val => criteria.bounds.min, :max_val => criteria.bounds.max From 0fa0b1a3f97c3e3f6d39139f6be769f113e961b4 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Fri, 23 May 2025 14:28:31 +1000 Subject: [PATCH 13/26] WIP some minor errors after refactor Signed-off-by: Peter Baker --- src/utility/assessment_interfaces.jl | 23 ++++++++++++++++++----- src/utility/helpers.jl | 3 ++- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/utility/assessment_interfaces.jl b/src/utility/assessment_interfaces.jl index 8c138c2..e47698a 100644 --- a/src/utility/assessment_interfaces.jl +++ b/src/utility/assessment_interfaces.jl @@ -161,15 +161,28 @@ function build_regional_assessment_parameters( !isnothing(input.threshold) ? input.threshold : DEFAULT_SUITABILITY_THRESHOLD regional_criteria::BoundedCriteriaDict = Dict() + regional_bounds::BoundedCriteriaDict = region_data.criteria for (criteria_id, possible_symbols) in PARAM_MAP - regional_bounds = get(region_data.criteria, criteria_id, nothing) - user_bounds = + bounds = get(regional_bounds, criteria_id, nothing) + user_min = isnothing(possible_symbols) ? nothing : - (get(input, first(possible_symbols)), get(input, first(possible_symbols))) - regional_criteria[criteria_id] = merge_bounds( - user_bounds[1], user_bounds[2], regional_bounds + getproperty(input, first(possible_symbols)) + user_max = + isnothing(possible_symbols) ? nothing : + getproperty(input, last(possible_symbols)) + + merged = merge_bounds( + user_min, + user_max, + bounds ) + if !isnothing(merged) + regional_criteria[criteria_id] = BoundedCriteria(; + metadata=ASSESSMENT_CRITERIA[criteria_id], + bounds=merged + ) + end end return RegionalAssessmentParameters(; diff --git a/src/utility/helpers.jl b/src/utility/helpers.jl index e4085e1..61379c9 100644 --- a/src/utility/helpers.jl +++ b/src/utility/helpers.jl @@ -11,7 +11,8 @@ Returns a hash component for bounded criteria function get_hash_components_from_regional_criteria( criteria::BoundedCriteriaDict )::Vector{String} - return [hash(criteria)] + # TODO Fix + return [String(hash(criteria))] end """ From 65610fd84c18737baf4910ef212f3269149317a3 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Fri, 23 May 2025 16:57:30 +1000 Subject: [PATCH 14/26] Finishing refactor Signed-off-by: Peter Baker --- src/job_worker/handlers.jl | 16 ++++++++-------- src/utility/assessment_interfaces.jl | 12 +++++------- src/utility/helpers.jl | 17 ++++++++++++++--- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/job_worker/handlers.jl b/src/job_worker/handlers.jl index db15747..a6057a9 100644 --- a/src/job_worker/handlers.jl +++ b/src/job_worker/handlers.jl @@ -286,27 +286,27 @@ function handle_job( @info "Configuration parsing complete." @info "Setting up regional assessment data" - reg_assess_data::RegionalData = get_regional_data(config) + data::RegionalData = get_regional_data(config) @info "Done setting up regional assessment data" @info "Compiling regional assessment parameters from regional data and input data" params = build_regional_assessment_parameters( input, - reg_assess_data + data ) @info "Done compiling parameters" @info "Performing regional assessment" - assessed_fn = build_regional_assessment_file_path(params; ext="tiff", config) - @debug "COG File name: $(assessed_fn)" + regional_assessment_filename = build_regional_assessment_file_path(params; ext="tiff", config) + @debug "COG File name: $(regional_assessment_filename)" - if !isfile(assessed_fn) + if !isfile(regional_assessment_filename) @debug "File system cache was not hit for this task" @debug "Assessing region $(params.region)" assessed = assess_region(params) - @debug now() "Writing COG of regional assessment to $(assessed_fn)" - _write_cog(assessed_fn, assessed, config) + @debug now() "Writing COG of regional assessment to $(regional_assessment_filename)" + _write_cog(regional_assessment_filename, assessed, config) @debug now() "Finished writing cog " else @info "Cache hit - skipping regional assessment process and re-uploading to output!" @@ -321,7 +321,7 @@ function handle_job( @debug "File paths:" relative = output_file_name_rel absolute = full_s3_target @debug now() "Initiating file upload" - upload_file(client, assessed_fn, full_s3_target) + upload_file(client, regional_assessment_filename, full_s3_target) @debug now() "File upload completed" @debug "Finished regional assessment job." diff --git a/src/utility/assessment_interfaces.jl b/src/utility/assessment_interfaces.jl index e47698a..6e7b502 100644 --- a/src/utility/assessment_interfaces.jl +++ b/src/utility/assessment_interfaces.jl @@ -252,14 +252,12 @@ function regional_assessment_params_hash(params::RegionalAssessmentParameters):: # Create hash input from key parameters hash_components = [ + # Region params.region, - string(params.suitability_threshold) - ] - - # Add criteria bounds to hash (only non-nothing criteria) - hash_components::Vector{String} = [ - hash_components; - get_hash_components_from_regional_criteria(params.regional_criteria) + # Suitability threshold + string(params.suitability_threshold), + # Criteria + get_hash_components_from_regional_criteria(params.regional_criteria)... ] # Create deterministic hash diff --git a/src/utility/helpers.jl b/src/utility/helpers.jl index 61379c9..6a9325b 100644 --- a/src/utility/helpers.jl +++ b/src/utility/helpers.jl @@ -6,13 +6,24 @@ function build_hash_from_components(components::Vector{String})::String end """ -Returns a hash component for bounded criteria +Returns a hash component for bounded criteria + +Carefully orders hash predictably and only includes present criteria """ function get_hash_components_from_regional_criteria( criteria::BoundedCriteriaDict )::Vector{String} - # TODO Fix - return [String(hash(criteria))] + @debug "Hashing criteria..." criteria + components::Vector{String} = [] + for id in keys(ASSESSMENT_CRITERIA) + components = vcat(components, + haskey(criteria, id) ? + [ + id, string(criteria[id].bounds.min), string(criteria[id].bounds.max) + ] : [] + ) + end + return components end """ From d4334d087d80df2941f0edb45ff2755237b7bd80 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Wed, 28 May 2025 16:55:07 +1000 Subject: [PATCH 15/26] Comment fixes Signed-off-by: Peter Baker --- src/criteria_assessment/query_thresholds.jl | 4 ++-- src/utility/deprecated.jl | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/criteria_assessment/query_thresholds.jl b/src/criteria_assessment/query_thresholds.jl index 9f1f3c2..9781da2 100644 --- a/src/criteria_assessment/query_thresholds.jl +++ b/src/criteria_assessment/query_thresholds.jl @@ -221,7 +221,7 @@ end """ apply_criteria_lookup( - raster_stack::RasterStack, + raster_stack::OldRegionalCriteria, rtype::Symbol, ruleset::Vector{CriteriaBounds{Function}} ) @@ -348,7 +348,7 @@ Handles threshold masking using the integrated assessment parameter struct function threshold_mask( params::RegionalAssessmentParameters )::Raster - # build out a set of criteria filters using the regional criteria + # Builds out a set of criteria filters using the regional criteria. # NOTE this will only filter over available criteria filters = build_criteria_bounds_from_regional_criteria(params.regional_criteria) diff --git a/src/utility/deprecated.jl b/src/utility/deprecated.jl index a0151aa..1133813 100644 --- a/src/utility/deprecated.jl +++ b/src/utility/deprecated.jl @@ -3,6 +3,7 @@ DEPRECATED ========== """ + function criteria_data_map() # TODO: Load from config? return OrderedDict( From e379178c06737088b59d67d0ee1aefebccd1250f Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Thu, 29 May 2025 13:14:19 +1000 Subject: [PATCH 16/26] Cleanup! Signed-off-by: Peter Baker --- src/ReefGuideAPI.jl | 96 +--- src/admin.jl | 6 - src/assessment_methods/apply_criteria.jl | 103 ++++ .../best_fit_polygons.jl | 63 +-- .../common_functions.jl | 24 - .../geom_ops.jl | 0 src/assessment_methods/index.jl | 5 + src/assessment_methods/site_identification.jl | 131 +++++ src/criteria_assessment/query_parser.jl | 54 --- src/criteria_assessment/query_thresholds.jl | 424 ---------------- .../regional_assessment.jl | 235 --------- .../site_identification.jl | 456 ------------------ src/criteria_assessment/tiles.jl | 274 ----------- src/file_io.jl | 115 ----- src/job_management/DiskService.jl | 124 ----- src/job_management/JobInterface.jl | 70 --- src/utility/config.jl | 45 ++ src/utility/deprecated.jl | 112 ----- src/utility/file_io.jl | 52 ++ src/utility/helpers.jl | 1 + src/utility/index.jl | 4 +- src/{Middleware.jl => utility/middleware.jl} | 6 + src/utility/regions_criteria_setup.jl | 37 -- src/utility/routes.jl | 14 +- 24 files changed, 379 insertions(+), 2072 deletions(-) delete mode 100644 src/admin.jl create mode 100644 src/assessment_methods/apply_criteria.jl rename src/{site_assessment => assessment_methods}/best_fit_polygons.jl (91%) rename src/{site_assessment => assessment_methods}/common_functions.jl (91%) rename src/{site_assessment => assessment_methods}/geom_ops.jl (100%) create mode 100644 src/assessment_methods/index.jl create mode 100644 src/assessment_methods/site_identification.jl delete mode 100644 src/criteria_assessment/query_parser.jl delete mode 100644 src/criteria_assessment/query_thresholds.jl delete mode 100644 src/criteria_assessment/regional_assessment.jl delete mode 100644 src/criteria_assessment/site_identification.jl delete mode 100644 src/criteria_assessment/tiles.jl delete mode 100644 src/file_io.jl delete mode 100644 src/job_management/DiskService.jl delete mode 100644 src/job_management/JobInterface.jl create mode 100644 src/utility/config.jl delete mode 100644 src/utility/deprecated.jl create mode 100644 src/utility/file_io.jl rename src/{Middleware.jl => utility/middleware.jl} (93%) diff --git a/src/ReefGuideAPI.jl b/src/ReefGuideAPI.jl index 17bd7ea..48652b8 100644 --- a/src/ReefGuideAPI.jl +++ b/src/ReefGuideAPI.jl @@ -1,56 +1,30 @@ module ReefGuideAPI -using Base.Threads -using - Glob, - TOML - -using Serialization - -using DataFrames -using OrderedCollections -using Memoization -using SparseArrays - -using FLoops, ThreadsX - import GeoDataFrames as GDF using + Base.Threads, + Glob, + TOML, ArchGDAL, GeoParquet, - Rasters - -using + Rasters, HTTP, - Oxygen - -# Worker system -include("job_worker/index.jl") + Oxygen, + Serialization, + DataFrames, + OrderedCollections, + Memoization, + SparseArrays, + FLoops, ThreadsX # Utilities and helpers for assessments include("utility/index.jl") -include("Middleware.jl") -include("admin.jl") -include("file_io.jl") - -# TODO Remove these due to deprecation -include("job_management/JobInterface.jl") -include("job_management/DiskService.jl") - -include("criteria_assessment/query_thresholds.jl") -include("criteria_assessment/regional_assessment.jl") -include("criteria_assessment/site_identification.jl") +# Assessment logic +include("assessment_methods/index.jl") -include("site_assessment/common_functions.jl") -include("site_assessment/best_fit_polygons.jl") -include("criteria_assessment/tiles.jl") - -function get_auth_router(config::Dict) - # Setup auth middleware - depends on config.toml - can return identity func - auth = setup_jwt_middleware(config) - return router(""; middleware=[auth]) -end +# Worker system +include("job_worker/index.jl") function start_server(config_path) @info "Launching server... please wait" @@ -64,23 +38,15 @@ function start_server(config_path) @info "Setting up auth middleware and router." auth = get_auth_router(config) - @info "Setting up criteria routes..." - setup_criteria_routes(config, auth) - - @info "Setting up region routes..." - setup_region_routes(config, auth) - - @info "Setting up tile routes..." - setup_tile_routes(config, auth) - @info "Setting up job routes..." setup_job_routes(config, auth) - @info "Setting up admin routes..." - setup_admin_routes(config) + @info "Setting up utility routes..." + setup_utility_routes(config, auth) # Which host should we listen on? host = config["server_config"]["HOST"] + # Which port should we listen on? port = 8000 @@ -112,30 +78,6 @@ function start_worker() @info "Worker closed itself..." end -export - OldRegionalCriteria, - criteria_data_map - -# Methods to assess/identify deployment "plots" of reef. -export - assess_reef_site, - identify_edge_aligned_sites, - filter_sites, - output_geojson - -# Geometry handling -export - create_poly, - create_bbox, - port_buffer_mask, - meters_to_degrees, - polygon_to_lines - -# Raster->Index interactions (defunct?) -export - valid_slope_lon_inds, - valid_slope_lat_inds, - valid_flat_lon_inds, - valid_flat_lat_inds +export start_worker, start_server end diff --git a/src/admin.jl b/src/admin.jl deleted file mode 100644 index 9daa2e0..0000000 --- a/src/admin.jl +++ /dev/null @@ -1,6 +0,0 @@ - -function setup_admin_routes(config) - @get "/health" function () - return json(Dict(:status => "healthy")) - end -end diff --git a/src/assessment_methods/apply_criteria.jl b/src/assessment_methods/apply_criteria.jl new file mode 100644 index 0000000..c25969b --- /dev/null +++ b/src/assessment_methods/apply_criteria.jl @@ -0,0 +1,103 @@ +"""Methods to filter criteria bounds over rasters and lookup tables""" + +struct CriteriaBounds{F<:Function} + "The field ID of the criteria" + name::Symbol + "min" + lower_bound::Float32 + "max" + upper_bound::Float32 + "A function which takes a value and returns if matches the criteria" + rule::F + + function CriteriaBounds(name::String, lb::S, ub::S)::CriteriaBounds where {S<:String} + lower_bound::Float32 = parse(Float32, lb) + upper_bound::Float32 = parse(Float32, ub) + func = (x) -> lower_bound .<= x .<= upper_bound + + return new{Function}(Symbol(name), lower_bound, upper_bound, func) + end + + function CriteriaBounds( + name::String, lb::Float32, ub::Float32 + )::CriteriaBounds + func = (x) -> lb .<= x .<= ub + return new{Function}(Symbol(name), lb, ub, func) + end +end + +"""Methods to support querying data layers.""" + +""" +Apply thresholds for each criteria. + +# Arguments +- `criteria_stack` : RasterStack of criteria data for a given region +- `lookup` : Lookup dataframe for the region +- `criteria_bounds` : A vector of CriteriaBounds which contains named criteria + with min/max ranges and a function to apply. + +# Returns +BitMatrix indicating locations within desired thresholds +""" +function filter_raster_by_criteria( + criteria_stack::RasterStack, + lookup::DataFrame, + criteria_bounds::Vector{CriteriaBounds} +)::Raster + # Result store + data = falses(size(criteria_stack)) + + # Apply criteria + res_lookup = trues(nrow(lookup)) + for filter::CriteriaBounds in criteria_bounds + res_lookup .= res_lookup .& filter.rule(lookup[!, rule_name]) + end + + tmp = lookup[res_lookup, [:lon_idx, :lat_idx]] + data[CartesianIndex.(tmp.lon_idx, tmp.lat_idx)] .= true + + res = Raster(criteria_stack.Depth; data=sparse(data), missingval=0) + return res +end + +""" +Filters the slope table (which contains raster param values too) by building a +bit mask AND'd for all thresholds +""" +function filter_lookup_table_by_criteria( + slope_table::DataFrame, + ruleset::Vector{CriteriaBounds} +)::DataFrame + slope_table.all_crit .= 1 + + for threshold in ruleset + slope_table.all_crit = + slope_table.all_crit .& threshold.rule(slope_table[!, threshold.name]) + end + + return slope_table[BitVector(slope_table.all_crit), :] +end + +""" + lookup_df_from_raster(raster::Raster, threshold::Union{Int64,Float64})::DataFrame + +Build a look up table identifying all pixels in a raster that meet a suitability threshold. + +# Arguments +- `raster` : Raster of regional data +- `threshold` : Suitability threshold value (greater or equal than) + +# Returns +DataFrame containing indices, lon and lat for each pixel that is intended for further +analysis. +""" +function lookup_df_from_raster(raster::Raster, threshold::Union{Int64,Float64})::DataFrame + criteria_matches::SparseMatrixCSC{Bool,Int64} = sparse(falses(size(raster))) + Rasters.read!(raster .>= threshold, criteria_matches) + indices::Vector{CartesianIndex{2}} = findall(criteria_matches) + indices_lon::Vector{Float64} = lookup(raster, X)[first.(Tuple.(indices))] + indices_lat::Vector{Float64} = lookup(raster, Y)[last.(Tuple.(indices))] + + return DataFrame(; indices=indices, lons=indices_lon, lats=indices_lat) +end diff --git a/src/site_assessment/best_fit_polygons.jl b/src/assessment_methods/best_fit_polygons.jl similarity index 91% rename from src/site_assessment/best_fit_polygons.jl rename to src/assessment_methods/best_fit_polygons.jl index 0b3532b..3017200 100644 --- a/src/site_assessment/best_fit_polygons.jl +++ b/src/assessment_methods/best_fit_polygons.jl @@ -85,6 +85,7 @@ function assess_reef_site( best_poly[argmax(score)], maximum(qc_flag) end + function assess_reef_site( rel_pix::DataFrame, rotated::Vector{GI.Wrappers.Polygon}, @@ -512,68 +513,6 @@ function find_optimal_site_alignment( ) end -# Raster based assessment methods - -# """ -# assess_reef_site( -# rst::Union{Raster,RasterStack}, -# geom::GI.Wrappers.Polygon, -# ruleset::Dict{Symbol,Function}; -# degree_step::Float64=15.0, -# start_rot::Float64=0.0, -# n_per_side::Int64=1 -# )::Tuple{Float64,Int64,GI.Wrappers.Polygon} - -# Assess given reef site for it's suitability score at different specified rotations around the -# initial reef-edge rotation. - -# # Arguments -# - `rst` : Raster of suitability scores. -# - `geom` : Initial site polygon with no rotation applied. -# - `ruleset` : Criteria ruleset to apply to `rst` pixels when assessing which pixels are suitable. -# - `degree_step` : Degree value to vary each rotation by. Default = 15 degrees. -# - `start_rot` : Initial rotation used to align the site polygon with the nearest reef edge. Default = 0 degrees. -# - `n_per_side` : Number of times to rotate polygon on each side (clockwise and anticlockwise). Default = 2 rotations on each side. - -# # Returns -# - Highest score identified with rotating polygons. -# - The index of the highest scoring rotation. -# - The polygon with the highest score out of the assessed rotated polygons. -# """ -# function assess_reef_site( -# rst::Union{Raster,RasterStack}, -# geom::GI.Wrappers.Polygon; -# degree_step::Float64=15.0, -# start_rot::Float64=0.0, -# n_per_side::Int64=2 -# )::Tuple{Float64,Int64,GI.Wrappers.Polygon} -# rotations = -# (start_rot - (degree_step * n_per_side)):degree_step:(start_rot + (degree_step * n_per_side)) -# n_rotations = length(rotations) -# score = zeros(n_rotations) -# best_poly = Vector(undef, n_rotations) - -# target_crs = convert(EPSG, GI.crs(rst)) -# for (j, r) in enumerate(rotations) -# rot_geom = rotate_geom(geom, r, target_crs) -# c_rst = crop(rst; to=rot_geom) -# if !all(size(c_rst) .> (0, 0)) -# @warn "No data found!" -# continue -# end - -# score[j] = mean(c_rst) -# best_poly[j] = rot_geom - -# if score[j] > 95.0 -# # Found a good enough rotation -# break -# end -# end - -# return score[argmax(score)], argmax(score) - (n_per_side + 1), best_poly[argmax(score)] -# end - """ assess_reef_site( rst::Union{Raster,RasterStack}, diff --git a/src/site_assessment/common_functions.jl b/src/assessment_methods/common_functions.jl similarity index 91% rename from src/site_assessment/common_functions.jl rename to src/assessment_methods/common_functions.jl index fab3beb..71fc4d3 100644 --- a/src/site_assessment/common_functions.jl +++ b/src/assessment_methods/common_functions.jl @@ -3,8 +3,6 @@ using LinearAlgebra using GeoDataFrames -include("geom_ops.jl") - """ meters_to_degrees(x, lat) @@ -283,28 +281,6 @@ function output_geojson( return nothing end -""" - search_lookup(raster::Raster, threshold::Union{Int64,Float64})::DataFrame - -Build a look up table identifying all pixels in a raster that meet a suitability threshold. - -# Arguments -- `raster` : Raster of regional data -- `threshold` : Suitability threshold value (greater or equal than) - -# Returns -DataFrame containing indices, lon and lat for each pixel that is intended for further -analysis. -""" -function search_lookup(raster::Raster, threshold::Union{Int64,Float64})::DataFrame - criteria_matches::SparseMatrixCSC{Bool,Int64} = sparse(falses(size(raster))) - Rasters.read!(raster .>= threshold, criteria_matches) - indices::Vector{CartesianIndex{2}} = findall(criteria_matches) - indices_lon::Vector{Float64} = lookup(raster, X)[first.(Tuple.(indices))] - indices_lat::Vector{Float64} = lookup(raster, Y)[last.(Tuple.(indices))] - - return DataFrame(; indices=indices, lons=indices_lon, lats=indices_lat) -end """ buffer_simplify( diff --git a/src/site_assessment/geom_ops.jl b/src/assessment_methods/geom_ops.jl similarity index 100% rename from src/site_assessment/geom_ops.jl rename to src/assessment_methods/geom_ops.jl diff --git a/src/assessment_methods/index.jl b/src/assessment_methods/index.jl new file mode 100644 index 0000000..c38ca2f --- /dev/null +++ b/src/assessment_methods/index.jl @@ -0,0 +1,5 @@ +include("apply_criteria.jl") +include("best_fit_polygons.jl") +include("common_functions.jl") +include("geom_ops.jl") +include("site_identification.jl") diff --git a/src/assessment_methods/site_identification.jl b/src/assessment_methods/site_identification.jl new file mode 100644 index 0000000..f63319e --- /dev/null +++ b/src/assessment_methods/site_identification.jl @@ -0,0 +1,131 @@ +"""Methods for identifying potential deployment locations.""" + +""" + proportion_suitable( + x::Union{BitMatrix,SparseMatrixCSC{Bool,Int64}}; square_offset::Tuple=(-4, 5) + )::SparseMatrixCSC{UInt8,Int64} + +Calculate the the proportion of the subsection that is suitable for deployments. +The `subsection` is the surrounding a rough hectare area centred on each cell of a raster +marked as being suitable according to user-selected criteria. + +Cells on the edges of a raster object are assessed using a smaller surrounding area, rather +than shifting the window inward. In usual applications, there will be no target pixel +close to the edge due to the use of buffer areas. + +# Arguments +- `x` : Matrix of boolean pixels after filtering with user criteria. +- `square_offset` : The number of pixels +/- around a center "target" pixel to assess as the + moving window. Defaults to (-4, 5). + Assuming a 10m² pixel, the default `square_offset` resolves to a one + hectare area. + +# Returns +Matrix of values 0 - 100 indicating the percentage of the area around the target pixel that +meet suitability criteria. +""" +function proportion_suitable( + x::Union{BitMatrix,SparseMatrixCSC{Bool,Int64}}; square_offset::Tuple=(-4, 5) +)::SparseMatrixCSC{UInt8,Int64} + subsection_dims = size(x) + target_area = spzeros(UInt8, subsection_dims) + + for row_col in findall(x) + (row, col) = Tuple(row_col) + x_left = max(col + square_offset[1], 1) + x_right = min(col + square_offset[2], subsection_dims[2]) + + y_top = max(row + square_offset[1], 1) + y_bottom = min(row + square_offset[2], subsection_dims[1]) + + target_area[row, col] = UInt8(sum(@views x[y_top:y_bottom, x_left:x_right])) + end + + return target_area +end + +""" + assess_region(reg_assess_data, reg, qp, rtype) + +Perform raster suitability assessment based on user-defined criteria. + +# Arguments +- params :: RegionalAssessmentParameters + +# Returns +GeoTiff file of surrounding hectare suitability (1-100%) based on the criteria bounds input +by a user. +""" +function assess_region(params::RegionalAssessmentParameters)::Raster + # Make mask of suitable locations + @debug "$(now()) : Creating mask for region" + + # Builds out a set of criteria filters using the regional criteria. + # NOTE this will only filter over available criteria + filters = build_criteria_bounds_from_regional_criteria(params.regional_criteria) + + # map our regional criteria + @debug "Applying criteria thresholds to generate mask layer" + mask_data = filter_raster_by_criteria( + # This is the raster stack + params.region_data.raster_stack, + # The slope table dataframe + params.region_data.slope_table, + # The list of criteria bounds + filters + ) + + # Assess remaining pixels for their suitability + @debug "$(now()) : Calculating proportional suitability score" + suitability_scores = proportion_suitable(mask_data.data) + + @debug "$(now()) : Rebuilding raster and returning results" + return rebuild(mask_data, suitability_scores) +end + +function assess_sites( + params::SuitabilityAssessmentParameters, + regional_raster::Raster +) + target_crs = convert(EPSG, crs(regional_raster)) + suitability_threshold = params.suitability_threshold + region = params.region + + @debug "$(now()) : Identifying search pixels for $(region)" + target_locs = lookup_df_from_raster(regional_raster, suitability_threshold) + + if size(target_locs, 1) == 0 + # No viable set of locations, return empty dataframe + return DataFrame(; + score=[], + orientation=[], + qc_flag=[], + geometry=[] + ) + end + + # Otherwise, create the file + @debug "$(now()) : Assessing criteria table for $(region)" + # Get criteria bounds list from criteria + filters = build_criteria_bounds_from_regional_criteria(params.regional_criteria) + + crit_pixels::DataFrame = filter_lookup_table_by_criteria( + # Slope table + params.region_data.slope_table, + filters + ) + + res = abs(step(dims(regional_raster, X))) + @debug "$(now()) : Assessing $(size(target_locs, 1)) candidate locations in $(region)." + @debug "Finding optimal site alignment" + initial_polygons = find_optimal_site_alignment( + crit_pixels, + target_locs, + res, + params.x_dist, + params.y_dist, + target_crs + ) + + return initial_polygons +end diff --git a/src/criteria_assessment/query_parser.jl b/src/criteria_assessment/query_parser.jl deleted file mode 100644 index 6f52d4c..0000000 --- a/src/criteria_assessment/query_parser.jl +++ /dev/null @@ -1,54 +0,0 @@ -""" - parse_criteria_query(qp)::Tuple - -Parse criteria values from request query. - -Queries should take the form of: -`Depth=-9.0:0.0&Slope=0.0:40.0&Rugosity=0.0:0.0` - -# Arguments -- `qp` : Parsed query string from request. - -# Returns -Tuple of criteria names, lower bounds, upper bounds -""" -function parse_criteria_query(qp::Dict)::Tuple - criteria = string.(keys(criteria_data_map())) - criteria_names = String[] - lbs = String[] - ubs = String[] - - for k in criteria - if k ∉ keys(qp) - continue - end - - lb, ub = string.(split(qp[k], ":")) - push!(criteria_names, k) - push!(lbs, lb) - push!(ubs, ub) - end - - return criteria_names, lbs, ubs -end - -""" - remove_rugosity(reg, criteria, lbs, ubs) - -Remove rugosity layer from consideration if region is not Townsville. -Rugosity data currently only exists for the Townsville region. -""" -function remove_rugosity(reg, criteria_names, lbs, ubs) - - # Need to specify `Base` as geospatial packages also define `contains()`. - if !Base.contains(reg, "Townsville") - # Remove rugosity layer from consideration as it doesn't exist for regions - # outside of Townsville. - pos = findfirst(lowercase.(criteria_names) .== "rugosity") - criteria_names = [cname for (i, cname) in enumerate(criteria_names) if i != pos] - lbs = [lb for (i, lb) in enumerate(lbs) if i != pos] - ubs = [ub for (i, ub) in enumerate(ubs) if i != pos] - end - - return criteria_names, lbs, ubs -end diff --git a/src/criteria_assessment/query_thresholds.jl b/src/criteria_assessment/query_thresholds.jl deleted file mode 100644 index 9781da2..0000000 --- a/src/criteria_assessment/query_thresholds.jl +++ /dev/null @@ -1,424 +0,0 @@ -"""Methods to support querying data layers.""" - -""" - within_thresholds(ctype::Val, data::Raster, lb::T, ub::T) where {T} - -Apply in-bound constraints. - -# Notes -Why is this a simple one line function? -Because we want to be able to cache results for each constraint type. -""" -function within_thresholds(reg::Val, ctype::Val, data::Raster, lb::T, ub::T) where {T} - return within_thresholds(data, lb, ub) -end -function within_thresholds(req, data::N, lb::T, ub::T) where {N,T} - return within_thresholds(data, lb, ub) -end -@memoize function within_thresholds(data::Raster, lb::T, ub::T) where {T} - return (lb .<= data .<= ub) -end -@memoize function within_thresholds(data::Vector, lb::T, ub::T) where {T} - return (lb .<= data .<= ub) -end - -function within_thresholds(data, lb::T, ub::T) where {T} - return (lb .<= data .<= ub) -end - -""" - port_buffer_mask(gdf::DataFrame, dist::Float64; unit::String="NM") - -Create a masking buffer around indicated port locations. - -# Arguments -- `gdf` : GeoDataFrame of port locations (given as long/lat points) -- `dist` : distance from port in degrees (deg), kilometers (km), or nautical miles (NM; default) -- `unit` : unit `dist` is in -""" -function port_buffer_mask(gdf::DataFrame, dist::Float64; unit::String="NM") - # Determine conversion factor (nautical miles or kilometers) - conv_factor = 1.0 - if unit == "NM" - conv_factor = 60.0 # 60 NM = 1 degree - elseif unit == "km" - conv_factor = 111.0 # 111 km = 1 degree - elseif unit != "deg" - error("Unknown distance unit requested. Can only be one of `NM` or `km` or `deg`") - end - - ports = gdf.geometry # TODO: Replace with `GI.geometrycolumns()` - - # Make buffer around ports - buffered_ports = GO.buffer.(ports, dist / conv_factor) - - # Combine all geoms into one - port_mask = reduce((x1, x2) -> LibGEOS.union(x1, x2), buffered_ports) - - return port_mask -end - -""" - filter_distances( - target_rast::Raster, - dist_buffer - )::Raster - -Apply a mask to exclude pixels that are outside the indicated distance buffer(s). - -`target_rast` and the `dist_buffer` should be in the same CRS (e.g., EPSG:7844 / GDA2020). - -# Arguments -- `target_rast` : Raster of suitable pixels (Bool) to filter pixels from. -- `dist_buffer` : Buffer geometry to use as the mask. - -# Returns -- Masked boolean raster indicating pixels that are within the target distance. -""" -function filter_distances(target_rast::Raster, dist_buffer)::Raster - # Mask out areas outside considered distance from port - return mask(Raster(target_rast; missingval=0); with=dist_buffer) -end - -""" - valid_lonlat_inds(data::DataFrame, criteria::Symbol, lb::T, ub::T) where {T} - -Retrieve the indices of valid data for a region. - -# Arguments -- `data` : -- `criteria` : -- `lb` : -- `ub` : - -# Returns -Tuple{Vector{Int64}, Vector{Int64}}, of lon and lat indices. -""" -function valid_lonlat_inds(data::DataFrame, criteria::Symbol, lb::T, ub::T) where {T} - valid_locs = within_thresholds(data[!, criteria], lb, ub) - - lon_pos = data[valid_locs, :lon_idx] - lat_pos = data[valid_locs, :lat_idx] - - return lon_pos, lat_pos -end - -""" - valid_pixel_positions(data::DataFrame, criteria::Symbol, lb::T, ub::T) where {T} - -Obtain the pixel positions of valid data. - -Intended for use in applications similar to [ImageryLayer - client side pixel filter](https://developers.arcgis.com/javascript/latest/sample-code/layers-imagery-pixelvalues/). - -# Arguments -- `data` : -- `criteria` : -- `lb` : lower bound -- `ub` : upper bound -""" -function valid_pixel_positions(data::DataFrame, criteria::Symbol, lb::T, ub::T) where {T} - lon_pos, lat_pos = valid_lonlat_inds(data, criteria, lb, ub) - pixel_pos = lon_pos .* lat_pos - - return pixel_pos -end - -function _create_filter(bounds::Tuple) - return (x) -> bounds[1] .< x .<= bounds[2] -end - -""" - apply_criteria_thresholds(criteria_stack::RasterStack, lookup::DataFrame, ruleset::Vector{CriteriaBounds{Function}})::Raster - apply_criteria_thresholds(criteria_stack::RasterStack, lookup::DataFrame, ruleset::Dict)::Raster - apply_criteria_thresholds(criteria_stack::RasterStack, lookup::DataFrame, ruleset::NamedTuple)::Raster - -Apply thresholds for each criteria. - -# Arguments -- `criteria_stack` : RasterStack of criteria data for a given region -- `lookup` : Lookup dataframe for the region -- `ruleset` : A set of CriteriaBounds, Dictionary or NamedTuple indicating a mapping of - criteria names to their lower and upper thresholds. - -# Returns -BitMatrix indicating locations within desired thresholds -""" -function apply_criteria_thresholds( - criteria_stack::RasterStack, - lookup::DataFrame, - ruleset::Dict -)::Raster - ruleset = NamedTuple{(keys(ruleset)...,)}( - Tuple(_create_filter.(values(ruleset))) - ) - - return apply_criteria_thresholds(criteria_stack, lookup, ruleset) -end -function apply_criteria_thresholds( - criteria_stack::RasterStack, - lookup::DataFrame, - ruleset::Vector{CriteriaBounds} -)::Raster - ruleset = NamedTuple{([c.name for c in ruleset]...,)}( - Tuple([c.rule for c in ruleset]) - ) - return apply_criteria_thresholds(criteria_stack, lookup, ruleset) -end - -function apply_criteria_thresholds( - criteria_stack::RasterStack, - lookup::DataFrame, - ruleset::NamedTuple -)::Raster - # Result store - data = falses(size(criteria_stack)) - - res_lookup = trues(nrow(lookup)) - for rule_name in keys(ruleset) - res_lookup .= res_lookup .& ruleset[rule_name](lookup[!, rule_name]) - end - - tmp = lookup[res_lookup, [:lon_idx, :lat_idx]] - data[CartesianIndex.(tmp.lon_idx, tmp.lat_idx)] .= true - - res = Raster(criteria_stack.Depth; data=sparse(data), missingval=0) - return res -end - -""" - apply_criteria_lookup( - reg_criteria::OldRegionalCriteria, - rtype::Symbol, - ruleset::Vector{CriteriaBounds{Function}} - ) - -Filter lookup table by applying user defined `ruleset` criteria. - -# Arguments -- `reg_criteria` : OldRegionalCriteria containing valid_rtype lookup table for filtering. -- `rtype` : Flats or slope category for assessment. -- `ruleset` : User defined ruleset for upper and lower bounds. - -# Returns -Filtered lookup table containing points that meet all criteria in `ruleset`. -""" -function apply_criteria_lookup( - reg_criteria::OldRegionalCriteria, - rtype::Symbol, - ruleset -)::DataFrame - lookup = getfield(reg_criteria, Symbol(:valid_, rtype)) - lookup.all_crit .= 1 - - for threshold in ruleset - lookup.all_crit = lookup.all_crit .& threshold.rule(lookup[!, threshold.name]) - end - - lookup = lookup[BitVector(lookup.all_crit), :] - - return lookup -end - -""" - apply_criteria_lookup( - raster_stack::OldRegionalCriteria, - rtype::Symbol, - ruleset::Vector{CriteriaBounds{Function}} - ) - -Filter lookup table by applying user defined `ruleset` criteria. - -# Arguments -- `reg_criteria` : OldRegionalCriteria containing valid_rtype lookup table for filtering. -- `rtype` : Flats or slope category for assessment. -- `ruleset` : User defined ruleset for upper and lower bounds. - -# Returns -Filtered lookup table containing points that meet all criteria in `ruleset`. -""" -function apply_criteria_lookup( - reg_criteria::OldRegionalCriteria, - rtype::Symbol, - ruleset -)::DataFrame - lookup = getfield(reg_criteria, Symbol(:valid_, rtype)) - lookup.all_crit .= 1 - - for threshold in ruleset - lookup.all_crit = lookup.all_crit .& threshold.rule(lookup[!, threshold.name]) - end - - lookup = lookup[BitVector(lookup.all_crit), :] - - return lookup -end - -""" -Filters the slope table (which contains raster param values too) by building a -bit mask AND'd for all thresholds -""" -function apply_criteria_lookup( - slope_table::DataFrame, - ruleset::Vector{CriteriaBounds} -)::DataFrame - slope_table.all_crit .= 1 - - for threshold in ruleset - slope_table.all_crit = - slope_table.all_crit .& threshold.rule(slope_table[!, threshold.name]) - end - - return slope_table[BitVector(slope_table.all_crit), :] -end - -""" - threshold_mask(params :: RegionalAssessmentParameters)::Raster - threshold_mask(reg_criteria, rtype::Symbol, crit_map)::Raster - threshold_mask(reg_criteria, rtype::Symbol, crit_map, lons::Tuple, lats::Tuple)::Raster - -Generate mask for a given region and reef type (slopes or flats) according to thresholds -applied to a set of criteria. - -# Notes -- Zeros indicate locations to mask **out**. -- Ones indicate locations to **keep**. - -# Arguments -- `reg_criteria` : OldRegionalCriteria to assess -- `rtype` : reef type to assess (`:slopes` or `:flats`) -- `crit_map` : List of criteria thresholds to apply (see `apply_criteria_thresholds()`) -- `lons` : Longitudinal extent (min and max, required when generating masks for tiles) -- `lats` : Latitudinal extent (min and max, required when generating masks for tiles) - -# Returns -True/false mask indicating locations within desired thresholds. -""" -function threshold_mask( - reg_criteria::OldRegionalCriteria, - rtype::Symbol, - crit_map::Vector{CriteriaBounds{Function}} -)::Raster - valid_lookup = getfield(reg_criteria, Symbol(:valid_, rtype)) - mask_layer = apply_criteria_thresholds( - reg_criteria.stack, - valid_lookup, - crit_map - ) - - return mask_layer -end -function threshold_mask( - reg_criteria::OldRegionalCriteria, - rtype::Symbol, - crit_map::Vector{CriteriaBounds{Function}}, - lons::Tuple, - lats::Tuple -)::Raster - lookup = getfield(reg_criteria, Symbol(:valid_, rtype)) - - lat1, lat2 = lats[1] > lats[2] ? (lats[2], lats[1]) : (lats[1], lats[2]) - - within_search = ( - (lons[1] .<= lookup.lons .<= lons[2]) .& - (lat1 .<= lookup.lats .<= lat2) - ) - - lookup = lookup[within_search, :] - if nrow(lookup) == 0 - # Exit as there is no data - return Raster(zeros(0, 0); dims=([0], [0])) - end - - # Need to pass in full representation of the raster as the lookup table relies on - # the original Cartesian indices. - res = apply_criteria_thresholds( - reg_criteria.stack, - lookup, - crit_map - ) - - # Extract data between lon/lats - view_of_data = view(res, X(lons[1] .. lons[2]), Y(lat1 .. lat2)) - return rebuild(view_of_data, sparse(convert.(UInt8, view_of_data))) -end - -""" -Handles threshold masking using the integrated assessment parameter struct -""" -function threshold_mask( - params::RegionalAssessmentParameters -)::Raster - # Builds out a set of criteria filters using the regional criteria. - # NOTE this will only filter over available criteria - filters = build_criteria_bounds_from_regional_criteria(params.regional_criteria) - - # map our regional criteria - @debug "Applying criteria thresholds to generate mask layer" - mask_layer = apply_criteria_thresholds( - # This is the raster stack - params.region_data.raster_stack, - # The slope table dataframe - params.region_data.slope_table, - # The list of criteria bounds - filters - ) - - return mask_layer -end - -""" - generate_criteria_mask!(fn::String, rst_stack::RasterStack, lookup::DataFrame, ruleset::Vector{CriteriaBounds{Function}}) - -Generate mask file for a given region and reef type (slopes or flats) according to thresholds -applied to a set of criteria. - -# Notes -- Zeros indicate locations to mask **out**. -- Ones indicate locations to **keep**. - -# Arguments -- `fn` : File to write geotiff to -- `reg_criteria` : OldRegionalCriteria to assess -- `rtype` : reef type to assess (`:slopes` or `:flats`) -- `crit_map` : List of criteria thresholds to apply (see `apply_criteria_thresholds()`) - -# Returns -Nothing -""" -function generate_criteria_mask!( - fn::String, - rst_stack::RasterStack, - lookup::DataFrame, - ruleset::Vector{CriteriaBounds{Function}} -)::Nothing - # Create the geotiff - res = spzeros(size(rst_stack)) - tmp_rst = Raster( - rst_stack[names(rst_stack)[1]]; - data=res, - missingval=0.0 - ) - - res_lookup = trues(nrow(lookup)) - for threshold in ruleset - res_lookup .= res_lookup .& threshold.rule(lookup[!, threshold.name]) - end - - tmp = lookup[res_lookup, [:lon_idx, :lat_idx]] - tmp_rst[CartesianIndex.(tmp.lon_idx, tmp.lat_idx)] .= 1.0 - - write( - fn, - UInt8.(tmp_rst); - ext=".tiff", - source="gdal", - driver="COG", # GTiff - options=Dict{String,String}( - "COMPRESS" => "LZW", - "SPARSE_OK" => "TRUE", - "OVERVIEW_COUNT" => "5" - ) - ) - - return nothing -end diff --git a/src/criteria_assessment/regional_assessment.jl b/src/criteria_assessment/regional_assessment.jl deleted file mode 100644 index bf942db..0000000 --- a/src/criteria_assessment/regional_assessment.jl +++ /dev/null @@ -1,235 +0,0 @@ -""" -DEPRECATED: To remove once jobs are migrated to new system fully -""" - -# HTTP response headers for COG files -const COG_HEADERS = [ - "Cache-Control" => "max-age=86400, no-transform" -] - -function setup_job_routes(config, auth) - reg_assess_data = get_regional_data(config) - - @get auth("/job/details/{job_id}") function (req::Request, job_id::String) - srv = DiskService(_cache_location(config)) - return json(job_details(srv, job_id)) - end - - @get auth("/job/result/{job_id}") function (req::Request, job_id::String) - srv = DiskService(_cache_location(config)) - return file(job_result(srv, job_id)) - end - - @get auth("/submit/region-assess/{reg}/{rtype}") function ( - req::Request, reg::String, rtype::String - ) - # 127.0.0.1:8000/submit/region-assess/Mackay-Capricorn/slopes?Depth=-12.0:-2.0&Slope=0.0:40.0&Rugosity=0.0:6.0&SuitabilityThreshold=95 - qp = queryparams(req) - srv = DiskService(_cache_location(config)) - job_id = create_job_id(qp) * "$(reg)_suitable" - - details = job_details(srv, job_id) - job_state = job_status(details) - if (job_state == "no job") && (job_state != "processing") - @debug "$(now()) : Submitting $(job_id)" - assessed_fn = cache_filename( - extract_criteria(qp, suitability_criteria()), - config, - "$(reg)_suitable", - "tiff" - ) - - details = submit_job(srv, job_id, assessed_fn) - - # Do job asyncronously... - @async assess_region( - config, - qp, - reg, - rtype, - reg_assess_data - ) - end - - if job_state == "processing" - @debug details - end - - @debug "$(now()) : Job submitted, return polling url" - return json(details.access_url) - end - - @get auth("/submit/site-assess/{reg}/{rtype}") function ( - req::Request, reg::String, rtype::String - ) - qp = queryparams(req) - suitable_sites_fn = cache_filename( - qp, config, "$(reg)_potential_sites", "geojson" - ) - - srv = DiskService(_cache_location(config)) - job_id = create_job_id(qp) * "$(reg)_potential_sites" - - details = job_details(srv, job_id) - job_state = job_status(details) - if (job_state != "no job") && (job_state != "completed") - @debug "$(now()) : Submitting $(job_id)" - details = submit_job(srv, job_id, suitable_sites_fn) - - # Do job asyncronously... - @async begin - assessed_fn = assess_region( - config, - qp, - reg, - rtype, - reg_assess_data - ) - - assessed = Raster(assessed_fn; missingval=0) - - # Extract criteria and assessment - pixel_criteria = extract_criteria(qp, search_criteria()) - deploy_site_criteria = extract_criteria(qp, site_criteria()) - - best_sites = filter_sites( - assess_sites( - reg_assess_data, reg, rtype, pixel_criteria, deploy_site_criteria, - assessed - ) - ) - - # Specifically clear from memory to invoke garbage collector - assessed = nothing - - if nrow(best_sites) == 0 - open(suitable_sites_fn, "w") do f - JSON.print(f, nothing) - end - else - output_geojson(suitable_sites_fn, best_sites) - end - - details.status = "completed" - update_job!(srv, job_id) - end - end - - @debug "$(now()) : Job submitted, return polling url" - return json(details.access_url) - end -end - -""" - setup_region_routes(config, auth) - -Set up endpoints for regional assessment. -""" -function setup_region_routes(config, auth) - reg_assess_data = get_regional_data(config) - - @get auth("/assess/{reg}/{rtype}") function (req::Request, reg::String, rtype::String) - qp = queryparams(req) - mask_path = cache_filename(qp, config, "", "tiff") - if isfile(mask_path) - return file(mask_path; headers=COG_HEADERS) - end - - # Otherwise, create the file - @debug "$(now()) : Assessing criteria" - mask_data = mask_region(reg_assess_data, reg, qp, rtype) - - @debug "$(now()) : Running on thread $(threadid())" - @debug "Writing to $(mask_path)" - # Writing time: ~10-25 seconds - _write_cog(mask_path, UInt8.(mask_data), config) - - return file(mask_path; headers=COG_HEADERS) - end - - @get auth("/suitability/assess/{reg}/{rtype}") function ( - req::Request, reg::String, rtype::String - ) - # somewhere:8000/suitability/assess/region-name/reeftype?criteria_names=Depth,Slope&lb=-9.0,0.0&ub=-2.0,40 - # 127.0.0.1:8000/suitability/assess/Cairns-Cooktown/slopes?Depth=-4.0:-2.0&Slope=0.0:40.0&Rugosity=0.0:6.0 - # 127.0.0.1:8000/suitability/assess/Cairns-Cooktown/slopes?Depth=-4.0:-2.0&Slope=0.0:40.0&Rugosity=0.0:6.0&SuitabilityThreshold=95 - # 127.0.0.1:8000/suitability/assess/Mackay-Capricorn/slopes?Depth=-12.0:-2.0&Slope=0.0:40.0&Rugosity=0.0:6.0&SuitabilityThreshold=95 - qp = queryparams(req) - assessed_fn = assess_region(config, qp, reg, rtype, reg_assess_data) - - return file(assessed_fn; headers=COG_HEADERS) - end - - @get auth("/suitability/site-suitability/{reg}/{rtype}") function ( - req::Request, reg::String, rtype::String - ) - # 127.0.0.1:8000/suitability/site-suitability/Cairns-Cooktown/slopes?Depth=-4.0:-2.0&Slope=0.0:40.0&Rugosity=0.0:6.0&SuitabilityThreshold=95&xdist=450&ydist=50 - # 127.0.0.1:8000/suitability/site-suitability/Mackay-Capricorn/slopes?Depth=-12.0:-2.0&Slope=0.0:40.0&Rugosity=0.0:6.0&SuitabilityThreshold=95&xdist=100&ydist=100 - qp = queryparams(req) - suitable_sites_fn = cache_filename( - qp, config, "$(reg)_potential_sites", "geojson" - ) - if isfile(suitable_sites_fn) - return file(suitable_sites_fn) - end - - # Assess location suitability if needed - assessed_fn = assess_region(config, qp, reg, rtype, reg_assess_data) - assessed = Raster(assessed_fn; lazy=true, missingval=0) - - # Extract criteria and assessment - pixel_criteria = extract_criteria(qp, search_criteria()) - deploy_site_criteria = extract_criteria(qp, site_criteria()) - - best_sites = filter_sites( - assess_sites( - reg_assess_data, reg, rtype, pixel_criteria, deploy_site_criteria, assessed - ) - ) - - # Specifically clear from memory to invoke garbage collector - assessed = nothing - - if nrow(best_sites) == 0 - open(suitable_sites_fn, "w") do f - JSON.print(f, nothing) - end - else - output_geojson(suitable_sites_fn, best_sites) - end - - return file(suitable_sites_fn) - end - - """Obtain the spatial bounds for a given region of interest""" - @get auth("/bounds/{reg}") function (req::Request, reg::String) - rst_stack = reg_assess_data[reg].stack - - return json(Rasters.bounds(rst_stack)) - end - - # Form for testing/dev - # https:://somewhere:8000/suitability/assess/region-name/reeftype?criteria_names=Depth,Slope&lb=-9.0,0.0&ub=-2.0,40 - @get "/" function () - return html(""" -
-
-
- -
-

- -
-

- - -
- """) - end - - # Parse the form data and return it - @post auth("/form") function (req) - data = formdata(req) - return data - end -end diff --git a/src/criteria_assessment/site_identification.jl b/src/criteria_assessment/site_identification.jl deleted file mode 100644 index edac031..0000000 --- a/src/criteria_assessment/site_identification.jl +++ /dev/null @@ -1,456 +0,0 @@ -"""Methods for identifying potential deployment locations.""" - -""" - proportion_suitable( - x::Union{BitMatrix,SparseMatrixCSC{Bool,Int64}}; square_offset::Tuple=(-4, 5) - )::SparseMatrixCSC{UInt8,Int64} - -Calculate the the proportion of the subsection that is suitable for deployments. -The `subsection` is the surrounding a rough hectare area centred on each cell of a raster -marked as being suitable according to user-selected criteria. - -Cells on the edges of a raster object are assessed using a smaller surrounding area, rather -than shifting the window inward. In usual applications, there will be no target pixel -close to the edge due to the use of buffer areas. - -# Arguments -- `x` : Matrix of boolean pixels after filtering with user criteria. -- `square_offset` : The number of pixels +/- around a center "target" pixel to assess as the - moving window. Defaults to (-4, 5). - Assuming a 10m² pixel, the default `square_offset` resolves to a one - hectare area. - -# Returns -Matrix of values 0 - 100 indicating the percentage of the area around the target pixel that -meet suitability criteria. -""" -function proportion_suitable( - x::Union{BitMatrix,SparseMatrixCSC{Bool,Int64}}; square_offset::Tuple=(-4, 5) -)::SparseMatrixCSC{UInt8,Int64} - subsection_dims = size(x) - target_area = spzeros(UInt8, subsection_dims) - - for row_col in findall(x) - (row, col) = Tuple(row_col) - x_left = max(col + square_offset[1], 1) - x_right = min(col + square_offset[2], subsection_dims[2]) - - y_top = max(row + square_offset[1], 1) - y_bottom = min(row + square_offset[2], subsection_dims[1]) - - target_area[row, col] = UInt8(sum(@views x[y_top:y_bottom, x_left:x_right])) - end - - return target_area -end - -""" - filter_distances( - target_rast::Raster, - gdf::DataFrame, - dist_nm - )::Raster - -Exclude pixels in `target_rast` that are beyond `dist_nm` (nautical miles) from a geometry -in `gdf`. `target_rast` and `gdf` should be in the same CRS (EPSG:7844 / GDA2020 for GBR-reef-guidance-assessment). - -# Arguments -- `target_rast` : Boolean raster of suitable pixels to filter. -- `gdf` : DataFrame with `geometry` column that contains vector objects of interest. -- `dist_nm` : Filtering distance from geometry object in nautical miles. - -# Returns -- `tmp_areas` : Raster of filtered pixels containing only pixels within target distance -from a geometry centroid. -""" -function filter_distances( - target_rast::Raster, gdf::DataFrame, dist; units::String="NM" -)::Raster - tmp_areas = copy(target_rast) - - # First dimension is the rows (latitude) - # Second dimension is the cols (longitude) - raster_lat = Vector{Float64}(tmp_areas.dims[1].val) - raster_lon = Vector{Float64}(tmp_areas.dims[2].val) - - @floop for row_col in ThreadsX.findall(tmp_areas) - (lat_ind, lon_ind) = Tuple(row_col) - point = AG.createpoint() - - lon = raster_lon[lon_ind] - lat = raster_lat[lat_ind] - AG.addpoint!(point, lon, lat) - - pixel_dists = AG.distance.([point], port_locs.geometry) - geom_point = gdf[argmin(pixel_dists), :geometry] - geom_point = (AG.getx(geom_point, 0), AG.gety(geom_point, 0)) - - dist_nearest = Distances.haversine(geom_point, (lon, lat)) - - # Convert from meters to nautical miles - if units == "NM" - dist_nearest = dist_nearest / 1852 - end - - # Convert from meters to kilometers - if units == "km" - dist_nearest = dist_nearest / 1000 - end - - tmp_areas.data[lon_ind, lat_ind] = dist_nearest < dist ? 1 : 0 - end - - return tmp_areas -end - -""" - mask_region(reg_assess_data, reg, qp, rtype) - -# Arguments -- `reg_assess_data` : Regional assessment data -- `reg` : The region name to assess -- `qp` : query parameters -- `rtype` : region type (one of `:slopes` or `:flats`) - -# Returns -Raster of region with locations that meet criteria masked. -""" -function mask_region(reg_assess_data, reg, qp, rtype) - criteria_names, lbs, ubs = remove_rugosity(reg, parse_criteria_query(qp)...) - - # Otherwise, create the file - @debug "$(now()) : Masking area based on criteria" - mask_data = threshold_mask( - reg_assess_data[reg], - Symbol(rtype), - CriteriaBounds.(criteria_names, lbs, ubs) - ) - - return mask_data -end - -""" -# Arguments -- params::RegionalAssessmentParameters - parameters needed to perform assessment - -# Returns -Raster of region with locations that meet criteria masked. -""" -function mask_region(params::RegionalAssessmentParameters) - @debug "$(now()) : Masking area based on criteria" - mask_data = threshold_mask( - params - ) - - return mask_data -end - -""" - lookup_assess_region(reg_assess_data, reg, qp, rtype; x_dist=100.0, y_dist=100.0) - -Perform suitability assessment with the lookup table based on user-defined criteria. -This is currently orders of magnitude slower than the raster-based approach, although -it uses significantly less memory. - -# Arguments -- `reg_assess_data` : Dictionary containing the regional data paths, reef outlines and full region names. -- `reg` : Name of the region being assessed (format `Cairns-Cooktown` rather than `Cairns/Cooktown Management Area`). -- `qp` : Dict containing bounds for each variable being filtered. -- `rtype` : Type of zone to assess (flats or slopes). -- `x_dist` : width of search polygon -- `y_dist` : height of search polygon - -# Returns -Raster of surrounding hectare suitability (1-100%) based on the criteria bounds input -by a user. -""" -function lookup_assess_region(reg_assess_data, reg, qp, rtype; x_dist=100.0, y_dist=100.0) - criteria_names, lbs, ubs = remove_rugosity(reg, parse_criteria_query(qp)...) - - assess_locs = getfield(reg_assess_data[reg], Symbol("valid_$(rtype)")) - - # Filter look up table down to locations that meet criteria - sel = true - for (crit, lb, ub) in zip(criteria_names, lbs, ubs) - lb = parse(Float32, lb) - ub = parse(Float32, ub) - - sel = sel .& (lb .<= assess_locs[:, crit] .<= ub) - end - - assess_locs = copy(assess_locs[sel, :]) - - target_crs = crs(assess_locs.geometry[1]) - if isnothing(target_crs) - target_crs = EPSG(4326) - end - - # Estimate maximum number of pixels in the target area - res = degrees_to_meters(step(dims(reg_assess_data[reg].stack)[1]), assess_locs.lats[1]) - max_count = floor(x_dist * y_dist / (res * res)) - - assess_locs[:, :suitability_score] .= 0.0 - - @debug "Assessing target area for $(nrow(assess_locs)) locations" - search_box = initial_search_box( - (assess_locs[1, :lons], assess_locs[1, :lats]), # center point - x_dist, # x distance in meters - y_dist, # y distance in meters - target_crs - ) - - # Create KD-tree to identify `n` nearest pixels - # Using the lookup method for very large numbers of locations is prohibitively slow - # hence why we use a KD-tree - lon_lats = Matrix(assess_locs[:, [:lons, :lats]])' - kdtree = KDTree(lon_lats; leafsize=25) - @debug "$(now()) : Assessing suitability for $(nrow(assess_locs)) locations" - for (i, coords) in enumerate(eachcol(lon_lats)) - # Retrieve the closest valid pixels - idx, _ = knn(kdtree, coords, ceil(Int64, max_count)) - - sb = move_geom(search_box, Tuple(coords)) - assess_locs[i, :suitability_score] = floor( - Int64, - (count(GO.contains.(Ref(sb), assess_locs[idx, :geometry])) / max_count) * 100 - ) - end - @debug "$(now()) : Finished suitability assessment" - - return assess_locs -end - -""" - assess_region(reg_assess_data, reg, qp, rtype) - -Perform raster suitability assessment based on user-defined criteria. - -# Arguments -- params :: RegionalAssessmentParameters - -# Returns -GeoTiff file of surrounding hectare suitability (1-100%) based on the criteria bounds input -by a user. -""" -function assess_region(params::RegionalAssessmentParameters)::Raster - # Make mask of suitable locations - @debug "$(now()) : Creating mask for region" - mask_data = mask_region(params::RegionalAssessmentParameters) - - # Assess remaining pixels for their suitability - @debug "$(now()) : Calculating proportional suitability score" - suitability_scores = proportion_suitable(mask_data.data) - - @debug "$(now()) : Rebuilding raster and returning results" - return rebuild(mask_data, suitability_scores) -end - -""" - assess_region(reg_assess_data, reg, qp, rtype) - -Perform raster suitability assessment based on user-defined criteria. - -# Arguments -- `reg_assess_data` : Dictionary containing the regional data paths, reef outlines and \ - full region names. -- `reg` : Name of the region being assessed (format `Cairns-Cooktown` rather than \ - `Cairns/Cooktown Management Area`). -- `qp` : Dict containing bounds for each variable being filtered. -- `rtype` : Type of zone to assess (flats or slopes). - -# Returns -GeoTiff file of surrounding hectare suitability (1-100%) based on the criteria bounds input -by a user. -""" -function assess_region(reg_assess_data, reg, qp, rtype)::Raster - # Make mask of suitable locations - @debug "$(now()) : Creating mask for region" - mask_data = mask_region(reg_assess_data, reg, qp, rtype) - - # Assess remaining pixels for their suitability - @debug "$(now()) : Calculating proportional suitability score" - suitability_scores = proportion_suitable(mask_data.data) - - @debug "$(now()) : Rebuilding raster and returning results" - return rebuild(mask_data, suitability_scores) -end - -""" - assess_region(config, qp::Dict, reg::String, rtype::String, reg_assess_data::OrderedDict) - -Convenience method wrapping around the analysis conducted by `assess_region()`. -Checks for previous assessment of indicated region and returns filename of cache if found. -If corresponding job is found, wait for results. - -""" -function assess_region( - config, qp::Dict, reg::String, rtype::String, reg_assess_data::OrderedDict -) - assessed_fn = cache_filename( - extract_criteria(qp, suitability_criteria()), config, "$(reg)_suitable", "tiff" - ) - if isfile(assessed_fn) - return assessed_fn - end - - srv = DiskService(_cache_location(config)) - job_id = create_job_id(qp) * "$(reg)_suitable" - - job_state = job_status(srv, job_id) - if (job_state != "no job") && (job_state != "completed") && (job_state != "error") - @debug "$(now()) : Waiting for $(reg) job to finish : ($(job_id))" - # Job exists, wait for job to finish - wait_time = 20.0 # seconds - max_wait = 60.0 # max time to wait per loop - - while true - st = job_status(srv, job_id) - if st ∈ ["completed", "error"] - break - end - - sleep(wait_time) - - # Exponential backoff (increase wait time every loop) - wait_time = min(wait_time * 2.0, max_wait) - end - - if job_status(srv, job_id) == "error" - throw(ArgumentError("Job $(job_id) errored.")) - end - else - @debug "$(now()) : Submitting job for $(reg)" - job_details = submit_job(srv, job_id, assessed_fn) - - @debug "$(now()) : Assessing region $(reg)" - assessed = assess_region(reg_assess_data, reg, qp, rtype) - - @debug "$(now()) : Writing to $(assessed_fn)" - _write_tiff(assessed_fn, assessed) - - @debug "$(now()) : Marking job for $(reg) as completed" - job_details.status = "completed" - update_job!(srv, job_id, job_details) - end - - return assessed_fn -end - -""" - assess_sites( - reg_assess_data::OrderedDict, - reg::String, - rtype::String, - pixel_criteria::Dict, - site_criteria::Dict, - assess_locs::Raster - ) - -# Arguments -- `reg_assess_data` : Regional assessment data -- `reg` : Short region name -- `rtype` : Slopes or Flats assessment type -- `pixel_criteria` : parameters to assess specific locations with -- `site_criteria` : parameters to assess sites based on their polygonal representation -- `assess_locs` : Raster of suitability scores for each valid pixel - -# Returns -GeoDataFrame of all potential sites -""" -function assess_sites( - reg_assess_data::OrderedDict, - reg::String, - rtype::String, - pixel_criteria::Dict, - site_criteria::Dict, - assess_locs::Raster -) - target_crs = convert(EPSG, crs(assess_locs)) - suitability_threshold = parse(Int64, site_criteria["SuitabilityThreshold"]) - - @debug "$(now()) : Identifying search pixels for $(reg)" - target_locs = search_lookup(assess_locs, suitability_threshold) - - if size(target_locs, 1) == 0 - # No viable set of locations, return empty dataframe - return DataFrame(; - score=[], - orientation=[], - qc_flag=[], - geometry=[] - ) - end - - criteria_names, lbs, ubs = remove_rugosity(reg, parse_criteria_query(pixel_criteria)...) - - # Otherwise, create the file - @debug "$(now()) : Assessing criteria table for $(reg)" - crit_pixels::DataFrame = apply_criteria_lookup( - reg_assess_data[reg], - Symbol(rtype), - CriteriaBounds.(criteria_names, lbs, ubs) - ) - - res = abs(step(dims(assess_locs, X))) - x_dist = parse(Int64, site_criteria["xdist"]) - y_dist = parse(Int64, site_criteria["ydist"]) - @debug "$(now()) : Assessing $(size(target_locs, 1)) candidate locations in $(reg)." - @debug "Finding optimal site alignment" - initial_polygons = find_optimal_site_alignment( - crit_pixels, - target_locs, - res, - x_dist, - y_dist, - target_crs - ) - - return initial_polygons -end - -function assess_sites( - params::SuitabilityAssessmentParameters, - regional_raster::Raster -) - target_crs = convert(EPSG, crs(regional_raster)) - suitability_threshold = params.suitability_threshold - region = params.region - - @debug "$(now()) : Identifying search pixels for $(region)" - target_locs = search_lookup(regional_raster, suitability_threshold) - - if size(target_locs, 1) == 0 - # No viable set of locations, return empty dataframe - return DataFrame(; - score=[], - orientation=[], - qc_flag=[], - geometry=[] - ) - end - - # Otherwise, create the file - @debug "$(now()) : Assessing criteria table for $(region)" - # Get criteria bounds list from criteria - filters = build_criteria_bounds_from_regional_criteria(params.regional_criteria) - - crit_pixels::DataFrame = apply_criteria_lookup( - # Slope table - params.region_data.slope_table, - filters - ) - - res = abs(step(dims(regional_raster, X))) - @debug "$(now()) : Assessing $(size(target_locs, 1)) candidate locations in $(region)." - @debug "Finding optimal site alignment" - initial_polygons = find_optimal_site_alignment( - crit_pixels, - target_locs, - res, - params.x_dist, - params.y_dist, - target_crs - ) - - return initial_polygons -end diff --git a/src/criteria_assessment/tiles.jl b/src/criteria_assessment/tiles.jl deleted file mode 100644 index 664f9a3..0000000 --- a/src/criteria_assessment/tiles.jl +++ /dev/null @@ -1,274 +0,0 @@ -"""Helper methods to support tiling.""" - -using Images, ImageIO, Interpolations - -# HTTP response headers for tile images -const TILE_HEADERS = [ - "Cache-Control" => "max-age=86400, no-transform" -] - -""" - tile_size(config::Dict)::Tuple - -Retrieve the configured size of map tiles in pixels (width and height / lon and lat). -""" -function tile_size(config::Dict)::Tuple - tile_dims = try - res = parse(Int, config["server_config"]["TILE_SIZE"]) - (res, res) - catch - (256, 256) # 256x256 - end - - return tile_dims -end - -""" - _tile_to_lon_lat(z::T, x::T, y::T) where {T<:Int64} - -Obtain lon/lat of top-left corner of a requested tile. - -# Returns -lon, lat -""" -function _tile_to_lon_lat(z::T, x::T, y::T) where {T<:Int64} - n = 2.0^z - lon = x / n * 360.0 - 180.0 - lat_rad = atan(sinh(π * (1 - 2 * y / n))) - lat = rad2deg(lat_rad) - - return (lon, lat) -end - -""" - _tile_bounds(z::T, x::T, y::T) where {T<:Int64} - -Obtain lon/lat bounds of a requested tile. - -# Returns -West, East, North South (min lon, max lon, lat max, lat min) -""" -function _tile_bounds(z::T, x::T, y::T) where {T<:Int64} - # Calculate the boundaries of the tile - n = 2.0^z - lat_min = atan(sinh(π * (1 - 2 * (y + 1) / n))) * 180.0 / π - lat_max = atan(sinh(π * (1 - 2 * y / n))) * 180.0 / π - lon_min = x / n * 360.0 - 180.0 - lon_max = (x + 1) / n * 360.0 - 180.0 - - # West, East, North, South - return lon_min, lon_max, lat_max, lat_min -end - -""" - _lon_lat_to_tile(zoom, lon, lat) - -Identify the corresponding tile coordinates for a given lon/lat. - -# Returns -x and y tile coordinates -""" -function _lon_lat_to_tile(zoom, lon, lat) - n = 2.0^zoom - x = floor(Int64, (lon + 180.0) / 360.0 * n) - - lat_rad = lat * π / 180.0 - y = floor(Int64, (1.0 - log(tan(lat_rad) + 1.0 / cos(lat_rad)) / π) / 2.0 * n) - - return x, y -end - -# Helper functions for Web Mercator projection -_lat_to_y(lat::Float64) = log(tan(π / 4 + lat * π / 360)) -_y_to_lat(y::Float64) = 360 / π * atan(exp(y)) - 90 - -""" - adjusted_nearest(rst::Raster, z::Int, x::Int, y::Int, tile_size::Tuple{Int,Int}, orig_rst_size::Tuple{Int,Int})::Matrix - -Resample a raster using nearest neighbor interpolation when the tile includes area outside -where data exists (e.g., viewing the globe where the data may appear in a small corner of -the tile). This approach attempts to account for planetary curvature while still maintaining -some performance. - -# Arguments -- `rst`: The input raster to be resampled. -- `z`: Tile zoom level requested. -- `x`: x coordinate for requested tile. -- `y`: y coordinate for the requested tile. -- `tile_size`: The desired dimensions of the tile (long, lat). - -# Returns -Matrix with the resampled data. -""" -function adjusted_nearest( - rst::Raster, - z::Int, - x::Int, - y::Int, - tile_size::Tuple{Int,Int} -)::Matrix - # Bounds for the requested tile - (t_lon_min, t_lon_max, t_lat_max, t_lat_min) = _tile_bounds(z, x, y) - - # Bounds for the area of interest (AOI; where we have data) - ((aoi_lon_min, aoi_lon_max), (aoi_lat_min, aoi_lat_max)) = Rasters.bounds(rst) - - # Create an empty tile (long/lat) - long_size, lat_size = tile_size - tile = zeros(lat_size, long_size) - - # Generate longitude and latitude arrays for the tile - lons = @. mod( - t_lon_min + (t_lon_max - t_lon_min) * ((1:long_size) - 1) / (long_size - 1) + 180, - 360 - ) - 180 - lats = @. _y_to_lat( - _lat_to_y(t_lat_max) - - (_lat_to_y(t_lat_max) - _lat_to_y(t_lat_min)) * ((1:lat_size) - 1) / (lat_size - 1) - ) - - # Determine which points are within the area of interest - in_lons = aoi_lon_min .<= lons .<= aoi_lon_max - in_lats = aoi_lat_min .<= lats .<= aoi_lat_max - - # Sample data that is within area of interest - for (i, lon) in enumerate(lons), (j, lat) in enumerate(lats) - if in_lons[i] && in_lats[j] - x_idx = - round( - Int, - (lon - aoi_lon_min) / (aoi_lon_max - aoi_lon_min) * (size(rst, 1) - 1) - ) + 1 - y_idx = - round( - Int, - (_lat_to_y(aoi_lat_max) - _lat_to_y(lat)) / - (_lat_to_y(aoi_lat_max) - _lat_to_y(aoi_lat_min)) * (size(rst, 2) - 1) - ) + 1 - x_idx = clamp(x_idx, 1, size(rst, 1)) - y_idx = clamp(y_idx, 1, size(rst, 2)) - tile[j, i] = rst[x_idx, y_idx] - end - end - - return tile -end - -function setup_tile_routes(config, auth) - @get auth("/to-tile/{zoom}/{lon}/{lat}") function ( - req::Request, zoom::Int64, lon::Float64, lat::Float64 - ) - x, y = _lon_lat_to_tile(zoom, lon, lat) - return json(Dict(:x => x, :y => y)) - end - - @get auth("/to-lonlat/{zoom}/{x}/{y}") function ( - req::Request, zoom::Int64, x::Int64, y::Int64 - ) - lon_min, lon_max, lat_max, lat_min = _tile_bounds(zoom, x, y) - return json( - Dict( - :lon_min => lon_min, - :lon_max => lon_max, - :lat_max => lat_max, - :lat_min => lat_min - ) - ) - end - - reg_assess_data = get_regional_data(config) - @get auth("/tile/{z}/{x}/{y}") function (req::Request, z::Int64, x::Int64, y::Int64) - # http://127.0.0.1:8000/tile/{z}/{x}/{y}?region=Cairns-Cooktown&rtype=slopes&Depth=-9.0:0.0&Slope=0.0:40.0&Rugosity=0.0:3.0 - # http://127.0.0.1:8000/tile/8/231/139?region=Cairns-Cooktown&rtype=slopes&Depth=-9.0:0.0&Slope=0.0:40.0&Rugosity=0.0:3.0 - # http://127.0.0.1:8000/tile/7/115/69?region=Cairns-Cooktown&rtype=slopes&Depth=-9.0:0.0&Slope=0.0:40.0&Rugosity=0.0:3.0 - # http://127.0.0.1:8000/tile/8/231/139?region=Cairns-Cooktown&rtype=slopes&Depth=-9.0:0.0&Slope=0.0:40.0&Rugosity=0.0:3.0 - - qp = queryparams(req) - mask_path = cache_filename(qp, config, "tile-zxy_$(z)_$(x)_$(y)", "png") - if isfile(mask_path) - return file(mask_path; headers=TILE_HEADERS) - end - - # Otherwise, create the file - thread_id = Threads.threadid() - @debug "Thread $(thread_id) - $(now()) : Assessing criteria" - # Filtering time: 0.6 - 7.0 seconds - reg = qp["region"] - rtype = qp["rtype"] - - # Calculate tile bounds - lon_min, lon_max, lat_max, lat_min = _tile_bounds(z, x, y) - lons, lats = (lon_min, lon_max), (lat_max, lat_min) - @debug "Thread $(thread_id) - $(now()) : Calculated bounds (z/x/y, lon bounds, lat bounds): $z $x $y | $(_tile_to_lon_lat(z, x, y)) | ($(lon_min), $(lon_max)), ($(lat_min), $(lat_max))" - - # Check if request is within target region - # return empty tile if not. - lookup = getfield(reg_assess_data[reg], Symbol(:valid_, rtype)) - lat1, lat2 = lats[1] > lats[2] ? (lats[2], lats[1]) : (lats[1], lats[2]) - - within_search = ( - (lons[1] .<= lookup.lons .<= lons[2]) .& - (lat1 .<= lookup.lats .<= lat2) - ) - - if !any(within_search) - no_data_path = cache_filename( - Dict("no_data" => "none"), config, "no_data", "png" - ) - - @debug "Thread $(thread_id) - No data for $reg ($rtype) at $z/$x/$y" - return file(no_data_path; headers=TILE_HEADERS) - end - - # TODO: Re-use pre-existing cache for entire region if available - # Get mask data if available - # assessed_fn = cache_filename( - # extract_criteria(qp, suitability_criteria()), config, "$(reg)_suitable", "tiff" - # ) - - # TODO: Use previously generated results - # assessed_fn = assess_region(config, qp, reg, rtype, reg_assess_data) - # if isfile(assessed_fn) - # # assessed = Raster(assessed_fn; lazy=true, missingval=0) - # # Load regional data, subset to area of interest and assess - # mask_data = - # else - # # Otherwise, assess the target area directly - # end - - criteria_names, lbs, ubs = remove_rugosity(reg, parse_criteria_query(qp)...) - - # Extract relevant data based on tile coordinates - @debug "Thread $(thread_id) - $(now()) : Extracting tile data" - mask_data = threshold_mask( - reg_assess_data[reg], - Symbol(rtype), - CriteriaBounds.(criteria_names, lbs, ubs), - (lon_min, lon_max), - (lat_min, lat_max) - ) - - @debug "Thread $(thread_id) - Extracted data size: $(size(mask_data))" - - @debug "Thread $(thread_id) - $(now()) : Creating PNG (with transparency)" - img = zeros(RGBA, tile_size(config)) - if (z < 12) - # Account for geographic positioning when zoomed out further than - # raster area - resampled = adjusted_nearest(mask_data, z, x, y, tile_size(config)) - else - # Zoomed in close so less need to account for curvature - # BSpline(Constant()) is equivalent to nearest neighbor. - # See details in: https://juliaimages.org/ImageTransformations.jl/stable/reference/#Low-level-warping-API - resampled = imresize( - mask_data.data', tile_size(config); method=BSpline(Constant()) - ) - end - - img[resampled .== 1] .= RGBA(0, 0, 0, 1) - - @debug "Thread $(thread_id) - $(now()) : Saving and serving file" - save(mask_path, img) - return file(mask_path; headers=TILE_HEADERS) - end -end diff --git a/src/file_io.jl b/src/file_io.jl deleted file mode 100644 index 4dafe7e..0000000 --- a/src/file_io.jl +++ /dev/null @@ -1,115 +0,0 @@ -"""Methods for common file I/O.""" - -""" - _cache_location(config::Dict)::String - -Retrieve cache location for geotiffs. -""" -function _cache_location(config::Dict)::String - cache_loc = try - in_debug = haskey(config["server_config"], "DEBUG_MODE") - if in_debug && lowercase(config["server_config"]["DEBUG_MODE"]) == "true" - if "DEBUG_CACHE_DIR" ∉ keys(ENV) - ENV["DEBUG_CACHE_DIR"] = mktempdir() - end - - ENV["DEBUG_CACHE_DIR"] - else - config["server_config"]["TIFF_CACHE_DIR"] - end - catch err - @warn "Encountered error:" err - if "DEBUG_CACHE_DIR" ∉ keys(ENV) - ENV["DEBUG_CACHE_DIR"] = mktempdir() - end - - ENV["DEBUG_CACHE_DIR"] - end - - return cache_loc -end - -""" - cache_filename(qp::Dict, config::Dict, suffix::String, ext::String) - -Generate a filename for a cache. - -# Arguments -- `qp` : Query parameters to hash -- `config` : app configuration (to extract cache parent directory from) -- `suffix` : a suffix to use in the filename (pass `""` if none required) -- `ext` : file extension to use -""" -function cache_filename(qp::Dict, config::Dict, suffix::String, ext::String) - file_id = create_job_id(qp) - temp_path = _cache_location(config) - cache_file_path = joinpath(temp_path, "$(file_id)$(suffix).$(ext)") - - return cache_file_path -end - -""" - n_gdal_threads(config::Dict)::String - -Retrieve the configured number of threads to use when writing COGs with GDAL. -""" -function n_gdal_threads(config::Dict)::String - n_cog_threads = try - config["server_config"]["COG_THREADS"] - catch - "1" # Default to using a single thread for GDAL write - end - - return n_cog_threads -end - -""" - _write_cog(file_path::String, data::Raster, config::Dict)::Nothing - -Write out a COG using common options. - -# Arguments -- `file_path` : Path to write data out to -- `data` : Raster data to write out -""" -function _write_cog(file_path::String, data::Raster, config::Dict)::Nothing - Rasters.write( - file_path, - data; - ext=".tiff", - source="gdal", - driver="COG", - options=Dict{String,String}( - "COMPRESS" => "DEFLATE", - "SPARSE_OK" => "TRUE", - "OVERVIEW_COUNT" => "5", - "BLOCKSIZE" => string(first(tile_size(config))), - "NUM_THREADS" => n_gdal_threads(config) - ), - force=true - ) - - return nothing -end - -""" - _write_tiff(file_path::String, data::Raster)::Nothing - -Write out a geotiff using common options. - -# Arguments -- `file_path` : Path to write data out to -- `data` : Raster data to write out -""" -function _write_tiff(file_path::String, data::Raster)::Nothing - Rasters.write( - file_path, - data; - ext=".tiff", - source="gdal", - driver="gtiff", - force=true - ) - - return nothing -end diff --git a/src/job_management/DiskService.jl b/src/job_management/DiskService.jl deleted file mode 100644 index bc1f71b..0000000 --- a/src/job_management/DiskService.jl +++ /dev/null @@ -1,124 +0,0 @@ -""" -DEPRECATED - kept around due to some dependencies, but should be removed soon -""" - -struct DiskService <: JobService - cache_dir::String -end - -function _job_location(srv::DiskService, job_id::String) - return joinpath(srv.cache_dir, job_id) -end - -""" - job_details(srv::DiskService, job_id::String) - -Retrieve details of a job from disk. - -# Arguments -- `srv` : DiskService -- `job_id` : ID of job, typically based on a hash of the search criteria - -# Returns -Job details in a dictionary -""" -function job_details(srv::DiskService, job_id::String)::JobAttributes - d = try - JSON.parsefile(_job_location(srv, job_id)) - catch err - if !(err isa SystemError) - rethrow(err) - end - - Dict( - "job_id" => "invalid", - "status" => "no job", - "result_loc" => "", - "access_url" => "" - ) - end - - return JobAttributes(d) -end - -""" - submit_job(srv::DiskService, job_id::String, result_loc::String)::JobAttributes - -Submit a job by writing state to disk. - -# Arguments -- `srv` : Service type -- `job_id` : ID of job, typically based on a hash of search criteria -- `result_loc` : Expected location of results -""" -function submit_job(srv::DiskService, job_id::String, result_loc::String)::JobAttributes - fn = _job_location(srv, job_id) - attr = JobAttributes(job_id, "processing", result_loc, create_job_url(job_id), now()) - - if isfile(fn) - rm(fn; force=true) - end - - open(fn, "w") do f - JSON.print(f, attr) - end - - return attr -end - -function update_job!(srv::DiskService, job_id::String, attrs::JobAttributes)::Nothing - fn = _job_location(srv, job_id) - open(fn, "w") do f - JSON.print(f, attrs) - end - - return nothing -end - -""" - job_status(srv::DiskService, job_id::String)::String - -Retrieve status of a job. - -# Arguments -- `srv` : Service type -- `job_id` : ID of job, typically based on a hash of search criteria -""" -function job_status(srv::DiskService, job_id::String)::String - details = job_details(srv, job_id) - - return job_status(details) -end - -function job_status(details::JobAttributes) - minutes_taken = (now() - details.init_datetime) / Millisecond(1000) / 60 - if minutes_taken > 10.0 - # Likely something has hung - return "error" - end - - return details.status -end - -""" - job_result(srv::DiskService, job_id::String)::Union{Bool,String} - -Retrieve the location of results for a job. -Raises ArgumentError in cases where the job does not exist. - -# Arguments -- `srv` : Service type -- `job_id` : ID of job, typically based on a hash of search criteria -""" -function job_result(srv::DiskService, job_id::String)::Union{Bool,String} - details = job_details(srv, job_id) - if details.status == "no job" - throw(ArgumentError("Job does not exist")) - end - - if details.status != "completed" - return false - end - - return details.result_loc -end \ No newline at end of file diff --git a/src/job_management/JobInterface.jl b/src/job_management/JobInterface.jl deleted file mode 100644 index c224353..0000000 --- a/src/job_management/JobInterface.jl +++ /dev/null @@ -1,70 +0,0 @@ -""" -DEPRECATED - kept around due to some dependencies, but should be removed soon -""" - -using Dates -using StructTypes - -abstract type JobService end - -mutable struct JobAttributes - const job_id::String - - # TODO: Should be an enum - status::String # One of "no job", "completed", "processing", "error" - const result_loc::String - const access_url::String - const init_datetime::DateTime - - # Add field for error message? -end - -# Define struct type definition to auto-serialize/deserialize to JSON -StructTypes.StructType(::Type{JobAttributes}) = StructTypes.Struct() - -""" - JobAttributes(attr::Dict)::JobAttributes - -Construct JobAttributes from a dictionary. -""" -function JobAttributes(attr::Dict)::JobAttributes - details::Dict{String,Union{String,DateTime}} = JSON.parse(JSON.json(attr)) - - if "init_datetime" ∉ keys(details) - details["init_datetime"] = DateTime(0) # empty datetime if not provided - elseif typeof(details["init_datetime"]) <: String - details["init_datetime"] = DateTime(details["init_datetime"]) - end - - return JobAttributes( - details["job_id"], - details["status"], - details["result_loc"], - details["access_url"], - details["init_datetime"] - ) -end - -""" - create_job_id(query_params::Dict)::String - -Generate a job id based on query parameters. -""" -function create_job_id(query_params::Dict)::String - return string(hash(query_params)) -end - -""" - create_job_url(job_id::String)::String - -Generate URL endpoint to retrieve results of a job. -""" -function create_job_url(job_id::String)::String - return "/job/result/$(job_id)" -end - -function job_details end -function job_status end -function submit_job end -function update_job! end -function job_result end \ No newline at end of file diff --git a/src/utility/config.jl b/src/utility/config.jl new file mode 100644 index 0000000..a27499f --- /dev/null +++ b/src/utility/config.jl @@ -0,0 +1,45 @@ +"""Methods for accessing properties of config.""" + +""" + _cache_location(config::Dict)::String + +Retrieve cache location for geotiffs. +""" +function _cache_location(config::Dict)::String + cache_loc = try + in_debug = haskey(config["server_config"], "DEBUG_MODE") + if in_debug && lowercase(config["server_config"]["DEBUG_MODE"]) == "true" + if "DEBUG_CACHE_DIR" ∉ keys(ENV) + ENV["DEBUG_CACHE_DIR"] = mktempdir() + end + + ENV["DEBUG_CACHE_DIR"] + else + config["server_config"]["TIFF_CACHE_DIR"] + end + catch err + @warn "Encountered error:" err + if "DEBUG_CACHE_DIR" ∉ keys(ENV) + ENV["DEBUG_CACHE_DIR"] = mktempdir() + end + + ENV["DEBUG_CACHE_DIR"] + end + + return cache_loc +end + +""" + n_gdal_threads(config::Dict)::String + +Retrieve the configured number of threads to use when writing COGs with GDAL. +""" +function _n_gdal_threads(config::Dict)::String + n_cog_threads = try + config["server_config"]["COG_THREADS"] + catch + "1" # Default to using a single thread for GDAL write + end + + return n_cog_threads +end diff --git a/src/utility/deprecated.jl b/src/utility/deprecated.jl deleted file mode 100644 index 1133813..0000000 --- a/src/utility/deprecated.jl +++ /dev/null @@ -1,112 +0,0 @@ -""" -========== -DEPRECATED -========== -""" - -function criteria_data_map() - # TODO: Load from config? - return OrderedDict( - :Depth => "_bathy", - :Benthic => "_benthic", - :Geomorphic => "_geomorphic", - :Slope => "_slope", - :Turbidity => "_turbid", - :WavesHs => "_waves_Hs", - :WavesTp => "_waves_Tp", - :Rugosity => "_rugosity", - :ValidSlopes => "_valid_slopes", - :ValidFlats => "_valid_flats" - ) -end - -function search_criteria()::Vector{String} - return string.(keys(criteria_data_map())) -end - -function site_criteria()::Vector{String} - return ["SuitabilityThreshold", "xdist", "ydist"] -end - -function suitability_criteria()::Vector{String} - return vcat(search_criteria(), ["SuitabilityThreshold"]) -end - -function criteria_data_map() - # TODO: Load from config? - return OrderedDict( - :Depth => "_bathy", - :Benthic => "_benthic", - :Geomorphic => "_geomorphic", - :Slope => "_slope", - :Turbidity => "_turbid", - :WavesHs => "_waves_Hs", - :WavesTp => "_waves_Tp", - :Rugosity => "_rugosity", - :ValidSlopes => "_valid_slopes", - :ValidFlats => "_valid_flats" - - # Unused datasets - # :PortDistSlopes => "_PortDistSlopes", - # :PortDistFlats => "_PortDistFlats" - ) -end - -function search_criteria()::Vector{String} - return string.(keys(criteria_data_map())) -end - -function site_criteria()::Vector{String} - return ["SuitabilityThreshold", "xdist", "ydist"] -end - -function suitability_criteria()::Vector{String} - return vcat(search_criteria(), ["SuitabilityThreshold"]) -end - -function extract_criteria(qp::T, criteria::Vector{String})::T where {T<:Dict{String,String}} - return filter( - k -> string(k.first) ∈ criteria, qp - ) -end - -struct OldRegionalCriteria{T} - stack::RasterStack - valid_slopes::T - valid_flats::T -end - -function valid_slope_lon_inds(reg::OldRegionalCriteria) - return reg.valid_slopes.lon_idx -end -function valid_slope_lat_inds(reg::OldRegionalCriteria) - return reg.valid_slopes.lat_idx -end -function valid_flat_lon_inds(reg::OldRegionalCriteria) - return reg.valid_flats.lon_idx -end -function valid_flat_lat_inds(reg::OldRegionalCriteria) - return reg.valid_flats.lat_idx -end - -struct CriteriaBounds{F<:Function} - name::Symbol - lower_bound::Float32 - upper_bound::Float32 - rule::F - - function CriteriaBounds(name::String, lb::S, ub::S)::CriteriaBounds where {S<:String} - lower_bound::Float32 = parse(Float32, lb) - upper_bound::Float32 = parse(Float32, ub) - func = (x) -> lower_bound .<= x .<= upper_bound - - return new{Function}(Symbol(name), lower_bound, upper_bound, func) - end - - function CriteriaBounds( - name::String, lb::Float32, ub::Float32 - )::CriteriaBounds - func = (x) -> lb .<= x .<= ub - return new{Function}(Symbol(name), lb, ub, func) - end -end diff --git a/src/utility/file_io.jl b/src/utility/file_io.jl new file mode 100644 index 0000000..72ca1a9 --- /dev/null +++ b/src/utility/file_io.jl @@ -0,0 +1,52 @@ +"""Methods for common file I/O.""" + +""" + _write_cog(file_path::String, data::Raster, config::Dict)::Nothing + +Write out a COG using common options. + +# Arguments +- `file_path` : Path to write data out to +- `data` : Raster data to write out +""" +function _write_cog(file_path::String, data::Raster, config::Dict)::Nothing + Rasters.write( + file_path, + data; + ext=".tiff", + source="gdal", + driver="COG", + options=Dict{String,String}( + "COMPRESS" => "DEFLATE", + "SPARSE_OK" => "TRUE", + "OVERVIEW_COUNT" => "5", + "BLOCKSIZE" => string(first(tile_size(config))), + "NUM_THREADS" => _n_gdal_threads(config) + ), + force=true + ) + + return nothing +end + +""" + _write_tiff(file_path::String, data::Raster)::Nothing + +Write out a geotiff using common options. + +# Arguments +- `file_path` : Path to write data out to +- `data` : Raster data to write out +""" +function _write_tiff(file_path::String, data::Raster)::Nothing + Rasters.write( + file_path, + data; + ext=".tiff", + source="gdal", + driver="gtiff", + force=true + ) + + return nothing +end diff --git a/src/utility/helpers.jl b/src/utility/helpers.jl index 6a9325b..6667b3d 100644 --- a/src/utility/helpers.jl +++ b/src/utility/helpers.jl @@ -1,3 +1,4 @@ + """ Builds a hash by combining strings and hashing result """ diff --git a/src/utility/index.jl b/src/utility/index.jl index 3f24c0f..fd2daba 100644 --- a/src/utility/index.jl +++ b/src/utility/index.jl @@ -1,5 +1,7 @@ include("regions_criteria_setup.jl") +include("config.jl") include("helpers.jl") -include("deprecated.jl") include("routes.jl") include("assessment_interfaces.jl") +include("file_io.jl") +include("middleware.jl") diff --git a/src/Middleware.jl b/src/utility/middleware.jl similarity index 93% rename from src/Middleware.jl rename to src/utility/middleware.jl index 7188914..f663a7c 100644 --- a/src/Middleware.jl +++ b/src/utility/middleware.jl @@ -88,3 +88,9 @@ function setup_jwt_middleware(config::Dict) return jwt_auth_middleware end + +function get_auth_router(config::Dict) + # Setup auth middleware - depends on config.toml - can return identity func + auth = setup_jwt_middleware(config) + return router(""; middleware=[auth]) +end diff --git a/src/utility/regions_criteria_setup.jl b/src/utility/regions_criteria_setup.jl index 7eaf855..5482042 100644 --- a/src/utility/regions_criteria_setup.jl +++ b/src/utility/regions_criteria_setup.jl @@ -527,40 +527,6 @@ function check_existing_regional_data_from_disk( return nothing end -""" -Get the file path for the empty tile cache. - -# Arguments -- `config::Dict` : Configuration dictionary containing cache settings - -# Returns -String path to the empty tile cache file. -""" -function get_empty_tile_path(config::Dict)::String - cache_location = _cache_location(config) - return joinpath(cache_location, "no_data_tile.png") -end - -""" -Initialize empty tile cache if it doesn't exist. - -Creates a blank PNG tile used for areas with no data coverage. - -# Arguments -- `config::Dict` : Configuration dictionary containing tile and cache settings -""" -function setup_empty_tile_cache(config::Dict)::Nothing - file_path = get_empty_tile_path(config) - if !isfile(file_path) - @info "Creating empty tile cache" file_path - # Create empty RGBA tile with configured dimensions - save(file_path, zeros(RGBA, tile_size(config))) - else - @debug "Empty tile cache already exists" file_path - end - return nothing -end - # ============================================================================= # Main Data Initialization Functions # ============================================================================= @@ -721,9 +687,6 @@ function initialise_data_with_cache(config::Dict; force_cache_invalidation::Bool # Update global cache REGIONAL_DATA = regional_data - # Initialize empty tile cache for map rendering - setup_empty_tile_cache(config) - # Save to disk for future use @info "Saving regional data to disk cache" cache_directory = regional_cache_directory try diff --git a/src/utility/routes.jl b/src/utility/routes.jl index 007e953..a5254d7 100644 --- a/src/utility/routes.jl +++ b/src/utility/routes.jl @@ -11,10 +11,15 @@ Creates REST endpoints for accessing regional criteria bounds and metadata. - `config` : Configuration object - `auth` : Authentication/authorization handler """ -function setup_criteria_routes(config, auth) +function setup_utility_routes(config, auth) @info "Setting up criteria routes" regional_data::RegionalData = get_regional_data(config) + # Health check + @get "/health" function () + return json(Dict(:status => "healthy")) + end + # Endpoint: GET /criteria/{region}/ranges # Returns JSON with min/max values for all criteria in specified region @get auth("/criteria/{region}/ranges") function (_::Request, region::String) @@ -45,5 +50,12 @@ function setup_criteria_routes(config, auth) return json(output_dict) end + """Obtain the spatial bounds for a given region of interest""" + @get auth("/bounds/{region}") function (req::Request, region::String) + rst_stack = regional_data.regions[region].raster_stack + + return json(Rasters.bounds(rst_stack)) + end + @info "Criteria routes setup completed" end From d88a439b171f143bda0eaf37c8b22c7b666d4b79 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Thu, 29 May 2025 13:36:39 +1000 Subject: [PATCH 17/26] Refactoring code around - working again Signed-off-by: Peter Baker --- src/ReefGuideAPI.jl | 5 - src/assessment_methods/best_fit_polygons.jl | 1 - src/assessment_methods/geom_ops.jl | 17 --- src/assessment_methods/index.jl | 16 +++ src/job_worker/config.jl | 2 - src/job_worker/ecs.jl | 4 - src/job_worker/handler_helpers.jl | 141 ++++++++++++++++++++ src/job_worker/handlers.jl | 6 - src/job_worker/http_client.jl | 5 - src/job_worker/index.jl | 10 ++ src/job_worker/storage_client.jl | 5 - src/job_worker/worker.jl | 6 - src/utility/assessment_interfaces.jl | 134 ------------------- src/utility/index.jl | 17 +++ src/utility/middleware.jl | 5 - src/utility/regions_criteria_setup.jl | 8 -- src/utility/types.jl | 3 + 17 files changed, 187 insertions(+), 198 deletions(-) create mode 100644 src/job_worker/handler_helpers.jl create mode 100644 src/utility/types.jl diff --git a/src/ReefGuideAPI.jl b/src/ReefGuideAPI.jl index 48652b8..d4a1485 100644 --- a/src/ReefGuideAPI.jl +++ b/src/ReefGuideAPI.jl @@ -1,6 +1,4 @@ module ReefGuideAPI - -import GeoDataFrames as GDF using Base.Threads, Glob, @@ -38,9 +36,6 @@ function start_server(config_path) @info "Setting up auth middleware and router." auth = get_auth_router(config) - @info "Setting up job routes..." - setup_job_routes(config, auth) - @info "Setting up utility routes..." setup_utility_routes(config, auth) diff --git a/src/assessment_methods/best_fit_polygons.jl b/src/assessment_methods/best_fit_polygons.jl index 3017200..efb7f39 100644 --- a/src/assessment_methods/best_fit_polygons.jl +++ b/src/assessment_methods/best_fit_polygons.jl @@ -1,6 +1,5 @@ """Geometry-based assessment methods.""" -using NearestNeighbors # Tabular data assessment methods diff --git a/src/assessment_methods/geom_ops.jl b/src/assessment_methods/geom_ops.jl index a29b6b4..9e0dfa2 100644 --- a/src/assessment_methods/geom_ops.jl +++ b/src/assessment_methods/geom_ops.jl @@ -1,23 +1,6 @@ """ Helper functions to support interaction with geometries. """ - -using Statistics - -import ArchGDAL as AG -import GeoInterface as GI -import GeoInterface.Wrappers as GIWrap - -import GeometryOps as GO -using Proj -using LibGEOS -using GeometryBasics - -using CoordinateTransformations - -using Rasters -using StaticArrays - function create_poly(verts, crs) sel_lines = GI.LineString(GI.Point.(verts)) ring = GI.LinearRing(GI.getpoint(sel_lines)) diff --git a/src/assessment_methods/index.jl b/src/assessment_methods/index.jl index c38ca2f..77a0307 100644 --- a/src/assessment_methods/index.jl +++ b/src/assessment_methods/index.jl @@ -1,3 +1,19 @@ +using + Statistics, + Proj, + LibGEOS, + GeometryBasics, + CoordinateTransformations, + Rasters, + StaticArrays, + NearestNeighbors + +import ArchGDAL as AG +import GeoInterface as GI +import GeoInterface.Wrappers as GIWrap +import GeometryOps as GO +import GeoDataFrames as GDF + include("apply_criteria.jl") include("best_fit_polygons.jl") include("common_functions.jl") diff --git a/src/job_worker/config.jl b/src/job_worker/config.jl index 0389f38..7cdc9cf 100644 --- a/src/job_worker/config.jl +++ b/src/job_worker/config.jl @@ -2,8 +2,6 @@ Manages config for the worker node. """ -using Dates - """ Configuration for the worker These are populated from the environment and are only loaded when the worker diff --git a/src/job_worker/ecs.jl b/src/job_worker/ecs.jl index d8b7dbf..5ab3de9 100644 --- a/src/job_worker/ecs.jl +++ b/src/job_worker/ecs.jl @@ -2,10 +2,6 @@ Manages the ECS specific runtime environment for the worker node. """ -using HTTP -using JSON3 -using Dates - """ Core identifiers and metadata for an ECS Fargate task """ diff --git a/src/job_worker/handler_helpers.jl b/src/job_worker/handler_helpers.jl new file mode 100644 index 0000000..6ab6ae3 --- /dev/null +++ b/src/job_worker/handler_helpers.jl @@ -0,0 +1,141 @@ +""" +Helpers for job handlers which interrupt main workflow. + +For example, converting between job system interfaces and assessment interfaces. +""" + +""" +Build regional assessment parameters from user input and regional data. + +Creates a complete parameter set for regional assessment by merging user-specified +criteria bounds with regional defaults. Validates that the specified region exists. + +# Arguments +- `input::RegionalAssessmentInput` : User input containing assessment parameters +- `regional_data::RegionalData` : Complete regional data for validation and defaults + +# Returns +`RegionalAssessmentParameters` struct ready for assessment execution. + +# Throws +- `ErrorException` : If specified region is not found in regional data +""" +function build_regional_assessment_parameters( + input::RegionalAssessmentInput, + regional_data::RegionalData +)::RegionalAssessmentParameters + @info "Building regional assessment parameters" region = input.region + + # Validate region exists + if !haskey(regional_data.regions, input.region) + available_regions = collect(keys(regional_data.regions)) + @error "Region not found in regional data" region = input.region available_regions + throw( + ErrorException( + "Regional data did not have data for region $(input.region). Available regions: $(join(available_regions, ", "))" + ) + ) + end + + region_data = regional_data.regions[input.region] + + # Extract threshold with default fallback + threshold = + !isnothing(input.threshold) ? input.threshold : DEFAULT_SUITABILITY_THRESHOLD + + regional_criteria::BoundedCriteriaDict = Dict() + regional_bounds::BoundedCriteriaDict = region_data.criteria + + for (criteria_id, possible_symbols) in PARAM_MAP + bounds = get(regional_bounds, criteria_id, nothing) + user_min = + isnothing(possible_symbols) ? nothing : + getproperty(input, first(possible_symbols)) + user_max = + isnothing(possible_symbols) ? nothing : + getproperty(input, last(possible_symbols)) + + merged = merge_bounds( + user_min, + user_max, + bounds + ) + if !isnothing(merged) + regional_criteria[criteria_id] = BoundedCriteria(; + metadata=ASSESSMENT_CRITERIA[criteria_id], + bounds=merged + ) + end + end + + return RegionalAssessmentParameters(; + region=input.region, + regional_criteria, + region_data, + suitability_threshold=Int64(threshold) + ) +end + +""" +Build suitability assessment parameters from user input and regional data. + +Creates a complete parameter set for suitability assessment by merging user-specified +criteria bounds with regional defaults. Includes spatial dimensions for polygon analysis. + +# Arguments +- `input::SuitabilityAssessmentInput` : User input containing assessment parameters and spatial dimensions +- `regional_data::RegionalData` : Complete regional data for validation and defaults + +# Returns +`SuitabilityAssessmentParameters` struct ready for assessment execution. + +# Throws +- `ErrorException` : If specified region is not found in regional data +""" +function build_suitability_assessment_parameters( + input::SuitabilityAssessmentInput, + regional_data::RegionalData +)::SuitabilityAssessmentParameters + @info "Building suitability assessment parameters" region = input.region x_dist = + input.x_dist y_dist = input.y_dist + + @debug "Building regional parameters first" + regional_input = regional_job_from_suitability_job(input) + regional_parameters = build_regional_assessment_parameters( + regional_input, + regional_data + ) + @debug "Extending regional parameters with suitability inputs x_dist and ydist" x = + input.x_dist y = input.y_dist + return SuitabilityAssessmentParameters(; + region=regional_parameters.region, + regional_criteria=regional_parameters.regional_criteria, + region_data=regional_parameters.region_data, + suitability_threshold=regional_parameters.suitability_threshold, + x_dist=input.x_dist, + y_dist=input.y_dist + ) +end + +""" +Converts parameters from a suitability job into a regional job +""" +function regional_job_from_suitability_job( + suitability_job::SuitabilityAssessmentInput +)::RegionalAssessmentInput + return RegionalAssessmentInput( + suitability_job.region, + suitability_job.reef_type, + suitability_job.depth_min, + suitability_job.depth_max, + suitability_job.slope_min, + suitability_job.slope_max, + suitability_job.rugosity_min, + suitability_job.rugosity_max, + suitability_job.waves_period_min, + suitability_job.waves_period_max, + suitability_job.waves_height_min, + suitability_job.waves_height_max, + suitability_job.threshold + ) +end diff --git a/src/job_worker/handlers.jl b/src/job_worker/handlers.jl index a6057a9..7a544c4 100644 --- a/src/job_worker/handlers.jl +++ b/src/job_worker/handlers.jl @@ -3,12 +3,6 @@ This is the file where handlers, input and output payloads are registered to handle jobs for this worker. """ -using JSON3 -using Logging -using Dates - -const OptionalValue{T} = Union{T,Nothing}; - # ================ # Type Definitions # ================ diff --git a/src/job_worker/http_client.jl b/src/job_worker/http_client.jl index d876f6a..14aa5f0 100644 --- a/src/job_worker/http_client.jl +++ b/src/job_worker/http_client.jl @@ -3,11 +3,6 @@ A HTTP client which wraps auth/headers to talk to the web API for the job system. """ -using HTTP -using JSON3 -using Dates -using JSONWebTokens - """ Custom error type for API errors """ diff --git a/src/job_worker/index.jl b/src/job_worker/index.jl index 60ab689..38aa55d 100644 --- a/src/job_worker/index.jl +++ b/src/job_worker/index.jl @@ -1,6 +1,16 @@ +using Dates +using HTTP +using JSON3 +using Logging +using AWSS3 +using AWS +using Random +using JSONWebTokens + include("config.jl") include("ecs.jl") include("http_client.jl") include("handlers.jl") include("storage_client.jl") include("worker.jl") +include("handler_helpers.jl") diff --git a/src/job_worker/storage_client.jl b/src/job_worker/storage_client.jl index 501dc4f..635b22f 100644 --- a/src/job_worker/storage_client.jl +++ b/src/job_worker/storage_client.jl @@ -3,11 +3,6 @@ An abstract storage client and implementation for S3 - which is untested/not used yet. """ -using Logging -using JSON3 -using AWSS3 -using AWS - """ Abstract type for storage clients All concrete storage clients should inherit from this diff --git a/src/job_worker/worker.jl b/src/job_worker/worker.jl index 99054b6..494cb97 100644 --- a/src/job_worker/worker.jl +++ b/src/job_worker/worker.jl @@ -4,12 +4,6 @@ components to orchestrate consuming jobs. Principally polls for, completes and reports back jobs done, on a loop, until idle for a configurable idle time. """ -using Dates -using Random -using Logging -using JSON3 - - """ Represents a job that needs to be processed """ diff --git a/src/utility/assessment_interfaces.jl b/src/utility/assessment_interfaces.jl index 6e7b502..ddbe3d0 100644 --- a/src/utility/assessment_interfaces.jl +++ b/src/utility/assessment_interfaces.jl @@ -121,118 +121,6 @@ const PARAM_MAP::Dict{String,OptionalValue{Tuple{Symbol,Symbol}}} = Dict( "Rugosity" => (:rugosity_min, :rugosity_max) ) -""" -Build regional assessment parameters from user input and regional data. - -Creates a complete parameter set for regional assessment by merging user-specified -criteria bounds with regional defaults. Validates that the specified region exists. - -# Arguments -- `input::RegionalAssessmentInput` : User input containing assessment parameters -- `regional_data::RegionalData` : Complete regional data for validation and defaults - -# Returns -`RegionalAssessmentParameters` struct ready for assessment execution. - -# Throws -- `ErrorException` : If specified region is not found in regional data -""" -function build_regional_assessment_parameters( - input::RegionalAssessmentInput, - regional_data::RegionalData -)::RegionalAssessmentParameters - @info "Building regional assessment parameters" region = input.region - - # Validate region exists - if !haskey(regional_data.regions, input.region) - available_regions = collect(keys(regional_data.regions)) - @error "Region not found in regional data" region = input.region available_regions - throw( - ErrorException( - "Regional data did not have data for region $(input.region). Available regions: $(join(available_regions, ", "))" - ) - ) - end - - region_data = regional_data.regions[input.region] - - # Extract threshold with default fallback - threshold = - !isnothing(input.threshold) ? input.threshold : DEFAULT_SUITABILITY_THRESHOLD - - regional_criteria::BoundedCriteriaDict = Dict() - regional_bounds::BoundedCriteriaDict = region_data.criteria - - for (criteria_id, possible_symbols) in PARAM_MAP - bounds = get(regional_bounds, criteria_id, nothing) - user_min = - isnothing(possible_symbols) ? nothing : - getproperty(input, first(possible_symbols)) - user_max = - isnothing(possible_symbols) ? nothing : - getproperty(input, last(possible_symbols)) - - merged = merge_bounds( - user_min, - user_max, - bounds - ) - if !isnothing(merged) - regional_criteria[criteria_id] = BoundedCriteria(; - metadata=ASSESSMENT_CRITERIA[criteria_id], - bounds=merged - ) - end - end - - return RegionalAssessmentParameters(; - region=input.region, - regional_criteria, - region_data, - suitability_threshold=Int64(threshold) - ) -end - -""" -Build suitability assessment parameters from user input and regional data. - -Creates a complete parameter set for suitability assessment by merging user-specified -criteria bounds with regional defaults. Includes spatial dimensions for polygon analysis. - -# Arguments -- `input::SuitabilityAssessmentInput` : User input containing assessment parameters and spatial dimensions -- `regional_data::RegionalData` : Complete regional data for validation and defaults - -# Returns -`SuitabilityAssessmentParameters` struct ready for assessment execution. - -# Throws -- `ErrorException` : If specified region is not found in regional data -""" -function build_suitability_assessment_parameters( - input::SuitabilityAssessmentInput, - regional_data::RegionalData -)::SuitabilityAssessmentParameters - @info "Building suitability assessment parameters" region = input.region x_dist = - input.x_dist y_dist = input.y_dist - - @debug "Building regional parameters first" - regional_input = regional_job_from_suitability_job(input) - regional_parameters = build_regional_assessment_parameters( - regional_input, - regional_data - ) - @debug "Extending regional parameters with suitability inputs x_dist and ydist" x = - input.x_dist y = input.y_dist - return SuitabilityAssessmentParameters(; - region=regional_parameters.region, - regional_criteria=regional_parameters.regional_criteria, - region_data=regional_parameters.region_data, - suitability_threshold=regional_parameters.suitability_threshold, - x_dist=input.x_dist, - y_dist=input.y_dist - ) -end """ Generate a deterministic hash string for RegionalAssessmentParameters. @@ -343,28 +231,6 @@ function build_regional_assessment_file_path( return file_path end -""" -Converts parameters from a suitability job into a regional job -""" -function regional_job_from_suitability_job( - suitability_job::SuitabilityAssessmentInput -)::RegionalAssessmentInput - return RegionalAssessmentInput( - suitability_job.region, - suitability_job.reef_type, - suitability_job.depth_min, - suitability_job.depth_max, - suitability_job.slope_min, - suitability_job.slope_max, - suitability_job.rugosity_min, - suitability_job.rugosity_max, - suitability_job.waves_period_min, - suitability_job.waves_period_max, - suitability_job.waves_height_min, - suitability_job.waves_height_max, - suitability_job.threshold - ) -end """ Converts parameters from a suitability assessment into a regional assessment diff --git a/src/utility/index.jl b/src/utility/index.jl index fd2daba..e1c4999 100644 --- a/src/utility/index.jl +++ b/src/utility/index.jl @@ -1,3 +1,20 @@ +using + JSONWebTokens, + HTTP, + Dates, + JSON, + Rasters, + Glob, + GeoParquet, + Serialization, + Logging, + Images, + ImageIO, + Interpolations +using Oxygen: json, Request +import GeoDataFrames as GDF + +include("types.jl") include("regions_criteria_setup.jl") include("config.jl") include("helpers.jl") diff --git a/src/utility/middleware.jl b/src/utility/middleware.jl index f663a7c..a3ad008 100644 --- a/src/utility/middleware.jl +++ b/src/utility/middleware.jl @@ -1,8 +1,3 @@ -using JSONWebTokens -using HTTP -using Dates -using JSON - # TODO tighten restrictions const CORS_HEADERS = [ "Access-Control-Allow-Origin" => "*", diff --git a/src/utility/regions_criteria_setup.jl b/src/utility/regions_criteria_setup.jl index 5482042..3b72ebe 100644 --- a/src/utility/regions_criteria_setup.jl +++ b/src/utility/regions_criteria_setup.jl @@ -1,11 +1,3 @@ -using Rasters -using Glob -using GeoParquet -using Serialization -using Oxygen: json, Request -using Logging -using Images, ImageIO, Interpolations - # ============================================================================= # Constants and Configuration # ============================================================================= diff --git a/src/utility/types.jl b/src/utility/types.jl new file mode 100644 index 0000000..9782eab --- /dev/null +++ b/src/utility/types.jl @@ -0,0 +1,3 @@ +"""Utility types""" + +const OptionalValue{T} = Union{T,Nothing}; From 22521bd7fe9c808f8fb3b85fca5fd7bc9a3fd2f3 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Thu, 29 May 2025 13:48:29 +1000 Subject: [PATCH 18/26] Couple of fixes Signed-off-by: Peter Baker --- src/assessment_methods/apply_criteria.jl | 2 +- src/utility/config.jl | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/assessment_methods/apply_criteria.jl b/src/assessment_methods/apply_criteria.jl index c25969b..c08e9dc 100644 --- a/src/assessment_methods/apply_criteria.jl +++ b/src/assessment_methods/apply_criteria.jl @@ -51,7 +51,7 @@ function filter_raster_by_criteria( # Apply criteria res_lookup = trues(nrow(lookup)) for filter::CriteriaBounds in criteria_bounds - res_lookup .= res_lookup .& filter.rule(lookup[!, rule_name]) + res_lookup .= res_lookup .& filter.rule(lookup[!, filter.name]) end tmp = lookup[res_lookup, [:lon_idx, :lat_idx]] diff --git a/src/utility/config.jl b/src/utility/config.jl index a27499f..9259912 100644 --- a/src/utility/config.jl +++ b/src/utility/config.jl @@ -43,3 +43,19 @@ function _n_gdal_threads(config::Dict)::String return n_cog_threads end + +""" + tile_size(config::Dict)::Tuple + +Retrieve the configured size of map tiles in pixels (width and height / lon and lat). +""" +function tile_size(config::Dict)::Tuple + tile_dims = try + res = parse(Int, config["server_config"]["TILE_SIZE"]) + (res, res) + catch + (256, 256) # 256x256 + end + + return tile_dims +end From f5f51880811dece20a21197a78c07298f81ab194 Mon Sep 17 00:00:00 2001 From: Peter Baker <87056634+PeterBaker0@users.noreply.github.com> Date: Fri, 30 May 2025 09:08:37 +1000 Subject: [PATCH 19/26] Update src/assessment_methods/apply_criteria.jl Co-authored-by: Takuya Iwanaga --- src/assessment_methods/apply_criteria.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/assessment_methods/apply_criteria.jl b/src/assessment_methods/apply_criteria.jl index c08e9dc..b1d73d7 100644 --- a/src/assessment_methods/apply_criteria.jl +++ b/src/assessment_methods/apply_criteria.jl @@ -10,7 +10,7 @@ struct CriteriaBounds{F<:Function} "A function which takes a value and returns if matches the criteria" rule::F - function CriteriaBounds(name::String, lb::S, ub::S)::CriteriaBounds where {S<:String} + function CriteriaBounds(name::S, lb::S, ub::S)::CriteriaBounds where {S<:String} lower_bound::Float32 = parse(Float32, lb) upper_bound::Float32 = parse(Float32, ub) func = (x) -> lower_bound .<= x .<= upper_bound From a120194ce6fe60c1978927d8f933a7e25edda28b Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Fri, 30 May 2025 11:57:15 +1000 Subject: [PATCH 20/26] PR changes Signed-off-by: Peter Baker --- src/ReefGuideAPI.jl | 39 ++++++++++--------- src/assessment_methods/apply_criteria.jl | 6 ++- .../{index.jl => assessment_methods.jl} | 0 src/job_worker/{index.jl => job_worker.jl} | 0 src/utility/{index.jl => utility.jl} | 0 5 files changed, 25 insertions(+), 20 deletions(-) rename src/assessment_methods/{index.jl => assessment_methods.jl} (100%) rename src/job_worker/{index.jl => job_worker.jl} (100%) rename src/utility/{index.jl => utility.jl} (100%) diff --git a/src/ReefGuideAPI.jl b/src/ReefGuideAPI.jl index d4a1485..9883d3d 100644 --- a/src/ReefGuideAPI.jl +++ b/src/ReefGuideAPI.jl @@ -1,28 +1,31 @@ module ReefGuideAPI -using - Base.Threads, - Glob, - TOML, - ArchGDAL, - GeoParquet, - Rasters, - HTTP, - Oxygen, - Serialization, - DataFrames, - OrderedCollections, - Memoization, - SparseArrays, - FLoops, ThreadsX + +# System imports +using Base.Threads + +# File IO +using Glob, TOML, Serialization + +# Geospatial +using ArchGDAL, GeoParquet, Rasters + +# Server +using HTTP, Oxygen + +# Collections +using DataFrames, OrderedCollections, SparseArrays + +# Multithreading +using FLoops, ThreadsX # Utilities and helpers for assessments -include("utility/index.jl") +include("utility/utility.jl") # Assessment logic -include("assessment_methods/index.jl") +include("assessment_methods/assessment_methods.jl") # Worker system -include("job_worker/index.jl") +include("job_worker/job_worker.jl") function start_server(config_path) @info "Launching server... please wait" diff --git a/src/assessment_methods/apply_criteria.jl b/src/assessment_methods/apply_criteria.jl index b1d73d7..7ceb142 100644 --- a/src/assessment_methods/apply_criteria.jl +++ b/src/assessment_methods/apply_criteria.jl @@ -1,5 +1,9 @@ """Methods to filter criteria bounds over rasters and lookup tables""" +""" +CriteriaBounds combine lookup information for a given criteria, bounds, and a +rule (function) which enforces it for a given value +""" struct CriteriaBounds{F<:Function} "The field ID of the criteria" name::Symbol @@ -26,8 +30,6 @@ struct CriteriaBounds{F<:Function} end end -"""Methods to support querying data layers.""" - """ Apply thresholds for each criteria. diff --git a/src/assessment_methods/index.jl b/src/assessment_methods/assessment_methods.jl similarity index 100% rename from src/assessment_methods/index.jl rename to src/assessment_methods/assessment_methods.jl diff --git a/src/job_worker/index.jl b/src/job_worker/job_worker.jl similarity index 100% rename from src/job_worker/index.jl rename to src/job_worker/job_worker.jl diff --git a/src/utility/index.jl b/src/utility/utility.jl similarity index 100% rename from src/utility/index.jl rename to src/utility/utility.jl From 25aa2c552431b33f1603c043efd3ebf852a3bee1 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Tue, 3 Jun 2025 15:01:46 +1000 Subject: [PATCH 21/26] Removing suitability threshold Signed-off-by: Peter Baker --- src/ReefGuideAPI.jl | 2 +- src/job_worker/handler_helpers.jl | 16 ++++++---------- src/job_worker/handlers.jl | 1 - src/utility/assessment_interfaces.jl | 15 +++------------ 4 files changed, 10 insertions(+), 24 deletions(-) diff --git a/src/ReefGuideAPI.jl b/src/ReefGuideAPI.jl index 9883d3d..91a4991 100644 --- a/src/ReefGuideAPI.jl +++ b/src/ReefGuideAPI.jl @@ -71,7 +71,7 @@ function start_worker() config = TOML.parsefile(worker.config.config_path) @info "Loading regional data" initialise_data_with_cache(config) - @info "Starting worker loop from ReefGuideAPI.jl" + @info "Starting worker loop from ReefGuideAPI.jl with $(Threads.nthreads()) threads." start(worker) @info "Worker closed itself..." end diff --git a/src/job_worker/handler_helpers.jl b/src/job_worker/handler_helpers.jl index 6ab6ae3..84fd521 100644 --- a/src/job_worker/handler_helpers.jl +++ b/src/job_worker/handler_helpers.jl @@ -38,11 +38,6 @@ function build_regional_assessment_parameters( end region_data = regional_data.regions[input.region] - - # Extract threshold with default fallback - threshold = - !isnothing(input.threshold) ? input.threshold : DEFAULT_SUITABILITY_THRESHOLD - regional_criteria::BoundedCriteriaDict = Dict() regional_bounds::BoundedCriteriaDict = region_data.criteria @@ -71,8 +66,7 @@ function build_regional_assessment_parameters( return RegionalAssessmentParameters(; region=input.region, regional_criteria, - region_data, - suitability_threshold=Int64(threshold) + region_data ) end @@ -105,13 +99,16 @@ function build_suitability_assessment_parameters( regional_input, regional_data ) + # Extract threshold with default fallback + threshold = + !isnothing(input.threshold) ? input.threshold : DEFAULT_SUITABILITY_THRESHOLD @debug "Extending regional parameters with suitability inputs x_dist and ydist" x = input.x_dist y = input.y_dist return SuitabilityAssessmentParameters(; region=regional_parameters.region, regional_criteria=regional_parameters.regional_criteria, region_data=regional_parameters.region_data, - suitability_threshold=regional_parameters.suitability_threshold, + suitability_threshold=Int64(threshold), x_dist=input.x_dist, y_dist=input.y_dist ) @@ -135,7 +132,6 @@ function regional_job_from_suitability_job( suitability_job.waves_period_min, suitability_job.waves_period_max, suitability_job.waves_height_min, - suitability_job.waves_height_max, - suitability_job.threshold + suitability_job.waves_height_max ) end diff --git a/src/job_worker/handlers.jl b/src/job_worker/handlers.jl index 7a544c4..deba39f 100644 --- a/src/job_worker/handlers.jl +++ b/src/job_worker/handlers.jl @@ -251,7 +251,6 @@ struct RegionalAssessmentInput <: AbstractJobInput waves_period_max::OptionalValue{Float64} waves_height_min::OptionalValue{Float64} waves_height_max::OptionalValue{Float64} - threshold::OptionalValue{Int64} end """ diff --git a/src/utility/assessment_interfaces.jl b/src/utility/assessment_interfaces.jl index ddbe3d0..6771dd4 100644 --- a/src/utility/assessment_interfaces.jl +++ b/src/utility/assessment_interfaces.jl @@ -16,22 +16,18 @@ regional data. - `region::String` : The region that is being assessed - `regional_criteria::BoundedCriteriaDict` : The criteria to assess, including user provided bounds - `region_data::RegionalDataEntry` : The data to consider for this region -- `suitability_threshold::Int64` : The cutoff to consider a site suitable """ struct RegionalAssessmentParameters region::String regional_criteria::BoundedCriteriaDict region_data::RegionalDataEntry - suitability_threshold::Int64 function RegionalAssessmentParameters(; region::String, regional_criteria::BoundedCriteriaDict, region_data::RegionalDataEntry, - suitability_threshold::Int64 ) - @debug "Created RegionalAssessmentParameters" region suitability_threshold - return new(region, regional_criteria, region_data, suitability_threshold) + return new(region, regional_criteria, region_data) end end @@ -135,16 +131,12 @@ for cache file naming. Same parameters will always produce the same hash. String hash suitable for use in cache file names. """ function regional_assessment_params_hash(params::RegionalAssessmentParameters)::String - @debug "Generating hash for regional assessment parameters" region = params.region threshold = - params.suitability_threshold + @debug "Generating hash for regional assessment parameters" region = params.region # Create hash input from key parameters hash_components = [ - # Region params.region, - # Suitability threshold - string(params.suitability_threshold), - # Criteria + # spread result list of components from regional criteria get_hash_components_from_regional_criteria(params.regional_criteria)... ] @@ -242,6 +234,5 @@ function regional_params_from_suitability_params( region=suitability_params.region, regional_criteria=suitability_params.regional_criteria, region_data=suitability_params.region_data, - suitability_threshold=suitability_params.suitability_threshold ) end From 4ed8251fc108c072e2bb551c08155b482039c989 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Tue, 3 Jun 2025 15:21:37 +1000 Subject: [PATCH 22/26] Adding other display information Signed-off-by: Peter Baker --- src/utility/regions_criteria_setup.jl | 37 +++++++++++++++++++++------ src/utility/routes.jl | 23 ++++++++++++++++- 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/src/utility/regions_criteria_setup.jl b/src/utility/regions_criteria_setup.jl index 3b72ebe..714e130 100644 --- a/src/utility/regions_criteria_setup.jl +++ b/src/utility/regions_criteria_setup.jl @@ -72,15 +72,27 @@ struct CriteriaMetadata display_label::String description::String units::String + payload_prefix::String + default_bounds::OptionalValue{Bounds} function CriteriaMetadata(; id::String, file_suffix::String, display_label::String, description::String, - units::String + units::String, + payload_prefix::String, + default_bounds::OptionalValue{Bounds}=nothing ) - return new(id, file_suffix, display_label, description, units) + return new( + id, + file_suffix, + display_label, + description, + units, + payload_prefix, + default_bounds + ) end end @@ -91,42 +103,51 @@ const ASSESSMENT_CRITERIA::Dict{String,CriteriaMetadata} = Dict( file_suffix="_bathy", display_label="Depth", description="TODO", - units="TODO" + units="TODO", + payload_prefix="depth_", + default_bounds=Bounds(; min=-10, max=-2) ), "Slope" => CriteriaMetadata(; id="Slope", file_suffix="_slope", display_label="Slope", description="TODO", - units="TODO" + units="TODO", + payload_prefix="slope_" ), "Turbidity" => CriteriaMetadata(; id="Turbidity", file_suffix="_turbid", display_label="Turbidity", description="TODO", - units="TODO" + units="TODO", + payload_prefix="turbidity_" ), "WavesHs" => CriteriaMetadata(; id="WavesHs", file_suffix="_waves_Hs", display_label="Wave Height (m)", description="TODO", - units="TODO" + units="TODO", + payload_prefix="waves_height_", + default_bounds=Bounds(; min=0, max=1) ), "WavesTp" => CriteriaMetadata(; id="WavesTp", file_suffix="_waves_Tp", display_label="Wave Period (s)", description="TODO", - units="TODO" + units="TODO", + payload_prefix="waves_period_", + default_bounds=Bounds(; min=0, max=6) ), "Rugosity" => CriteriaMetadata(; id="Rugosity", file_suffix="_rugosity", display_label="Rugosity", description="TODO", - units="TODO" + units="TODO", + payload_prefix="rugosity_" ) ) diff --git a/src/utility/routes.jl b/src/utility/routes.jl index a5254d7..1162bc9 100644 --- a/src/utility/routes.jl +++ b/src/utility/routes.jl @@ -40,9 +40,30 @@ function setup_utility_routes(config, auth) # Format each criteria with min/max bounds for (id::String, criteria::BoundedCriteria) in regional_criteria_lookup + # build default min/max + default_bounds::Bounds = something( + criteria.metadata.default_bounds, criteria.bounds + ) + output_dict[id] = OrderedDict( + # Unique ID (and data field name) + :id => id, + + # min/max bounds :min_val => criteria.bounds.min, - :max_val => criteria.bounds.max + :max_val => criteria.bounds.max, + + # display info + :display_title => criteria.metadata.display_label, + :display_subtitle => criteria.metadata.description, + :units => criteria.metadata.units, + + # default min/max + :default_min_val => default_bounds.min, + :default_max_val => default_bounds.max, + + # how to build a job payload (prefix of job i.e. depth_) then build depth_min depth_max + :payload_property_prefix => criteria.metadata.payload_prefix ) end From 4a52f66e821aaf21eed98ddaab9fd4c3a465d66b Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Tue, 3 Jun 2025 15:42:30 +1000 Subject: [PATCH 23/26] Addming min/max tooltip Signed-off-by: Peter Baker --- src/utility/regions_criteria_setup.jl | 34 ++++++++++++++++++++------- src/utility/routes.jl | 2 ++ 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/src/utility/regions_criteria_setup.jl b/src/utility/regions_criteria_setup.jl index 714e130..0f490c0 100644 --- a/src/utility/regions_criteria_setup.jl +++ b/src/utility/regions_criteria_setup.jl @@ -74,6 +74,8 @@ struct CriteriaMetadata units::String payload_prefix::String default_bounds::OptionalValue{Bounds} + min_tooltip::String + max_tooltip::String function CriteriaMetadata(; id::String, @@ -82,7 +84,9 @@ struct CriteriaMetadata description::String, units::String, payload_prefix::String, - default_bounds::OptionalValue{Bounds}=nothing + default_bounds::OptionalValue{Bounds}=nothing, + min_tooltip::String, + max_tooltip::String ) return new( id, @@ -91,7 +95,9 @@ struct CriteriaMetadata description, units, payload_prefix, - default_bounds + default_bounds, + min_tooltip, + max_tooltip ) end end @@ -105,7 +111,9 @@ const ASSESSMENT_CRITERIA::Dict{String,CriteriaMetadata} = Dict( description="TODO", units="TODO", payload_prefix="depth_", - default_bounds=Bounds(; min=-10, max=-2) + default_bounds=Bounds(; min=-10, max=-2), + min_tooltip="TODO", + max_tooltip="TODO" ), "Slope" => CriteriaMetadata(; id="Slope", @@ -113,7 +121,9 @@ const ASSESSMENT_CRITERIA::Dict{String,CriteriaMetadata} = Dict( display_label="Slope", description="TODO", units="TODO", - payload_prefix="slope_" + payload_prefix="slope_", + min_tooltip="TODO", + max_tooltip="TODO" ), "Turbidity" => CriteriaMetadata(; id="Turbidity", @@ -121,7 +131,9 @@ const ASSESSMENT_CRITERIA::Dict{String,CriteriaMetadata} = Dict( display_label="Turbidity", description="TODO", units="TODO", - payload_prefix="turbidity_" + payload_prefix="turbidity_", + min_tooltip="TODO", + max_tooltip="TODO" ), "WavesHs" => CriteriaMetadata(; id="WavesHs", @@ -130,7 +142,9 @@ const ASSESSMENT_CRITERIA::Dict{String,CriteriaMetadata} = Dict( description="TODO", units="TODO", payload_prefix="waves_height_", - default_bounds=Bounds(; min=0, max=1) + default_bounds=Bounds(; min=0, max=1), + min_tooltip="TODO", + max_tooltip="TODO" ), "WavesTp" => CriteriaMetadata(; id="WavesTp", @@ -139,7 +153,9 @@ const ASSESSMENT_CRITERIA::Dict{String,CriteriaMetadata} = Dict( description="TODO", units="TODO", payload_prefix="waves_period_", - default_bounds=Bounds(; min=0, max=6) + default_bounds=Bounds(; min=0, max=6), + min_tooltip="TODO", + max_tooltip="TODO" ), "Rugosity" => CriteriaMetadata(; id="Rugosity", @@ -147,7 +163,9 @@ const ASSESSMENT_CRITERIA::Dict{String,CriteriaMetadata} = Dict( display_label="Rugosity", description="TODO", units="TODO", - payload_prefix="rugosity_" + payload_prefix="rugosity_", + min_tooltip="TODO", + max_tooltip="TODO" ) ) diff --git a/src/utility/routes.jl b/src/utility/routes.jl index 1162bc9..6731fc0 100644 --- a/src/utility/routes.jl +++ b/src/utility/routes.jl @@ -57,6 +57,8 @@ function setup_utility_routes(config, auth) :display_title => criteria.metadata.display_label, :display_subtitle => criteria.metadata.description, :units => criteria.metadata.units, + :min_tooltip => criteria.metadata.min_tooltip, + :max_tooltip => criteria.metadata.max_tooltip, # default min/max :default_min_val => default_bounds.min, From 1de582fc635f1548f870eecb382be4275063fa4f Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Tue, 3 Jun 2025 15:46:16 +1000 Subject: [PATCH 24/26] Changing descriptoin to subtitle Signed-off-by: Peter Baker --- src/utility/regions_criteria_setup.jl | 24 ++++++++++++++---------- src/utility/routes.jl | 2 +- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/utility/regions_criteria_setup.jl b/src/utility/regions_criteria_setup.jl index 0f490c0..affa165 100644 --- a/src/utility/regions_criteria_setup.jl +++ b/src/utility/regions_criteria_setup.jl @@ -63,14 +63,18 @@ Metadata for assessment criteria including file naming conventions. - `id::String` : Unique system identifier for the criteria - `file_suffix::String` : File suffix pattern for data files - `display_label::String` : Human-readable label for UI display -- `description::String` : Human-readable info about this criteria +- `subtitle::String` : Human-readable info about this criteria on subtitle of slider - `units::String` : Human-readable info about relevant units +- `payload_prefix::String` : The prefix for building the job payload +- `default_bounds::OptionalValue{Bounds}` : The default bounds for the parameter sliders +- `min_tooltip::String` : Tooltip text on min slider +- `max_tooltip::String` : Tooltip text on max slider """ struct CriteriaMetadata id::String file_suffix::String display_label::String - description::String + subtitle::String units::String payload_prefix::String default_bounds::OptionalValue{Bounds} @@ -81,7 +85,7 @@ struct CriteriaMetadata id::String, file_suffix::String, display_label::String, - description::String, + subtitle::String, units::String, payload_prefix::String, default_bounds::OptionalValue{Bounds}=nothing, @@ -92,7 +96,7 @@ struct CriteriaMetadata id, file_suffix, display_label, - description, + subtitle, units, payload_prefix, default_bounds, @@ -108,7 +112,7 @@ const ASSESSMENT_CRITERIA::Dict{String,CriteriaMetadata} = Dict( id="Depth", file_suffix="_bathy", display_label="Depth", - description="TODO", + subtitle="TODO", units="TODO", payload_prefix="depth_", default_bounds=Bounds(; min=-10, max=-2), @@ -119,7 +123,7 @@ const ASSESSMENT_CRITERIA::Dict{String,CriteriaMetadata} = Dict( id="Slope", file_suffix="_slope", display_label="Slope", - description="TODO", + subtitle="TODO", units="TODO", payload_prefix="slope_", min_tooltip="TODO", @@ -129,7 +133,7 @@ const ASSESSMENT_CRITERIA::Dict{String,CriteriaMetadata} = Dict( id="Turbidity", file_suffix="_turbid", display_label="Turbidity", - description="TODO", + subtitle="TODO", units="TODO", payload_prefix="turbidity_", min_tooltip="TODO", @@ -139,7 +143,7 @@ const ASSESSMENT_CRITERIA::Dict{String,CriteriaMetadata} = Dict( id="WavesHs", file_suffix="_waves_Hs", display_label="Wave Height (m)", - description="TODO", + subtitle="TODO", units="TODO", payload_prefix="waves_height_", default_bounds=Bounds(; min=0, max=1), @@ -150,7 +154,7 @@ const ASSESSMENT_CRITERIA::Dict{String,CriteriaMetadata} = Dict( id="WavesTp", file_suffix="_waves_Tp", display_label="Wave Period (s)", - description="TODO", + subtitle="TODO", units="TODO", payload_prefix="waves_period_", default_bounds=Bounds(; min=0, max=6), @@ -161,7 +165,7 @@ const ASSESSMENT_CRITERIA::Dict{String,CriteriaMetadata} = Dict( id="Rugosity", file_suffix="_rugosity", display_label="Rugosity", - description="TODO", + subtitle="TODO", units="TODO", payload_prefix="rugosity_", min_tooltip="TODO", diff --git a/src/utility/routes.jl b/src/utility/routes.jl index 6731fc0..cb7aae0 100644 --- a/src/utility/routes.jl +++ b/src/utility/routes.jl @@ -55,7 +55,7 @@ function setup_utility_routes(config, auth) # display info :display_title => criteria.metadata.display_label, - :display_subtitle => criteria.metadata.description, + :display_subtitle => criteria.metadata.subtitle, :units => criteria.metadata.units, :min_tooltip => criteria.metadata.min_tooltip, :max_tooltip => criteria.metadata.max_tooltip, From b87b738d0ec7bf87fd2ba1551822e70cfa9286f1 Mon Sep 17 00:00:00 2001 From: Takuya Iwanaga Date: Tue, 3 Jun 2025 17:44:10 +1000 Subject: [PATCH 25/26] Filled in criteria metadata TODO: Double check the percentile values for Hs and Tp (waves) --- src/utility/regions_criteria_setup.jl | 56 +++++++++++++-------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/src/utility/regions_criteria_setup.jl b/src/utility/regions_criteria_setup.jl index affa165..7789760 100644 --- a/src/utility/regions_criteria_setup.jl +++ b/src/utility/regions_criteria_setup.jl @@ -111,65 +111,65 @@ const ASSESSMENT_CRITERIA::Dict{String,CriteriaMetadata} = Dict( "Depth" => CriteriaMetadata(; id="Depth", file_suffix="_bathy", - display_label="Depth", - subtitle="TODO", - units="TODO", + display_label="Depth [m]", + subtitle="Depth from Mean Astronomical Tide", + units="meters", payload_prefix="depth_", default_bounds=Bounds(; min=-10, max=-2), - min_tooltip="TODO", - max_tooltip="TODO" + min_tooltip="Maximum depth", + max_tooltip="Minimum depth" ), "Slope" => CriteriaMetadata(; id="Slope", file_suffix="_slope", - display_label="Slope", - subtitle="TODO", - units="TODO", + display_label="Slope [degrees]", + subtitle="Slope of reef", + units="degrees", payload_prefix="slope_", - min_tooltip="TODO", - max_tooltip="TODO" + min_tooltip="Minimum slope angle (0 is flat)", + max_tooltip="Maximum slope angle" ), "Turbidity" => CriteriaMetadata(; id="Turbidity", file_suffix="_turbid", display_label="Turbidity", - subtitle="TODO", - units="TODO", + subtitle="Usual clarity of water", + units="Secchi depth meters", payload_prefix="turbidity_", - min_tooltip="TODO", - max_tooltip="TODO" + min_tooltip="Minimum Secchi depth", + max_tooltip="Maximum Secchi depth" ), "WavesHs" => CriteriaMetadata(; id="WavesHs", file_suffix="_waves_Hs", - display_label="Wave Height (m)", - subtitle="TODO", - units="TODO", + display_label="Wave Height [m]", + subtitle="Significant Wave Height (90th percentile)", + units="meters", payload_prefix="waves_height_", default_bounds=Bounds(; min=0, max=1), - min_tooltip="TODO", - max_tooltip="TODO" + min_tooltip="Minimum wave height", + max_tooltip="Maximum wave height" ), "WavesTp" => CriteriaMetadata(; id="WavesTp", file_suffix="_waves_Tp", - display_label="Wave Period (s)", - subtitle="TODO", - units="TODO", + display_label="Wave Period [s]", + subtitle="Time between waves in seconds (90th percentile)", + units="seconds", payload_prefix="waves_period_", default_bounds=Bounds(; min=0, max=6), - min_tooltip="TODO", - max_tooltip="TODO" + min_tooltip="Minimum periodicity", + max_tooltip="Maximum periodicity" ), "Rugosity" => CriteriaMetadata(; id="Rugosity", file_suffix="_rugosity", display_label="Rugosity", - subtitle="TODO", - units="TODO", + subtitle="Roughness of the sea floor", + units="stdev", payload_prefix="rugosity_", - min_tooltip="TODO", - max_tooltip="TODO" + min_tooltip="Minimum variability", + max_tooltip="Maximum variability" ) ) From 9bc18929f20c3872bd8e14a19516e753c647635a Mon Sep 17 00:00:00 2001 From: Takuya Iwanaga Date: Wed, 4 Jun 2025 09:51:35 +1000 Subject: [PATCH 26/26] Update src/utility/regions_criteria_setup.jl Swap tooltip text Co-authored-by: Peter Baker <87056634+PeterBaker0@users.noreply.github.com> --- src/utility/regions_criteria_setup.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utility/regions_criteria_setup.jl b/src/utility/regions_criteria_setup.jl index 7789760..b57eef1 100644 --- a/src/utility/regions_criteria_setup.jl +++ b/src/utility/regions_criteria_setup.jl @@ -116,8 +116,8 @@ const ASSESSMENT_CRITERIA::Dict{String,CriteriaMetadata} = Dict( units="meters", payload_prefix="depth_", default_bounds=Bounds(; min=-10, max=-2), - min_tooltip="Maximum depth", - max_tooltip="Minimum depth" + min_tooltip="Minimum depth", + max_tooltip="Maximum depth" ), "Slope" => CriteriaMetadata(; id="Slope",