Merge pull request #55 from open-AIMS/perf-check

Range of performance changes

Author: ConnectedSystems

Manifest-v1.11.toml

@@ -49,6 +49,7 @@ ThreadsX = "ac1d9e8a-700a-412c-b207-f0111f4b6c0d"
 YAXArrays = "c21b50f5-aa40-41ea-b809-c0f5e47bfa5c"
 
 [compat]
-DiskArrays = "=0.3.23, ^0.4.5"
+DiskArrays = "0.4.9"
 MKL_jll = "2024.2.0"
 NearestNeighbors = "0.4.20"
+YAXArrays = "0.5"

Project.toml

@@ -56,6 +56,7 @@ function initialize_regional_data_cache(reef_data_path::String, reg_cache_fn::St
         String,Union{RegionalCriteria,DataFrame,Dict}
     }()
     for reg in get_regions()
+        @debug "$(now()) : Initializing cache for $reg"
         data_paths = String[]
         data_names = String[]
 
@@ -77,7 +78,8 @@ function initialize_regional_data_cache(reef_data_path::String, reg_cache_fn::St
                 if occursin("slope", string(dp))
                     slope_table = GeoParquet.read(parq_file)
                 elseif occursin("flat", string(dp))
-                    flat_table = GeoParquet.read(parq_file)
+                    @warn "Skipping data for reef flats as it is currently unused"
+                    # flat_table = GeoParquet.read(parq_file)
                 else
                     msg = "Unknown lookup found: $(parq_file). Must be 'slope' or 'flat'"
                     throw(ArgumentError(msg))
@@ -90,9 +92,9 @@ function initialize_regional_data_cache(reef_data_path::String, reg_cache_fn::St
         slope_table[!, :lons] .= first.(coords)
         slope_table[!, :lats] .= last.(coords)
 
-        coords = GI.coordinates.(flat_table.geometry)
-        flat_table[!, :lons] .= first.(coords)
-        flat_table[!, :lats] .= last.(coords)
+        # coords = GI.coordinates.(flat_table.geometry)
+        # flat_table[!, :lons] .= first.(coords)
+        # flat_table[!, :lats] .= last.(coords)
 
         rst_stack = RasterStack(data_paths; name=data_names, lazy=true)
 
@@ -106,8 +108,10 @@ function initialize_regional_data_cache(reef_data_path::String, reg_cache_fn::St
         regional_assessment_data[reg] = RegionalCriteria(
             rst_stack,
             slope_table,
-            flat_table
+            slope_table[[1], :]  # flat_table
         )
+
+        @debug "$(now()) : Finished initialization for $reg"
     end
 
     regional_assessment_data["region_long_names"] = Dict(

src/ReefGuideAPI.jl

@@ -35,6 +35,24 @@ function criteria_data_map()
     )
 end
 
+function search_criteria()::Vector{String}
+    return string.(keys(criteria_data_map()))
+end
+
+function site_criteria()::Vector{String}
+    return ["SuitabilityThreshold", "xdist", "ydist"]
+end
+
+function suitability_criteria()::Vector{String}
+    return vcat(search_criteria(), ["SuitabilityThreshold"])
+end
+
+function extract_criteria(qp::T, criteria::Vector{String})::T where {T<:Dict{String,String}}
+    return filter(
+        k -> string(k.first) ∈ criteria, qp
+    )
+end
+
 struct RegionalCriteria{T}
     stack::RasterStack
     valid_slopes::T
@@ -175,18 +193,9 @@ function setup_region_routes(config, auth)
         # somewhere:8000/suitability/assess/region-name/reeftype?criteria_names=Depth,Slope&lb=-9.0,0.0&ub=-2.0,40
         # 127.0.0.1:8000/suitability/assess/Cairns-Cooktown/slopes?Depth=-4.0:-2.0&Slope=0.0:40.0&Rugosity=0.0:6.0
         # 127.0.0.1:8000/suitability/assess/Cairns-Cooktown/slopes?Depth=-4.0:-2.0&Slope=0.0:40.0&Rugosity=0.0:6.0&SuitabilityThreshold=95
-
+        # 127.0.0.1:8000/suitability/assess/Mackay-Capricorn/slopes?Depth=-12.0:-2.0&Slope=0.0:40.0&Rugosity=0.0:6.0&SuitabilityThreshold=95
         qp = queryparams(req)
-        assessed_fn = cache_filename(qp, config, "$(reg)_suitable", "tiff")
-        if isfile(assessed_fn)
-            return file(assessed_fn; headers=COG_HEADERS)
-        end
-
-        @debug "$(now()) : Assessing region $(reg)"
-        assessed = assess_region(reg_assess_data, reg, qp, rtype)
-
-        @debug "$(now()) : Writing to $(assessed_fn)"
-        _write_tiff(assessed_fn, assessed)
+        assessed_fn = assess_region(config, qp, reg, rtype, reg_assess_data)
 
         return file(assessed_fn; headers=COG_HEADERS)
     end
@@ -195,6 +204,7 @@ function setup_region_routes(config, auth)
         req::Request, reg::String, rtype::String
     )
         # 127.0.0.1:8000/suitability/site-suitability/Cairns-Cooktown/slopes?Depth=-4.0:-2.0&Slope=0.0:40.0&Rugosity=0.0:6.0&SuitabilityThreshold=95&xdist=450&ydist=50
+        # 127.0.0.1:8000/suitability/site-suitability/Mackay-Capricorn/slopes?Depth=-12.0:-2.0&Slope=0.0:40.0&Rugosity=0.0:6.0&SuitabilityThreshold=95&xdist=100&ydist=100
         qp = queryparams(req)
         suitable_sites_fn = cache_filename(
             qp, config, "$(reg)_potential_sites", "geojson"
@@ -204,24 +214,16 @@ function setup_region_routes(config, auth)
         end
 
         # Assess location suitability if needed
-        assessed_fn = cache_filename(qp, config, "$(reg)_suitable", "tiff")
-        if isfile(assessed_fn)
-            assessed = Raster(assessed_fn)
-        else
-            assessed = assess_region(reg_assess_data, reg, qp, rtype)
-            _write_tiff(assessed_fn, assessed)
-        end
+        assessed_fn = assess_region(config, qp, reg, rtype, reg_assess_data)
+        assessed = Raster(assessed_fn; lazy=true)
 
         # Extract criteria and assessment
-        criteria_names = string.(keys(criteria_data_map()))
-        pixel_criteria = filter(k -> k.first ∈ criteria_names, qp)
-        site_criteria = filter(
-            k -> string(k.first) ∈ ["SuitabilityThreshold", "xdist", "ydist"], qp
-        )
+        pixel_criteria = extract_criteria(qp, search_criteria())
+        deploy_site_criteria = extract_criteria(qp, site_criteria())
 
         best_sites = filter_sites(
             assess_sites(
-                reg_assess_data, reg, rtype, pixel_criteria, site_criteria, assessed
+                reg_assess_data, reg, rtype, pixel_criteria, deploy_site_criteria, assessed
             )
         )
 

src/criteria_assessment/criteria.jl

@@ -161,16 +161,17 @@ function apply_criteria_thresholds(
     ruleset::NamedTuple
 )::Raster
     # Result store
-    res = Raster(criteria_stack.Depth; data=falses(size(criteria_stack)), missingval=0)
+    data = falses(size(criteria_stack))
 
     res_lookup = trues(nrow(lookup))
     for rule_name in keys(ruleset)
         res_lookup .= res_lookup .& ruleset[rule_name](lookup[!, rule_name])
     end
 
     tmp = lookup[res_lookup, [:lon_idx, :lat_idx]]
-    res[CartesianIndex.(tmp.lon_idx, tmp.lat_idx)] .= true
+    data[CartesianIndex.(tmp.lon_idx, tmp.lat_idx)] .= true
 
+    res = Raster(criteria_stack.Depth; data=sparse(data), missingval=0)
     return res
 end
 function apply_criteria_thresholds(
@@ -179,22 +180,24 @@ function apply_criteria_thresholds(
     ruleset::Vector{CriteriaBounds{Function}}
 )::Raster where {T}
     # Result store
-    res = Raster(criteria_stack.Depth; data=falses(size(criteria_stack)), missingval=0)
+    data = falses(size(criteria_stack))
 
     res_lookup = trues(nrow(lookup))
     for threshold in ruleset
         res_lookup .= res_lookup .& threshold.rule(lookup[!, threshold.name])
     end
 
     tmp = lookup[res_lookup, [:lon_idx, :lat_idx]]
-    res[CartesianIndex.(tmp.lon_idx, tmp.lat_idx)] .= true
+    data[CartesianIndex.(tmp.lon_idx, tmp.lat_idx)] .= true
+
+    res = Raster(criteria_stack.Depth; data=sparse(data), missingval=0)
 
     return res
 end
 
 """
     apply_criteria_lookup(
-        reg_criteria,
+        reg_criteria::RegionalCriteria,
         rtype::Symbol,
         ruleset::Vector{CriteriaBounds{Function}}
     )
@@ -247,7 +250,11 @@ applied to a set of criteria.
 # Returns
 True/false mask indicating locations within desired thresholds.
 """
-function threshold_mask(reg_criteria, rtype::Symbol, crit_map)::Raster
+function threshold_mask(
+    reg_criteria::RegionalCriteria,
+    rtype::Symbol,
+    crit_map::Vector{CriteriaBounds{Function}}
+)::Raster
     valid_lookup = getfield(reg_criteria, Symbol(:valid_, rtype))
     mask_layer = apply_criteria_thresholds(
         reg_criteria.stack,
@@ -258,9 +265,9 @@ function threshold_mask(reg_criteria, rtype::Symbol, crit_map)::Raster
     return mask_layer
 end
 function threshold_mask(
-    reg_criteria,
+    reg_criteria::RegionalCriteria,
     rtype::Symbol,
-    crit_map,
+    crit_map::Vector{CriteriaBounds{Function}},
     lons::Tuple,
     lats::Tuple
 )::Raster