review comments [skip ci]

Author: kmdeck

Project.toml

@@ -31,15 +31,14 @@ ClimaAnalysis = "29b5916a-a76c-4e73-9657-3c8fd22e65e6"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab"
 Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
-Format = "1fa38f19-a742-5d3f-a2b9-30dd87b9d5f8"
 GeoMakie = "db073c08-6b98-4ee5-b6a4-5efafb3259c6"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 Poppler_jll = "9c32591e-4766-534b-9725-b71a8799265b"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
 
 [extensions]
-FluxnetSimulations = ["DelimitedFiles", "Format"]
+FluxnetSimulations = ["DelimitedFiles",]
 LandSimulationVisualization = ["CairoMakie", "ClimaAnalysis", "GeoMakie", "Poppler_jll", "Printf", "StatsBase"]
 NeuralSnowExt = ["CSV", "DataFrames", "HTTP", "Flux", "StatsBase", "BSON"]
 
@@ -59,7 +58,6 @@ Dates = "1"
 DelimitedFiles = "1"
 DocStringExtensions = "0.9"
 Flux = "0.15, 0.16"
-Format = "1"
 GeoMakie = "< 0.7.13"
 HTTP = "1.10"
 Insolation = "0.9.2"

experiments/integrated/fluxnet/run_fluxnet.jl

@@ -22,7 +22,6 @@ import ClimaUtilities.OutputPathGenerator: generate_output_path
 using ClimaDiagnostics
 using ClimaUtilities
 using DelimitedFiles
-using Format
 FluxnetSimulations =
     Base.get_extension(ClimaLand, :FluxnetSimulations).FluxnetSimulations;
 
@@ -500,7 +499,7 @@ lines!(
     label = "Ice, 1.25cm",
 )
 
-if comparison_data.SWC.absent == false
+if !comparison_data.SWC.absent
     lines!(
         ax1,
         data_times ./ 3600 ./ 24,
@@ -562,7 +561,7 @@ lines!(
     color = "blue",
 )
 
-if comparison_data.TS.absent == false
+if !comparison_data.TS.absent
     lines!(
         ax12,
         data_times ./ 3600 ./ 24,

ext/FluxnetSimulations.jl

@@ -5,7 +5,6 @@ import ClimaUtilities.TimeManager: ITime, date
 using Thermodynamics
 using Dates
 using DelimitedFiles
-using Format
 using DocStringExtensions
 using Insolation
 import ClimaLand.Parameters as LP

ext/fluxnet_sims/data_processing.jl

@@ -1,11 +1,30 @@
-var_missing(array; val = -9999) = array == val
+"""
+    var_missing(x; val = -9999)
+
+A function that checks if the value of `x` is equal to 
+-9999, which is the value that Fluxnet uses for missing 
+data. Returns true if x == -9999.
+"""
+var_missing(x; val = -9999) = x == val
 
+"""
+   mask_data(t, v; val = -9999)
+
+Returns t[id], v[id], where id corresponds
+to the indices where v is not equal `val`,
+which indicates missing data.
+
+Note that `v` can either be a 1d array or a 2d array;
+in the latter case, id refers to rows where
+none of the columns of `v` are missing. The returned
+array is 2d: v[id, :]
+"""
 function mask_data(t, v; val = -9999)
     if ndims(v) == 1
-        not_missing_mask = .~var_missing.(v)
+        not_missing_mask = .~var_missing.(v; val)
         return t[not_missing_mask], v[not_missing_mask]
     elseif ndims(v) == 2
-        not_missing = .~var_missing.(v)
+        not_missing = .~var_missing.(v; val)
         not_missing_mask = all(not_missing, dims = 2)[:]
         return t[not_missing_mask], v[not_missing_mask, :]
     else
@@ -14,74 +33,157 @@ function mask_data(t, v; val = -9999)
 end
 
 """
+    time_varying_input_from_data(
+        driver_data,
+        varname::String,
+        column_name_map,
+        time_in_seconds::Vector;
+        preprocess_func = identity,
+        val=-9999,
+    )
+
+Returns the TimeVaryingInput object corresponding
+to `varname` in the driver_data matrix, the `column_name_map`
+should map between varname and column id, and the `time_in_seconds`
+should be the timestamp in seconds relative to the start_date of the
+simulation corresponding to each row of driver_data.
+
+If you need to preprocess the data (e.g., unit conversion), you must pass 
+a pointwise function preprocess_func(var) as a keyword argument.
+
+Note that this function handles missing data by removing it (assuming it is marked by missing by a given value equal to `val`), because the 
+TimeVaryingInput object is an interpolating object (in time).
 """
 function time_varying_input_from_data(
     driver_data,
     varname::String,
     column_name_map,
     time_in_seconds::Vector;
-    skip_header = true,
     preprocess_func = identity,
+    val = -9999
 )
-    first_row = skip_header ? 2 : 1
-    var_data = driver_data[first_row:end, column_name_map[varname]]
+    var_data = driver_data[:, column_name_map[varname]]
     # The time varying input object interpolates over gaps
     # as needed, so we remove data that is marked as missing here
-    t, v = mask_data(time_in_seconds, var_data)
-    return TimeVaryingInput(Float64.(t), Float64.(preprocess_func.(v)))
+    t, v = mask_data(time_in_seconds, var_data; val)
+    return TimeVaryingInput(t, preprocess_func.(v))
 end
 
 """
+    time_varying_input_from_data(
+        driver_data,
+        varnames::Vector{String},
+        column_name_map,
+        time_in_seconds::Vector,
+        preprocess_func,
+        val=-9999,
+    )
+
+Returns a TimeVaryingInput object which is computing using
+`preprocess_func` as a pointwise function of -in order - the columns in
+`driver_data` specified by `varnames`. 
+
+For example, if you wish to compute specific humidity from temperature,
+pressure, and vpd, you would do:
+varnames = ["TA_F", "PA_F", "VPD_F"] along with a preprocess function of
+the form
+function preprocess_func(T,P,VPD)
+# carries out unit conversion
+# computes q from T, P, VPD
+#returns q
+end
+
+The `column_name_map`
+should map between varname and column id, and the `time_in_seconds`
+should be the timestamp in seconds relative to the start_date of the
+simulation corresponding to each row of driver_data.
+
+Note that this function handles missing data by removing it (assuming it is marked by missing by a given value equal to `val`), because the 
+TimeVaryingInput object is an interpolating object (in time).
 """
 function time_varying_input_from_data(
     driver_data,
     varnames::Vector{String},
     column_name_map,
     time_in_seconds::Vector;
-    skip_header = true,
     preprocess_func = identity,
-)
-    first_row = skip_header ? 2 : 1
+    val = -9999
+    )
     var_ids = [column_name_map[varname] for varname in varnames]
-    var_data = driver_data[first_row:end, var_ids]
+    var_data = driver_data[:, var_ids]
     # The time varying input object interpolates over gaps
     # as needed, so we remove data that is marked as missing here
-    t, v = mask_data(time_in_seconds, var_data)
+    t, v = mask_data(time_in_seconds, var_data; val)
     return TimeVaryingInput(
-        Float64.(t),
-        Float64.(preprocess_func.(eachcol(v)...)),
+        t,
+        preprocess_func.(eachcol(v)...),
     )
 end
 
 """
+     get_data_at_start_date(
+        v::Vector,
+        Δ_date::Vector;
+        preprocess_func = identity,
+        val = -9999,
+    )
+
+Returns the value in the raw data `v` closest to where
+|Δ_date| = 0, after preprocessing the data using preprocess_func
+(a pointwise function) and removing missing values.
+
+If Δ_date corresponds to a vector of dates relative to the start_date
+of the simulation, the returned value can be used as an initial condition.
 """
 function get_data_at_start_date(
     v::Vector,
     Δ_date::Vector;
-    skip_header = true,
     preprocess_func = identity,
+    val = -9999
 )
-    Δ_date, v = mask_data(Δ_date, v)
+    Δ_date, v = mask_data(Δ_date, v; val)
     idx_start = argmin(abs.(Δ_date))
     return preprocess_func(v[idx_start])
 end
 
+"""
+    get_comparison_data(
+        driver_data,
+        varname,
+        column_name_map;
+        preprocess_func = identity,
+
+Gets and returns the a NamedTuple with the data identified 
+by `varname` in the `driver_data` matrix by looking up the 
+column index of varname
+using the column_name_map, replacing missing data with the mean
+of the non-missing data, and preprocessing the data using the
+`preprocess_func`, which should be a pointwise function.
+
+The NamedTuple has two keys: absent, and values. If the data column
+is missing completely, the value of absent is true, and no values
+are returned. If the data column is present, absent is set to false,
+and the data is returned with the key values.
+
+In the future, we can explore dropping the missing values to be
+consistent with what we do above, but this is consistent with the
+current Fluxnet runs.
+)
+"""
 function get_comparison_data(
     driver_data,
     varname,
     column_name_map;
-    skip_header = true,
     preprocess_func = identity,
 )
-    first_row = skip_header ? 2 : 1
     idx = column_name_map[varname]
-    if idx isa Nothing
+    if isnothing(idx)
         return (; absent = true)
     else
-        v = driver_data[first_row:end, idx]
+        v = driver_data[:, idx]
         missing_mask = var_missing.(v)
         not_missing_mask = .~missing_mask
         v[missing_mask] .= sum(v[not_missing_mask]) / sum(not_missing_mask)
-        return (; absent = false, values = Float64.(preprocess_func.(v)))
+        return (; absent = false, values = preprocess_func.(v))
     end
 end