From 94669d47165029c1b9db3b0cfc700e409172b56f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 17 Sep 2024 01:12:56 -0400 Subject: [PATCH 01/18] fix jra55 and bathymetry --- test/runtests_setup.jl | 1 + test/test_bathymetry.jl | 13 ++++++++++--- test/test_jra55.jl | 17 ++++++++++++----- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/test/runtests_setup.jl b/test/runtests_setup.jl index b5002b4bd..cb71f9416 100644 --- a/test/runtests_setup.jl +++ b/test/runtests_setup.jl @@ -15,3 +15,4 @@ using ClimaOcean gpu_test = parse(Bool, get(ENV, "GPU_TEST", "false")) test_architectures = gpu_test ? [GPU()] : [CPU()] +data_directory = gpu_test ? "GPU_data" : "CPU_data" diff --git a/test/test_bathymetry.jl b/test/test_bathymetry.jl index 0547e8d9b..162e5b1f1 100644 --- a/test/test_bathymetry.jl +++ b/test/test_bathymetry.jl @@ -6,6 +6,13 @@ using Statistics @testset "Availability of Bathymetry" begin @info "Testing Bathymetry utils..." for arch in test_architectures + + # Make data directory if it doesn't exist + try + mkdir(data_directory) + catch + end + grid = LatitudeLongitudeGrid(arch; size = (100, 100, 10), longitude = (0, 100), @@ -13,7 +20,7 @@ using Statistics z = (-6000, 0)) # Test that remove_minor_basins!(Z, Inf) does nothing - control_bottom_height = regrid_bathymetry(grid) + control_bottom_height = regrid_bathymetry(grid; dir=data_directory) bottom_height = deepcopy(control_bottom_height) @test_throws ArgumentError remove_minor_basins!(bottom_height, Inf) @@ -52,8 +59,8 @@ using Statistics latitude = (-10, 50), z = (-6000, 0)) - control_bottom_height = regrid_bathymetry(grid) - interpolated_bottom_height = regrid_bathymetry(grid; interpolation_passes = 100) + control_bottom_height = regrid_bathymetry(grid; dir=data_directory) + interpolated_bottom_height = regrid_bathymetry(grid; interpolation_passes=100, dir=data_directory) # Testing that multiple passes do not change the solution when refining the grid @test parent(control_bottom_height) == parent(interpolated_bottom_height) diff --git a/test/test_jra55.jl b/test/test_jra55.jl index 2c6ec39af..2febceffb 100644 --- a/test/test_jra55.jl +++ b/test/test_jra55.jl @@ -11,10 +11,16 @@ using ClimaOcean.OceanSeaIceModels: PrescribedAtmosphere test_name = :downwelling_shortwave_radiation time_indices = 1:3 + # Make data directory if it doesn't exist + try + mkdir(data_directory) + catch + end + # This should download a file called "RYF.rsds.1990_1991.nc" - jra55_fts = JRA55_field_time_series(test_name; architecture=arch, time_indices) + jra55_fts = JRA55_field_time_series(test_name; architecture=arch, time_indices, dir = data_directory) - test_filename = joinpath(download_jra55_cache, "RYF.rsds.1990_1991.nc") + test_filename = joinpath(data_directory, "RYF.rsds.1990_1991.nc") @test jra55_fts isa FieldTimeSeries @test jra55_fts.grid isa LatitudeLongitudeGrid @@ -39,7 +45,8 @@ using ClimaOcean.OceanSeaIceModels: PrescribedAtmosphere in_memory_jra55_fts = JRA55_field_time_series(test_name; time_indices, architecture = arch, - backend = InMemory(2)) + backend = InMemory(2), + dir = data_directory) @test in_memory_jra55_fts isa FieldTimeSeries @@ -101,11 +108,11 @@ using ClimaOcean.OceanSeaIceModels: PrescribedAtmosphere ##### backend = JRA55NetCDFBackend(2) - atmosphere = JRA55_prescribed_atmosphere(arch; backend, include_rivers_and_icebergs=false) + atmosphere = JRA55_prescribed_atmosphere(arch; backend, include_rivers_and_icebergs=false, dir=data_directory) @test atmosphere isa PrescribedAtmosphere @test isnothing(atmosphere.auxiliary_freshwater_flux) - atmosphere = JRA55_prescribed_atmosphere(arch; backend, include_rivers_and_icebergs=true) + atmosphere = JRA55_prescribed_atmosphere(arch; backend, include_rivers_and_icebergs=true, dir=data_directory) @test haskey(atmosphere.auxiliary_freshwater_flux, :rivers) @test haskey(atmosphere.auxiliary_freshwater_flux, :icebergs) end From a2026747e7e3b64a77424cabff4d3571752290e3 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 17 Sep 2024 01:18:56 -0400 Subject: [PATCH 02/18] add correct dirs --- test/runtests_setup.jl | 3 ++- test/test_bathymetry.jl | 11 +++++++---- test/test_jra55.jl | 13 ++++++++----- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/test/runtests_setup.jl b/test/runtests_setup.jl index cb71f9416..b5684e30c 100644 --- a/test/runtests_setup.jl +++ b/test/runtests_setup.jl @@ -15,4 +15,5 @@ using ClimaOcean gpu_test = parse(Bool, get(ENV, "GPU_TEST", "false")) test_architectures = gpu_test ? [GPU()] : [CPU()] -data_directory = gpu_test ? "GPU_data" : "CPU_data" +JRA55_data_directory = gpu_test ? "GPU_JRA55_data" : "CPU_JRA55_data" +bathymetry_data_directory = gpu_test ? "GPU_Bathymetry_data" : "CPU_Bathymetry_data" diff --git a/test/test_bathymetry.jl b/test/test_bathymetry.jl index 162e5b1f1..ff805c2fc 100644 --- a/test/test_bathymetry.jl +++ b/test/test_bathymetry.jl @@ -9,7 +9,7 @@ using Statistics # Make data directory if it doesn't exist try - mkdir(data_directory) + mkdir(bathymetry_data_directory) catch end @@ -20,7 +20,7 @@ using Statistics z = (-6000, 0)) # Test that remove_minor_basins!(Z, Inf) does nothing - control_bottom_height = regrid_bathymetry(grid; dir=data_directory) + control_bottom_height = regrid_bathymetry(grid; dir=bathymetry_data_directory) bottom_height = deepcopy(control_bottom_height) @test_throws ArgumentError remove_minor_basins!(bottom_height, Inf) @@ -59,10 +59,13 @@ using Statistics latitude = (-10, 50), z = (-6000, 0)) - control_bottom_height = regrid_bathymetry(grid; dir=data_directory) - interpolated_bottom_height = regrid_bathymetry(grid; interpolation_passes=100, dir=data_directory) + control_bottom_height = regrid_bathymetry(grid; dir=bathymetry_data_directory) + interpolated_bottom_height = regrid_bathymetry(grid; interpolation_passes=100, dir=bathymetry_data_directory) # Testing that multiple passes do not change the solution when refining the grid @test parent(control_bottom_height) == parent(interpolated_bottom_height) + + # Remove the data directory + rm(bathymetry_data_directory; recursive=true) end end \ No newline at end of file diff --git a/test/test_jra55.jl b/test/test_jra55.jl index 2febceffb..9a98a97c5 100644 --- a/test/test_jra55.jl +++ b/test/test_jra55.jl @@ -13,12 +13,12 @@ using ClimaOcean.OceanSeaIceModels: PrescribedAtmosphere # Make data directory if it doesn't exist try - mkdir(data_directory) + mkdir(JRA55_data_directory) catch end # This should download a file called "RYF.rsds.1990_1991.nc" - jra55_fts = JRA55_field_time_series(test_name; architecture=arch, time_indices, dir = data_directory) + jra55_fts = JRA55_field_time_series(test_name; architecture=arch, time_indices, dir=JRA55_data_directory) test_filename = joinpath(data_directory, "RYF.rsds.1990_1991.nc") @@ -46,7 +46,7 @@ using ClimaOcean.OceanSeaIceModels: PrescribedAtmosphere time_indices, architecture = arch, backend = InMemory(2), - dir = data_directory) + dir = JRA55_data_directory) @test in_memory_jra55_fts isa FieldTimeSeries @@ -108,13 +108,16 @@ using ClimaOcean.OceanSeaIceModels: PrescribedAtmosphere ##### backend = JRA55NetCDFBackend(2) - atmosphere = JRA55_prescribed_atmosphere(arch; backend, include_rivers_and_icebergs=false, dir=data_directory) + atmosphere = JRA55_prescribed_atmosphere(arch; backend, include_rivers_and_icebergs=false, dir=JRA55_data_directory) @test atmosphere isa PrescribedAtmosphere @test isnothing(atmosphere.auxiliary_freshwater_flux) - atmosphere = JRA55_prescribed_atmosphere(arch; backend, include_rivers_and_icebergs=true, dir=data_directory) + atmosphere = JRA55_prescribed_atmosphere(arch; backend, include_rivers_and_icebergs=true, dir=JRA55_data_directory) @test haskey(atmosphere.auxiliary_freshwater_flux, :rivers) @test haskey(atmosphere.auxiliary_freshwater_flux, :icebergs) + + # Remove the data directory + rm(JRA55_data_directory; recursive=true) end end From 3321f2e60e464688a577b16ea3b44313eef76154 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 17 Sep 2024 01:45:56 -0400 Subject: [PATCH 03/18] fix jra55 tests --- test/test_jra55.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_jra55.jl b/test/test_jra55.jl index 9a98a97c5..fe43ff8c2 100644 --- a/test/test_jra55.jl +++ b/test/test_jra55.jl @@ -20,7 +20,7 @@ using ClimaOcean.OceanSeaIceModels: PrescribedAtmosphere # This should download a file called "RYF.rsds.1990_1991.nc" jra55_fts = JRA55_field_time_series(test_name; architecture=arch, time_indices, dir=JRA55_data_directory) - test_filename = joinpath(data_directory, "RYF.rsds.1990_1991.nc") + test_filename = joinpath(JRA55_data_directory, "RYF.rsds.1990_1991.nc") @test jra55_fts isa FieldTimeSeries @test jra55_fts.grid isa LatitudeLongitudeGrid From a6ecdca914540e061349d714e20a26da80bc84ce Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 17 Sep 2024 08:27:55 -0400 Subject: [PATCH 04/18] dowload directory also for ecco --- src/DataWrangling/ECCO.jl | 2 +- src/DataWrangling/ecco_metadata.jl | 25 ++++++++++++++++++------- src/DataWrangling/ecco_restoring.jl | 2 +- test/runtests_setup.jl | 1 + test/test_ecco.jl | 8 ++++---- 5 files changed, 25 insertions(+), 13 deletions(-) diff --git a/src/DataWrangling/ECCO.jl b/src/DataWrangling/ECCO.jl index 7f09808fa..a7ee2dd0f 100644 --- a/src/DataWrangling/ECCO.jl +++ b/src/DataWrangling/ECCO.jl @@ -139,7 +139,7 @@ function ecco_field(metadata::ECCOMetadata; download_dataset!(metadata) - ds = Dataset(filename) + ds = Dataset(joinpath(metadata.dir, filename)) if variable_is_three_dimensional(metadata) data = ds[shortname][:, :, :, 1] # The surface layer in three-dimensional ECCO fields is at `k = 1` diff --git a/src/DataWrangling/ecco_metadata.jl b/src/DataWrangling/ecco_metadata.jl index ceb1d30cc..6d8c13259 100644 --- a/src/DataWrangling/ecco_metadata.jl +++ b/src/DataWrangling/ecco_metadata.jl @@ -1,9 +1,15 @@ using CFTime using Dates +using Scratch import Oceananigans.Fields: set! import Base +download_ECCO_cache::String = "" +function __init__() + global download_ECCO_cache = @get_scratch!("ECCO") +end + struct ECCO2Monthly end struct ECCO2Daily end struct ECCO4Monthly end @@ -16,6 +22,7 @@ struct ECCOMetadata{D, V} name :: Symbol dates :: D version :: V + dir :: String end Base.show(io::IO, metadata::ECCOMetadata) = @@ -27,18 +34,21 @@ Base.show(io::IO, metadata::ECCOMetadata) = # The default is the ECCO2Daily dataset at 1993-01-01. function ECCOMetadata(name::Symbol; date = DateTimeProlepticGregorian(1993, 1, 1), - version = ECCO2Daily()) + version = ECCO2Daily(), + dir = download_ECCO_cache) - return ECCOMetadata(name, date, version) + return ECCOMetadata(name, date, version, dir) end +ECCOMetadata(name::Symbol, date, version=ECCO4Monthly(); dir = download_ECCO_cache) = ECCOMetadata(name, date, version, dir) + # Treat ECCOMetadata as an array to allow iteration over the dates. -Base.getindex(metadata::ECCOMetadata, i::Int) = @inbounds ECCOMetadata(metadata.name, metadata.dates[i], metadata.version) +Base.getindex(metadata::ECCOMetadata, i::Int) = @inbounds ECCOMetadata(metadata.name, metadata.dates[i], metadata.version, metadata.dir) Base.length(metadata::ECCOMetadata) = length(metadata.dates) Base.eltype(metadata::ECCOMetadata) = Base.eltype(metadata.dates) -Base.first(metadata::ECCOMetadata) = @inbounds ECCOMetadata(metadata.name, metadata.dates[1], metadata.version) -Base.last(metadata::ECCOMetadata) = @inbounds ECCOMetadata(metadata.name, metadata.dates[end], metadata.version) -Base.iterate(metadata::ECCOMetadata, i=1) = (@inline; (i % UInt) - 1 < length(metadata) ? (@inbounds ECCOMetadata(metadata.name, metadata.dates[i], metadata.version), i + 1) : nothing) +Base.first(metadata::ECCOMetadata) = @inbounds ECCOMetadata(metadata.name, metadata.dates[1], metadata.version, metadata.dir) +Base.last(metadata::ECCOMetadata) = @inbounds ECCOMetadata(metadata.name, metadata.dates[end], metadata.version, metadata.dir) +Base.iterate(metadata::ECCOMetadata, i=1) = (@inline; (i % UInt) - 1 < length(metadata) ? (@inbounds ECCOMetadata(metadata.name, metadata.dates[i], metadata.version, metadata.dir), i + 1) : nothing) Base.axes(metadata::ECCOMetadata{<:AbstractCFDateTime}) = 1 Base.first(metadata::ECCOMetadata{<:AbstractCFDateTime}) = metadata @@ -146,6 +156,7 @@ function download_dataset!(metadata::ECCOMetadata; username = get(ENV, "ECCO_USERNAME", nothing) password = get(ENV, "ECCO_PASSWORD", nothing) + dir = metadata.dir for data in metadata filename = metadata_filename(data) @@ -164,7 +175,7 @@ function download_dataset!(metadata::ECCOMetadata; fileurl = joinpath(url, shortname, year, filename) end - cmd = `wget --http-user=$(username) --http-passwd=$(password) $(fileurl)` + cmd = `wget --http-user=$(username) --http-passwd=$(password) --directory-prefix=$(dir) $(fileurl)` run(cmd) end diff --git a/src/DataWrangling/ecco_restoring.jl b/src/DataWrangling/ecco_restoring.jl index 006ff6ef5..d4a9465c4 100644 --- a/src/DataWrangling/ecco_restoring.jl +++ b/src/DataWrangling/ecco_restoring.jl @@ -52,7 +52,7 @@ function set!(fts::ECCONetCDFFTS, path::ECCOMetadata=fts.path, name::String=fts. metadata = @inbounds path[t] arch = architecture(fts) - f = inpainted_ecco_field(metadata; architecture = arch) + f = inpainted_ecco_field(metadata; architecture=arch) if on_native_grid(backend) set!(fts[t], f) else diff --git a/test/runtests_setup.jl b/test/runtests_setup.jl index b5684e30c..e86d55d59 100644 --- a/test/runtests_setup.jl +++ b/test/runtests_setup.jl @@ -17,3 +17,4 @@ gpu_test = parse(Bool, get(ENV, "GPU_TEST", "false")) test_architectures = gpu_test ? [GPU()] : [CPU()] JRA55_data_directory = gpu_test ? "GPU_JRA55_data" : "CPU_JRA55_data" bathymetry_data_directory = gpu_test ? "GPU_Bathymetry_data" : "CPU_Bathymetry_data" +ECCO_data_directory = gpu_test ? "GPU_ECCO_data" : "CPU_ECCO_data" \ No newline at end of file diff --git a/test/test_ecco.jl b/test/test_ecco.jl index ee59de09e..6cda4176e 100644 --- a/test/test_ecco.jl +++ b/test/test_ecco.jl @@ -17,14 +17,14 @@ using Dates end_date = DateTimeProlepticGregorian(1993, 4, 1) dates = start_date : Month(1) : end_date - temperature = ECCOMetadata(:temperature, dates, ECCO4Monthly()) + temperature = ECCOMetadata(:temperature, dates; dir=ECCO_data_directory) t_restoring = ECCO_restoring_forcing(temperature; timescale = 1000.0) ecco_fts = t_restoring.func.ecco_fts for metadata in temperature temperature_filename = metadata_filename(metadata) - @test isfile(temperature_filename) + @test isfile(joinparth(ECCO_data_directory, temperature_filename)) end @test ecco_fts isa FieldTimeSeries @@ -45,7 +45,7 @@ end for arch in test_architectures grid = LatitudeLongitudeGrid(size = (10, 10, 10), latitude = (-60, -40), longitude = (10, 15), z = (-200, 0)) field = CenterField(grid) - set!(field, ECCOMetadata(:temperature)) - set!(field, ECCOMetadata(:salinity)) + set!(field, ECCOMetadata(:temperature; dir=ECCO_data_directory)) + set!(field, ECCOMetadata(:salinity; dir=ECCO_data_directory)) end end From 5ab2157cca7d24abee7ea6232875cb30467c616f Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 17 Sep 2024 08:41:06 -0400 Subject: [PATCH 05/18] make and delete ECCO directory --- test/test_ecco.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/test_ecco.jl b/test/test_ecco.jl index 6cda4176e..021b97b1d 100644 --- a/test/test_ecco.jl +++ b/test/test_ecco.jl @@ -8,6 +8,12 @@ using Oceananigans.Grids: topology using CFTime using Dates +# Make data directory if it doesn't exist +try + mkdir(ECCO_data_directory) +catch +end + @testset "ECCO fields utilities" begin for arch in test_architectures A = typeof(arch) @@ -49,3 +55,5 @@ end set!(field, ECCOMetadata(:salinity; dir=ECCO_data_directory)) end end + +rm(ECCO_data_directory; recursive=true) \ No newline at end of file From d3cc3c3b23295b91423e2d773062e551ac420efd Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 17 Sep 2024 09:32:57 -0400 Subject: [PATCH 06/18] correct --- test/test_ecco.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_ecco.jl b/test/test_ecco.jl index 021b97b1d..40d097771 100644 --- a/test/test_ecco.jl +++ b/test/test_ecco.jl @@ -30,7 +30,7 @@ end for metadata in temperature temperature_filename = metadata_filename(metadata) - @test isfile(joinparth(ECCO_data_directory, temperature_filename)) + @test isfile(joinpath(ECCO_data_directory, temperature_filename)) end @test ecco_fts isa FieldTimeSeries From ac244e33e90b55cbd994035af4b1480747bf5798 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 18 Sep 2024 08:23:45 -0400 Subject: [PATCH 07/18] try downloading in init --- test/runtests.jl | 18 ++++++++++++++++++ test/test_bathymetry.jl | 15 +++------------ test/test_ecco.jl | 14 +++----------- test/test_jra55.jl | 17 ++++------------- 4 files changed, 28 insertions(+), 36 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 4a1a4b90a..f9d512691 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,9 +4,27 @@ include("runtests_setup.jl") test_group = get(ENV, "TEST_GROUP", :all) test_group = Symbol(test_group) +# Fictitious grid that triggers bathymetry download +function download_bathymetry() + grid = LatitudeLongitudeGrid(size = (10, 10, 1), + longitude = (0, 100), + latitude = (0, 50), + z = (-6000, 0)) + + bottom = regrid_bathymetry(grid) + + return nothing +end + if test_group == :init || test_group == :all using CUDA CUDA.precompile_runtime() + + # Download bathymetry data + download_bathymetry() + + # Download JRA55 data + atmosphere = JRA55_prescribed_atmosphere() end # Tests JRA55 utilities, plus some DataWrangling utilities diff --git a/test/test_bathymetry.jl b/test/test_bathymetry.jl index ff805c2fc..9071f6c0a 100644 --- a/test/test_bathymetry.jl +++ b/test/test_bathymetry.jl @@ -7,12 +7,6 @@ using Statistics @info "Testing Bathymetry utils..." for arch in test_architectures - # Make data directory if it doesn't exist - try - mkdir(bathymetry_data_directory) - catch - end - grid = LatitudeLongitudeGrid(arch; size = (100, 100, 10), longitude = (0, 100), @@ -20,7 +14,7 @@ using Statistics z = (-6000, 0)) # Test that remove_minor_basins!(Z, Inf) does nothing - control_bottom_height = regrid_bathymetry(grid; dir=bathymetry_data_directory) + control_bottom_height = regrid_bathymetry(grid) bottom_height = deepcopy(control_bottom_height) @test_throws ArgumentError remove_minor_basins!(bottom_height, Inf) @@ -59,13 +53,10 @@ using Statistics latitude = (-10, 50), z = (-6000, 0)) - control_bottom_height = regrid_bathymetry(grid; dir=bathymetry_data_directory) - interpolated_bottom_height = regrid_bathymetry(grid; interpolation_passes=100, dir=bathymetry_data_directory) + control_bottom_height = regrid_bathymetry(grid) + interpolated_bottom_height = regrid_bathymetry(grid; interpolation_passes=100) # Testing that multiple passes do not change the solution when refining the grid @test parent(control_bottom_height) == parent(interpolated_bottom_height) - - # Remove the data directory - rm(bathymetry_data_directory; recursive=true) end end \ No newline at end of file diff --git a/test/test_ecco.jl b/test/test_ecco.jl index 40d097771..615eaf366 100644 --- a/test/test_ecco.jl +++ b/test/test_ecco.jl @@ -8,12 +8,6 @@ using Oceananigans.Grids: topology using CFTime using Dates -# Make data directory if it doesn't exist -try - mkdir(ECCO_data_directory) -catch -end - @testset "ECCO fields utilities" begin for arch in test_architectures A = typeof(arch) @@ -23,7 +17,7 @@ end end_date = DateTimeProlepticGregorian(1993, 4, 1) dates = start_date : Month(1) : end_date - temperature = ECCOMetadata(:temperature, dates; dir=ECCO_data_directory) + temperature = ECCOMetadata(:temperature, dates) t_restoring = ECCO_restoring_forcing(temperature; timescale = 1000.0) ecco_fts = t_restoring.func.ecco_fts @@ -51,9 +45,7 @@ end for arch in test_architectures grid = LatitudeLongitudeGrid(size = (10, 10, 10), latitude = (-60, -40), longitude = (10, 15), z = (-200, 0)) field = CenterField(grid) - set!(field, ECCOMetadata(:temperature; dir=ECCO_data_directory)) - set!(field, ECCOMetadata(:salinity; dir=ECCO_data_directory)) + set!(field, ECCOMetadata(:temperature)) + set!(field, ECCOMetadata(:salinity)) end end - -rm(ECCO_data_directory; recursive=true) \ No newline at end of file diff --git a/test/test_jra55.jl b/test/test_jra55.jl index fe43ff8c2..bd2b61832 100644 --- a/test/test_jra55.jl +++ b/test/test_jra55.jl @@ -11,16 +11,10 @@ using ClimaOcean.OceanSeaIceModels: PrescribedAtmosphere test_name = :downwelling_shortwave_radiation time_indices = 1:3 - # Make data directory if it doesn't exist - try - mkdir(JRA55_data_directory) - catch - end - # This should download a file called "RYF.rsds.1990_1991.nc" - jra55_fts = JRA55_field_time_series(test_name; architecture=arch, time_indices, dir=JRA55_data_directory) + jra55_fts = JRA55_field_time_series(test_name; architecture=arch, time_indices) - test_filename = joinpath(JRA55_data_directory, "RYF.rsds.1990_1991.nc") + test_filename = joinpath(download_jra55_cache, "RYF.rsds.1990_1991.nc") @test jra55_fts isa FieldTimeSeries @test jra55_fts.grid isa LatitudeLongitudeGrid @@ -108,16 +102,13 @@ using ClimaOcean.OceanSeaIceModels: PrescribedAtmosphere ##### backend = JRA55NetCDFBackend(2) - atmosphere = JRA55_prescribed_atmosphere(arch; backend, include_rivers_and_icebergs=false, dir=JRA55_data_directory) + atmosphere = JRA55_prescribed_atmosphere(arch; backend, include_rivers_and_icebergs=false) @test atmosphere isa PrescribedAtmosphere @test isnothing(atmosphere.auxiliary_freshwater_flux) - atmosphere = JRA55_prescribed_atmosphere(arch; backend, include_rivers_and_icebergs=true, dir=JRA55_data_directory) + atmosphere = JRA55_prescribed_atmosphere(arch; backend, include_rivers_and_icebergs=true) @test haskey(atmosphere.auxiliary_freshwater_flux, :rivers) @test haskey(atmosphere.auxiliary_freshwater_flux, :icebergs) - - # Remove the data directory - rm(JRA55_data_directory; recursive=true) end end From 61f6703cfe91302af3ab47ebcdadf581e562434d Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 18 Sep 2024 09:11:12 -0400 Subject: [PATCH 08/18] call it "path" instead of dir --- src/DataWrangling/ECCO.jl | 8 ++++---- src/DataWrangling/ecco_metadata.jl | 23 ++++++++++++----------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/DataWrangling/ECCO.jl b/src/DataWrangling/ECCO.jl index a7ee2dd0f..984abe2bd 100644 --- a/src/DataWrangling/ECCO.jl +++ b/src/DataWrangling/ECCO.jl @@ -121,7 +121,6 @@ end horizontal_halo = (1, 1), user_data = nothing, url = ecco_urls[variable_name], - filename = ecco_metadata_filenames[variable_name], short_name = ecco_short_names[variable_name]) Retrieve the ecco field corresponding to `variable_name`. @@ -132,14 +131,15 @@ The data is either: """ function ecco_field(metadata::ECCOMetadata; architecture = CPU(), - horizontal_halo = (3, 3), - filename = metadata_filename(metadata)) + horizontal_halo = (3, 3)) + filename = metadata_filename(metadata) + path = metadata.path shortname = short_name(metadata) download_dataset!(metadata) - ds = Dataset(joinpath(metadata.dir, filename)) + ds = Dataset(joinpath(path, filename)) if variable_is_three_dimensional(metadata) data = ds[shortname][:, :, :, 1] # The surface layer in three-dimensional ECCO fields is at `k = 1` diff --git a/src/DataWrangling/ecco_metadata.jl b/src/DataWrangling/ecco_metadata.jl index 6d8c13259..cbcec316b 100644 --- a/src/DataWrangling/ecco_metadata.jl +++ b/src/DataWrangling/ecco_metadata.jl @@ -22,33 +22,34 @@ struct ECCOMetadata{D, V} name :: Symbol dates :: D version :: V - dir :: String + path :: String end Base.show(io::IO, metadata::ECCOMetadata) = print(io, "ECCOMetadata:", '\n', "├── field: $(metadata.name)", '\n', "├── dates: $(metadata.dates)", '\n', - "└── data version: $(metadata.version)") + "├── version: $(metadata.version)", '\n', + "└── file path: $(metadata.path)") # The default is the ECCO2Daily dataset at 1993-01-01. function ECCOMetadata(name::Symbol; date = DateTimeProlepticGregorian(1993, 1, 1), version = ECCO2Daily(), - dir = download_ECCO_cache) + path = download_ECCO_cache) - return ECCOMetadata(name, date, version, dir) + return ECCOMetadata(name, date, version, path) end -ECCOMetadata(name::Symbol, date, version=ECCO4Monthly(); dir = download_ECCO_cache) = ECCOMetadata(name, date, version, dir) +ECCOMetadata(name::Symbol, date, version=ECCO4Monthly(); path = download_ECCO_cache) = ECCOMetadata(name, date, version, path) # Treat ECCOMetadata as an array to allow iteration over the dates. -Base.getindex(metadata::ECCOMetadata, i::Int) = @inbounds ECCOMetadata(metadata.name, metadata.dates[i], metadata.version, metadata.dir) +Base.getindex(metadata::ECCOMetadata, i::Int) = @inbounds ECCOMetadata(metadata.name, metadata.dates[i], metadata.version, metadata.path) Base.length(metadata::ECCOMetadata) = length(metadata.dates) Base.eltype(metadata::ECCOMetadata) = Base.eltype(metadata.dates) -Base.first(metadata::ECCOMetadata) = @inbounds ECCOMetadata(metadata.name, metadata.dates[1], metadata.version, metadata.dir) -Base.last(metadata::ECCOMetadata) = @inbounds ECCOMetadata(metadata.name, metadata.dates[end], metadata.version, metadata.dir) -Base.iterate(metadata::ECCOMetadata, i=1) = (@inline; (i % UInt) - 1 < length(metadata) ? (@inbounds ECCOMetadata(metadata.name, metadata.dates[i], metadata.version, metadata.dir), i + 1) : nothing) +Base.first(metadata::ECCOMetadata) = @inbounds ECCOMetadata(metadata.name, metadata.dates[1], metadata.version, metadata.path) +Base.last(metadata::ECCOMetadata) = @inbounds ECCOMetadata(metadata.name, metadata.dates[end], metadata.version, metadata.path) +Base.iterate(metadata::ECCOMetadata, i=1) = (@inline; (i % UInt) - 1 < length(metadata) ? (@inbounds ECCOMetadata(metadata.name, metadata.dates[i], metadata.version, metadata.path), i + 1) : nothing) Base.axes(metadata::ECCOMetadata{<:AbstractCFDateTime}) = 1 Base.first(metadata::ECCOMetadata{<:AbstractCFDateTime}) = metadata @@ -156,7 +157,7 @@ function download_dataset!(metadata::ECCOMetadata; username = get(ENV, "ECCO_USERNAME", nothing) password = get(ENV, "ECCO_PASSWORD", nothing) - dir = metadata.dir + path = metadata.path for data in metadata filename = metadata_filename(data) @@ -175,7 +176,7 @@ function download_dataset!(metadata::ECCOMetadata; fileurl = joinpath(url, shortname, year, filename) end - cmd = `wget --http-user=$(username) --http-passwd=$(password) --directory-prefix=$(dir) $(fileurl)` + cmd = `wget --http-user=$(username) --http-passwd=$(password) --directory-prefix=$(path) $(fileurl)` run(cmd) end From 1fce1de4c7b76fb239747dcf34ef65d96ed254a1 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 18 Sep 2024 09:13:47 -0400 Subject: [PATCH 09/18] remove data directories --- test/runtests_setup.jl | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/runtests_setup.jl b/test/runtests_setup.jl index e86d55d59..10d0b1cdd 100644 --- a/test/runtests_setup.jl +++ b/test/runtests_setup.jl @@ -14,7 +14,4 @@ using Oceananigans.OutputReaders: interpolate! using ClimaOcean gpu_test = parse(Bool, get(ENV, "GPU_TEST", "false")) -test_architectures = gpu_test ? [GPU()] : [CPU()] -JRA55_data_directory = gpu_test ? "GPU_JRA55_data" : "CPU_JRA55_data" -bathymetry_data_directory = gpu_test ? "GPU_Bathymetry_data" : "CPU_Bathymetry_data" -ECCO_data_directory = gpu_test ? "GPU_ECCO_data" : "CPU_ECCO_data" \ No newline at end of file +test_architectures = gpu_test ? [GPU()] : [CPU()] \ No newline at end of file From 3bb7369f553ae4016575a3a78a6ef6972ce4e786 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 18 Sep 2024 09:17:39 -0400 Subject: [PATCH 10/18] docstring --- src/DataWrangling/ecco_metadata.jl | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/DataWrangling/ecco_metadata.jl b/src/DataWrangling/ecco_metadata.jl index cbcec316b..e38ec3b1c 100644 --- a/src/DataWrangling/ecco_metadata.jl +++ b/src/DataWrangling/ecco_metadata.jl @@ -18,6 +18,7 @@ struct ECCO4Monthly end # - `name`: The name of the dataset. # - `dates`: The dates of the dataset, in a `AbstractCFDateTime` format. # - `version`: The version of the dataset, could be ECCO2Monthly, ECCO2Daily, or ECCO4Monthly. +# - `path`: The path where the dataset is stored. struct ECCOMetadata{D, V} name :: Symbol dates :: D @@ -32,7 +33,24 @@ Base.show(io::IO, metadata::ECCOMetadata) = "├── version: $(metadata.version)", '\n', "└── file path: $(metadata.path)") -# The default is the ECCO2Daily dataset at 1993-01-01. +""" + ECCOMetadata(name::Symbol; + date = DateTimeProlepticGregorian(1993, 1, 1), + version = ECCO2Daily(), + path = download_ECCO_cache) + +Constructs an `ECCOMetadata` object with the specified parameters. + +# Arguments +============ +- `name::Symbol`: The name of the metadata. + +# Keyword Arguments +=================== +- `date`: The date of the metadata (default: DateTimeProlepticGregorian(1993, 1, 1)). +- `version`: The version of the metadata (for the moment the choices are ECCO2Monthly(), ECCO2Daily(), or ECCO4Monthly()). +- `path`: The path to the datafile (default: download_ECCO_cache). +""" function ECCOMetadata(name::Symbol; date = DateTimeProlepticGregorian(1993, 1, 1), version = ECCO2Daily(), @@ -138,7 +156,8 @@ urls(::ECCOMetadata{<:Any, <:ECCO2Daily}) = "https://ecco.jpl.nasa.gov/drive/f urls(::ECCOMetadata{<:Any, <:ECCO4Monthly}) = "https://ecco.jpl.nasa.gov/drive/files/Version4/Release4/interp_monthly/" """ - download_dataset!(metadata::ECCOMetadata) + download_dataset!(metadata::ECCOMetadata; + url = urls(metadata)) Download the dataset specified by the given metadata. If the metadata contains a single date, the dataset is downloaded directly. If the metadata contains multiple dates, the dataset is From e6cac0fa54fa8506328851c6a36839900fb9a1b5 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 18 Sep 2024 10:55:05 -0400 Subject: [PATCH 11/18] removed filename --- src/DataWrangling/ECCO.jl | 12 +++--------- test/test_jra55.jl | 3 +-- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/src/DataWrangling/ECCO.jl b/src/DataWrangling/ECCO.jl index 984abe2bd..de9a539c7 100644 --- a/src/DataWrangling/ECCO.jl +++ b/src/DataWrangling/ECCO.jl @@ -198,10 +198,9 @@ A boolean field where `true` represents a missing value in the ECCO dataset. """ function ecco_mask(metadata, architecture = CPU(); minimum_value = Float32(-1e5), - maximum_value = Float32(1e5), - filename = metadata_filename(metadata)) + maximum_value = Float32(1e5)) - field = ecco_field(metadata; architecture, filename) + field = ecco_field(metadata; architecture) mask = Field{location(field)...}(field.grid, Bool) # ECCO4 has zeros in place of the missing values, while @@ -220,7 +219,6 @@ ecco_mask() = ecco_mask(ECCOMetadata(:temperature)) """ inpainted_ecco_field(variable_name; architecture = CPU(), - filename = "./inpainted_ecco_fields.nc", mask = ecco_mask(architecture)) Retrieve the ECCO field corresponding to `variable_name` inpainted to fill all the @@ -236,9 +234,6 @@ Keyword Arguments: - `architecture`: either `CPU()` or `GPU()`. -- `filename`: the path where to retrieve the data from. If the file does not exist, - the data will be downloaded from the ECCO dataset. - - `mask`: the mask used to inpaint the field (see `inpaint_mask!`). - `maxiter`: the maximum number of iterations to inpaint the field (see `inpaint_mask!`). @@ -246,12 +241,11 @@ Keyword Arguments: """ function inpainted_ecco_field(metadata::ECCOMetadata; architecture = CPU(), - filename = metadata_filename(metadata), mask = ecco_mask(metadata, architecture), maxiter = Inf, kw...) - f = ecco_field(metadata; architecture, filename, kw...) + f = ecco_field(metadata; architecture, kw...) # Make sure all values are extended properly @info "In-painting ecco $(metadata.name)" diff --git a/test/test_jra55.jl b/test/test_jra55.jl index bd2b61832..2c6ec39af 100644 --- a/test/test_jra55.jl +++ b/test/test_jra55.jl @@ -39,8 +39,7 @@ using ClimaOcean.OceanSeaIceModels: PrescribedAtmosphere in_memory_jra55_fts = JRA55_field_time_series(test_name; time_indices, architecture = arch, - backend = InMemory(2), - dir = JRA55_data_directory) + backend = InMemory(2)) @test in_memory_jra55_fts isa FieldTimeSeries From dd77c95e01e2c91f1648ef1e5d2e38182fc5672c Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Thu, 19 Sep 2024 09:05:20 -0400 Subject: [PATCH 12/18] fix another bug --- test/test_ecco.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_ecco.jl b/test/test_ecco.jl index 615eaf366..7339e2b09 100644 --- a/test/test_ecco.jl +++ b/test/test_ecco.jl @@ -24,7 +24,7 @@ using Dates for metadata in temperature temperature_filename = metadata_filename(metadata) - @test isfile(joinpath(ECCO_data_directory, temperature_filename)) + @test isfile(joinpath(metadata.path, temperature_filename)) end @test ecco_fts isa FieldTimeSeries From 00cc01d5c29ace29c7b33a249b94548b714f2c21 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Fri, 20 Sep 2024 17:43:30 -0400 Subject: [PATCH 13/18] remove filename --- .gitignore | 2 ++ src/DataWrangling/ECCO/ECCO_mask.jl | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index a9fdf43b1..997ed814c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +/Manifest.toml + # Files generated by invoking Julia with --code-coverage *.jl.cov *.jl.*.cov diff --git a/src/DataWrangling/ECCO/ECCO_mask.jl b/src/DataWrangling/ECCO/ECCO_mask.jl index 154dba050..0e7736228 100644 --- a/src/DataWrangling/ECCO/ECCO_mask.jl +++ b/src/DataWrangling/ECCO/ECCO_mask.jl @@ -7,10 +7,9 @@ A boolean field where `true` represents a missing value in the ECCO dataset. """ function ECCO_mask(metadata, architecture = CPU(); minimum_value = Float32(-1e5), - maximum_value = Float32(1e5), - filename = metadata_filename(metadata)) + maximum_value = Float32(1e5)) - field = ECCO_field(metadata; architecture, filename) + field = ECCO_field(metadata; architecture) mask = Field{location(field)...}(field.grid, Bool) # ECCO4 has zeros in place of the missing values, while From a139b8a719b43193c1a6af30bb984f0a2529abc0 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 24 Sep 2024 09:45:56 -0400 Subject: [PATCH 14/18] fix tests --- src/DataWrangling/ECCO/ECCO.jl | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/DataWrangling/ECCO/ECCO.jl b/src/DataWrangling/ECCO/ECCO.jl index 5ea5690fa..af2db8376 100644 --- a/src/DataWrangling/ECCO/ECCO.jl +++ b/src/DataWrangling/ECCO/ECCO.jl @@ -183,8 +183,8 @@ ECCO_field(var_name::Symbol; kw...) = ECCO_field(ECCOMetadata(var_name); kw...) """ inpainted_ECCO_field(variable_name; architecture = CPU(), - filename = "./inpainted_ECCO_fields.nc", - mask = ECCO_mask(architecture)) + mask = ECCO_mask(architecture), + maxiter = Inf) Retrieve the ECCO field corresponding to `variable_name` inpainted to fill all the missing values in the original dataset. @@ -198,20 +198,17 @@ Keyword Arguments: ================== - `architecture`: either `CPU()` or `GPU()`. - - `mask`: the mask used to inpaint the field (see `inpaint_mask!`). - - `maxiter`: the maximum number of iterations to inpaint the field (see `inpaint_mask!`). """ function inpainted_ECCO_field(metadata::ECCOMetadata; architecture = CPU(), - filename = metadata_filename(metadata), mask = ECCO_mask(metadata, architecture), maxiter = Inf, kw...) - f = ECCO_field(metadata; architecture, filename, kw...) + f = ECCO_field(metadata; architecture, kw...) # Make sure all values are extended properly @info "In-painting ECCO $(metadata.name)" From 80113e314e00bc1a77ccf48408cbd4e9f4e88d99 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 8 Oct 2024 00:16:37 +0200 Subject: [PATCH 15/18] change directory --- src/DataWrangling/ECCO/ECCO.jl | 5 +++++ src/DataWrangling/ECCO/ECCO_metadata.jl | 5 ----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/DataWrangling/ECCO/ECCO.jl b/src/DataWrangling/ECCO/ECCO.jl index af2db8376..fd34f4c53 100644 --- a/src/DataWrangling/ECCO/ECCO.jl +++ b/src/DataWrangling/ECCO/ECCO.jl @@ -19,6 +19,11 @@ using Downloads: download using Dates using Adapt +download_ECCO_cache::String = "" +function __init__() + global download_ECCO_cache = @get_scratch!("ECCO") +end + include("ECCO_metadata.jl") include("ECCO_mask.jl") include("ECCO_restoring.jl") diff --git a/src/DataWrangling/ECCO/ECCO_metadata.jl b/src/DataWrangling/ECCO/ECCO_metadata.jl index 4118ecc56..7b709f33f 100644 --- a/src/DataWrangling/ECCO/ECCO_metadata.jl +++ b/src/DataWrangling/ECCO/ECCO_metadata.jl @@ -7,11 +7,6 @@ using Base: @propagate_inbounds import Oceananigans.Fields: set! import Base -download_ECCO_cache::String = "" -function __init__() - global download_ECCO_cache = @get_scratch!("ECCO") -end - struct ECCO2Monthly end struct ECCO2Daily end struct ECCO4Monthly end From 9baa1f6a0de4cb16623af61963259d24587e3b9e Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Tue, 8 Oct 2024 09:16:35 +0200 Subject: [PATCH 16/18] bugfix --- src/DataWrangling/ECCO/ECCO.jl | 1 + src/DataWrangling/ECCO/ECCO_metadata.jl | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DataWrangling/ECCO/ECCO.jl b/src/DataWrangling/ECCO/ECCO.jl index fd34f4c53..7fff90d72 100644 --- a/src/DataWrangling/ECCO/ECCO.jl +++ b/src/DataWrangling/ECCO/ECCO.jl @@ -18,6 +18,7 @@ using NCDatasets using Downloads: download using Dates using Adapt +using Scratch download_ECCO_cache::String = "" function __init__() diff --git a/src/DataWrangling/ECCO/ECCO_metadata.jl b/src/DataWrangling/ECCO/ECCO_metadata.jl index 7b709f33f..52d4b990a 100644 --- a/src/DataWrangling/ECCO/ECCO_metadata.jl +++ b/src/DataWrangling/ECCO/ECCO_metadata.jl @@ -1,6 +1,5 @@ using CFTime using Dates -using Scratch using Base: @propagate_inbounds From 63ea456a9938a024464b57580bd8fc28d6a57c4c Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 10:35:33 +0200 Subject: [PATCH 17/18] initialize cuda runtime --- .buildkite/pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 10586c169..e3fb4e2a9 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -27,6 +27,7 @@ steps: # Force the initialization of the CUDA runtime as it is lazily loaded by default: - "echo '--- Initialize the CUDA runtime'" + - "julia --project -e 'using CUDA; CUDA.precompile_runtime()'" - "julia --project -e 'using Pkg; Pkg.test()'" agents: slurm_gpus: 1 From 97cc7608ce004fd1017182282f510cf3a4986c01 Mon Sep 17 00:00:00 2001 From: Simone Silvestri Date: Wed, 9 Oct 2024 10:37:42 +0200 Subject: [PATCH 18/18] retry tests --- test/runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index f9d512691..80cfaff5a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -5,7 +5,7 @@ test_group = get(ENV, "TEST_GROUP", :all) test_group = Symbol(test_group) # Fictitious grid that triggers bathymetry download -function download_bathymetry() +function download_bathymetry() grid = LatitudeLongitudeGrid(size = (10, 10, 1), longitude = (0, 100), latitude = (0, 50),