diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 10586c169..e3fb4e2a9 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -27,6 +27,7 @@ steps: # Force the initialization of the CUDA runtime as it is lazily loaded by default: - "echo '--- Initialize the CUDA runtime'" + - "julia --project -e 'using CUDA; CUDA.precompile_runtime()'" - "julia --project -e 'using Pkg; Pkg.test()'" agents: slurm_gpus: 1 diff --git a/.gitignore b/.gitignore index a9fdf43b1..997ed814c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +/Manifest.toml + # Files generated by invoking Julia with --code-coverage *.jl.cov *.jl.*.cov diff --git a/src/DataWrangling/ECCO/ECCO.jl b/src/DataWrangling/ECCO/ECCO.jl index c3e13deb2..7fff90d72 100644 --- a/src/DataWrangling/ECCO/ECCO.jl +++ b/src/DataWrangling/ECCO/ECCO.jl @@ -18,6 +18,12 @@ using NCDatasets using Downloads: download using Dates using Adapt +using Scratch + +download_ECCO_cache::String = "" +function __init__() + global download_ECCO_cache = @get_scratch!("ECCO") +end include("ECCO_metadata.jl") include("ECCO_mask.jl") @@ -122,9 +128,8 @@ end architecture = CPU(), horizontal_halo = (1, 1), user_data = nothing, - url = ECCO_urls[variable_name], - filename = ECCO_metadata_filenames[variable_name], - short_name = ECCO_short_names[variable_name]) + url = ecco_urls[variable_name], + short_name = ecco_short_names[variable_name]) Retrieve the ECCO field corresponding to `variable_name`. The data is either: @@ -134,14 +139,15 @@ The data is either: """ function ECCO_field(metadata::ECCOMetadata; architecture = CPU(), - horizontal_halo = (3, 3), - filename = metadata_filename(metadata)) + horizontal_halo = (3, 3)) + filename = metadata_filename(metadata) + path = metadata.path shortname = short_name(metadata) download_dataset!(metadata) - ds = Dataset(filename) + ds = Dataset(joinpath(path, filename)) if variable_is_three_dimensional(metadata) data = ds[shortname][:, :, :, 1] # The surface layer in three-dimensional ECCO fields is at `k = 1` @@ -183,8 +189,8 @@ ECCO_field(var_name::Symbol; kw...) = ECCO_field(ECCOMetadata(var_name); kw...) """ inpainted_ECCO_field(variable_name; architecture = CPU(), - filename = "./inpainted_ECCO_fields.nc", - mask = ECCO_mask(architecture)) + mask = ECCO_mask(architecture), + maxiter = Inf) Retrieve the ECCO field corresponding to `variable_name` inpainted to fill all the missing values in the original dataset. @@ -198,23 +204,17 @@ Keyword Arguments: ================== - `architecture`: either `CPU()` or `GPU()`. - -- `filename`: the path where to retrieve the data from. If the file does not exist, - the data will be downloaded from the ECCO dataset. - - `mask`: the mask used to inpaint the field (see `inpaint_mask!`). - - `maxiter`: the maximum number of iterations to inpaint the field (see `inpaint_mask!`). """ function inpainted_ECCO_field(metadata::ECCOMetadata; architecture = CPU(), - filename = metadata_filename(metadata), mask = ECCO_mask(metadata, architecture), maxiter = Inf, kw...) - f = ECCO_field(metadata; architecture, filename, kw...) + f = ECCO_field(metadata; architecture, kw...) # Make sure all values are extended properly @info "In-painting ECCO $(metadata.name)" diff --git a/src/DataWrangling/ECCO/ECCO_mask.jl b/src/DataWrangling/ECCO/ECCO_mask.jl index 154dba050..0e7736228 100644 --- a/src/DataWrangling/ECCO/ECCO_mask.jl +++ b/src/DataWrangling/ECCO/ECCO_mask.jl @@ -7,10 +7,9 @@ A boolean field where `true` represents a missing value in the ECCO dataset. """ function ECCO_mask(metadata, architecture = CPU(); minimum_value = Float32(-1e5), - maximum_value = Float32(1e5), - filename = metadata_filename(metadata)) + maximum_value = Float32(1e5)) - field = ECCO_field(metadata; architecture, filename) + field = ECCO_field(metadata; architecture) mask = Field{location(field)...}(field.grid, Bool) # ECCO4 has zeros in place of the missing values, while diff --git a/src/DataWrangling/ECCO/ECCO_metadata.jl b/src/DataWrangling/ECCO/ECCO_metadata.jl index ae792201d..52d4b990a 100644 --- a/src/DataWrangling/ECCO/ECCO_metadata.jl +++ b/src/DataWrangling/ECCO/ECCO_metadata.jl @@ -14,36 +14,59 @@ struct ECCO4Monthly end # - `name`: The name of the dataset. # - `dates`: The dates of the dataset, in a `AbstractCFDateTime` format. # - `version`: The version of the dataset, could be ECCO2Monthly, ECCO2Daily, or ECCO4Monthly. +# - `path`: The path where the dataset is stored. struct ECCOMetadata{D, V} name :: Symbol dates :: D version :: V + path :: String end Base.show(io::IO, metadata::ECCOMetadata) = print(io, "ECCOMetadata:", '\n', "├── field: $(metadata.name)", '\n', "├── dates: $(metadata.dates)", '\n', - "└── data version: $(metadata.version)") + "├── version: $(metadata.version)", '\n', + "└── file path: $(metadata.path)") -# The default is the ECCO2Daily dataset at 1993-01-01. +""" + ECCOMetadata(name::Symbol; + date = DateTimeProlepticGregorian(1993, 1, 1), + version = ECCO2Daily(), + path = download_ECCO_cache) + +Constructs an `ECCOMetadata` object with the specified parameters. + +# Arguments +============ +- `name::Symbol`: The name of the metadata. + +# Keyword Arguments +=================== +- `date`: The date of the metadata (default: DateTimeProlepticGregorian(1993, 1, 1)). +- `version`: The version of the metadata (for the moment the choices are ECCO2Monthly(), ECCO2Daily(), or ECCO4Monthly()). +- `path`: The path to the datafile (default: download_ECCO_cache). +""" function ECCOMetadata(name::Symbol; date = DateTimeProlepticGregorian(1993, 1, 1), - version = ECCO2Daily()) + version = ECCO2Daily(), + path = download_ECCO_cache) - return ECCOMetadata(name, date, version) + return ECCOMetadata(name, date, version, path) end +ECCOMetadata(name::Symbol, date, version=ECCO4Monthly(); path = download_ECCO_cache) = ECCOMetadata(name, date, version, path) + # Treat ECCOMetadata as an array to allow iteration over the dates. Base.length(metadata::ECCOMetadata) = length(metadata.dates) Base.eltype(metadata::ECCOMetadata) = Base.eltype(metadata.dates) -@propagate_inbounds Base.getindex(m::ECCOMetadata, i::Int) = ECCOMetadata(m.name, m.dates[i], m.version) -@propagate_inbounds Base.first(m::ECCOMetadata) = ECCOMetadata(m.name, m.dates[1], m.version) -@propagate_inbounds Base.last(m::ECCOMetadata) = ECCOMetadata(m.name, m.dates[end], m.version) +@propagate_inbounds Base.getindex(m::ECCOMetadata, i::Int) = ECCOMetadata(m.name, m.dates[i], m.version, m.path) +@propagate_inbounds Base.first(m::ECCOMetadata) = ECCOMetadata(m.name, m.dates[1], m.version, m.path) +@propagate_inbounds Base.last(m::ECCOMetadata) = ECCOMetadata(m.name, m.dates[end], m.version, m.path) @inline function Base.iterate(m::ECCOMetadata, i=1) if (i % UInt) - 1 < length(m) - return ECCOMetadata(m.name, m.dates[i], m.version), i + 1 + return ECCOMetadata(m.name, m.dates[i], m.version, m.path), i + 1 else return nothing end @@ -136,7 +159,8 @@ urls(::ECCOMetadata{<:Any, <:ECCO2Daily}) = "https://ecco.jpl.nasa.gov/drive/f urls(::ECCOMetadata{<:Any, <:ECCO4Monthly}) = "https://ecco.jpl.nasa.gov/drive/files/Version4/Release4/interp_monthly/" """ - download_dataset!(metadata::ECCOMetadata) + download_dataset!(metadata::ECCOMetadata; + url = urls(metadata)) Download the dataset specified by the given metadata. If the metadata contains a single date, the dataset is downloaded directly. If the metadata contains multiple dates, the dataset is @@ -155,6 +179,7 @@ function download_dataset!(metadata::ECCOMetadata; username = get(ENV, "ECCO_USERNAME", nothing) password = get(ENV, "ECCO_PASSWORD", nothing) + path = metadata.path for data in metadata filename = metadata_filename(data) @@ -174,7 +199,7 @@ function download_dataset!(metadata::ECCOMetadata; fileurl = joinpath(url, shortname, year, filename) end - cmd = `wget --http-user=$(username) --http-passwd=$(password) $(fileurl)` + cmd = `wget --http-user=$(username) --http-passwd=$(password) --directory-prefix=$(path) $(fileurl)` run(cmd) end diff --git a/src/DataWrangling/ECCO/ECCO_restoring.jl b/src/DataWrangling/ECCO/ECCO_restoring.jl index 1cce75c1d..92b1765a4 100644 --- a/src/DataWrangling/ECCO/ECCO_restoring.jl +++ b/src/DataWrangling/ECCO/ECCO_restoring.jl @@ -49,14 +49,7 @@ function set!(fts::ECCONetCDFFTS, path::ECCOMetadata=fts.path, name::String=fts. # find the file associated with the time index metadata = @inbounds path[t] - - arch = architecture(fts) - f = inpainted_ECCO_field(metadata; architecture = arch) - if on_native_grid(backend) - set!(fts[t], f) - else - interpolate!(fts[t], f) - end + set!(fts[t], metadata) end fill_halo_regions!(fts) diff --git a/test/runtests.jl b/test/runtests.jl index 4a1a4b90a..80cfaff5a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,9 +4,27 @@ include("runtests_setup.jl") test_group = get(ENV, "TEST_GROUP", :all) test_group = Symbol(test_group) +# Fictitious grid that triggers bathymetry download +function download_bathymetry() + grid = LatitudeLongitudeGrid(size = (10, 10, 1), + longitude = (0, 100), + latitude = (0, 50), + z = (-6000, 0)) + + bottom = regrid_bathymetry(grid) + + return nothing +end + if test_group == :init || test_group == :all using CUDA CUDA.precompile_runtime() + + # Download bathymetry data + download_bathymetry() + + # Download JRA55 data + atmosphere = JRA55_prescribed_atmosphere() end # Tests JRA55 utilities, plus some DataWrangling utilities diff --git a/test/runtests_setup.jl b/test/runtests_setup.jl index b5002b4bd..10d0b1cdd 100644 --- a/test/runtests_setup.jl +++ b/test/runtests_setup.jl @@ -14,4 +14,4 @@ using Oceananigans.OutputReaders: interpolate! using ClimaOcean gpu_test = parse(Bool, get(ENV, "GPU_TEST", "false")) -test_architectures = gpu_test ? [GPU()] : [CPU()] +test_architectures = gpu_test ? [GPU()] : [CPU()] \ No newline at end of file diff --git a/test/test_bathymetry.jl b/test/test_bathymetry.jl index 0547e8d9b..9071f6c0a 100644 --- a/test/test_bathymetry.jl +++ b/test/test_bathymetry.jl @@ -6,6 +6,7 @@ using Statistics @testset "Availability of Bathymetry" begin @info "Testing Bathymetry utils..." for arch in test_architectures + grid = LatitudeLongitudeGrid(arch; size = (100, 100, 10), longitude = (0, 100), @@ -53,7 +54,7 @@ using Statistics z = (-6000, 0)) control_bottom_height = regrid_bathymetry(grid) - interpolated_bottom_height = regrid_bathymetry(grid; interpolation_passes = 100) + interpolated_bottom_height = regrid_bathymetry(grid; interpolation_passes=100) # Testing that multiple passes do not change the solution when refining the grid @test parent(control_bottom_height) == parent(interpolated_bottom_height) diff --git a/test/test_ecco.jl b/test/test_ecco.jl index 1674b9607..0e83bec41 100644 --- a/test/test_ecco.jl +++ b/test/test_ecco.jl @@ -17,14 +17,14 @@ using Dates end_date = DateTimeProlepticGregorian(1993, 4, 1) dates = start_date : Month(1) : end_date - temperature = ECCOMetadata(:temperature, dates, ECCO4Monthly()) + temperature = ECCOMetadata(:temperature, dates) t_restoring = ECCO_restoring_forcing(temperature; timescale = 1000.0) ECCO_fts = t_restoring.func.ECCO_fts for metadata in temperature temperature_filename = metadata_filename(metadata) - @test isfile(temperature_filename) + @test isfile(joinpath(metadata.path, temperature_filename)) end @test ECCO_fts isa FieldTimeSeries