Add different download directories for JRA55 and Bathymetry

.gitignore

@@ -1,3 +1,5 @@
+/Manifest.toml
+
 # Files generated by invoking Julia with --code-coverage
 *.jl.cov
 *.jl.*.cov

src/DataWrangling/ECCO/ECCO.jl

@@ -122,9 +122,8 @@ end
                 architecture = CPU(),
                 horizontal_halo = (1, 1),
                 user_data = nothing,
-                url = ECCO_urls[variable_name],
-                filename = ECCO_metadata_filenames[variable_name],
-                short_name = ECCO_short_names[variable_name])
+                url = ecco_urls[variable_name],
+                short_name = ecco_short_names[variable_name])
 
 Retrieve the ECCO field corresponding to `variable_name`. 
 The data is either:
@@ -134,14 +133,15 @@ The data is either:
 """
 function ECCO_field(metadata::ECCOMetadata;
                     architecture = CPU(),
-                    horizontal_halo = (3, 3),
-                    filename = metadata_filename(metadata))
+                    horizontal_halo = (3, 3))
 
+    filename  = metadata_filename(metadata)
+    path      = metadata.path
     shortname = short_name(metadata)
 
     download_dataset!(metadata)
 
-    ds = Dataset(filename)
+    ds = Dataset(joinpath(path, filename))
     if variable_is_three_dimensional(metadata)
         data = ds[shortname][:, :, :, 1]
         # The surface layer in three-dimensional ECCO fields is at `k = 1`
@@ -183,8 +183,8 @@ ECCO_field(var_name::Symbol; kw...) = ECCO_field(ECCOMetadata(var_name); kw...)
 """
     inpainted_ECCO_field(variable_name; 
                          architecture = CPU(),
-                         filename = "./inpainted_ECCO_fields.nc",
-                         mask = ECCO_mask(architecture))
+                         mask = ECCO_mask(architecture),
+                         maxiter = Inf)
 
 Retrieve the ECCO field corresponding to `variable_name` inpainted to fill all the
 missing values in the original dataset.
@@ -198,23 +198,17 @@ Keyword Arguments:
 ==================
 
 - `architecture`: either `CPU()` or `GPU()`.
-
-- `filename`: the path where to retrieve the data from. If the file does not exist,
-              the data will be downloaded from the ECCO dataset.
-
 - `mask`: the mask used to inpaint the field (see `inpaint_mask!`).
-
 - `maxiter`: the maximum number of iterations to inpaint the field (see `inpaint_mask!`).
 
 """
 function inpainted_ECCO_field(metadata::ECCOMetadata; 
                               architecture = CPU(),
-                              filename = metadata_filename(metadata),
                               mask = ECCO_mask(metadata, architecture),
                               maxiter = Inf,
                               kw...)
 
-    f = ECCO_field(metadata; architecture, filename, kw...)
+    f = ECCO_field(metadata; architecture, kw...)
 
     # Make sure all values are extended properly
     @info "In-painting ECCO $(metadata.name)"

src/DataWrangling/ECCO/ECCO_mask.jl

@@ -7,10 +7,9 @@ A boolean field where `true` represents a missing value in the ECCO dataset.
 """
 function ECCO_mask(metadata, architecture = CPU(); 
                    minimum_value = Float32(-1e5),
-                   maximum_value = Float32(1e5),
-                   filename = metadata_filename(metadata))
+                   maximum_value = Float32(1e5))
 
-    field = ECCO_field(metadata; architecture, filename)
+    field = ECCO_field(metadata; architecture)
     mask  = Field{location(field)...}(field.grid, Bool)
 
     # ECCO4 has zeros in place of the missing values, while

src/DataWrangling/ECCO/ECCO_metadata.jl

@@ -1,11 +1,17 @@
 using CFTime
 using Dates
+using Scratch
 
 using Base: @propagate_inbounds
 
 import Oceananigans.Fields: set!
 import Base
 
+download_ECCO_cache::String = ""
+function __init__()
+    global download_ECCO_cache = @get_scratch!("ECCO")
+end
+
 struct ECCO2Monthly end
 struct ECCO2Daily end
 struct ECCO4Monthly end
@@ -14,36 +20,59 @@ struct ECCO4Monthly end
 # - `name`: The name of the dataset.
 # - `dates`: The dates of the dataset, in a `AbstractCFDateTime` format.
 # - `version`: The version of the dataset, could be ECCO2Monthly, ECCO2Daily, or ECCO4Monthly.
+# - `path`: The path where the dataset is stored.
 struct ECCOMetadata{D, V} 
     name  :: Symbol
     dates :: D
     version :: V
+    path :: String
 end
 
 Base.show(io::IO, metadata::ECCOMetadata) = 
     print(io, "ECCOMetadata:", '\n',
     "├── field: $(metadata.name)", '\n',
     "├── dates: $(metadata.dates)", '\n',
-    "└── data version: $(metadata.version)")
+    "├── version: $(metadata.version)", '\n',
+    "└── file path: $(metadata.path)")
+
+"""
+    ECCOMetadata(name::Symbol; 
+                 date = DateTimeProlepticGregorian(1993, 1, 1), 
+                 version = ECCO2Daily(), 
+                 path = download_ECCO_cache)
 
-# The default is the ECCO2Daily dataset at 1993-01-01.
+Constructs an `ECCOMetadata` object with the specified parameters.
+
+# Arguments
+============
+- `name::Symbol`: The name of the metadata.
+
+# Keyword Arguments
+===================
+- `date`: The date of the metadata (default: DateTimeProlepticGregorian(1993, 1, 1)).
+- `version`: The version of the metadata (for the moment the choices are ECCO2Monthly(), ECCO2Daily(), or ECCO4Monthly()).
+- `path`: The path to the datafile (default: download_ECCO_cache).
+"""
 function ECCOMetadata(name::Symbol; 
                       date = DateTimeProlepticGregorian(1993, 1, 1),
-                      version = ECCO2Daily()) 
+                   version = ECCO2Daily(),
+                      path = download_ECCO_cache) 
 
-    return ECCOMetadata(name, date, version) 
+    return ECCOMetadata(name, date, version, path) 
 end
 
+ECCOMetadata(name::Symbol, date, version=ECCO4Monthly(); path = download_ECCO_cache) = ECCOMetadata(name, date, version, path)
+
 # Treat ECCOMetadata as an array to allow iteration over the dates.
 Base.length(metadata::ECCOMetadata)       = length(metadata.dates)
 Base.eltype(metadata::ECCOMetadata)       = Base.eltype(metadata.dates)
-@propagate_inbounds Base.getindex(m::ECCOMetadata, i::Int) = ECCOMetadata(m.name, m.dates[i],   m.version)
-@propagate_inbounds Base.first(m::ECCOMetadata)            = ECCOMetadata(m.name, m.dates[1],   m.version)
-@propagate_inbounds Base.last(m::ECCOMetadata)             = ECCOMetadata(m.name, m.dates[end], m.version)
+@propagate_inbounds Base.getindex(m::ECCOMetadata, i::Int) = ECCOMetadata(m.name, m.dates[i],   m.version, m.path)
+@propagate_inbounds Base.first(m::ECCOMetadata)            = ECCOMetadata(m.name, m.dates[1],   m.version, m.path)
+@propagate_inbounds Base.last(m::ECCOMetadata)             = ECCOMetadata(m.name, m.dates[end], m.version, m.path)
 
 @inline function Base.iterate(m::ECCOMetadata, i=1)
     if (i % UInt) - 1 < length(m)
-        return ECCOMetadata(m.name, m.dates[i], m.version), i + 1
+        return ECCOMetadata(m.name, m.dates[i], m.version, m.path), i + 1
     else
         return nothing
     end
@@ -136,7 +165,8 @@ urls(::ECCOMetadata{<:Any, <:ECCO2Daily})   = "https://ecco.jpl.nasa.gov/drive/f
 urls(::ECCOMetadata{<:Any, <:ECCO4Monthly}) = "https://ecco.jpl.nasa.gov/drive/files/Version4/Release4/interp_monthly/"
 
 """
-    download_dataset!(metadata::ECCOMetadata)
+    download_dataset!(metadata::ECCOMetadata;
+                      url = urls(metadata))
 
 Download the dataset specified by the given metadata. If the metadata contains a single date, 
 the dataset is downloaded directly. If the metadata contains multiple dates, the dataset is 
@@ -155,6 +185,7 @@ function download_dataset!(metadata::ECCOMetadata;
 
     username = get(ENV, "ECCO_USERNAME", nothing)
     password = get(ENV, "ECCO_PASSWORD", nothing)
+    path     = metadata.path
 
     for data in metadata
         filename  = metadata_filename(data)
@@ -174,7 +205,7 @@ function download_dataset!(metadata::ECCOMetadata;
                 fileurl = joinpath(url, shortname, year, filename)
             end
 
-            cmd = `wget --http-user=$(username) --http-passwd=$(password) $(fileurl)`
+            cmd = `wget --http-user=$(username) --http-passwd=$(password)  --directory-prefix=$(path) $(fileurl)`
 
             run(cmd)
         end

src/DataWrangling/ECCO/ECCO_restoring.jl

@@ -49,14 +49,7 @@ function set!(fts::ECCONetCDFFTS, path::ECCOMetadata=fts.path, name::String=fts.
 
         # find the file associated with the time index
         metadata = @inbounds path[t] 
-
-        arch = architecture(fts)
-        f = inpainted_ECCO_field(metadata; architecture = arch)
-        if on_native_grid(backend)
-            set!(fts[t], f)
-        else
-            interpolate!(fts[t], f)
-        end
+        set!(fts[t], metadata)
     end
 
     fill_halo_regions!(fts)

test/runtests.jl

@@ -4,9 +4,27 @@ include("runtests_setup.jl")
 test_group = get(ENV, "TEST_GROUP", :all)
 test_group = Symbol(test_group)
 
+# Fictitious grid that triggers bathymetry download
+function download_bathymetry() 
+    grid = LatitudeLongitudeGrid(size = (10, 10, 1), 
+                                 longitude = (0, 100), 
+                                 latitude = (0, 50),
+                                 z = (-6000, 0))
+
+    bottom = regrid_bathymetry(grid)
+
+    return nothing
+end
+
 if test_group == :init || test_group == :all
     using CUDA
     CUDA.precompile_runtime()
+
+    # Download bathymetry data
+    download_bathymetry() 
+
+    # Download JRA55 data
+    atmosphere = JRA55_prescribed_atmosphere()
 end
 
 # Tests JRA55 utilities, plus some DataWrangling utilities

test/runtests_setup.jl

@@ -14,4 +14,4 @@ using Oceananigans.OutputReaders: interpolate!
 using ClimaOcean
 
 gpu_test = parse(Bool, get(ENV, "GPU_TEST", "false"))
-test_architectures = gpu_test ? [GPU()] : [CPU()]
+test_architectures = gpu_test ? [GPU()] : [CPU()]

test/test_bathymetry.jl

@@ -6,6 +6,7 @@ using Statistics
 @testset "Availability of Bathymetry" begin
     @info "Testing Bathymetry utils..."
     for arch in test_architectures
+
         grid = LatitudeLongitudeGrid(arch;
                                      size = (100, 100, 10), 
                                      longitude = (0, 100), 
@@ -53,7 +54,7 @@ using Statistics
                                      z = (-6000, 0))
 
         control_bottom_height = regrid_bathymetry(grid)
-        interpolated_bottom_height = regrid_bathymetry(grid; interpolation_passes = 100)
+        interpolated_bottom_height = regrid_bathymetry(grid; interpolation_passes=100)
 
         # Testing that multiple passes do not change the solution when refining the grid
         @test parent(control_bottom_height) == parent(interpolated_bottom_height)

test/test_ecco.jl

@@ -17,14 +17,14 @@ using Dates
         end_date = DateTimeProlepticGregorian(1993, 4, 1)
         dates = start_date : Month(1) : end_date
 
-        temperature = ECCOMetadata(:temperature, dates, ECCO4Monthly())
+        temperature = ECCOMetadata(:temperature, dates)
         t_restoring = ECCO_restoring_forcing(temperature; timescale = 1000.0)
 
         ECCO_fts = t_restoring.func.ECCO_fts
 
         for metadata in temperature
             temperature_filename = metadata_filename(metadata)
-            @test isfile(temperature_filename)
+            @test isfile(joinpath(metadata.path, temperature_filename))
         end
 
         @test ECCO_fts isa FieldTimeSeries