Merge pull request #112 from alan-turing-institute/dev

For a 0.6.1 release

Author: ablaom

Project.toml

@@ -1,7 +1,7 @@
 name = "MLJTuning"
 uuid = "03970b2e-30c4-11ea-3135-d1576263f10f"
 authors = ["Anthony D. Blaom <anthony.blaom@gmail.com>"]
-version = "0.6.0"
+version = "0.6.1"
 
 [deps]
 ComputationalResources = "ed09eef8-17a6-5b46-8889-db040fac31e3"
@@ -18,8 +18,8 @@ RecipesBase = "3cdcf5f2-1ef4-517c-9805-6587b60abb01"
 ComputationalResources = "^0.3"
 Distributions = "^0.22,^0.23,^0.24"
 LatinHypercubeSampling = "^1.7.2"
-MLJBase = "^0.15,^0.16"
-MLJModelInterface = "^0.3"
+MLJBase = "^0.17"
+MLJModelInterface = "^0.3.7,^0.4"
 ProgressMeter = "^1.3"
 RecipesBase = "^0.8,^0.9,^1"
 julia = "^1"

src/tuned_models.jl

@@ -16,6 +16,7 @@ mutable struct DeterministicTunedModel{T,M<:Deterministic} <: MLJBase.Determinis
     acceleration::AbstractResource
     acceleration_resampling::AbstractResource
     check_measure::Bool
+    cache::Bool
 end
 
 mutable struct ProbabilisticTunedModel{T,M<:Probabilistic} <: MLJBase.Probabilistic
@@ -33,6 +34,7 @@ mutable struct ProbabilisticTunedModel{T,M<:Probabilistic} <: MLJBase.Probabilis
     acceleration::AbstractResource
     acceleration_resampling::AbstractResource
     check_measure::Bool
+    cache::Bool
 end
 
 const EitherTunedModel{T,M} =
@@ -55,7 +57,8 @@ MLJBase.is_wrapper(::Type{<:EitherTunedModel}) = true
                              train_best=true,
                              acceleration=default_resource(),
                              acceleration_resampling=CPU1(),
-                             check_measure=true)
+                             check_measure=true,
+                             cache=true)
 
 Construct a model wrapper for hyperparameter optimization of a
 supervised learner.
@@ -171,9 +174,13 @@ plus other key/value pairs specific to the `tuning` strategy.
 - `acceleration_resampling=CPU1()`: mode of parallelization for
   resampling
 
-- `check_measure`: whether to check `measure` is compatible with the
+- `check_measure=true`: whether to check `measure` is compatible with the
   specified `model` and `operation`)
 
+- `cache=true`: whether to cache model-specific representations of
+  user-suplied data; set to `false` to conserve memory. Speed gains
+  likely limited to the case `resampling isa Holdout`.
+
 """
 function TunedModel(; model=nothing,
                     tuning=Grid(),
@@ -190,7 +197,8 @@ function TunedModel(; model=nothing,
                     n=nothing,
                     acceleration=default_resource(),
                     acceleration_resampling=CPU1(),
-                    check_measure=true)
+                    check_measure=true,
+                    cache=true)
 
     range === nothing && error("You need to specify `range=...`.")
     model == nothing && error("You need to specify model=... .\n"*
@@ -204,15 +212,17 @@ function TunedModel(; model=nothing,
                                               train_best, repeats, n,
                                               acceleration,
                                               acceleration_resampling,
-                                              check_measure)
+                                              check_measure,
+                                              cache)
     elseif model isa Probabilistic
         tuned_model = ProbabilisticTunedModel(model, tuning, resampling,
                                               measure, weights, operation,
                                               range, selection_heuristic,
                                               train_best, repeats, n,
                                               acceleration,
                                               acceleration_resampling,
-                                              check_measure)
+                                              check_measure,
+                                              cache)
     else
         error("Only `Deterministic` and `Probabilistic` "*
               "model types supported.")
@@ -432,7 +442,7 @@ function assemble_events(metamodels,
                 ProgressMeter.updateProgress!(p)
             end
         end
-        # One tresampling_machine per task
+        # One resampling_machine per task
          machs = [resampling_machine,
                  [machine(Resampler(
                      model= resampling_machine.model.model,
@@ -442,8 +452,9 @@ function assemble_events(metamodels,
                      operation     = resampling_machine.model.operation,
                      check_measure = resampling_machine.model.check_measure,
                      repeats       = resampling_machine.model.repeats,
-                     acceleration  = resampling_machine.model.acceleration),
-                          resampling_machine.args...) for _ in 2:length(partitions)]...]
+                     acceleration  = resampling_machine.model.acceleration,
+                     cache         = resampling_machine.model.cache),
+                  resampling_machine.args...) for _ in 2:length(partitions)]...]
 
         @sync for (i, parts) in enumerate(partitions)
             Threads.@spawn begin
@@ -566,7 +577,8 @@ function MLJBase.fit(tuned_model::EitherTunedModel{T,M},
                           operation     = tuned_model.operation,
                           check_measure = tuned_model.check_measure,
                           repeats       = tuned_model.repeats,
-                          acceleration  = tuned_model.acceleration_resampling)
+                          acceleration  = tuned_model.acceleration_resampling,
+                          cache         = tuned_model.cache)
     resampling_machine = machine(resampler, data...)
     history, state = build(nothing, n, tuning, model, state,
                            verbosity, acceleration, resampling_machine)

test/models.jl

@@ -20,4 +20,3 @@ include("models/simple_composite_model.jl")
 include("models/ensembles.jl")
 
 end
-

test/models/Constant.jl

@@ -1,10 +1,10 @@
 ## THE CONSTANT REGRESSOR
 
+const MMI = MLJModelInterface
 export ConstantClassifier, ConstantRegressor,
-    DeterministicConstantClassifier,
-    ProbabilisticConstantClassifer
+       DeterministicConstantClassifier,
+       ProbabilisticConstantClassifer
 
-import MLJBase
 import Distributions
 
 """
@@ -14,49 +14,61 @@ A regressor that, for any new input pattern, predicts the univariate
 probability distribution best fitting the training target data. Use
 `predict_mean` to predict the mean value instead.
 """
-struct ConstantRegressor{D} <: MLJBase.Probabilistic
-    distribution_type::Type{D}
-end
+struct ConstantRegressor{D} <: MMI.Probabilistic end
 
 function ConstantRegressor(; distribution_type=Distributions.Normal)
-    model   = ConstantRegressor(distribution_type)
+    model   = ConstantRegressor{distribution_type}()
     message = clean!(model)
     isempty(message) || @warn message
     return model
 end
 
-function clean!(model::ConstantRegressor)
+function MMI.clean!(model::ConstantRegressor{D}) where D
     message = ""
-    MLJBase.isdistribution(model.distribution_type) ||
+    D <: Distributions.Sampleable ||
         error("$model.distribution_type is not a valid distribution_type.")
     return message
 end
 
-function MLJBase.fit(::ConstantRegressor{D}, verbosity::Int, X, y) where D
+MMI.reformat(::ConstantRegressor, X) = (MMI.matrix(X),)
+MMI.reformat(::ConstantRegressor, X, y) = (MMI.matrix(X), y)
+MMI.selectrows(::ConstantRegressor, I, A) = (view(A, I, :),)
+MMI.selectrows(::ConstantRegressor, I, A, y) = (view(A, I, :), y[I])
+
+function MMI.fit(::ConstantRegressor{D}, verbosity::Int, A, y) where D
     fitresult = Distributions.fit(D, y)
     cache     = nothing
     report    = NamedTuple()
     return fitresult, cache, report
 end
 
-MLJBase.fitted_params(::ConstantRegressor, fitresult) = (target_distribution=fitresult,)
+MMI.fitted_params(::ConstantRegressor, fitresult) =
+    (target_distribution=fitresult,)
 
-MLJBase.predict(::ConstantRegressor, fitresult, Xnew) = fill(fitresult, nrows(Xnew))
+MMI.predict(::ConstantRegressor, fitresult, Xnew) =
+    fill(fitresult, nrows(Xnew))
 
 ##
 ## THE CONSTANT DETERMINISTIC REGRESSOR (FOR TESTING)
 ##
 
-struct DeterministicConstantRegressor <: MLJBase.Deterministic end
+struct DeterministicConstantRegressor <: MMI.Deterministic end
 
-function MLJBase.fit(::DeterministicConstantRegressor, verbosity::Int, X, y)
+function MMI.fit(::DeterministicConstantRegressor, verbosity::Int, X, y)
     fitresult = mean(y)
     cache     = nothing
     report    = NamedTuple()
     return fitresult, cache, report
 end
 
-MLJBase.predict(::DeterministicConstantRegressor, fitresult, Xnew) = fill(fitresult, nrows(Xnew))
+MMI.reformat(::DeterministicConstantRegressor, X) = (MMI.matrix(X),)
+MMI.reformat(::DeterministicConstantRegressor, X, y) = (MMI.matrix(X), y)
+MMI.selectrows(::DeterministicConstantRegressor, I, A) = (view(A, I, :),)
+MMI.selectrows(::DeterministicConstantRegressor, I, A, y) =
+    (view(A, I, :), y[I])
+
+MMI.predict(::DeterministicConstantRegressor, fitresult, Xnew) =
+    fill(fitresult, nrows(Xnew))
 
 ##
 ## THE CONSTANT CLASSIFIER
@@ -71,39 +83,89 @@ training target data. So, `pdf(d, level)` is the proportion of levels
 in the training data coinciding with `level`. Use `predict_mode` to
 obtain the training target mode instead.
 """
-struct ConstantClassifier <: MLJBase.Probabilistic end
+mutable struct ConstantClassifier <: MMI.Probabilistic
+    testing::Bool
+    bogus::Int
+end
+
+ConstantClassifier(; testing=false, bogus=0) =
+    ConstantClassifier(testing, bogus)
+
+function MMI.reformat(model::ConstantClassifier, X)
+    model.testing && @info "reformatting X"
+    return (MMI.matrix(X),)
+end
+
+function MMI.reformat(model::ConstantClassifier, X, y)
+    model.testing && @info "reformatting X, y"
+    return (MMI.matrix(X), y)
+end
+
+function MMI.reformat(model::ConstantClassifier, X, y, w)
+    model.testing && @info "reformatting X, y, w"
+    return (MMI.matrix(X), y, w)
+end
+
+function MMI.selectrows(model::ConstantClassifier, I, A)
+    model.testing && @info "resampling X"
+    return (view(A, I, :),)
+end
+
+function MMI.selectrows(model::ConstantClassifier, I, A, y)
+    model.testing && @info "resampling X, y"
+    return (view(A, I, :), y[I])
+end
+
+function MMI.selectrows(model::ConstantClassifier, I, A, y, ::Nothing)
+    model.testing && @info "resampling X, y, nothing"
+    return (view(A, I, :), y[I], nothing)
+end
+
+function MMI.selectrows(model::ConstantClassifier, I, A, y, w)
+    model.testing && @info "resampling X, y, nothing"
+    return (view(A, I, :), y[I], w[I])
+end
 
 # here `args` is `y` or `y, w`:
-function MLJBase.fit(::ConstantClassifier, verbosity::Int, X, y, w=nothing)
+function MMI.fit(::ConstantClassifier, verbosity::Int, A, y, w=nothing)
     fitresult = Distributions.fit(MLJBase.UnivariateFinite, y, w)
     cache     = nothing
     report    = NamedTuple
     return fitresult, cache, report
 end
 
-MLJBase.fitted_params(::ConstantClassifier, fitresult) = (target_distribution=fitresult,)
+MMI.fitted_params(::ConstantClassifier, fitresult) =
+    (target_distribution=fitresult,)
 
-MLJBase.predict(::ConstantClassifier, fitresult, Xnew) = fill(fitresult, nrows(Xnew))
+MMI.predict(::ConstantClassifier, fitresult, Xnew) =
+    fill(fitresult, nrows(Xnew))
 
 ##
 ## DETERMINISTIC CONSTANT CLASSIFIER (FOR TESTING)
 ##
 
-struct DeterministicConstantClassifier <: MLJBase.Deterministic end
+struct DeterministicConstantClassifier <: MMI.Deterministic end
 
-function MLJBase.fit(::DeterministicConstantClassifier, verbosity::Int, X, y)
+function MMI.fit(::DeterministicConstantClassifier, verbosity::Int, X, y)
     # dump missing target values and make into a regular array:
-    fitresult = mode(skipmissing(y) |> collect) # a CategoricalValue or CategoricalString
+    fitresult = mode(skipmissing(y) |> collect) # a CategoricalValue
     cache     = nothing
     report    = NamedTuple()
     return fitresult, cache, report
 end
 
-MLJBase.predict(::DeterministicConstantClassifier, fitresult, Xnew) = fill(fitresult, nrows(Xnew))
+MMI.reformat(::DeterministicConstantClassifier, X) = (MMI.matrix(X),)
+MMI.reformat(::DeterministicConstantClassifier, X, y) = (MMI.matrix(X), y)
+MMI.selectrows(::DeterministicConstantClassifier, I, A) = (view(A, I, :),)
+MMI.selectrows(::DeterministicConstantClassifier, I, A, y) =
+    (view(A, I, :), y[I])
 
-##
-## METADATA
-##
+MMI.predict(::DeterministicConstantClassifier, fitresult, Xnew) =
+    fill(fitresult, nrows(Xnew))
+
+#
+# METADATA
+#
 
 metadata_pkg.((ConstantRegressor, ConstantClassifier,
                DeterministicConstantRegressor, DeterministicConstantClassifier),
@@ -115,29 +177,29 @@ metadata_pkg.((ConstantRegressor, ConstantClassifier,
               is_wrapper=false)
 
 metadata_model(ConstantRegressor,
-               input=MLJBase.Table(MLJBase.Scientific),
-               target=AbstractVector{MLJBase.Continuous},
+               input=MMI.Table,
+               target=AbstractVector{MMI.Continuous},
                weights=false,
                descr="Constant regressor (Probabilistic).",
                path="MLJModels.ConstantRegressor")
 
 metadata_model(DeterministicConstantRegressor,
-               input=MLJBase.Table(MLJBase.Scientific),
-               target=AbstractVector{MLJBase.Continuous},
+               input=MMI.Table,
+               target=AbstractVector{MMI.Continuous},
                weights=false,
                descr="Constant regressor (Deterministic).",
                path="MLJModels.DeterministicConstantRegressor")
 
 metadata_model(ConstantClassifier,
-               input=MLJBase.Table(MLJBase.Scientific),
-               target=AbstractVector{<:MLJBase.Finite},
+               input=MMI.Table,
+               target=AbstractVector{<:MMI.Finite},
                weights=true,
                descr="Constant classifier (Probabilistic).",
                path="MLJModels.ConstantClassifier")
 
 metadata_model(DeterministicConstantClassifier,
-               input=MLJBase.Table(MLJBase.Scientific),
-               target=AbstractVector{<:MLJBase.Finite},
+               input=MMI.Table,
+               target=AbstractVector{<:MMI.Finite},
                weights=false,
                descr="Constant classifier (Deterministic).",
                path="MLJModels.DeterministicConstantClassifier")

test/models/DecisionTree.jl

@@ -4,7 +4,7 @@ using MLJScientificTypes
 
 using CategoricalArrays
 
-import DecisionTree 
+import DecisionTree
 
 ## DESCRIPTIONS
 
@@ -50,7 +50,7 @@ from the DecisionTree.jl algorithm).
 
 For post-fit pruning, set `post-prune=true` and set
 `min_purity_threshold` appropriately. Other hyperparameters as per
-package documentation cited above. 
+package documentation cited above.
 
 
 """
@@ -214,4 +214,3 @@ metadata_model(DecisionTreeRegressor,
                target=AbstractVector{MLJBase.Continuous},
                weights=false,
                descr=DTR_DESCR)
-

test/strategies/grid.jl

@@ -147,8 +147,6 @@ end
                              resampling=holdout, measure=rms,
                              range=r)
 
-    MLJBase.info_dict(tuned_model)
-
     tuned = machine(tuned_model, X, y)
 
     fit!(tuned, verbosity=0)