From 2c85ed49928633ef9cfbedb23951f6d54d34b932 Mon Sep 17 00:00:00 2001 From: Manikya Date: Sun, 13 Jun 2021 20:40:04 +0530 Subject: [PATCH 01/24] added tabular model --- src/models/Models.jl | 4 ++- src/models/tabularmodel.jl | 59 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 src/models/tabularmodel.jl diff --git a/src/models/Models.jl b/src/models/Models.jl index 65c70ae4d5..957fc8242a 100644 --- a/src/models/Models.jl +++ b/src/models/Models.jl @@ -1,5 +1,6 @@ module Models +using Base: Bool using ..FastAI using BSON @@ -13,9 +14,10 @@ include("blocks.jl") include("xresnet.jl") include("unet.jl") +include("tabularmodel.jl") -export xresnet18, xresnet50, UNetDynamic +export xresnet18, xresnet50, UNetDynamic, TabularModel end diff --git a/src/models/tabularmodel.jl b/src/models/tabularmodel.jl new file mode 100644 index 0000000000..51e224c91d --- /dev/null +++ b/src/models/tabularmodel.jl @@ -0,0 +1,59 @@ +struct TabularModel + embeds + emb_drop + bn_cont + n_emb + n_cont + layers +end + +function TabularModel( + layers; + emb_szs, + n_cont, + out_sz, + ps::Union{Tuple, Vector, Number, Nothing}=nothing, + embed_p::Float64=0., + y_range=nothing, + use_bn::Bool=true, + bn_final::Bool=false, + bn_cont::Bool=true, + act_cls=Flux.relu, + lin_first::Bool=true) + + n_cont = Int64(n_cont) + if isnothing(ps) + ps = zeros(length(layers)) + end + if ps isa Number + ps = fill(ps, length(layers)) + end + embedslist = [Embedding(ni, nf) for (ni, nf) in emb_szs] + emb_drop = Dropout(embed_p) + bn_cont = bn_cont && BatchNorm(n_cont) + n_emb = sum(size(embedlayer.weight)[1] for embedlayer in embedslist) + sizes = append!(zeros(0), [n_emb+n_cont], layers, [out_sz]) + actns = append!([], [act_cls for i in 1:(length(sizes)-1)], [nothing]) + _layers = [linbndrop(Int64(sizes[i]), Int64(sizes[i+1]), use_bn=(use_bn && ((i!=(length(actns)-1)) || bn_final)), p=p, act=a, lin_first=lin_first) for (i, (p, a)) in enumerate(zip(push!(ps, 0.), actns))] + if !isnothing(y_range) + push!(_layers, Chain(@. x->Flux.sigmoid(x) * (y_range[2] - y_range[1]) + y_range[1])) + end + layers = Chain(_layers...) + TabularModel(embedslist, emb_drop, bn_cont, n_emb, n_cont, layers) +end + +function (tm::TabularModel)(x) + x_cat, x_cont = x + if tm.n_emb != 0 + x = [e(x_cat[i, :]) for (i, e) in enumerate(tm.embeds)] + x = vcat(x...) + x = tm.emb_drop(x) + end + if tm.n_cont != 0 + if !isnothing(tm.bn_cont) + x_cont = tm.bn_cont(x_cont) + end + x = tm.n_emb!=0 ? vcat(x, x_cont) : x_cont + end + tm.layers(x) +end \ No newline at end of file From 97546c7ddc6c7608f0e8b669c8daf07ec527ed3c Mon Sep 17 00:00:00 2001 From: Manikya Date: Sun, 13 Jun 2021 22:49:14 +0530 Subject: [PATCH 02/24] fixed batchnorm in tabular model --- src/models/tabularmodel.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/models/tabularmodel.jl b/src/models/tabularmodel.jl index 51e224c91d..46e755bacb 100644 --- a/src/models/tabularmodel.jl +++ b/src/models/tabularmodel.jl @@ -30,7 +30,7 @@ function TabularModel( end embedslist = [Embedding(ni, nf) for (ni, nf) in emb_szs] emb_drop = Dropout(embed_p) - bn_cont = bn_cont && BatchNorm(n_cont) + bn_cont = bn_cont ? BatchNorm(n_cont) : false n_emb = sum(size(embedlayer.weight)[1] for embedlayer in embedslist) sizes = append!(zeros(0), [n_emb+n_cont], layers, [out_sz]) actns = append!([], [act_cls for i in 1:(length(sizes)-1)], [nothing]) @@ -50,7 +50,7 @@ function (tm::TabularModel)(x) x = tm.emb_drop(x) end if tm.n_cont != 0 - if !isnothing(tm.bn_cont) + if (tm.bn_cont != false) x_cont = tm.bn_cont(x_cont) end x = tm.n_emb!=0 ? vcat(x, x_cont) : x_cont From c1bd73bfcb944b4bf53c1dd7834371e745d3e6ae Mon Sep 17 00:00:00 2001 From: Manikya Date: Mon, 14 Jun 2021 02:40:14 +0530 Subject: [PATCH 03/24] added function for calculating embedding dimensions --- src/models/Models.jl | 5 +++-- src/models/tabularmodel.jl | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/models/Models.jl b/src/models/Models.jl index 957fc8242a..afe7614ef8 100644 --- a/src/models/Models.jl +++ b/src/models/Models.jl @@ -1,6 +1,6 @@ module Models -using Base: Bool +using Base: Bool, Symbol using ..FastAI using BSON @@ -17,7 +17,8 @@ include("unet.jl") include("tabularmodel.jl") -export xresnet18, xresnet50, UNetDynamic, TabularModel +export xresnet18, xresnet50, UNetDynamic, TabularModel, +get_emb_sz end diff --git a/src/models/tabularmodel.jl b/src/models/tabularmodel.jl index 46e755bacb..c99ca11359 100644 --- a/src/models/tabularmodel.jl +++ b/src/models/tabularmodel.jl @@ -1,3 +1,23 @@ +function emb_sz_rule(n_cat) + min(600, round(1.6 * n_cat^0.56)) +end + +function _one_emb_sz(catdict, catcol::Symbol, sz_dict=nothing) + sz_dict = isnothing(sz_dict) ? Dict() : sz_dict + n_cat = length(catdict[catcol]) + sz = catcol in keys(sz_dict) ? sz_dict[catcol] : emb_sz_rule(n_cat) + n_cat, sz +end + +function get_emb_sz(catdict, cols, sz_dict=nothing) + [_one_emb_sz(catdict, catcol, sz_dict) for catcol in cols] +end + +# function get_emb_sz(td::TableDataset, sz_dict=nothing) +# cols = Tables.columnaccess(td.table) ? Tables.columnnames(td.table) : Tables.columnnames(Tables.rows(td.table)[1]) +# [_one_emb_sz(catdict, catcol, sz_dict) for catcol in cols] +# end + struct TabularModel embeds emb_drop From 2551fbb8072b447dd288ef47782d6ade935a1265 Mon Sep 17 00:00:00 2001 From: Manikya Date: Thu, 17 Jun 2021 16:56:09 +0530 Subject: [PATCH 04/24] updated tabular model --- src/models/tabularmodel.jl | 60 +++++++++++--------------------------- 1 file changed, 17 insertions(+), 43 deletions(-) diff --git a/src/models/tabularmodel.jl b/src/models/tabularmodel.jl index c99ca11359..17d8356c5d 100644 --- a/src/models/tabularmodel.jl +++ b/src/models/tabularmodel.jl @@ -6,10 +6,10 @@ function _one_emb_sz(catdict, catcol::Symbol, sz_dict=nothing) sz_dict = isnothing(sz_dict) ? Dict() : sz_dict n_cat = length(catdict[catcol]) sz = catcol in keys(sz_dict) ? sz_dict[catcol] : emb_sz_rule(n_cat) - n_cat, sz + Int64(n_cat), Int64(sz) end -function get_emb_sz(catdict, cols, sz_dict=nothing) +function get_emb_sz(catdict, cols; sz_dict=nothing) [_one_emb_sz(catdict, catcol, sz_dict) for catcol in cols] end @@ -18,21 +18,12 @@ end # [_one_emb_sz(catdict, catcol, sz_dict) for catcol in cols] # end -struct TabularModel - embeds - emb_drop - bn_cont - n_emb - n_cont - layers -end - function TabularModel( layers; emb_szs, - n_cont, + n_cont::Int64, out_sz, - ps::Union{Tuple, Vector, Number, Nothing}=nothing, + ps::Union{Tuple, Vector, Number}=0, embed_p::Float64=0., y_range=nothing, use_bn::Bool=true, @@ -41,39 +32,22 @@ function TabularModel( act_cls=Flux.relu, lin_first::Bool=true) - n_cont = Int64(n_cont) - if isnothing(ps) - ps = zeros(length(layers)) - end - if ps isa Number - ps = fill(ps, length(layers)) - end embedslist = [Embedding(ni, nf) for (ni, nf) in emb_szs] emb_drop = Dropout(embed_p) - bn_cont = bn_cont ? BatchNorm(n_cont) : false + embeds = Chain(x -> ntuple(i -> x[i, :], length(emb_szs)), Parallel(vcat, embedslist...), emb_drop) + + bn_cont = bn_cont ? BatchNorm(n_cont) : identity + n_emb = sum(size(embedlayer.weight)[1] for embedlayer in embedslist) - sizes = append!(zeros(0), [n_emb+n_cont], layers, [out_sz]) - actns = append!([], [act_cls for i in 1:(length(sizes)-1)], [nothing]) - _layers = [linbndrop(Int64(sizes[i]), Int64(sizes[i+1]), use_bn=(use_bn && ((i!=(length(actns)-1)) || bn_final)), p=p, act=a, lin_first=lin_first) for (i, (p, a)) in enumerate(zip(push!(ps, 0.), actns))] - if !isnothing(y_range) - push!(_layers, Chain(@. x->Flux.sigmoid(x) * (y_range[2] - y_range[1]) + y_range[1])) - end - layers = Chain(_layers...) - TabularModel(embedslist, emb_drop, bn_cont, n_emb, n_cont, layers) -end + sizes = append!(zeros(0), [n_emb+n_cont], layers) + actns = append!([], [act_cls for i in 1:(length(sizes)-1)]) -function (tm::TabularModel)(x) - x_cat, x_cont = x - if tm.n_emb != 0 - x = [e(x_cat[i, :]) for (i, e) in enumerate(tm.embeds)] - x = vcat(x...) - x = tm.emb_drop(x) + _layers = [] + for (i, (p, a)) in enumerate(zip(Iterators.cycle(ps), actns)) + layer = linbndrop(Int64(sizes[i]), Int64(sizes[i+1]), use_bn=use_bn, p=p, act=a, lin_first=lin_first) + push!(_layers, layer) end - if tm.n_cont != 0 - if (tm.bn_cont != false) - x_cont = tm.bn_cont(x_cont) - end - x = tm.n_emb!=0 ? vcat(x, x_cont) : x_cont - end - tm.layers(x) + push!(_layers, linbndrop(Int64(last(sizes)), Int64(out_sz), use_bn=bn_final, lin_first=lin_first)) + layers = isnothing(y_range) ? Chain(Parallel(vcat, embeds, bn_cont), _layers...) : Chain(Parallel(vcat, embeds, bn_cont), _layers..., @. x->Flux.sigmoid(x) * (y_range[2] - y_range[1]) + y_range[1]) + layers end \ No newline at end of file From ef11450e35309a98f083dcd280648eb0d52bfab5 Mon Sep 17 00:00:00 2001 From: Manikya Bardhan Date: Sat, 19 Jun 2021 14:21:18 +0530 Subject: [PATCH 05/24] Apply suggestions from code review Co-authored-by: Kyle Daruwalla --- src/models/tabularmodel.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/models/tabularmodel.jl b/src/models/tabularmodel.jl index 17d8356c5d..ed9c816630 100644 --- a/src/models/tabularmodel.jl +++ b/src/models/tabularmodel.jl @@ -40,14 +40,13 @@ function TabularModel( n_emb = sum(size(embedlayer.weight)[1] for embedlayer in embedslist) sizes = append!(zeros(0), [n_emb+n_cont], layers) - actns = append!([], [act_cls for i in 1:(length(sizes)-1)]) _layers = [] for (i, (p, a)) in enumerate(zip(Iterators.cycle(ps), actns)) - layer = linbndrop(Int64(sizes[i]), Int64(sizes[i+1]), use_bn=use_bn, p=p, act=a, lin_first=lin_first) + layer = linbndrop(Int64(sizes[i]), Int64(sizes[i+1]), use_bn=use_bn, p=p, act=act_cls, lin_first=lin_first) push!(_layers, layer) end push!(_layers, linbndrop(Int64(last(sizes)), Int64(out_sz), use_bn=bn_final, lin_first=lin_first)) layers = isnothing(y_range) ? Chain(Parallel(vcat, embeds, bn_cont), _layers...) : Chain(Parallel(vcat, embeds, bn_cont), _layers..., @. x->Flux.sigmoid(x) * (y_range[2] - y_range[1]) + y_range[1]) layers -end \ No newline at end of file +end From c0b2922f27996763f03b3360e83354a4aaa39c85 Mon Sep 17 00:00:00 2001 From: Manikya Date: Sun, 20 Jun 2021 01:53:01 +0530 Subject: [PATCH 06/24] simplified tabular model --- src/models/tabularmodel.jl | 64 ++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/src/models/tabularmodel.jl b/src/models/tabularmodel.jl index ed9c816630..3a1262aaff 100644 --- a/src/models/tabularmodel.jl +++ b/src/models/tabularmodel.jl @@ -19,34 +19,38 @@ end # end function TabularModel( - layers; - emb_szs, - n_cont::Int64, - out_sz, - ps::Union{Tuple, Vector, Number}=0, - embed_p::Float64=0., - y_range=nothing, - use_bn::Bool=true, - bn_final::Bool=false, - bn_cont::Bool=true, - act_cls=Flux.relu, - lin_first::Bool=true) - - embedslist = [Embedding(ni, nf) for (ni, nf) in emb_szs] - emb_drop = Dropout(embed_p) - embeds = Chain(x -> ntuple(i -> x[i, :], length(emb_szs)), Parallel(vcat, embedslist...), emb_drop) - - bn_cont = bn_cont ? BatchNorm(n_cont) : identity - - n_emb = sum(size(embedlayer.weight)[1] for embedlayer in embedslist) - sizes = append!(zeros(0), [n_emb+n_cont], layers) - - _layers = [] - for (i, (p, a)) in enumerate(zip(Iterators.cycle(ps), actns)) - layer = linbndrop(Int64(sizes[i]), Int64(sizes[i+1]), use_bn=use_bn, p=p, act=act_cls, lin_first=lin_first) - push!(_layers, layer) - end - push!(_layers, linbndrop(Int64(last(sizes)), Int64(out_sz), use_bn=bn_final, lin_first=lin_first)) - layers = isnothing(y_range) ? Chain(Parallel(vcat, embeds, bn_cont), _layers...) : Chain(Parallel(vcat, embeds, bn_cont), _layers..., @. x->Flux.sigmoid(x) * (y_range[2] - y_range[1]) + y_range[1]) - layers + layers; + emb_szs, + n_cont::Int64, + out_sz, + ps::Union{Tuple, Vector, Number}=0, + embed_p::Float64=0., + y_range=nothing, + use_bn::Bool=true, + bn_final::Bool=false, + bn_cont::Bool=true, + act_cls=Flux.relu, + lin_first::Bool=true) + + embedslist = [Embedding(ni, nf) for (ni, nf) in emb_szs] + emb_drop = Dropout(embed_p) + embeds = Chain(x -> ntuple(i -> x[i, :], length(emb_szs)), Parallel(vcat, embedslist...), emb_drop) + + bn_cont = bn_cont ? BatchNorm(n_cont) : identity + + n_emb = sum(size(embedlayer.weight)[1] for embedlayer in embedslist) + + ps = Iterators.cycle(ps) + classifiers = [] + + first_ps, ps = Iterators.peel(ps) + push!(classifiers, linbndrop(n_emb+n_cont, first(layers); use_bn=use_bn, p=first_ps, lin_first=lin_first, act=act_cls)) + for (isize, osize, p) in zip(layers[1:(end-1)], layers[2:(end)], ps) + layer = linbndrop(isize, osize; use_bn=use_bn, p=p, act=act_cls, lin_first=lin_first) + push!(classifiers, layer) + end + push!(classifiers, linbndrop(last(layers), out_sz; use_bn=bn_final, lin_first=lin_first)) + + layers = isnothing(y_range) ? Chain(Parallel(vcat, embeds, bn_cont), classifiers...) : Chain(Parallel(vcat, embeds, bn_cont), classifiers..., @. x->Flux.sigmoid(x) * (y_range[2] - y_range[1]) + y_range[1]) + layers end From c2c95d54987206533108525219b4757494606ab7 Mon Sep 17 00:00:00 2001 From: Manikya Date: Mon, 28 Jun 2021 01:14:29 +0530 Subject: [PATCH 07/24] updated tabular model --- src/models/tabularmodel.jl | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/src/models/tabularmodel.jl b/src/models/tabularmodel.jl index 3a1262aaff..8ea09d24ee 100644 --- a/src/models/tabularmodel.jl +++ b/src/models/tabularmodel.jl @@ -21,24 +21,29 @@ end function TabularModel( layers; emb_szs, - n_cont::Int64, + n_cont, out_sz, - ps::Union{Tuple, Vector, Number}=0, - embed_p::Float64=0., - y_range=nothing, - use_bn::Bool=true, - bn_final::Bool=false, - bn_cont::Bool=true, + ps=0, + embed_p=0., + use_bn=true, + bn_final=false, + bn_cont=true, act_cls=Flux.relu, - lin_first::Bool=true) + lin_first=true, + final_activation=identity) embedslist = [Embedding(ni, nf) for (ni, nf) in emb_szs] + n_emb = sum(size(embedlayer.weight)[1] for embedlayer in embedslist) + # n_emb = first(Flux.outputsize(embeds, (length(emb_szs), 1))) emb_drop = Dropout(embed_p) - embeds = Chain(x -> ntuple(i -> x[i, :], length(emb_szs)), Parallel(vcat, embedslist...), emb_drop) - - bn_cont = bn_cont ? BatchNorm(n_cont) : identity + embeds = Chain( + x -> collect(eachrow(x)), + x -> ntuple(i -> x[i], length(x)), + Parallel(vcat, embedslist), + emb_drop + ) - n_emb = sum(size(embedlayer.weight)[1] for embedlayer in embedslist) + bn_cont = bn_cont && n_cont>0 ? BatchNorm(n_cont) : identity ps = Iterators.cycle(ps) classifiers = [] @@ -50,7 +55,11 @@ function TabularModel( push!(classifiers, layer) end push!(classifiers, linbndrop(last(layers), out_sz; use_bn=bn_final, lin_first=lin_first)) - - layers = isnothing(y_range) ? Chain(Parallel(vcat, embeds, bn_cont), classifiers...) : Chain(Parallel(vcat, embeds, bn_cont), classifiers..., @. x->Flux.sigmoid(x) * (y_range[2] - y_range[1]) + y_range[1]) + layers = Chain( + x -> tuple(x...), + Parallel(vcat, embeds, Chain(x -> ndims(x)==1 ? Flux.unsqueeze(x, 2) : x, bn_cont)), + classifiers..., + final_activation + ) layers end From a081616f8bd36d89b3a060f0bfcb7c5b05093074 Mon Sep 17 00:00:00 2001 From: Manikya Date: Sun, 4 Jul 2021 15:08:50 +0530 Subject: [PATCH 08/24] refactored TabularModel --- src/models/Models.jl | 4 ++-- src/models/tabularmodel.jl | 46 +++++++++++++++++++++----------------- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/src/models/Models.jl b/src/models/Models.jl index afe7614ef8..5e49e6d401 100644 --- a/src/models/Models.jl +++ b/src/models/Models.jl @@ -17,8 +17,8 @@ include("unet.jl") include("tabularmodel.jl") -export xresnet18, xresnet50, UNetDynamic, TabularModel, -get_emb_sz +export xresnet18, xresnet50, UNetDynamic, +TabularModel, get_emb_sz, embeddingbackbone, continuousbackbone end diff --git a/src/models/tabularmodel.jl b/src/models/tabularmodel.jl index 8ea09d24ee..49c90fda17 100644 --- a/src/models/tabularmodel.jl +++ b/src/models/tabularmodel.jl @@ -18,48 +18,54 @@ end # [_one_emb_sz(catdict, catcol, sz_dict) for catcol in cols] # end +function embeddingbackbone(embedding_sizes, dropoutprob) + embedslist = [Embedding(ni, nf) for (ni, nf) in embedding_sizes] + emb_drop = Dropout(dropoutprob) + Chain( + x -> tuple(eachrow(x)...), + Parallel(vcat, embedslist), + emb_drop + ) +end + +function continuousbackbone(n_cont) + n_cont > 0 ? BatchNorm(n_cont) : identity +end + function TabularModel( + catbackbone, + contbackbone, layers; - emb_szs, + n_cat, n_cont, out_sz, ps=0, - embed_p=0., use_bn=true, bn_final=false, - bn_cont=true, act_cls=Flux.relu, lin_first=true, - final_activation=identity) - - embedslist = [Embedding(ni, nf) for (ni, nf) in emb_szs] - n_emb = sum(size(embedlayer.weight)[1] for embedlayer in embedslist) - # n_emb = first(Flux.outputsize(embeds, (length(emb_szs), 1))) - emb_drop = Dropout(embed_p) - embeds = Chain( - x -> collect(eachrow(x)), - x -> ntuple(i -> x[i], length(x)), - Parallel(vcat, embedslist), - emb_drop + final_activation=identity ) - bn_cont = bn_cont && n_cont>0 ? BatchNorm(n_cont) : identity - + tabularbackbone = Parallel(vcat, catbackbone, contbackbone) + + catoutsize = first(Flux.outputsize(catbackbone, (n_cat, 1))) ps = Iterators.cycle(ps) classifiers = [] first_ps, ps = Iterators.peel(ps) - push!(classifiers, linbndrop(n_emb+n_cont, first(layers); use_bn=use_bn, p=first_ps, lin_first=lin_first, act=act_cls)) + push!(classifiers, linbndrop(catoutsize+n_cont, first(layers); use_bn=use_bn, p=first_ps, lin_first=lin_first, act=act_cls)) + for (isize, osize, p) in zip(layers[1:(end-1)], layers[2:(end)], ps) layer = linbndrop(isize, osize; use_bn=use_bn, p=p, act=act_cls, lin_first=lin_first) push!(classifiers, layer) end + push!(classifiers, linbndrop(last(layers), out_sz; use_bn=bn_final, lin_first=lin_first)) + layers = Chain( - x -> tuple(x...), - Parallel(vcat, embeds, Chain(x -> ndims(x)==1 ? Flux.unsqueeze(x, 2) : x, bn_cont)), + tabularbackbone, classifiers..., final_activation ) - layers end From f565675f73b5d812a20733ef8d50138d21319ffd Mon Sep 17 00:00:00 2001 From: Manikya Date: Mon, 12 Jul 2021 18:16:34 +0530 Subject: [PATCH 09/24] updated tabular model, and added tests --- src/models/Models.jl | 2 +- src/models/tabularmodel.jl | 10 +++++--- test/imports.jl | 1 + test/models/tabularmodel.jl | 51 +++++++++++++++++++++++++++++++++++++ test/runtests.jl | 6 +++++ 5 files changed, 66 insertions(+), 4 deletions(-) create mode 100644 test/models/tabularmodel.jl diff --git a/src/models/Models.jl b/src/models/Models.jl index 5e49e6d401..b75693348b 100644 --- a/src/models/Models.jl +++ b/src/models/Models.jl @@ -18,7 +18,7 @@ include("tabularmodel.jl") export xresnet18, xresnet50, UNetDynamic, -TabularModel, get_emb_sz, embeddingbackbone, continuousbackbone +TabularModel, get_emb_sz, embeddingbackbone, continuousbackbone, sigmoidrange end diff --git a/src/models/tabularmodel.jl b/src/models/tabularmodel.jl index 49c90fda17..8921ec1d41 100644 --- a/src/models/tabularmodel.jl +++ b/src/models/tabularmodel.jl @@ -18,8 +18,12 @@ end # [_one_emb_sz(catdict, catcol, sz_dict) for catcol in cols] # end -function embeddingbackbone(embedding_sizes, dropoutprob) - embedslist = [Embedding(ni, nf) for (ni, nf) in embedding_sizes] +function sigmoidrange(x, low, high) + @. Flux.sigmoid(x) * (high - low) + low +end + +function embeddingbackbone(embedding_sizes, dropoutprob=0.) + embedslist = [Embedding(ni => nf) for (ni, nf) in embedding_sizes] emb_drop = Dropout(dropoutprob) Chain( x -> tuple(eachrow(x)...), @@ -35,7 +39,7 @@ end function TabularModel( catbackbone, contbackbone, - layers; + layers=[200, 100]; n_cat, n_cont, out_sz, diff --git a/test/imports.jl b/test/imports.jl index 53acd55674..de7ca3fc02 100644 --- a/test/imports.jl +++ b/test/imports.jl @@ -6,6 +6,7 @@ using FastAI: Image, Keypoints, Mask, testencoding, Label, OneHot, ProjectiveTra encodedblock, decodedblock, encode, decode, mockblock using FilePathsBase using FastAI.Datasets +using FastAI.Models using DLPipelines import DataAugmentation import DataAugmentation: getbounds diff --git a/test/models/tabularmodel.jl b/test/models/tabularmodel.jl new file mode 100644 index 0000000000..cfbb398c91 --- /dev/null +++ b/test/models/tabularmodel.jl @@ -0,0 +1,51 @@ +include("../imports.jl") + +@testset ExtendedTestSet "TabularModel Components" begin + @testset ExtendedTestSet "embeddingbackbone" begin + embed_szs = [(5, 10), (100, 30), (2, 30)] + embeds = embeddingbackbone(embed_szs, 0.) + x = [rand(1:n) for (n, _) in embed_szs] + + @test size(embeds(x)) == (70, 1) + end + + @testset ExtendedTestSet "continuousbackbone" begin + n = 5 + contback = continuousbackbone(n) + x = rand(5, 1) + @test size(contback(x)) == (5, 1) + end + + @testset ExtendedTestSet "TabularModel" begin + n = 5 + embed_szs = [(5, 10), (100, 30), (2, 30)] + + embeds = embeddingbackbone(embed_szs, 0.) + contback = continuousbackbone(n) + + tm = TabularModel( + embeds, + contback, + [200, 100], + n_cat=3, + n_cont=5, + out_sz=4 + ) + x = ([rand(1:n) for (n, _) in embed_szs], rand(5, 1)) + @test size(tm(x)) == (4, 1) + + tm2 = TabularModel( + embeds, + contback, + [200, 100], + n_cat=3, + n_cont=5, + out_sz=4, + final_activation=x->FastAI.sigmoidrange(x, 2, 5) + ) + y2 = tm2(x) + @test all(y2.> 2) && all(y2.<5) + end +end + + diff --git a/test/runtests.jl b/test/runtests.jl index e5d7beeb79..05a034f473 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -55,4 +55,10 @@ include("imports.jl") end # TODO: test learning rate finder end + + @testset ExtendedTestSet "models/" begin + @testset ExtendedTestSet "tabularmodel.jl" begin + include("models/tabularmodel.jl") + end + end end From 04d27d41a5d0b84739af55c7917f799fa2d9586a Mon Sep 17 00:00:00 2001 From: Manikya Date: Thu, 12 Aug 2021 14:03:42 +0530 Subject: [PATCH 10/24] added classifierbackbone --- src/models/tabularmodel.jl | 78 ++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 36 deletions(-) diff --git a/src/models/tabularmodel.jl b/src/models/tabularmodel.jl index 8921ec1d41..3638be489d 100644 --- a/src/models/tabularmodel.jl +++ b/src/models/tabularmodel.jl @@ -1,30 +1,25 @@ function emb_sz_rule(n_cat) - min(600, round(1.6 * n_cat^0.56)) + min(600, round(1.6 * n_cat^0.56)) end function _one_emb_sz(catdict, catcol::Symbol, sz_dict=nothing) - sz_dict = isnothing(sz_dict) ? Dict() : sz_dict - n_cat = length(catdict[catcol]) - sz = catcol in keys(sz_dict) ? sz_dict[catcol] : emb_sz_rule(n_cat) - Int64(n_cat), Int64(sz) + sz_dict = isnothing(sz_dict) ? Dict() : sz_dict + n_cat = length(catdict[catcol]) + sz = catcol in keys(sz_dict) ? sz_dict[catcol] : emb_sz_rule(n_cat) + Int64(n_cat)+1, Int64(sz) end function get_emb_sz(catdict, cols; sz_dict=nothing) - [_one_emb_sz(catdict, catcol, sz_dict) for catcol in cols] + [_one_emb_sz(catdict, catcol, sz_dict) for catcol in cols] end -# function get_emb_sz(td::TableDataset, sz_dict=nothing) -# cols = Tables.columnaccess(td.table) ? Tables.columnnames(td.table) : Tables.columnnames(Tables.rows(td.table)[1]) -# [_one_emb_sz(catdict, catcol, sz_dict) for catcol in cols] -# end - function sigmoidrange(x, low, high) - @. Flux.sigmoid(x) * (high - low) + low + @. Flux.sigmoid(x) * (high - low) + low end function embeddingbackbone(embedding_sizes, dropoutprob=0.) embedslist = [Embedding(ni => nf) for (ni, nf) in embedding_sizes] - emb_drop = Dropout(dropoutprob) + emb_drop = dropoutprob==0. ? identity : Dropout(dropoutprob) Chain( x -> tuple(eachrow(x)...), Parallel(vcat, embedslist), @@ -36,40 +31,51 @@ function continuousbackbone(n_cont) n_cont > 0 ? BatchNorm(n_cont) : identity end -function TabularModel( - catbackbone, - contbackbone, - layers=[200, 100]; - n_cat, - n_cont, - out_sz, +function classifierbackbone( + layers; ps=0, use_bn=true, bn_final=false, act_cls=Flux.relu, - lin_first=true, - final_activation=identity - ) - - tabularbackbone = Parallel(vcat, catbackbone, contbackbone) - - catoutsize = first(Flux.outputsize(catbackbone, (n_cat, 1))) + lin_first=true) ps = Iterators.cycle(ps) classifiers = [] - first_ps, ps = Iterators.peel(ps) - push!(classifiers, linbndrop(catoutsize+n_cont, first(layers); use_bn=use_bn, p=first_ps, lin_first=lin_first, act=act_cls)) - - for (isize, osize, p) in zip(layers[1:(end-1)], layers[2:(end)], ps) + for (isize, osize, p) in zip(layers[1:(end-1)], layers[2:end], ps) layer = linbndrop(isize, osize; use_bn=use_bn, p=p, act=act_cls, lin_first=lin_first) push!(classifiers, layer) end - - push!(classifiers, linbndrop(last(layers), out_sz; use_bn=bn_final, lin_first=lin_first)) - - layers = Chain( + Chain(classifiers...) +end + +function TabularModel( + catbackbone, + contbackbone, + classifierbackbone; + final_activation=identity) + tabularbackbone = Parallel(vcat, catbackbone, contbackbone) + Chain( tabularbackbone, - classifiers..., + classifierbackbone, final_activation ) end + +function TabularModel( + catcols, + n_cont::Number, + out_sz::Number, + layers=[200, 100]; + catdict, + embszs=nothing, + ps=0.) + embedszs = get_emb_sz(catdict, catcols, sz_dict=embszs) + catback = embeddingbackbone(embedszs) + contback = continuousbackbone(n_cont) + + classifierin = mapreduce(layer -> size(layer.weight)[1], +, catback[2].layers, init = n_cont) + layers = append!([classifierin], layers, [out_sz]) + classback = classifierbackbone(layers, ps=ps) + + TabularModel(catback, contback, classback) +end From e1c22631b78057f81c547b6fedd9e730745a40eb Mon Sep 17 00:00:00 2001 From: Manikya Date: Thu, 12 Aug 2021 15:02:57 +0530 Subject: [PATCH 11/24] update tablemodel tests --- src/models/tabularmodel.jl | 6 +++--- test/models/tabularmodel.jl | 38 +++++++++++++++++-------------------- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/src/models/tabularmodel.jl b/src/models/tabularmodel.jl index 3638be489d..5ecb0311ce 100644 --- a/src/models/tabularmodel.jl +++ b/src/models/tabularmodel.jl @@ -18,7 +18,7 @@ function sigmoidrange(x, low, high) end function embeddingbackbone(embedding_sizes, dropoutprob=0.) - embedslist = [Embedding(ni => nf) for (ni, nf) in embedding_sizes] + embedslist = [Flux.Embedding(ni, nf) for (ni, nf) in embedding_sizes] emb_drop = dropoutprob==0. ? identity : Dropout(dropoutprob) Chain( x -> tuple(eachrow(x)...), @@ -67,9 +67,9 @@ function TabularModel( out_sz::Number, layers=[200, 100]; catdict, - embszs=nothing, + sz_dict=nothing, ps=0.) - embedszs = get_emb_sz(catdict, catcols, sz_dict=embszs) + embedszs = get_emb_sz(catdict, catcols, sz_dict=sz_dict) catback = embeddingbackbone(embedszs) contback = continuousbackbone(n_cont) diff --git a/test/models/tabularmodel.jl b/test/models/tabularmodel.jl index cfbb398c91..d4c3a9ce7a 100644 --- a/test/models/tabularmodel.jl +++ b/test/models/tabularmodel.jl @@ -16,35 +16,31 @@ include("../imports.jl") @test size(contback(x)) == (5, 1) end + @testset ExtendedTestSet "classifierbackbone" begin + classback = classifierbackbone([10, 200, 100, 2]) + x = rand(10, 2) + @test size(classback(x)) == (2, 2) + end + @testset ExtendedTestSet "TabularModel" begin n = 5 embed_szs = [(5, 10), (100, 30), (2, 30)] embeds = embeddingbackbone(embed_szs, 0.) contback = continuousbackbone(n) + classback = classifierbackbone([75, 200, 100, 4]) + + tm = TabularModel(embeds, contback, classback, final_activation = x->FastAI.sigmoidrange(x, 2, 5)) - tm = TabularModel( - embeds, - contback, - [200, 100], - n_cat=3, - n_cont=5, - out_sz=4 - ) x = ([rand(1:n) for (n, _) in embed_szs], rand(5, 1)) - @test size(tm(x)) == (4, 1) - - tm2 = TabularModel( - embeds, - contback, - [200, 100], - n_cat=3, - n_cont=5, - out_sz=4, - final_activation=x->FastAI.sigmoidrange(x, 2, 5) - ) - y2 = tm2(x) - @test all(y2.> 2) && all(y2.<5) + y1 = tm(x) + @test size(y1) == (4, 1) + @test all(y1.> 2) && all(y1.<5) + + catcols = [:a, :b, :c] + catdict = Dict(:a => rand(4), :b => rand(99), :c => rand(1)) + tm2 = TabularModel(catcols, n, 4, [200, 100], catdict = catdict, sz_dict = Dict(:a=>10, :b=>30, :c=>30)) + @test size(tm2(x)) == (4, 1) end end From b4d7149fa2c9ea57bf67667f0ae780f362e93060 Mon Sep 17 00:00:00 2001 From: Manikya Date: Thu, 12 Aug 2021 16:37:21 +0530 Subject: [PATCH 12/24] export classifierbackbone --- src/models/Models.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/models/Models.jl b/src/models/Models.jl index b75693348b..3c55d38f29 100644 --- a/src/models/Models.jl +++ b/src/models/Models.jl @@ -18,7 +18,7 @@ include("tabularmodel.jl") export xresnet18, xresnet50, UNetDynamic, -TabularModel, get_emb_sz, embeddingbackbone, continuousbackbone, sigmoidrange +TabularModel, get_emb_sz, embeddingbackbone, continuousbackbone, classifierbackbone, sigmoidrange end From bc250a1c5a0d706eac9aea4185bea5099c5f1011 Mon Sep 17 00:00:00 2001 From: Manikya Date: Sat, 14 Aug 2021 11:59:21 +0530 Subject: [PATCH 13/24] refactored TabularModel methods --- Project.toml | 2 +- src/models/Models.jl | 4 +-- src/models/tabularmodel.jl | 73 ++++++++++++++++++++------------------ 3 files changed, 41 insertions(+), 38 deletions(-) diff --git a/Project.toml b/Project.toml index 5ceb560da6..0a24e9a423 100644 --- a/Project.toml +++ b/Project.toml @@ -53,8 +53,8 @@ Flux = "0.12" FluxTraining = "0.2" Glob = "1" IndirectArrays = "0.5" -LearnBase = "0.3, 0.4" JLD2 = "0.4" +LearnBase = "0.3, 0.4" MLDataPattern = "0.5" Makie = "0.15" MosaicViews = "0.2, 0.3" diff --git a/src/models/Models.jl b/src/models/Models.jl index 3c55d38f29..9401596839 100644 --- a/src/models/Models.jl +++ b/src/models/Models.jl @@ -17,8 +17,6 @@ include("unet.jl") include("tabularmodel.jl") -export xresnet18, xresnet50, UNetDynamic, -TabularModel, get_emb_sz, embeddingbackbone, continuousbackbone, classifierbackbone, sigmoidrange - +export xresnet18, xresnet50, UNetDynamic, TabularModel end diff --git a/src/models/tabularmodel.jl b/src/models/tabularmodel.jl index 5ecb0311ce..71afe72720 100644 --- a/src/models/tabularmodel.jl +++ b/src/models/tabularmodel.jl @@ -2,22 +2,22 @@ function emb_sz_rule(n_cat) min(600, round(1.6 * n_cat^0.56)) end -function _one_emb_sz(catdict, catcol::Symbol, sz_dict=nothing) +function _one_emb_sz(cardinalitydict, catcol, sz_dict=nothing) sz_dict = isnothing(sz_dict) ? Dict() : sz_dict - n_cat = length(catdict[catcol]) + n_cat = cardinalitydict[catcol] sz = catcol in keys(sz_dict) ? sz_dict[catcol] : emb_sz_rule(n_cat) Int64(n_cat)+1, Int64(sz) end -function get_emb_sz(catdict, cols; sz_dict=nothing) - [_one_emb_sz(catdict, catcol, sz_dict) for catcol in cols] +function get_emb_sz(cardinalitydict, cols; sz_dict=nothing) + map(catcol -> _one_emb_sz(cardinalitydict, catcol, sz_dict), cols) end function sigmoidrange(x, low, high) @. Flux.sigmoid(x) * (high - low) + low end -function embeddingbackbone(embedding_sizes, dropoutprob=0.) +function tabular_embedding_backbone(embedding_sizes, dropoutprob=0.) embedslist = [Flux.Embedding(ni, nf) for (ni, nf) in embedding_sizes] emb_drop = dropoutprob==0. ? identity : Dropout(dropoutprob) Chain( @@ -27,55 +27,60 @@ function embeddingbackbone(embedding_sizes, dropoutprob=0.) ) end -function continuousbackbone(n_cont) - n_cont > 0 ? BatchNorm(n_cont) : identity +function tabular_continuous_backbone(n_cont) + BatchNorm(n_cont) end -function classifierbackbone( - layers; - ps=0, +function TabularModel( + catbackbone, + contbackbone; + outsz, + layers=[200, 100], + kwargs...) + TabularModel(catbackbone, contbackbone, Dense(layers[end], outsz); layers=layers, kwargs...) +end + +function TabularModel( + catbackbone, + contbackbone, + finalclassifier; + layers=[200, 100], + ps=0., use_bn=true, - bn_final=false, act_cls=Flux.relu, lin_first=true) + + tabularbackbone = Parallel(vcat, catbackbone, contbackbone) + + classifierin = mapreduce(layer -> size(layer.weight)[1], +, catbackbone[2].layers, init = contbackbone.chs) ps = Iterators.cycle(ps) classifiers = [] + first_ps, ps = Iterators.peel(ps) + push!(classifiers, linbndrop(classifierin, first(layers); use_bn=use_bn, p=first_ps, lin_first=lin_first, act=act_cls)) + for (isize, osize, p) in zip(layers[1:(end-1)], layers[2:end], ps) layer = linbndrop(isize, osize; use_bn=use_bn, p=p, act=act_cls, lin_first=lin_first) push!(classifiers, layer) end - Chain(classifiers...) -end - -function TabularModel( - catbackbone, - contbackbone, - classifierbackbone; - final_activation=identity) - tabularbackbone = Parallel(vcat, catbackbone, contbackbone) + Chain( tabularbackbone, - classifierbackbone, - final_activation + classifiers..., + finalclassifier ) end function TabularModel( catcols, n_cont::Number, - out_sz::Number, + outsz::Number, layers=[200, 100]; - catdict, - sz_dict=nothing, - ps=0.) - embedszs = get_emb_sz(catdict, catcols, sz_dict=sz_dict) - catback = embeddingbackbone(embedszs) - contback = continuousbackbone(n_cont) - - classifierin = mapreduce(layer -> size(layer.weight)[1], +, catback[2].layers, init = n_cont) - layers = append!([classifierin], layers, [out_sz]) - classback = classifierbackbone(layers, ps=ps) + cardinalitydict, + sz_dict=nothing) + embedszs = get_emb_sz(cardinalitydict, catcols, sz_dict=sz_dict) + catback = tabular_embedding_backbone(embedszs) + contback = tabular_continuous_backbone(n_cont) - TabularModel(catback, contback, classback) + TabularModel(catback, contback; layers=layers, outsz=outsz) end From 506f889653cbb90d618076911f3338649816f2c7 Mon Sep 17 00:00:00 2001 From: Manikya Date: Sat, 14 Aug 2021 11:59:33 +0530 Subject: [PATCH 14/24] updated tabular model tests --- test/models/tabularmodel.jl | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/test/models/tabularmodel.jl b/test/models/tabularmodel.jl index d4c3a9ce7a..979a0dc499 100644 --- a/test/models/tabularmodel.jl +++ b/test/models/tabularmodel.jl @@ -3,7 +3,7 @@ include("../imports.jl") @testset ExtendedTestSet "TabularModel Components" begin @testset ExtendedTestSet "embeddingbackbone" begin embed_szs = [(5, 10), (100, 30), (2, 30)] - embeds = embeddingbackbone(embed_szs, 0.) + embeds = FastAI.Models.tabular_embedding_backbone(embed_szs, 0.) x = [rand(1:n) for (n, _) in embed_szs] @test size(embeds(x)) == (70, 1) @@ -11,36 +11,31 @@ include("../imports.jl") @testset ExtendedTestSet "continuousbackbone" begin n = 5 - contback = continuousbackbone(n) + contback = FastAI.Models.tabular_continuous_backbone(n) x = rand(5, 1) @test size(contback(x)) == (5, 1) end - @testset ExtendedTestSet "classifierbackbone" begin - classback = classifierbackbone([10, 200, 100, 2]) - x = rand(10, 2) - @test size(classback(x)) == (2, 2) - end - @testset ExtendedTestSet "TabularModel" begin n = 5 embed_szs = [(5, 10), (100, 30), (2, 30)] - embeds = embeddingbackbone(embed_szs, 0.) - contback = continuousbackbone(n) - classback = classifierbackbone([75, 200, 100, 4]) - - tm = TabularModel(embeds, contback, classback, final_activation = x->FastAI.sigmoidrange(x, 2, 5)) + embeds = FastAI.Models.tabular_embedding_backbone(embed_szs, 0.) + contback = FastAI.Models.tabular_continuous_backbone(n) x = ([rand(1:n) for (n, _) in embed_szs], rand(5, 1)) - y1 = tm(x) - @test size(y1) == (4, 1) - @test all(y1.> 2) && all(y1.<5) + + tm = TabularModel(embeds, contback; outsz=4) + @test size(tm(x)) == (4, 1) + + tm2 = TabularModel(embeds, contback, Chain(Dense(100, 4), x->FastAI.Models.sigmoidrange(x, 2, 5))) + y2 = tm2(x) + @test all(y2.> 2) && all(y2.<5) catcols = [:a, :b, :c] - catdict = Dict(:a => rand(4), :b => rand(99), :c => rand(1)) - tm2 = TabularModel(catcols, n, 4, [200, 100], catdict = catdict, sz_dict = Dict(:a=>10, :b=>30, :c=>30)) - @test size(tm2(x)) == (4, 1) + cardict = Dict(:a => 4, :b => 99, :c => 1) + tm3 = TabularModel(catcols, n, 4, [200, 100], cardinalitydict = cardict, sz_dict = Dict(:a=>10, :b=>30, :c=>30)) + @test size(tm3(x)) == (4, 1) end end From 725c6ddb4d159cfd459a313ddcdc4a4a407adfac Mon Sep 17 00:00:00 2001 From: Manikya Date: Sat, 14 Aug 2021 13:33:36 +0530 Subject: [PATCH 15/24] add TabularModel docstring --- src/models/tabularmodel.jl | 43 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/models/tabularmodel.jl b/src/models/tabularmodel.jl index 71afe72720..6d1ef18219 100644 --- a/src/models/tabularmodel.jl +++ b/src/models/tabularmodel.jl @@ -9,6 +9,20 @@ function _one_emb_sz(cardinalitydict, catcol, sz_dict=nothing) Int64(n_cat)+1, Int64(sz) end +""" + get_emb_sz(cardinalitydict, cols; sz_dict=nothing) + +Returns a collection of tuples containing embedding dimensions for each column +in `cols`. `cardinalitydict` is an indexable collection which maps to the cardinality +for each column present in `cols`. + +## Keyword arguments + +- `sz_dict`: An indexable collection which may contain the required embedding + size for a particular column present in `cols`. If not passed, then the size is + calculated using fastai's rule of thumb for embedding dimensions. +""" + function get_emb_sz(cardinalitydict, cols; sz_dict=nothing) map(catcol -> _one_emb_sz(cardinalitydict, catcol, sz_dict), cols) end @@ -31,6 +45,35 @@ function tabular_continuous_backbone(n_cont) BatchNorm(n_cont) end +""" + TabularModel(catbackbone, contbackbone, [finalclassifier]; kwargs...) + TabularModel(catcols, `n_cont::Number, outsz::Number[; kwargs...) + +Create a tabular model which takes in a tuple of categorical values +(label or one-hot encoded) and continuous values. The default categorical backbone is +a Parallel of Embedding layers corresponding to each categorical variable, and continuous +variables are just BatchNormed. The output from these backbones is then passed through +a final classifier block. + +## Keyword arguments + +- `outsz`: The output size of the final classifier block. For single classification tasks, + this would just be the number of classes and for regression tasks, this could be the + number of target continuous variables. +- `layers`: The sizes of the hidden layers in the classifier block. +- `ps`: Dropout probability. This could either be a single number which would be used for + for all the classifier layers, or a collection of numbers which are cycled through + for each layer. +- `use_bn`: Boolean variable which controls whether to use batch normalization in the classifier. +- `act_cls`: The activation function to use in the classifier layers. +- `lin_first`: Controls if the linear layer comes before or after BatchNorm and Dropout. +- `cardinalitydict`: An indexable collection which maps to the cardinality for each column present + in `catcols`. +- `sz_dict`: An indexable collection which may contain the required embedding + size for a particular column present in `cols`. If not passed, then the size is + calculated using fastai's rule of thumb for embedding dimensions. +""" + function TabularModel( catbackbone, contbackbone; From 59eb66ae0da0d984b934f0b6e57010c2d2ab1b80 Mon Sep 17 00:00:00 2001 From: Manikya Bardhan Date: Mon, 16 Aug 2021 12:26:38 +0530 Subject: [PATCH 16/24] renamed args Co-authored-by: Kyle Daruwalla --- src/models/tabularmodel.jl | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/models/tabularmodel.jl b/src/models/tabularmodel.jl index 6d1ef18219..86149dcad9 100644 --- a/src/models/tabularmodel.jl +++ b/src/models/tabularmodel.jl @@ -77,7 +77,7 @@ a final classifier block. function TabularModel( catbackbone, contbackbone; - outsz, + outsize, layers=[200, 100], kwargs...) TabularModel(catbackbone, contbackbone, Dense(layers[end], outsz); layers=layers, kwargs...) @@ -87,15 +87,16 @@ function TabularModel( catbackbone, contbackbone, finalclassifier; - layers=[200, 100], - ps=0., - use_bn=true, - act_cls=Flux.relu, - lin_first=true) + layersizes=[200, 100], + dropout_rates=0., + batchnorm=true, + activation=Flux.relu, + linear_first=true) tabularbackbone = Parallel(vcat, catbackbone, contbackbone) - classifierin = mapreduce(layer -> size(layer.weight)[1], +, catbackbone[2].layers, init = contbackbone.chs) + classifierin = mapreduce(layer -> size(layer.weight)[1], +, catbackbone[2].layers; + init = contbackbone.chs) ps = Iterators.cycle(ps) classifiers = [] From d4fded075325b0d200af6f37d5d163b40a435ec6 Mon Sep 17 00:00:00 2001 From: Manikya Date: Mon, 16 Aug 2021 18:28:47 +0530 Subject: [PATCH 17/24] updated docstrings and embed dims calculation, made args usage consistent --- src/models/tabularmodel.jl | 95 ++++++++++++++++++++----------------- test/models/tabularmodel.jl | 7 ++- 2 files changed, 54 insertions(+), 48 deletions(-) diff --git a/src/models/tabularmodel.jl b/src/models/tabularmodel.jl index 86149dcad9..9781414444 100644 --- a/src/models/tabularmodel.jl +++ b/src/models/tabularmodel.jl @@ -1,30 +1,39 @@ +""" + emb_sz_rule(n_cat) + +Returns an embedding size corresponding to the number of classes for a +categorical variable using the rule of thumb present in python fastai. +(see https://github.com/fastai/fastai/blob/2742fe844573d06e700f869839fb9ec5f3a9bca9/fastai/tabular/model.py#L12) +""" function emb_sz_rule(n_cat) min(600, round(1.6 * n_cat^0.56)) end -function _one_emb_sz(cardinalitydict, catcol, sz_dict=nothing) - sz_dict = isnothing(sz_dict) ? Dict() : sz_dict - n_cat = cardinalitydict[catcol] - sz = catcol in keys(sz_dict) ? sz_dict[catcol] : emb_sz_rule(n_cat) - Int64(n_cat)+1, Int64(sz) -end - """ - get_emb_sz(cardinalitydict, cols; sz_dict=nothing) + get_emb_sz(cardinalities, size_overrides=nothing) -Returns a collection of tuples containing embedding dimensions for each column -in `cols`. `cardinalitydict` is an indexable collection which maps to the cardinality -for each column present in `cols`. +Returns a collection of tuples containing embedding dimensions corresponding to +number of classes in categorical columns present in `cardinalities` and adjusting for nans. ## Keyword arguments -- `sz_dict`: An indexable collection which may contain the required embedding - size for a particular column present in `cols`. If not passed, then the size is - calculated using fastai's rule of thumb for embedding dimensions. +- `size_overrides`: A collection of Integers and `nothing`. The integer present at + any index will be used to override the rule of thumb for getting embedding sizes. """ -function get_emb_sz(cardinalitydict, cols; sz_dict=nothing) - map(catcol -> _one_emb_sz(cardinalitydict, catcol, sz_dict), cols) +function get_emb_sz(cardinalities, size_overrides = fill(nothing, length(cardinalities))) + map(Iterators.enumerate(cardinalities)) do (i, cardinality) + emb_dim = isnothing(size_overrides[i]) ? emb_sz_rule(cardinality+1) : size_overrides[i] + (Int64(cardinality)+1, Int64(emb_dim)) + end +end + +function get_emb_sz(cardinalities; catcols, size_overrides=Dict()) + keylist = keys(size_overrides) + overrides = map(catcols) do col + col in keylist ? size_overrides[col] : nothing + end + get_emb_sz(cardinalities, overrides) end function sigmoidrange(x, low, high) @@ -47,7 +56,7 @@ end """ TabularModel(catbackbone, contbackbone, [finalclassifier]; kwargs...) - TabularModel(catcols, `n_cont::Number, outsz::Number[; kwargs...) + TabularModel(catcols, `n_cont::Number, outsize::Number[; kwargs...) Create a tabular model which takes in a tuple of categorical values (label or one-hot encoded) and continuous values. The default categorical backbone is @@ -57,30 +66,29 @@ a final classifier block. ## Keyword arguments -- `outsz`: The output size of the final classifier block. For single classification tasks, +- `outsize`: The output size of the final classifier block. For single classification tasks, this would just be the number of classes and for regression tasks, this could be the number of target continuous variables. -- `layers`: The sizes of the hidden layers in the classifier block. -- `ps`: Dropout probability. This could either be a single number which would be used for - for all the classifier layers, or a collection of numbers which are cycled through - for each layer. -- `use_bn`: Boolean variable which controls whether to use batch normalization in the classifier. -- `act_cls`: The activation function to use in the classifier layers. -- `lin_first`: Controls if the linear layer comes before or after BatchNorm and Dropout. -- `cardinalitydict`: An indexable collection which maps to the cardinality for each column present - in `catcols`. -- `sz_dict`: An indexable collection which may contain the required embedding - size for a particular column present in `cols`. If not passed, then the size is - calculated using fastai's rule of thumb for embedding dimensions. +- `layersizes`: The sizes of the hidden layers in the classifier block. +- `dropout_rates`: Dropout probability. This could either be a single number which would be + used for for all the classifier layers, or a collection of numbers which are cycled through + for each layer. +- `batchnorm`: Boolean variable which controls whether to use batch normalization in the classifier. +- `activation`: The activation function to use in the classifier layers. +- `linear_first`: Controls if the linear layer comes before or after BatchNorm and Dropout. +- `cardinalities`: A collection of sizes (number of classes) for each categorical column. +- `size_overrides`: An optional argument which corresponds to a collection containing + embedding sizes to override the value returned by the "rule of thumb" for a particular index + corresponding to `cardinalities`, or `nothing`. """ function TabularModel( catbackbone, contbackbone; outsize, - layers=[200, 100], + layersizes=[200, 100], kwargs...) - TabularModel(catbackbone, contbackbone, Dense(layers[end], outsz); layers=layers, kwargs...) + TabularModel(catbackbone, contbackbone, Dense(layersizes[end], outsize); layersizes=layersizes, kwargs...) end function TabularModel( @@ -97,14 +105,14 @@ function TabularModel( classifierin = mapreduce(layer -> size(layer.weight)[1], +, catbackbone[2].layers; init = contbackbone.chs) - ps = Iterators.cycle(ps) + dropout_rates = Iterators.cycle(dropout_rates) classifiers = [] - first_ps, ps = Iterators.peel(ps) - push!(classifiers, linbndrop(classifierin, first(layers); use_bn=use_bn, p=first_ps, lin_first=lin_first, act=act_cls)) + first_ps, dropout_rates = Iterators.peel(dropout_rates) + push!(classifiers, linbndrop(classifierin, first(layersizes); use_bn=batchnorm, p=first_ps, lin_first=linear_first, act=activation)) - for (isize, osize, p) in zip(layers[1:(end-1)], layers[2:end], ps) - layer = linbndrop(isize, osize; use_bn=use_bn, p=p, act=act_cls, lin_first=lin_first) + for (isize, osize, p) in zip(layersizes[1:(end-1)], layersizes[2:end], dropout_rates) + layer = linbndrop(isize, osize; use_bn=batchnorm, p=p, act=activation, lin_first=linear_first) push!(classifiers, layer) end @@ -116,15 +124,14 @@ function TabularModel( end function TabularModel( - catcols, n_cont::Number, - outsz::Number, - layers=[200, 100]; - cardinalitydict, - sz_dict=nothing) - embedszs = get_emb_sz(cardinalitydict, catcols, sz_dict=sz_dict) + outsize::Number, + layersizes=[200, 100]; + cardinalities, + size_overrides=fill(nothing, length(cardinalities))) + embedszs = get_emb_sz(cardinalities, size_overrides) catback = tabular_embedding_backbone(embedszs) contback = tabular_continuous_backbone(n_cont) - TabularModel(catback, contback; layers=layers, outsz=outsz) + TabularModel(catback, contback; layersizes=layersizes, outsize=outsize) end diff --git a/test/models/tabularmodel.jl b/test/models/tabularmodel.jl index 979a0dc499..91b39db05e 100644 --- a/test/models/tabularmodel.jl +++ b/test/models/tabularmodel.jl @@ -25,16 +25,15 @@ include("../imports.jl") x = ([rand(1:n) for (n, _) in embed_szs], rand(5, 1)) - tm = TabularModel(embeds, contback; outsz=4) + tm = TabularModel(embeds, contback; outsize=4) @test size(tm(x)) == (4, 1) tm2 = TabularModel(embeds, contback, Chain(Dense(100, 4), x->FastAI.Models.sigmoidrange(x, 2, 5))) y2 = tm2(x) @test all(y2.> 2) && all(y2.<5) - catcols = [:a, :b, :c] - cardict = Dict(:a => 4, :b => 99, :c => 1) - tm3 = TabularModel(catcols, n, 4, [200, 100], cardinalitydict = cardict, sz_dict = Dict(:a=>10, :b=>30, :c=>30)) + cardinalities = (4, 99, 1) + tm3 = TabularModel(n, 4, [200, 100], cardinalities = cardinalities, size_overrides = (10, 30, 30)) @test size(tm3(x)) == (4, 1) end end From 96564f03cdd6ae4f4229d9c09fdc45fe4cbcb540 Mon Sep 17 00:00:00 2001 From: Manikya Date: Mon, 16 Aug 2021 19:05:54 +0530 Subject: [PATCH 18/24] docstring fixes --- src/models/tabularmodel.jl | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/models/tabularmodel.jl b/src/models/tabularmodel.jl index 9781414444..c943a83b5c 100644 --- a/src/models/tabularmodel.jl +++ b/src/models/tabularmodel.jl @@ -10,18 +10,22 @@ function emb_sz_rule(n_cat) end """ - get_emb_sz(cardinalities, size_overrides=nothing) + get_emb_sz(cardinalities, [size_overrides]) + get_emb_sz(cardinalities; catcols, [size_overrides]) Returns a collection of tuples containing embedding dimensions corresponding to number of classes in categorical columns present in `cardinalities` and adjusting for nans. ## Keyword arguments -- `size_overrides`: A collection of Integers and `nothing`. The integer present at - any index will be used to override the rule of thumb for getting embedding sizes. +- `size_overrides`: Depending on the method used, this could either be a collection of + Integers and `nothing` or an indexable collection with column name as key and size + to override it with as the value. In the first case, the integer present at any index + will be used to override the rule of thumb for getting embedding sizes. +- `categorical_cols`: A collection of categorical column names. """ -function get_emb_sz(cardinalities, size_overrides = fill(nothing, length(cardinalities))) +function get_emb_sz(cardinalities, size_overrides=fill(nothing, length(cardinalities))) map(Iterators.enumerate(cardinalities)) do (i, cardinality) emb_dim = isnothing(size_overrides[i]) ? emb_sz_rule(cardinality+1) : size_overrides[i] (Int64(cardinality)+1, Int64(emb_dim)) @@ -56,7 +60,7 @@ end """ TabularModel(catbackbone, contbackbone, [finalclassifier]; kwargs...) - TabularModel(catcols, `n_cont::Number, outsize::Number[; kwargs...) + TabularModel(`n_cont::Number, outsize::Number[; kwargs...]) Create a tabular model which takes in a tuple of categorical values (label or one-hot encoded) and continuous values. The default categorical backbone is From ddb4d620d71867e26008f306612624d968ebd81c Mon Sep 17 00:00:00 2001 From: Manikya Bardhan Date: Tue, 17 Aug 2021 07:35:22 +0530 Subject: [PATCH 19/24] made methods concise Co-authored-by: Kyle Daruwalla --- src/models/tabularmodel.jl | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/src/models/tabularmodel.jl b/src/models/tabularmodel.jl index c943a83b5c..bffa71c7ae 100644 --- a/src/models/tabularmodel.jl +++ b/src/models/tabularmodel.jl @@ -5,16 +5,14 @@ Returns an embedding size corresponding to the number of classes for a categorical variable using the rule of thumb present in python fastai. (see https://github.com/fastai/fastai/blob/2742fe844573d06e700f869839fb9ec5f3a9bca9/fastai/tabular/model.py#L12) """ -function emb_sz_rule(n_cat) - min(600, round(1.6 * n_cat^0.56)) -end +emb_sz_rule(n_cat) = min(600, round(Int, 1.6 * n_cat^0.56)) """ get_emb_sz(cardinalities, [size_overrides]) get_emb_sz(cardinalities; catcols, [size_overrides]) Returns a collection of tuples containing embedding dimensions corresponding to -number of classes in categorical columns present in `cardinalities` and adjusting for nans. +number of classes in categorical columns present in `cardinalities` and adjusting for NaNs. ## Keyword arguments @@ -25,12 +23,11 @@ number of classes in categorical columns present in `cardinalities` and adjustin - `categorical_cols`: A collection of categorical column names. """ -function get_emb_sz(cardinalities, size_overrides=fill(nothing, length(cardinalities))) - map(Iterators.enumerate(cardinalities)) do (i, cardinality) - emb_dim = isnothing(size_overrides[i]) ? emb_sz_rule(cardinality+1) : size_overrides[i] - (Int64(cardinality)+1, Int64(emb_dim)) +get_emb_sz(cardinalities::AbstractVector{<:Integer}, size_overrides=fill(nothing, length(cardinalities))) = + map(zip(cardinalities, size_overrides)) do (cardinality, override) + emb_dim = isnothing(override) ? emb_sz_rule(cardinality + 1) : Int64(override) + return (cardinality + 1, emb_dim) end -end function get_emb_sz(cardinalities; catcols, size_overrides=Dict()) keylist = keys(size_overrides) @@ -40,9 +37,7 @@ function get_emb_sz(cardinalities; catcols, size_overrides=Dict()) get_emb_sz(cardinalities, overrides) end -function sigmoidrange(x, low, high) - @. Flux.sigmoid(x) * (high - low) + low -end +sigmoidrange(x, low, high) = @. Flux.sigmoid(x) * (high - low) + low function tabular_embedding_backbone(embedding_sizes, dropoutprob=0.) embedslist = [Flux.Embedding(ni, nf) for (ni, nf) in embedding_sizes] @@ -54,13 +49,11 @@ function tabular_embedding_backbone(embedding_sizes, dropoutprob=0.) ) end -function tabular_continuous_backbone(n_cont) - BatchNorm(n_cont) -end +tabular_continuous_backbone(n_cont) = BatchNorm(n_cont) """ TabularModel(catbackbone, contbackbone, [finalclassifier]; kwargs...) - TabularModel(`n_cont::Number, outsize::Number[; kwargs...]) + TabularModel(n_cont, outsize [; kwargs...]) Create a tabular model which takes in a tuple of categorical values (label or one-hot encoded) and continuous values. The default categorical backbone is @@ -90,7 +83,7 @@ function TabularModel( catbackbone, contbackbone; outsize, - layersizes=[200, 100], + layersizes=(200, 100), kwargs...) TabularModel(catbackbone, contbackbone, Dense(layersizes[end], outsize); layersizes=layersizes, kwargs...) end @@ -113,7 +106,8 @@ function TabularModel( classifiers = [] first_ps, dropout_rates = Iterators.peel(dropout_rates) - push!(classifiers, linbndrop(classifierin, first(layersizes); use_bn=batchnorm, p=first_ps, lin_first=linear_first, act=activation)) + push!(classifiers, linbndrop(classifierin, first(layersizes); + use_bn=batchnorm, p=first_ps, lin_first=linear_first, act=activation)) for (isize, osize, p) in zip(layersizes[1:(end-1)], layersizes[2:end], dropout_rates) layer = linbndrop(isize, osize; use_bn=batchnorm, p=p, act=activation, lin_first=linear_first) @@ -130,7 +124,7 @@ end function TabularModel( n_cont::Number, outsize::Number, - layersizes=[200, 100]; + layersizes=(200, 100); cardinalities, size_overrides=fill(nothing, length(cardinalities))) embedszs = get_emb_sz(cardinalities, size_overrides) From 825d146759704a2f8b2666d5f4c13c5e054e45d6 Mon Sep 17 00:00:00 2001 From: Manikya Date: Tue, 17 Aug 2021 18:45:01 +0530 Subject: [PATCH 20/24] updated docstrings and get_emb_sz --- src/models/tabularmodel.jl | 59 ++++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/src/models/tabularmodel.jl b/src/models/tabularmodel.jl index bffa71c7ae..d5dff14e21 100644 --- a/src/models/tabularmodel.jl +++ b/src/models/tabularmodel.jl @@ -9,18 +9,14 @@ emb_sz_rule(n_cat) = min(600, round(Int, 1.6 * n_cat^0.56)) """ get_emb_sz(cardinalities, [size_overrides]) - get_emb_sz(cardinalities; catcols, [size_overrides]) Returns a collection of tuples containing embedding dimensions corresponding to number of classes in categorical columns present in `cardinalities` and adjusting for NaNs. ## Keyword arguments -- `size_overrides`: Depending on the method used, this could either be a collection of - Integers and `nothing` or an indexable collection with column name as key and size - to override it with as the value. In the first case, the integer present at any index +- `size_overrides`: A collection of Integers and `nothing` where the integer present at any index will be used to override the rule of thumb for getting embedding sizes. -- `categorical_cols`: A collection of categorical column names. """ get_emb_sz(cardinalities::AbstractVector{<:Integer}, size_overrides=fill(nothing, length(cardinalities))) = @@ -29,19 +25,32 @@ get_emb_sz(cardinalities::AbstractVector{<:Integer}, size_overrides=fill(nothing return (cardinality + 1, emb_dim) end -function get_emb_sz(cardinalities; catcols, size_overrides=Dict()) +""" + get_emb_sz(cardinalities, categorical_cols, [size_overrides]) + +Returns a collection of tuples containing embedding dimensions corresponding to +number of classes in categorical columns present in `cardinalities` and adjusting for NaNs. + +## Keyword arguments + +- `size_overrides`: An indexable collection with column name as key and size + to override it with as the value. +- `categorical_cols`: A collection of categorical column names. +""" + +function get_emb_sz(cardinalities::AbstractVector{<:Integer}, categorical_cols::Tuple, size_overrides=Dict()) keylist = keys(size_overrides) - overrides = map(catcols) do col + overrides = collect(map(categorical_cols) do col col in keylist ? size_overrides[col] : nothing - end + end) get_emb_sz(cardinalities, overrides) end sigmoidrange(x, low, high) = @. Flux.sigmoid(x) * (high - low) + low -function tabular_embedding_backbone(embedding_sizes, dropoutprob=0.) +function tabular_embedding_backbone(embedding_sizes, dropout_rate=0.) embedslist = [Flux.Embedding(ni, nf) for (ni, nf) in embedding_sizes] - emb_drop = dropoutprob==0. ? identity : Dropout(dropoutprob) + emb_drop = iszero(dropout_rate) ? identity : Dropout(dropout_rate) Chain( x -> tuple(eachrow(x)...), Parallel(vcat, embedslist), @@ -53,13 +62,12 @@ tabular_continuous_backbone(n_cont) = BatchNorm(n_cont) """ TabularModel(catbackbone, contbackbone, [finalclassifier]; kwargs...) - TabularModel(n_cont, outsize [; kwargs...]) Create a tabular model which takes in a tuple of categorical values -(label or one-hot encoded) and continuous values. The default categorical backbone is +(label or one-hot encoded) and continuous values. The default categorical backbone or `catbackbone` is a Parallel of Embedding layers corresponding to each categorical variable, and continuous -variables are just BatchNormed. The output from these backbones is then passed through -a final classifier block. +variables are just BatchNormed using `contbackbone`. The output from these backbones is then passed through +a `finalclassifier` block. ## Keyword arguments @@ -73,10 +81,6 @@ a final classifier block. - `batchnorm`: Boolean variable which controls whether to use batch normalization in the classifier. - `activation`: The activation function to use in the classifier layers. - `linear_first`: Controls if the linear layer comes before or after BatchNorm and Dropout. -- `cardinalities`: A collection of sizes (number of classes) for each categorical column. -- `size_overrides`: An optional argument which corresponds to a collection containing - embedding sizes to override the value returned by the "rule of thumb" for a particular index - corresponding to `cardinalities`, or `nothing`. """ function TabularModel( @@ -121,6 +125,25 @@ function TabularModel( ) end +""" + TabularModel(n_cont, outsize, [layersizes; kwargs...]) + +Create a tabular model which takes in a tuple of categorical values +(label or one-hot encoded) and continuous values. The default categorical backbone is +a Parallel of Embedding layers corresponding to each categorical variable, and continuous +variables are just BatchNormed. The output from these backbones is then passed through +a final classifier block. Uses `n_cont` the number of continuous columns, `outsize` which +is the output size of the final classifier block, and `layersizes` which is a collection of +classifier layer sizes, to create the model. + +## Keyword arguments + +- `cardinalities`: A collection of sizes (number of classes) for each categorical column. +- `size_overrides`: An optional argument which corresponds to a collection containing + embedding sizes to override the value returned by the "rule of thumb" for a particular index + corresponding to `cardinalities`, or `nothing`. +""" + function TabularModel( n_cont::Number, outsize::Number, From 979c9ba3f18d4b517f904e2588abcacf6546b4e7 Mon Sep 17 00:00:00 2001 From: Manikya Date: Tue, 17 Aug 2021 18:48:58 +0530 Subject: [PATCH 21/24] updated model test --- test/models/tabularmodel.jl | 2 +- test/runtests.jl | 96 ++++++++++++++++++------------------- 2 files changed, 49 insertions(+), 49 deletions(-) diff --git a/test/models/tabularmodel.jl b/test/models/tabularmodel.jl index 91b39db05e..b2aad087ee 100644 --- a/test/models/tabularmodel.jl +++ b/test/models/tabularmodel.jl @@ -32,7 +32,7 @@ include("../imports.jl") y2 = tm2(x) @test all(y2.> 2) && all(y2.<5) - cardinalities = (4, 99, 1) + cardinalities = [4, 99, 1] tm3 = TabularModel(n, 4, [200, 100], cardinalities = cardinalities, size_overrides = (10, 30, 30)) @test size(tm3(x)) == (4, 1) end diff --git a/test/runtests.jl b/test/runtests.jl index 05a034f473..8b9ac0ffd2 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,57 +4,57 @@ include("imports.jl") ## @testset ExtendedTestSet "FastAI.jl" begin - @testset ExtendedTestSet "encodings/" begin - @testset ExtendedTestSet "projective.jl" begin - include("encodings/projective.jl") - end - @testset ExtendedTestSet "imagepreprocessing.jl" begin - include("encodings/imagepreprocessing.jl") - end - @testset ExtendedTestSet "keypointpreprocessing.jl" begin - include("encodings/keypointpreprocessing.jl") - end - end + # @testset ExtendedTestSet "encodings/" begin + # @testset ExtendedTestSet "projective.jl" begin + # include("encodings/projective.jl") + # end + # @testset ExtendedTestSet "imagepreprocessing.jl" begin + # include("encodings/imagepreprocessing.jl") + # end + # @testset ExtendedTestSet "keypointpreprocessing.jl" begin + # include("encodings/keypointpreprocessing.jl") + # end + # end - @testset ExtendedTestSet "methods/" begin - @testset ExtendedTestSet "imageclassification.jl" begin - include("methods/imageclassification.jl") - end - @testset ExtendedTestSet "imagesegmentation.jl" begin - include("methods/imagesegmentation.jl") - end - @testset ExtendedTestSet "singlekeypointregression.jl" begin - include("methods/singlekeypointregression.jl") - end - end + # @testset ExtendedTestSet "methods/" begin + # @testset ExtendedTestSet "imageclassification.jl" begin + # include("methods/imageclassification.jl") + # end + # @testset ExtendedTestSet "imagesegmentation.jl" begin + # include("methods/imagesegmentation.jl") + # end + # @testset ExtendedTestSet "singlekeypointregression.jl" begin + # include("methods/singlekeypointregression.jl") + # end + # end - @testset ExtendedTestSet "datasets/" begin - @testset ExtendedTestSet "transformations.jl" begin - include("datasets/transformations.jl") - end - @testset ExtendedTestSet "containers.jl" begin - include("datasets/containers.jl") - end - end + # @testset ExtendedTestSet "datasets/" begin + # @testset ExtendedTestSet "transformations.jl" begin + # include("datasets/transformations.jl") + # end + # @testset ExtendedTestSet "containers.jl" begin + # include("datasets/containers.jl") + # end + # end - @testset ExtendedTestSet "training/" begin - @testset ExtendedTestSet "paramgroups.jl" begin - include("training/paramgroups.jl") - end - @testset ExtendedTestSet "discriminativelrs.jl" begin - include("training/discriminativelrs.jl") - end - @testset ExtendedTestSet "fitonecycle.jl" begin - include("training/fitonecycle.jl") - end - @testset ExtendedTestSet "finetune.jl" begin - include("training/finetune.jl") - end - @testset ExtendedTestSet "lrfind.jl" begin - include("training/lrfind.jl") - end - # TODO: test learning rate finder - end + # @testset ExtendedTestSet "training/" begin + # @testset ExtendedTestSet "paramgroups.jl" begin + # include("training/paramgroups.jl") + # end + # @testset ExtendedTestSet "discriminativelrs.jl" begin + # include("training/discriminativelrs.jl") + # end + # @testset ExtendedTestSet "fitonecycle.jl" begin + # include("training/fitonecycle.jl") + # end + # @testset ExtendedTestSet "finetune.jl" begin + # include("training/finetune.jl") + # end + # @testset ExtendedTestSet "lrfind.jl" begin + # include("training/lrfind.jl") + # end + # # TODO: test learning rate finder + # end @testset ExtendedTestSet "models/" begin @testset ExtendedTestSet "tabularmodel.jl" begin From 8f8c65a37bc5a31e42cb8d36f39fa34daeb15c79 Mon Sep 17 00:00:00 2001 From: Manikya Date: Thu, 19 Aug 2021 09:31:13 +0530 Subject: [PATCH 22/24] undo unintentional comments --- test/runtests.jl | 96 ++++++++++++++++++++++++------------------------ 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 8b9ac0ffd2..05a034f473 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,57 +4,57 @@ include("imports.jl") ## @testset ExtendedTestSet "FastAI.jl" begin - # @testset ExtendedTestSet "encodings/" begin - # @testset ExtendedTestSet "projective.jl" begin - # include("encodings/projective.jl") - # end - # @testset ExtendedTestSet "imagepreprocessing.jl" begin - # include("encodings/imagepreprocessing.jl") - # end - # @testset ExtendedTestSet "keypointpreprocessing.jl" begin - # include("encodings/keypointpreprocessing.jl") - # end - # end + @testset ExtendedTestSet "encodings/" begin + @testset ExtendedTestSet "projective.jl" begin + include("encodings/projective.jl") + end + @testset ExtendedTestSet "imagepreprocessing.jl" begin + include("encodings/imagepreprocessing.jl") + end + @testset ExtendedTestSet "keypointpreprocessing.jl" begin + include("encodings/keypointpreprocessing.jl") + end + end - # @testset ExtendedTestSet "methods/" begin - # @testset ExtendedTestSet "imageclassification.jl" begin - # include("methods/imageclassification.jl") - # end - # @testset ExtendedTestSet "imagesegmentation.jl" begin - # include("methods/imagesegmentation.jl") - # end - # @testset ExtendedTestSet "singlekeypointregression.jl" begin - # include("methods/singlekeypointregression.jl") - # end - # end + @testset ExtendedTestSet "methods/" begin + @testset ExtendedTestSet "imageclassification.jl" begin + include("methods/imageclassification.jl") + end + @testset ExtendedTestSet "imagesegmentation.jl" begin + include("methods/imagesegmentation.jl") + end + @testset ExtendedTestSet "singlekeypointregression.jl" begin + include("methods/singlekeypointregression.jl") + end + end - # @testset ExtendedTestSet "datasets/" begin - # @testset ExtendedTestSet "transformations.jl" begin - # include("datasets/transformations.jl") - # end - # @testset ExtendedTestSet "containers.jl" begin - # include("datasets/containers.jl") - # end - # end + @testset ExtendedTestSet "datasets/" begin + @testset ExtendedTestSet "transformations.jl" begin + include("datasets/transformations.jl") + end + @testset ExtendedTestSet "containers.jl" begin + include("datasets/containers.jl") + end + end - # @testset ExtendedTestSet "training/" begin - # @testset ExtendedTestSet "paramgroups.jl" begin - # include("training/paramgroups.jl") - # end - # @testset ExtendedTestSet "discriminativelrs.jl" begin - # include("training/discriminativelrs.jl") - # end - # @testset ExtendedTestSet "fitonecycle.jl" begin - # include("training/fitonecycle.jl") - # end - # @testset ExtendedTestSet "finetune.jl" begin - # include("training/finetune.jl") - # end - # @testset ExtendedTestSet "lrfind.jl" begin - # include("training/lrfind.jl") - # end - # # TODO: test learning rate finder - # end + @testset ExtendedTestSet "training/" begin + @testset ExtendedTestSet "paramgroups.jl" begin + include("training/paramgroups.jl") + end + @testset ExtendedTestSet "discriminativelrs.jl" begin + include("training/discriminativelrs.jl") + end + @testset ExtendedTestSet "fitonecycle.jl" begin + include("training/fitonecycle.jl") + end + @testset ExtendedTestSet "finetune.jl" begin + include("training/finetune.jl") + end + @testset ExtendedTestSet "lrfind.jl" begin + include("training/lrfind.jl") + end + # TODO: test learning rate finder + end @testset ExtendedTestSet "models/" begin @testset ExtendedTestSet "tabularmodel.jl" begin From f2042a4dd2b4cddeeffbd55198979b7387ea0da6 Mon Sep 17 00:00:00 2001 From: Manikya Bardhan Date: Fri, 20 Aug 2021 09:26:00 +0530 Subject: [PATCH 23/24] Docstring updates Co-authored-by: Kyle Daruwalla --- src/models/tabularmodel.jl | 83 +++++++++++++++++++------------------- 1 file changed, 41 insertions(+), 42 deletions(-) diff --git a/src/models/tabularmodel.jl b/src/models/tabularmodel.jl index d5dff14e21..5fb897361a 100644 --- a/src/models/tabularmodel.jl +++ b/src/models/tabularmodel.jl @@ -1,24 +1,25 @@ """ emb_sz_rule(n_cat) -Returns an embedding size corresponding to the number of classes for a +Compute an embedding size corresponding to the number of classes for a categorical variable using the rule of thumb present in python fastai. (see https://github.com/fastai/fastai/blob/2742fe844573d06e700f869839fb9ec5f3a9bca9/fastai/tabular/model.py#L12) """ emb_sz_rule(n_cat) = min(600, round(Int, 1.6 * n_cat^0.56)) """ - get_emb_sz(cardinalities, [size_overrides]) + get_emb_sz(cardinalities::AbstractVector, [size_overrides::AbstractVector]) -Returns a collection of tuples containing embedding dimensions corresponding to -number of classes in categorical columns present in `cardinalities` and adjusting for NaNs. +Given a vector of `cardinalities` of each categorical column +(i.e. each element of `cardinalities` is the number of classes in that categorical column), +compute the output embedding size according to [`emb_sz_rule`](#). +Return a vector of tuples where each element is `(in_size, out_size)` for an embedding layer. ## Keyword arguments -- `size_overrides`: A collection of Integers and `nothing` where the integer present at any index - will be used to override the rule of thumb for getting embedding sizes. +- `size_overrides`: A collection of integers (or `nothing` to skip override) where the value present at any index + will be used to as the output embedding size for that column. """ - get_emb_sz(cardinalities::AbstractVector{<:Integer}, size_overrides=fill(nothing, length(cardinalities))) = map(zip(cardinalities, size_overrides)) do (cardinality, override) emb_dim = isnothing(override) ? emb_sz_rule(cardinality + 1) : Int64(override) @@ -26,24 +27,21 @@ get_emb_sz(cardinalities::AbstractVector{<:Integer}, size_overrides=fill(nothing end """ - get_emb_sz(cardinalities, categorical_cols, [size_overrides]) + get_emb_sz(cardinalities::Dict, [size_overrides::Dict]) -Returns a collection of tuples containing embedding dimensions corresponding to -number of classes in categorical columns present in `cardinalities` and adjusting for NaNs. +Given a map from columns to `cardinalities`, compute the output embedding size according to [`emb_sz_rule`](#). +Return a vector of tuples where each element is `(in_size, out_size)` for an embedding layer. ## Keyword arguments -- `size_overrides`: An indexable collection with column name as key and size - to override it with as the value. -- `categorical_cols`: A collection of categorical column names. +- `size_overrides`: An map of output embedding size overrides + (i.e. `size_overrides[col]` is the output embedding size for `col`). """ - -function get_emb_sz(cardinalities::AbstractVector{<:Integer}, categorical_cols::Tuple, size_overrides=Dict()) - keylist = keys(size_overrides) - overrides = collect(map(categorical_cols) do col - col in keylist ? size_overrides[col] : nothing - end) - get_emb_sz(cardinalities, overrides) +function get_emb_sz(cardinalities::Dict{<:Any, <:Integer}, size_overrides=Dict()) + values_and_overrides = map(pairs(cardinalities)) do (col, cardinality) + cardinality, get(size_overrides, col, nothing) + end + get_emb_sz(first.(values_and_overrides), last.(values_and_overrides)) end sigmoidrange(x, low, high) = @. Flux.sigmoid(x) * (high - low) + low @@ -63,26 +61,24 @@ tabular_continuous_backbone(n_cont) = BatchNorm(n_cont) """ TabularModel(catbackbone, contbackbone, [finalclassifier]; kwargs...) -Create a tabular model which takes in a tuple of categorical values -(label or one-hot encoded) and continuous values. The default categorical backbone or `catbackbone` is -a Parallel of Embedding layers corresponding to each categorical variable, and continuous -variables are just BatchNormed using `contbackbone`. The output from these backbones is then passed through -a `finalclassifier` block. +Create a tabular model which operates on a tuple of categorical values +(label or one-hot encoded) and continuous values. +The categorical backbones (`catbackbone`) and continuous backbone (`contbackbone`) operate on each element of the input tuple. +The output from these backbones is then passed through a series of linear-batch norm-dropout layers before a `finalclassifier` block. ## Keyword arguments - `outsize`: The output size of the final classifier block. For single classification tasks, - this would just be the number of classes and for regression tasks, this could be the + this would be the number of classes, and for regression tasks, this would be the number of target continuous variables. -- `layersizes`: The sizes of the hidden layers in the classifier block. -- `dropout_rates`: Dropout probability. This could either be a single number which would be - used for for all the classifier layers, or a collection of numbers which are cycled through - for each layer. -- `batchnorm`: Boolean variable which controls whether to use batch normalization in the classifier. +- `layersizes`: A vector of sizes for each hidden layer in the sequence of linear layers. +- `dropout_rates`: Dropout probabilities for the linear-batch norm-dropout layers. + This could either be a single number which would be used for for all the layers, + or a collection of numbers which are cycled through for each layer. +- `batchnorm`: Set to `false` to skip each batch norm in the linear-batch norm-dropout sequence. - `activation`: The activation function to use in the classifier layers. -- `linear_first`: Controls if the linear layer comes before or after BatchNorm and Dropout. +- `linear_first`: Controls if the linear layer comes before or after batch norm and dropout. """ - function TabularModel( catbackbone, contbackbone; @@ -96,7 +92,7 @@ function TabularModel( catbackbone, contbackbone, finalclassifier; - layersizes=[200, 100], + layersizes=(200, 100), dropout_rates=0., batchnorm=true, activation=Flux.relu, @@ -128,13 +124,17 @@ end """ TabularModel(n_cont, outsize, [layersizes; kwargs...]) -Create a tabular model which takes in a tuple of categorical values -(label or one-hot encoded) and continuous values. The default categorical backbone is -a Parallel of Embedding layers corresponding to each categorical variable, and continuous -variables are just BatchNormed. The output from these backbones is then passed through -a final classifier block. Uses `n_cont` the number of continuous columns, `outsize` which -is the output size of the final classifier block, and `layersizes` which is a collection of -classifier layer sizes, to create the model. +Create a tabular model which operates on a tuple of categorical values +(label or one-hot encoded) and continuous values. The default categorical backbone (`catbackbone`) is +a [`Flux.Parallel`](https://fluxml.ai/Flux.jl/stable/models/layers/#Flux.Parallel) set of `Flux.Embedding` layers corresponding to each categorical variable. +The default continuous backbone (`contbackbone`) is a single [`Flux.BatchNorm`](https://fluxml.ai/Flux.jl/stable/models/layers/#Flux.BatchNorm). +The output from these backbones is concatenated then passed through a series of linear-batch norm-dropout layers before a `finalclassifier` block. + +## Arguments + +- `n_cont`: The number of continuous columns. +- `outsize`: The output size of the model. +- `layersizes`: A vector of sizes for each hidden layer in the sequence of linear layers. ## Keyword arguments @@ -143,7 +143,6 @@ classifier layer sizes, to create the model. embedding sizes to override the value returned by the "rule of thumb" for a particular index corresponding to `cardinalities`, or `nothing`. """ - function TabularModel( n_cont::Number, outsize::Number, From 928509bf847367227b36e5a9a84d62e7cb28a2d5 Mon Sep 17 00:00:00 2001 From: Manikya Date: Sun, 22 Aug 2021 15:47:36 +0530 Subject: [PATCH 24/24] minor docstring fix --- src/models/tabularmodel.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/models/tabularmodel.jl b/src/models/tabularmodel.jl index 5fb897361a..626293e6f5 100644 --- a/src/models/tabularmodel.jl +++ b/src/models/tabularmodel.jl @@ -34,7 +34,7 @@ Return a vector of tuples where each element is `(in_size, out_size)` for an emb ## Keyword arguments -- `size_overrides`: An map of output embedding size overrides +- `size_overrides`: A map of output embedding size overrides (i.e. `size_overrides[col]` is the output embedding size for `col`). """ function get_emb_sz(cardinalities::Dict{<:Any, <:Integer}, size_overrides=Dict())