From 0c35d775f3a352d1f4e0ec80b73a0e5d6046ea70 Mon Sep 17 00:00:00 2001 From: lorenzoh Date: Mon, 20 Dec 2021 13:08:00 +0100 Subject: [PATCH 01/23] Make block learning methods more modular --- src/FastAI.jl | 3 +- src/Tabular/Tabular.jl | 2 +- src/Tabular/learningmethods/classification.jl | 8 +- src/Tabular/learningmethods/regression.jl | 10 +- src/Vision/Vision.jl | 2 +- src/Vision/learningmethods/classification.jl | 4 +- src/Vision/learningmethods/segmentation.jl | 4 +- src/datablock/describe.jl | 80 ++++++---- src/datablock/method.jl | 119 ++++++++------ src/interpretation/learner.jl | 2 +- src/interpretation/method.jl | 146 +++++++++++------- src/learner.jl | 2 +- src/training/onecycle.jl | 7 +- 13 files changed, 232 insertions(+), 157 deletions(-) diff --git a/src/FastAI.jl b/src/FastAI.jl index 1b83cb8b9f..b08bc904eb 100644 --- a/src/FastAI.jl +++ b/src/FastAI.jl @@ -165,7 +165,8 @@ export augs_projection, augs_lighting, TabularPreprocessing, - BlockMethod, + SupervisedMethod, + AbstractBlockMethod, describemethod, checkblock, makebatch, diff --git a/src/Tabular/Tabular.jl b/src/Tabular/Tabular.jl index 80ef277950..a288138b59 100644 --- a/src/Tabular/Tabular.jl +++ b/src/Tabular/Tabular.jl @@ -5,7 +5,7 @@ using ..FastAI using ..FastAI: # blocks Block, WrapperBlock, AbstractBlock, OneHotTensor, OneHotTensorMulti, Label, - LabelMulti, wrapped, Continuous, + LabelMulti, wrapped, Continuous, getencodings, getblocks, # encodings Encoding, StatefulEncoding, OneHot, # visualization diff --git a/src/Tabular/learningmethods/classification.jl b/src/Tabular/learningmethods/classification.jl index 6b6e5a5e71..6869ffe0ed 100644 --- a/src/Tabular/learningmethods/classification.jl +++ b/src/Tabular/learningmethods/classification.jl @@ -49,17 +49,17 @@ end td = TableDataset(df) method = TabularClassificationSingle(["P", "F"], td; catcols=(:B,), contcols=(:A,), ) - testencoding(method.encodings, method.blocks) + testencoding(getencodings(method), getblocks(method).sample) DLPipelines.checkmethod_core(method) @test_nowarn methodlossfn(method) @test_nowarn methodmodel(method) @testset "`encodeinput`" begin - row = mockblock(method.blocks[1]) + row = mockblock(getblocks(method)[1]) xtrain = encodeinput(method, Training(), row) - @test length(xtrain[1]) == length(method.blocks[1].catcols) - @test length(xtrain[2]) == length(method.blocks[1].contcols) + @test length(xtrain[1]) == length(getblocks(method).input.catcols) + @test length(xtrain[2]) == length(getblocks(method).input.contcols) @test eltype(xtrain[1]) <: Number end diff --git a/src/Tabular/learningmethods/regression.jl b/src/Tabular/learningmethods/regression.jl index f53f8ea4a7..664a602787 100644 --- a/src/Tabular/learningmethods/regression.jl +++ b/src/Tabular/learningmethods/regression.jl @@ -7,7 +7,7 @@ function TabularRegression( return BlockMethod( blocks, (setup(TabularPreprocessing, blocks[1], tabledata),), - outputblock=blocks[2] + ŷblock=blocks[2], ) end @@ -45,17 +45,17 @@ end td = TableDataset(df) targets = [rand(2) for _ in 1:4] method = TabularRegression(2, td; catcols=(:B,), contcols=(:A,)) - testencoding(method.encodings, method.blocks) + testencoding(getencodings(method), getblocks(method).sample) DLPipelines.checkmethod_core(method) @test_nowarn methodlossfn(method) @test_nowarn methodmodel(method) @testset "`encodeinput`" begin - row = mockblock(method.blocks[1]) + row = mockblock(getblocks(method).input) xtrain = encodeinput(method, Training(), row) - @test length(xtrain[1]) == length(method.blocks[1].catcols) - @test length(xtrain[2]) == length(method.blocks[1].contcols) + @test length(xtrain[1]) == length(getblocks(method).input.catcols) + @test length(xtrain[2]) == length(getblocks(method).input.contcols) @test eltype(xtrain[1]) <: Number end diff --git a/src/Vision/Vision.jl b/src/Vision/Vision.jl index afd2f21330..35545bb772 100644 --- a/src/Vision/Vision.jl +++ b/src/Vision/Vision.jl @@ -29,7 +29,7 @@ using ..FastAI using ..FastAI: # blocks Block, WrapperBlock, AbstractBlock, OneHotTensor, OneHotTensorMulti, Label, - LabelMulti, wrapped, + LabelMulti, wrapped, getencodings, getblocks, # encodings Encoding, StatefulEncoding, OneHot, # visualization diff --git a/src/Vision/learningmethods/classification.jl b/src/Vision/learningmethods/classification.jl index 41c188be69..5ad53333a8 100644 --- a/src/Vision/learningmethods/classification.jl +++ b/src/Vision/learningmethods/classification.jl @@ -100,7 +100,7 @@ registerlearningmethod!(FASTAI_METHOD_REGISTRY, ImageClassificationMulti, (Image @testset "ImageClassificationSingle [method]" begin method = ImageClassificationSingle((16, 16), [1, 2]) - testencoding(method.encodings, method.blocks) + testencoding(getencodings(method), getblocks(method).sample) DLPipelines.checkmethod_core(method) @test_nowarn methodlossfn(method) @test_nowarn methodmodel(method, Models.xresnet18()) @@ -140,7 +140,7 @@ end method = ImageClassificationMulti((16, 16), [1, 2]) - testencoding(method.encodings, method.blocks) + testencoding(getencodings(method), getblocks(method).sample) DLPipelines.checkmethod_core(method) @test_nowarn methodlossfn(method) @test_nowarn methodmodel(method, Models.xresnet18()) diff --git a/src/Vision/learningmethods/segmentation.jl b/src/Vision/learningmethods/segmentation.jl index 78544a4678..b5cbc33829 100644 --- a/src/Vision/learningmethods/segmentation.jl +++ b/src/Vision/learningmethods/segmentation.jl @@ -49,7 +49,7 @@ registerlearningmethod!(FASTAI_METHOD_REGISTRY, ImageSegmentation, (Image, Mask) @testset "ImageSegmentation [method]" begin @testset "2D" begin method = ImageSegmentation((16, 16), 1:4) - testencoding(method.encodings, method.blocks) + testencoding(getencodings(method), getblocks(method).sample) DLPipelines.checkmethod_core(method) @test_nowarn methodlossfn(method) @test_nowarn methodmodel(method, Models.xresnet18()) @@ -68,7 +68,7 @@ registerlearningmethod!(FASTAI_METHOD_REGISTRY, ImageSegmentation, (Image, Mask) FastAI.OneHot() ) ) - testencoding(method.encodings, method.blocks) + testencoding(getencodings(method), getblocks(method).sample) DLPipelines.checkmethod_core(method) @test_nowarn methodlossfn(method) end diff --git a/src/datablock/describe.jl b/src/datablock/describe.jl index 37c72de729..6e7f59bf56 100644 --- a/src/datablock/describe.jl +++ b/src/datablock/describe.jl @@ -26,69 +26,85 @@ tuplemap(f, args...) = f(args...) tuplemap(f, args::Vararg{Tuple}) = map((as...) -> tuplemap(f, as...), args...) function blockcolumn(encodings, block; decode = false) - blocks, changed = decode ? listdecodeblocks(encodings, block) : listencodeblocks(encodings, block) - blockscol = [tuplemap((b, c) -> c ? "**`$(typeof(b))`**" : "`$(typeof(b))`", bs, ch) for (bs, ch) in zip(blocks, changed)] + blocks, changed = + decode ? listdecodeblocks(encodings, block) : listencodeblocks(encodings, block) + blockscol = [ + tuplemap((b, c) -> c ? "**`$(typeof(b))`**" : "`$(typeof(b))`", bs, ch) for + (bs, ch) in zip(blocks, changed) + ] if block isa Tuple blockscol = [join(row, ", ") for row in blockscol] end return reshape(blockscol, :, 1) end -encodingscolumn(encodings) = reshape( - ["", ["`$(typeof(enc).name.name)`" for enc in encodings]...], :, 1) +encodingscolumn(encodings) = + reshape(["", ["`$(typeof(enc).name.name)`" for enc in encodings]...], :, 1) function describeencodings( - encodings, - blocks::Tuple; - inname="Input", - outname="Output", - blocknames=repeat([""], length(blocks)), - decode=false, - markdown=false, - tf=tf_markdown) - namescol = reshape([inname, ["" for _ in 2:length(encodings)]..., outname], :, 1) + encodings, + blocks::Tuple; + inname = "Input", + outname = "Output", + blocknames = repeat([""], length(blocks)), + decode = false, + markdown = false, + tf = tf_markdown, +) + namescol = reshape([inname, ["" for _ = 2:length(encodings)]..., outname], :, 1) data = hcat( encodingscolumn(decode ? reverse(encodings) : encodings), namescol, - [blockcolumn(encodings, block; decode=decode) for block in blocks]..., + [blockcolumn(encodings, block; decode = decode) for block in blocks]..., ) s = pretty_table( String, data, - header=[decode ? "Decoding" : "Encoding", "Name", blocknames...], - alignment=[:r, :r, [:l for _ in 1:length(blocknames)]...], - tf=tf, - ) + header = [decode ? "Decoding" : "Encoding", "Name", blocknames...], + alignment = [:r, :r, [:l for _ = 1:length(blocknames)]...], + tf = tf, + ) return markdown ? Markdown.parse(s) : s end -function describemethod(method::BlockMethod) - xblock = encodedblock(method.encodings, method.blocks[1]) +function describemethod(method::SupervisedMethod) + blocks = getblocks(method) + input, target, x, ŷ = blocks.input, blocks.target, blocks.x + + encoding = describeencodings( + getencodings(method), + getblocks(method).sample, + blocknames = ["`$(summary(input))`", "`$(summary(input))`"], + inname = "`(input, target)`", + outname = "`(x, y)`", + ) + + decoding = describeencodings( + getencodings(method), + (ŷ,), + blocknames = ["`getblocks(method).ŷ`"], + inname = "`ŷ`", + outname = "`target_pred`", + decode = true, + ) + s = """ #### `LearningMethod` summary - - Task: `$(typeof(method.blocks[1])) -> $(typeof(method.blocks[2]))` - - Model blocks: `$(typeof(xblock)) -> $(typeof(method.outputblock))` + - Task: `$(summary(input)) -> $(summary(target))` + - Model blocks: `$(summary(x)) -> $(summary(ŷ))` Encoding a sample (`encode(method, context, sample)`) - $(describeencodings( - method.encodings, - method.blocks, - blocknames=["`method.blocks[1]`", "`method.blocks[2]`"], - inname="`(input, target)`", outname="`(x, y)`")) + $encoding Decoding a model output (`decode(method, context, ŷ)`) - $(describeencodings( - method.encodings, - (method.outputblock,), - blocknames=["`method.outputblock`"], - inname="`ŷ`", outname="`target_pred`", decode=true)) + $decoding """ return Markdown.parse(s) diff --git a/src/datablock/method.jl b/src/datablock/method.jl index 3b0640bbea..4ca0676872 100644 --- a/src/datablock/method.jl +++ b/src/datablock/method.jl @@ -1,83 +1,110 @@ -""" - BlockMethod(blocks, encodings) <: LearningMethod - -Learning method based on the `Block` and `Encoding` interfaces. -""" -struct BlockMethod{B, E, O} <: LearningMethod - blocks::B - encodings::E - outputblock::O -end -function BlockMethod(blocks, encodings; outputblock = encodedblockfilled(encodings, blocks[2])) - return BlockMethod(blocks, encodings, outputblock) -end +abstract type AbstractBlockMethod <: LearningMethod end +getblocks(method::AbstractBlockMethod) = method.blocks +getencodings(method::AbstractBlockMethod) = method.encodings # Core interface -function encode(method::BlockMethod, context, sample) - encode(method.encodings, context, method.blocks, sample) +function encode(method::AbstractBlockMethod, context, sample) + encode(getencodings(method), context, getblocks(method).sample, sample) end -function encodeinput(method::BlockMethod, context, input) - encode(method.encodings, context, method.blocks[1], input) +function encodeinput(method::AbstractBlockMethod, context, input) + encode(getencodings(method), context, getblocks(method).input, input) end -function encodetarget(method::BlockMethod, context, target) - encode(method.encodings, context, method.blocks[2], target) +function encodetarget(method::AbstractBlockMethod, context, target) + encode(getencodings(method), context, getblocks(method).target, target) end -function decode(method::BlockMethod, context, xy) - xyblock = encodedblock(method.encodings, method.blocks) - decode(method.encodings, context, xyblock, xy) +function decode(method::AbstractBlockMethod, context, encodedsample) + xyblock = encodedblock(getencodings(method), getblocks(method)) + decode(getencodings(method), context, getblocks(method).encodedsample, encodedsample) end -function decodeŷ(method::BlockMethod, context, ŷ) - decode(method.encodings, context, method.outputblock, ŷ) +function decodeŷ(method::AbstractBlockMethod, context, ŷ) + decode(getencodings(method), context, getblocks(method).ŷ, ŷ) end -function decodey(method::BlockMethod, context, y) - yblock = encodedblock(method.encodings, method.blocks[2]) - decode(method.encodings, context, yblock, y) +function decodey(method::AbstractBlockMethod, context, y) + decode(getencodings(method), context, getblocks(method).y, y) end # Training interface -function methodmodel(method::BlockMethod, backbone) - xblock = encodedblockfilled(method.encodings, method.blocks[1]) - return blockmodel(xblock, method.outputblock, backbone) +function methodmodel(method::AbstractBlockMethod, backbone) + return blockmodel(getblocks(method).x, getblocks(method).ŷ, backbone) end -function methodmodel(method::BlockMethod) - xblock = encodedblockfilled(method.encodings, method.blocks[1]) - return blockmodel(xblock, method.outputblock, blockbackbone(xblock)) +function methodmodel(method::AbstractBlockMethod) + backbone = blockbackbone(getblocks(method).x) + return blockmodel(getblocks(method).x, getblocks(method).ŷ, backbone) end -function methodlossfn(method::BlockMethod) - yblock = encodedblockfilled(method.encodings, method.blocks[2]) - return blocklossfn(method.outputblock, yblock) +function methodlossfn(method::AbstractBlockMethod) + return blocklossfn(getblocks(method).ŷ, getblocks(method).y) end # Testing interface -mocksample(method::BlockMethod) = mockblock(method.blocks) +mocksample(method::AbstractBlockMethod) = mockblock(method, :sample) +mockblock(method::AbstractBlockMethod, name::Symbol) = mockblock(getblocks(method)[name]) -mockmodel(method::BlockMethod) = mockmodel( - encodedblock(method.encodings, method.blocks[1]), - method.outputblock -) +mockmodel(method::AbstractBlockMethod) = mockmodel(getblocks(method).x, getblocks(method).ŷ) -function mockmodel(inblock::AbstractBlock, outblock::AbstractBlock) +function mockmodel(xblock, ŷblock) return function mockmodel_block(xs) - out = mockblock(outblock) + out = mockblock(ŷblock) DataLoaders.collate([out]) end end -# Pretty-printing -function Base.show(io::IO, method::BlockMethod) - print(io, "BlockMethod(", typeof(method.blocks[1]), " -> ", typeof(method.blocks[2]), ")") +# ## Supervised learning method + +""" + SupervisedMethod((inputblock, targetblock), encodings) <: LearningMethod + +Learning method for the supervised task of learning to predict a `target` +given an `input`. `encodings` are applied to samples before being input to +the model. Model outputs are decoded using those same encodings to get +a target prediction. +""" +struct SupervisedMethod{B<:NamedTuple,E} <: AbstractBlockMethod + blocks::B + encodings::E +end + + +function SupervisedMethod(blocks::Tuple{Any, Any}, encodings; ŷblock = nothing) + sample = input, target = blocks + x, y = encodedsample = encodedblockfilled(encodings, sample) + ŷ = isnothing(ŷblock) ? y : ŷblock + pred = decodedblockfilled(encodings, ŷ) + blocks = (; input, target, sample, encodedsample, x, y, ŷ, pred) + SupervisedMethod(blocks, encodings) +end + + +function Base.show(io::IO, method::SupervisedMethod) + print( + io, + "SupervisedMethod(", + summary(getblocks(method).input), + " -> ", + summary(getblocks(method).target), + ")", + ) end + + +# ## Deprecations + +BlockMethod(args...; kwargs...) = SupervisedMethod(args...; kwargs...) +Base.@deprecate BlockMethod(blocks, encodings; kwargs...) SupervisedMethod( + blocks, + encodings; + kwargs..., +) diff --git a/src/interpretation/learner.jl b/src/interpretation/learner.jl index 17c14910cb..19332456ac 100644 --- a/src/interpretation/learner.jl +++ b/src/interpretation/learner.jl @@ -7,7 +7,7 @@ Run a trained model in `learner` on `n` samples and visualize the outputs. """ -function showoutputs(method::BlockMethod, learner::Learner; n=4, context=Validation(), backend = default_showbackend()) +function showoutputs(method::AbstractBlockMethod, learner::Learner; n=4, context=Validation(), backend = default_showbackend()) cb = FluxTraining.getcallback(learner, ToDevice) devicefn = isnothing(cb) ? identity : cb.movedatafn backfn = isnothing(cb) ? identity : cpu diff --git a/src/interpretation/method.jl b/src/interpretation/method.jl index e739896886..c38035e2ba 100644 --- a/src/interpretation/method.jl +++ b/src/interpretation/method.jl @@ -16,11 +16,12 @@ showsample(method, sample) # select backend automatically showsample(ShowText(), method, sample) ``` """ -function showsample(backend::ShowBackend, method::BlockMethod, sample) - blocks = ("Input" => method.blocks[1], "Target" => method.blocks[2]) +function showsample(backend::ShowBackend, method::AbstractBlockMethod, sample) + blocks = ("Input" => getblocks(method)[1], "Target" => getblocks(method)[2]) showblock(backend, blocks, sample) end -showsample(method::BlockMethod, sample) = showsample(default_showbackend(), method, sample) +showsample(method::AbstractBlockMethod, sample) = + showsample(default_showbackend(), method, sample) """ @@ -39,39 +40,45 @@ showsamples(method, samples) # select backend automatically showsamples(ShowText(), method, samples) ``` """ -function showsamples(backend::ShowBackend, method::BlockMethod, samples) - blocks = ("Input" => method.blocks[1], "Target" => method.blocks[2]) - showblocks(backend, blocks, samples) +function showsamples(backend::ShowBackend, method::AbstractBlockMethod, samples) + showblocks(backend, "Sample" => getblocks(method).sample, samples) end -showsamples(method::BlockMethod, samples) = showsamples(default_showbackend(), method, sample) +showsamples(method::AbstractBlockMethod, samples) = + showsamples(default_showbackend(), method, sample) """ showencodedsample([backend], method, encsample) Show an encoded sample `encsample` to `backend`. """ -function showencodedsample(backend::ShowBackend, method::BlockMethod, encsample) - xblock, yblock = encodedblockfilled(method.encodings, method.blocks) +function showencodedsample(backend::ShowBackend, method::AbstractBlockMethod, encsample) showblockinterpretable( backend, - method.encodings, - ("x" => xblock, "y" => yblock), - encsample) + getencodings(method), + getblocks(method).encodedsample, + encsample, + ) end -showencodedsample(method, encsample) = showencodedsample(default_showbackend(), method, encsample) +showencodedsample(method, encsample) = + showencodedsample(default_showbackend(), method, encsample) """ showencodedsamples([backend], method, encsamples) Show a vector of encoded samples `encsamples` to `backend`. """ -function showencodedsamples(backend::ShowBackend, method::BlockMethod, encsamples::AbstractVector) - xblock, yblock = encodedblockfilled(method.encodings, method.blocks) +function showencodedsamples( + backend::ShowBackend, + method::AbstractBlockMethod, + encsamples::AbstractVector, +) + xblock, yblock = encodedblockfilled(getencodings(method), getblocks(method)) showblocksinterpretable( backend, - method.encodings, + getencodings(method), ("x" => xblock, "y" => yblock), - encsamples) + encsamples, + ) end """ @@ -79,7 +86,7 @@ end Show a collated batch of encoded samples to `backend`. """ -function showbatch(backend::ShowBackend, method::BlockMethod, batch) +function showbatch(backend::ShowBackend, method::AbstractBlockMethod, batch) encsamples = collect(DataLoaders.obsslices(batch)) showencodedsamples(backend, method, encsamples) end @@ -90,24 +97,23 @@ showbatch(method, batch) = showbatch(default_showbackend(), method, batch) showprediction([backend], method, sample, pred) Show a prediction `pred`. If a `sample` is also given, show it next to -the prediction. +the prediction. ŷ """ -function showprediction(backend::ShowBackend, method::BlockMethod, pred) - predblock = decodedblockfilled(method.encodings, method.outputblock) - showblock(backend, "Prediction" => predblock, pred) +function showprediction(backend::ShowBackend, method::AbstractBlockMethod, pred) + showblock(backend, "Prediction" => getblocks(method).pred, pred) end -function showprediction(backend::ShowBackend, method::BlockMethod, sample, pred) - predblock = decodedblockfilled(method.encodings, method.outputblock) +function showprediction(backend::ShowBackend, method::AbstractBlockMethod, sample, pred) + blocks = getblocks(method) showblock( backend, - ("Sample" => method.blocks, "Prediction" => predblock), - (sample, pred) + ("Sample" => blocks.sample, "Prediction" => blocks.pred), + (sample, pred), ) end -showprediction(method::BlockMethod, args...) = +showprediction(method::AbstractBlockMethod, args...) = showprediction(default_showbackend(), method, args...) """ @@ -117,21 +123,21 @@ showprediction(method::BlockMethod, args...) = Show predictions `pred`. If `samples` are also given, show them next to the prediction. """ -function showpredictions(backend::ShowBackend, method::BlockMethod, preds) - predblock = decodedblockfilled(method.encodings, method.outputblock) +function showpredictions(backend::ShowBackend, method::AbstractBlockMethod, preds) + predblock = decodedblockfilled(getencodings(method), getblocks(method).ŷ) showblocks(backend, "Prediction" => predblock, preds) end -function showpredictions(backend::ShowBackend, method::BlockMethod, samples, preds) - predblock = decodedblockfilled(method.encodings, method.outputblock) +function showpredictions(backend::ShowBackend, method::AbstractBlockMethod, samples, preds) + predblock = decodedblockfilled(getencodings(method), getblocks(method).ŷ) showblocks( backend, - ("Sample" => method.blocks, "Prediction" => predblock), + ("Sample" => getblocks(method), "Prediction" => predblock), collect(zip(samples, preds)), ) end -showpredictions(method::BlockMethod, args...) = +showpredictions(method::AbstractBlockMethod, args...) = showpredictions(default_showbackend(), method, args...) """ @@ -141,19 +147,25 @@ showpredictions(method::BlockMethod, args...) = Show a model output to `backend`. If an encoded sample `encsample` is also given, show it next to the output. """ -function showoutput(backend::ShowBackend, method::BlockMethod, output) - showblockinterpretable(backend, method.encodings, "Output" => method.outputblock, output) +function showoutput(backend::ShowBackend, method::AbstractBlockMethod, output) + showblockinterpretable( + backend, + getencodings(method), + "Output" => getblocks(method).ŷ, + output, + ) end -function showoutput(backend::ShowBackend, method::BlockMethod, encsample, output) - encsampleblock = encodedblockfilled(method.encodings, method.blocks) - outblock = method.outputblock +function showoutput(backend::ShowBackend, method::AbstractBlockMethod, encsample, output) + blocks = getblocks(method) showblockinterpretable( backend, - method.encodings, - ("Encoded sample" => encsampleblock, "Output" => outblock), - (encsample, output)) + getencodings(method), + ("Encoded sample" => blocks.encodedsample, "Output" => blocks.ŷ), + (encsample, output), + ) end -showoutput(method::BlockMethod, args...) = showoutput(default_showbackend(), method, args...) +showoutput(method::AbstractBlockMethod, args...) = + showoutput(default_showbackend(), method, args...) """ showoutputs([backend], method, outputs) @@ -163,20 +175,26 @@ Show model outputs to `backend`. If a vector of encoded samples `encsamples` is given, show them next to the outputs. Use [`showoutputbatch`](#) to show collated batches of outputs. """ -function showoutputs(backend::ShowBackend, method::BlockMethod, outputs) - showblocks(backend, "Output" => method.outputblock, outputs) +function showoutputs(backend::ShowBackend, method::AbstractBlockMethod, outputs) + showblocksinterpretable( + backend, + getencodings(method), + "Output" => getblocks(method).ŷ, + outputs, + ) end -function showoutputs(backend::ShowBackend, method::BlockMethod, encsamples, outputs) - encsampleblock = encodedblockfilled(method.encodings, method.blocks) - outblock = method.outputblock +function showoutputs(backend::ShowBackend, method::AbstractBlockMethod, encsamples, outputs) + blocks = getblocks(method) showblocksinterpretable( backend, - method.encodings, - ("Encoded sample" => encsampleblock, "Output" => outblock), - collect(zip(encsamples, outputs))) + getencodings(method), + ("Encoded sample" => blocks.encodedsample, "Output" => blocks.ŷ), + collect(zip(encsamples, outputs)), + ) end -showoutputs(method::BlockMethod, args...) = showoutputs(default_showbackend(), method, args...) +showoutputs(method::AbstractBlockMethod, args...) = + showoutputs(default_showbackend(), method, args...) """ @@ -187,17 +205,23 @@ Show collated batch of outputs to `backend`. If a collated batch of encoded samp `batch` is also given, show them next to the outputs. See [`showoutputs`](#) if you have vectors of outputs and not collated batches. """ -function showoutputbatch(backend::ShowBackend, method::BlockMethod, outputbatch) +function showoutputbatch(backend::ShowBackend, method::AbstractBlockMethod, outputbatch) outputs = collect(DataLoaders.obsslices(outputbatch)) return showoutputs(backend, method, outputs) end -function showoutputbatch(backend::ShowBackend, method::BlockMethod, batch, outputbatch) +function showoutputbatch( + backend::ShowBackend, + method::AbstractBlockMethod, + batch, + outputbatch, +) encsamples = collect(DataLoaders.obsslices(batch)) outputs = collect(DataLoaders.obsslices(outputbatch)) return showoutputs(backend, method, encsamples, outputs) end -showoutputbatch(method::BlockMethod, args...) = showoutputbatch(default_showbackend(), method, args...) +showoutputbatch(method::AbstractBlockMethod, args...) = + showoutputbatch(default_showbackend(), method, args...) # Testing helper @@ -209,14 +233,16 @@ work for `backend` ## Keyword arguments -- `sample = mockblock(method.blocks)`: Sample data to use for tests. -- `output = mockblock(method.outputblock)`: Model output data to use for tests. +- `sample = mockblock(getblocks(method))`: Sample data to use for tests. +- `output = mockblock(getblocks(method).ŷ)`: Model output data to use for tests. """ function test_method_show( - method::LearningMethod, backend::ShowBackend; - sample = mockblock(method.blocks), - output = mockblock(method.outputblock), - context = Training()) + method::LearningMethod, + backend::ShowBackend; + sample = mockblock(getblocks(method).sample), + output = mockblock(getblocks(method).ŷ), + context = Training(), +) encsample = encode(method, context, sample) pred = decodeŷ(method, context, output) diff --git a/src/learner.jl b/src/learner.jl index 7ac8a554a6..a1459337a5 100644 --- a/src/learner.jl +++ b/src/learner.jl @@ -97,7 +97,7 @@ end @testset "methodlearner" begin - method = BlockMethod((Label(1:2), Label(1:2)), (OneHot(),)) + method = SupervisedMethod((Label(1:2), Label(1:2)), (OneHot(),)) data = (rand(1:2, 1000), rand(1:2, 1000)) @test_nowarn learner = methodlearner(method, data, model=identity) diff --git a/src/training/onecycle.jl b/src/training/onecycle.jl index c43a5ffb71..190a348e14 100644 --- a/src/training/onecycle.jl +++ b/src/training/onecycle.jl @@ -15,6 +15,7 @@ to `lrmax` and then goes down to `lrmax/div_final` over the remaining duration. """ function fitonecycle!( learner::Learner, nepochs::Int, maxlr=0.1; + phases = (TrainingPhase(), ValidationPhase()), dataiters=(learner.data.training, learner.data.validation), wd=0., kwargs...) @@ -28,7 +29,11 @@ function fitonecycle!( wdoptim = wd > 0 ? decay_optim(learner.optimizer, wd) : learner.optimizer withfields(learner, optimizer=wdoptim) do withcallbacks(learner, scheduler) do - fit!(learner, nepochs, dataiters) + for _ in 1:nepochs + for (phase, data) in zip(phases, dataiters) + epoch!(learner, phase, data) + end + end end end end From 8d978ad532e69399fa9bf10690f5002df8209ad7 Mon Sep 17 00:00:00 2001 From: lorenzoh Date: Mon, 20 Dec 2021 13:09:40 +0100 Subject: [PATCH 02/23] Add WIP VAE notebook --- notebooks/vae.ipynb | 666 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 666 insertions(+) create mode 100644 notebooks/vae.ipynb diff --git a/notebooks/vae.ipynb b/notebooks/vae.ipynb new file mode 100644 index 0000000000..55efef3685 --- /dev/null +++ b/notebooks/vae.ipynb @@ -0,0 +1,666 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: using Makie.Label in module FastAI conflicts with an existing identifier.\n" + ] + } + ], + "source": [ + "using FastAI, StaticArrays, Colors\n", + "using FastAI: FluxTraining\n", + "import CairoMakie" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfQAAAImCAIAAAAqqu9kAAAABmJLR0QA/wD/AP+gvaeTAAALkUlEQVR4nO3cP6iW5QOH8bc0I6pDYFObOUgUDp6glpbGFMEQjoM4NaRjuGRSQUO5BA1BEAUNbUUgCEoggn/iEBLUUISDg0WQVFbUIp7f0vYbvB/wfU/n6vOZvzzvzRkubs5w37W2tjYDoOXu9T4AAHeeuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0DQ5gX/3ssvv3z58uUF/yjA+lpeXn7zzTcX+YuLjvvly5c///zzBf8owH+Nf8sABIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEbV7vA9D3zDPPTNpfunRpfLxjx47x8Z49e8bHu3fvHh+fOnVqfDzVF198MT4+f/78/E7CBuLmDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBBnvzlH0tLS+Pjjz/+eHz87LPPTjrJ33//PT7esmXL+PiBBx6YdJJxU581nmTSH+Svv/4aHx8+fHh8/Mknn4yPWXdu7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4Q5Mlf/nHixInx8e7du+d3kvvuu298/O23346Pf/755/Hx77//Pj6e6u67J9yrnnvuufHxpL/eBx98MD7+/vvvx8ez2ezrr7+etOfOcnMHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwjynnvZ448/Pj7ev3//nI5x7dq1SftDhw6Nj69cuTI+/u2338bHf/755/h4qknvub/66qvj4+PHj4+Pl5aWxsevvfba+Hg2m73wwgvj419//XXSx7ktN3eAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCPLkb9mDDz44Pt66dev4eG1tbXx84sSJ8fFsNjt37tyk/UZ069at8fHrr78+Pt6yZcv4+OjRo+Pjffv2jY9ns9mHH344Pj516tSkj3Nbbu4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOEOTJ37J77713Tl/+6KOPxsfvvvvunI7B/zt27Nj4eGVlZXy8bdu2SSd5/vnnx8ee/L3j3NwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIMiTv2VvvPHGnL68uro6py+zSGfOnBkfv/jii5M+/vTTT088DneSmztAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBHnyd4N59NFHx8ePPPLI+PjGjRvj42+++WZ8zL/W2bNnx8dTn/xlfbm5AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEec99gzl48OD4eNLj759++un4+NKlS+NjYPHc3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gyJO/G8yBAwfGxzdu3Bgfv/POO9OPA/xLubkDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJAnf8u+++678fGFCxfmdxJgwdzcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSDIk7/r7P7775+0v+eee+Z0EqDEzR0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgjz5u85WVlYm7bdv3z4+vn79+sTj8N+yd+/e+X385s2b8/s4t+XmDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4Q5D13SFleXh4f79mzZ34nOXbs2Pw+zm25uQMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkCd/4d9u0iu+L7300vj4oYceGh9fvHhxfDybzc6cOTNpz53l5g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQZ78XWdXr16dtP/jjz/mcxAWatOmTePjo0ePjo9XVlbGxz/88MOcjjGbzW7evDlpz53l5g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQZ78XWdnz56dtJ/0RuvS0tL4+OGHHx4fX79+fXy8Qe3cuXN8fOTIkUkf37Vr1/j4ySefnPTxcQcPHhwfr66uzukYzIObO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEefK37LHHHhsfnz59enz8008/TT/OBvPUU0+Nj7du3Tq/k0x6YPnkyZPj4y+//HL6cdgY3NwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYK8577BvPLKK+Pj48ePj4937do1/Tj849atW5P2v/zyy/j47bffHh+/9dZbk05ClZs7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwR58neD+eyzz8bHq6ur4+PTp0+Pj5944onx8Qb1/vvvj4+/+uqrSR9/7733Jh4HpnFzBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgT/6W/fjjj+PjnTt3zu8kwIK5uQMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwRtXvDvLS8vL/gXAdbd4tN319ra2oJ/EoB5828ZgCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gKD/AU1d8UvOdiFWAAAAAElFTkSuQmCC", + "text/plain": [ + "Scene (500px, 550px):\n", + " 18 Plots:\n", + " ├ MakieCore.Combined{Makie.poly, Tuple{Vector{Vector{GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " └ MakieCore.Text{Tuple{String}}\n", + " 1 Child Scene:\n", + " └ Scene (468px, 468px)" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "path = datasetpath(\"mnist_png\")\n", + "data = Datasets.loadfolderdata(\n", + " path,\n", + " filterfn = isimagefile,\n", + " loadfn = loadfile,\n", + ")\n", + "showblock(Image{2}(), getobs(data, 1))" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfQAAAImCAIAAAAqqu9kAAAABmJLR0QA/wD/AP+gvaeTAAALkUlEQVR4nO3cP6iW5QOH8bc0I6pDYFObOUgUDp6glpbGFMEQjoM4NaRjuGRSQUO5BA1BEAUNbUUgCEoggn/iEBLUUISDg0WQVFbUIp7f0vYbvB/wfU/n6vOZvzzvzRkubs5w37W2tjYDoOXu9T4AAHeeuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0DQ5gX/3ssvv3z58uUF/yjA+lpeXn7zzTcX+YuLjvvly5c///zzBf8owH+Nf8sABIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEbV7vA9D3zDPPTNpfunRpfLxjx47x8Z49e8bHu3fvHh+fOnVqfDzVF198MT4+f/78/E7CBuLmDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBBnvzlH0tLS+Pjjz/+eHz87LPPTjrJ33//PT7esmXL+PiBBx6YdJJxU581nmTSH+Svv/4aHx8+fHh8/Mknn4yPWXdu7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4Q5Mlf/nHixInx8e7du+d3kvvuu298/O23346Pf/755/Hx77//Pj6e6u67J9yrnnvuufHxpL/eBx98MD7+/vvvx8ez2ezrr7+etOfOcnMHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwjynnvZ448/Pj7ev3//nI5x7dq1SftDhw6Nj69cuTI+/u2338bHf/755/h4qknvub/66qvj4+PHj4+Pl5aWxsevvfba+Hg2m73wwgvj419//XXSx7ktN3eAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCPLkb9mDDz44Pt66dev4eG1tbXx84sSJ8fFsNjt37tyk/UZ069at8fHrr78+Pt6yZcv4+OjRo+Pjffv2jY9ns9mHH344Pj516tSkj3Nbbu4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOEOTJ37J77713Tl/+6KOPxsfvvvvunI7B/zt27Nj4eGVlZXy8bdu2SSd5/vnnx8ee/L3j3NwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIMiTv2VvvPHGnL68uro6py+zSGfOnBkfv/jii5M+/vTTT088DneSmztAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBHnyd4N59NFHx8ePPPLI+PjGjRvj42+++WZ8zL/W2bNnx8dTn/xlfbm5AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEec99gzl48OD4eNLj759++un4+NKlS+NjYPHc3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gyJO/G8yBAwfGxzdu3Bgfv/POO9OPA/xLubkDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJAnf8u+++678fGFCxfmdxJgwdzcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSDIk7/r7P7775+0v+eee+Z0EqDEzR0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgjz5u85WVlYm7bdv3z4+vn79+sTj8N+yd+/e+X385s2b8/s4t+XmDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4Q5D13SFleXh4f79mzZ34nOXbs2Pw+zm25uQMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkCd/4d9u0iu+L7300vj4oYceGh9fvHhxfDybzc6cOTNpz53l5g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQZ78XWdXr16dtP/jjz/mcxAWatOmTePjo0ePjo9XVlbGxz/88MOcjjGbzW7evDlpz53l5g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQZ78XWdnz56dtJ/0RuvS0tL4+OGHHx4fX79+fXy8Qe3cuXN8fOTIkUkf37Vr1/j4ySefnPTxcQcPHhwfr66uzukYzIObO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEefK37LHHHhsfnz59enz8008/TT/OBvPUU0+Nj7du3Tq/k0x6YPnkyZPj4y+//HL6cdgY3NwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYK8577BvPLKK+Pj48ePj4937do1/Tj849atW5P2v/zyy/j47bffHh+/9dZbk05ClZs7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwR58neD+eyzz8bHq6ur4+PTp0+Pj5944onx8Qb1/vvvj4+/+uqrSR9/7733Jh4HpnFzBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgT/6W/fjjj+PjnTt3zu8kwIK5uQMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwRtXvDvLS8vL/gXAdbd4tN319ra2oJ/EoB5828ZgCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gKD/AU1d8UvOdiFWAAAAAElFTkSuQmCC", + "text/plain": [ + "Scene (500px, 550px):\n", + " 18 Plots:\n", + " ├ MakieCore.Combined{Makie.poly, Tuple{Vector{Vector{GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " └ MakieCore.Text{Tuple{String}}\n", + " 1 Child Scene:\n", + " └ Scene (468px, 468px)" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "\n", + "using FastAI: AbstractBlockMethod, encodedblockfilled, decodedblockfilled\n", + "struct EmbeddingMethod{B<:NamedTuple,E} <: AbstractBlockMethod\n", + " blocks::B\n", + " encodings::E\n", + "end\n", + "\n", + "function EmbeddingMethod(block, encodings)\n", + " sample = input = target = block\n", + " encodedsample = x = y = ŷ = encodedblockfilled(encodings, sample)\n", + " pred = decodedblockfilled(encodings, ŷ)\n", + " blocks = (; sample, x, y, ŷ, encodedsample, pred, input, target)\n", + " EmbeddingMethod(blocks, encodings)\n", + "end\n", + "\n", + "\n", + "Base.show(io::IO, task::EmbeddingMethod) =\n", + " print(io, \"EmbeddingMethod($(summary(task.blocks.sample)))\")\n", + "\n", + "\n", + "method = EmbeddingMethod(\n", + " Image{2}(),\n", + " (ImagePreprocessing(means = SVector(0.0), stds = SVector(1.0), C = Gray{Float32}),),\n", + ")\n", + "\n", + "x = encode(method, Training(), getobs(data, 1))\n", + "showencodedsample(method, x)" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "βELBO (generic function with 1 method)" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "\n", + "struct VAE{E, D}\n", + " encoder::E\n", + " decoder::D\n", + "end\n", + "\n", + "Flux.@functor VAE\n", + "\n", + "function (vae::VAE)(xs)\n", + " μ, logσ² = learner.model.encoder(xs)\n", + " zs = sample_latent(μ, logσ²)\n", + " x̄s = learner.model.decoder(zs)\n", + " return x̄s, (; μ, logσ²)\n", + "end\n", + "\n", + "\n", + "using Random: randn!\n", + "using Statistics: mean\n", + "\n", + "sample_latent(μ::AbstractArray{T}, logσ²::AbstractArray{T}) where {T} =\n", + " μ .+ exp.(logσ² ./ 2) .* randn!(similar(logσ²))\n", + "\n", + "function βELBO(x, x̄, μ, logσ²; β = 1)\n", + " reconstruction_error = mean(sum(@.((x̄ - x)^2); dims = 1))\n", + " # D(N(μ, Σ)||N(0, I)) = 1/2 * (μᵀμ + tr(Σ) - length(μ) - log(|Σ|))\n", + " kl_divergence = mean(sum(@.((μ^2 + exp(logσ²) - 1 - logσ²) / 2); dims = 1))\n", + "\n", + " return reconstruction_error + β * kl_divergence\n", + "end" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "SIZE = (28, 28, 1)\n", + "Din = prod(SIZE)\n", + "Dhidden = 512\n", + "Dlatent = 2\n", + "\n", + "encoder =\n", + " Chain(\n", + " flatten,\n", + " Dense(Din, Dhidden, relu), # backbone\n", + " Parallel(\n", + " tuple,\n", + " Dense(Dhidden, Dlatent), # μ\n", + " Dense(Dhidden, Dlatent), # logσ²\n", + " ),\n", + " ) |> gpu\n", + "\n", + "decoder = Chain(Dense(Dlatent, Dhidden, relu), Dense(Dhidden, Din, sigmoid), xs -> reshape(xs, SIZE..., :)) |> gpu\n", + "\n", + "model = VAE(encoder, decoder);" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "traindl, validdl =\n", + " methoddataloaders(data, data, method, 1024, buffered = false)\n", + "traindl = collect(traindl);" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "struct VAETrainingPhase <: FluxTraining.AbstractTrainingPhase end\n", + "\n", + "function FluxTraining.step!(learner, phase::VAETrainingPhase, batch)\n", + " FluxTraining.runstep(learner, phase, (x = batch,)) do handle, state\n", + " x = state.x\n", + " gs = gradient(learner.params) do\n", + " # get encode, sample latent space, decode\n", + " x̄, (; μ, logσ²) = learner.model(x)\n", + " handle(FluxTraining.LossBegin())\n", + " state.loss = learner.lossfn(flatten(x), flatten(x̄), μ, logσ²)\n", + "\n", + " handle(FluxTraining.BackwardBegin())\n", + " return state.loss\n", + " end\n", + " handle(FluxTraining.BackwardEnd())\n", + " Flux.Optimise.update!(learner.optimizer, learner.params, gs)\n", + " end\n", + "end\n", + "\n", + "# for ToGPU to work\n", + "function FluxTraining.on(\n", + " ::FluxTraining.StepBegin,\n", + " ::VAETrainingPhase,\n", + " cb::ToDevice,\n", + " learner,\n", + ")\n", + " learner.step.x = cb.movedatafn(learner.step.x)\n", + "end" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "ProgressPrinter()" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "opt = ADAM(1e-3)\n", + "learner = Learner(model, (training = traindl,), opt, βELBO, ToGPU())\n", + "FluxTraining.removecallback!(learner, ProgressPrinter)" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "┌──────────────────┬───────┬─────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼─────────┤\n", + "│ VAETrainingPhase │ 1.0 │ 100.814 │\n", + "└──────────────────┴───────┴─────────┘\n", + "┌──────────────────┬───────┬─────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼─────────┤\n", + "│ VAETrainingPhase │ 2.0 │ 54.5541 │\n", + "└──────────────────┴───────┴─────────┘\n", + "┌──────────────────┬───────┬─────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼─────────┤\n", + "│ VAETrainingPhase │ 3.0 │ 49.6636 │\n", + "└──────────────────┴───────┴─────────┘\n", + "┌──────────────────┬───────┬────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼────────┤\n", + "│ VAETrainingPhase │ 4.0 │ 45.739 │\n", + "└──────────────────┴───────┴────────┘\n", + "┌──────────────────┬───────┬─────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼─────────┤\n", + "│ VAETrainingPhase │ 5.0 │ 43.4472 │\n", + "└──────────────────┴───────┴─────────┘\n", + "┌──────────────────┬───────┬─────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼─────────┤\n", + "│ VAETrainingPhase │ 6.0 │ 42.2355 │\n", + "└──────────────────┴───────┴─────────┘\n", + "┌──────────────────┬───────┬─────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼─────────┤\n", + "│ VAETrainingPhase │ 7.0 │ 41.5158 │\n", + "└──────────────────┴───────┴─────────┘\n", + "┌──────────────────┬───────┬─────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼─────────┤\n", + "│ VAETrainingPhase │ 8.0 │ 40.8949 │\n", + "└──────────────────┴───────┴─────────┘\n", + "┌──────────────────┬───────┬─────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼─────────┤\n", + "│ VAETrainingPhase │ 9.0 │ 40.3357 │\n", + "└──────────────────┴───────┴─────────┘\n", + "┌──────────────────┬───────┬─────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼─────────┤\n", + "│ VAETrainingPhase │ 10.0 │ 39.9624 │\n", + "└──────────────────┴───────┴─────────┘\n", + "┌──────────────────┬───────┬─────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼─────────┤\n", + "│ VAETrainingPhase │ 11.0 │ 39.5658 │\n", + "└──────────────────┴───────┴─────────┘\n", + "┌──────────────────┬───────┬─────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼─────────┤\n", + "│ VAETrainingPhase │ 12.0 │ 39.2665 │\n", + "└──────────────────┴───────┴─────────┘\n", + "┌──────────────────┬───────┬─────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼─────────┤\n", + "│ VAETrainingPhase │ 13.0 │ 38.9916 │\n", + "└──────────────────┴───────┴─────────┘\n", + "┌──────────────────┬───────┬─────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼─────────┤\n", + "│ VAETrainingPhase │ 14.0 │ 38.7168 │\n", + "└──────────────────┴───────┴─────────┘\n", + "┌──────────────────┬───────┬─────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼─────────┤\n", + "│ VAETrainingPhase │ 15.0 │ 38.5706 │\n", + "└──────────────────┴───────┴─────────┘\n", + "┌──────────────────┬───────┬─────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼─────────┤\n", + "│ VAETrainingPhase │ 16.0 │ 38.3802 │\n", + "└──────────────────┴───────┴─────────┘\n", + "┌──────────────────┬───────┬─────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼─────────┤\n", + "│ VAETrainingPhase │ 17.0 │ 38.2334 │\n", + "└──────────────────┴───────┴─────────┘\n", + "┌──────────────────┬───────┬─────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼─────────┤\n", + "│ VAETrainingPhase │ 18.0 │ 38.0221 │\n", + "└──────────────────┴───────┴─────────┘\n", + "┌──────────────────┬───────┬─────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼─────────┤\n", + "│ VAETrainingPhase │ 19.0 │ 37.8736 │\n", + "└──────────────────┴───────┴─────────┘\n", + "┌──────────────────┬───────┬─────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼─────────┤\n", + "│ VAETrainingPhase │ 20.0 │ 37.7326 │\n", + "└──────────────────┴───────┴─────────┘\n", + "┌──────────────────┬───────┬─────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼─────────┤\n", + "│ VAETrainingPhase │ 21.0 │ 37.6447 │\n", + "└──────────────────┴───────┴─────────┘\n", + "┌──────────────────┬───────┬─────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼─────────┤\n", + "│ VAETrainingPhase │ 22.0 │ 37.5145 │\n", + "└──────────────────┴───────┴─────────┘\n", + "┌──────────────────┬───────┬─────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼─────────┤\n", + "│ VAETrainingPhase │ 23.0 │ 37.3824 │\n", + "└──────────────────┴───────┴─────────┘\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "┌──────────────────┬───────┬─────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼─────────┤\n", + "│ VAETrainingPhase │ 24.0 │ 37.2937 │\n", + "└──────────────────┴───────┴─────────┘\n", + "┌──────────────────┬───────┬─────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼─────────┤\n", + "│ VAETrainingPhase │ 25.0 │ 37.2094 │\n", + "└──────────────────┴───────┴─────────┘\n", + "┌──────────────────┬───────┬────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼────────┤\n", + "│ VAETrainingPhase │ 26.0 │ 37.139 │\n", + "└──────────────────┴───────┴────────┘\n", + "┌──────────────────┬───────┬─────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼─────────┤\n", + "│ VAETrainingPhase │ 27.0 │ 37.0836 │\n", + "└──────────────────┴───────┴─────────┘\n", + "┌──────────────────┬───────┬─────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼─────────┤\n", + "│ VAETrainingPhase │ 28.0 │ 37.0444 │\n", + "└──────────────────┴───────┴─────────┘\n", + "┌──────────────────┬───────┬────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼────────┤\n", + "│ VAETrainingPhase │ 29.0 │ 37.021 │\n", + "└──────────────────┴───────┴────────┘\n", + "┌──────────────────┬───────┬─────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼─────────┤\n", + "│ VAETrainingPhase │ 30.0 │ 36.9836 │\n", + "└──────────────────┴───────┴─────────┘\n" + ] + } + ], + "source": [ + "fitonecycle!(\n", + " learner,\n", + " 30,\n", + " 0.01,\n", + " dataiters = (learner.data.training,),\n", + " phases = (VAETrainingPhase(),),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABLAAAAl0CAIAAACP7Rn6AAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nOzdeZDX9X3H8c9v98e13IIHeMd6QDyikGhJR4JSTKJVJmirkqikzjQmMVXTqhmtjiaZkDZVYz1iNFOvFjXGI7Y6iEGJifcKNvFAMRgUD04VWJa9fv3jN9nZ2YUNn4Xf/oD34/GX/nj9fr8PP3GX5373KJRKpQQAAEA8NdU+AAAAANUhCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQghG3Ol7/85UKh8LWvfW07ffxtVvk3fvbZZ1f7IAAA2wpByI7gn/7pnwrdKhaL1T4jAGTbsGHDLbfcctJJJ+21114DBgwYOnToQQcddPbZZz/22GNb5fGnTp1aKBS++c1vbpVH276eHSjzt2QAgG3RE088ceaZZy5ZsqT9lsbGxo8//njhwoU/+9nPJk+e/F//9V+77LJLFU8I7ABcIWTHMWLEiNImtLS0VPt0AJDhgQcemDJlypIlS4YOHfr973//lVdeaWhoWL169bx5884444xCofDYY48deeSR7777brVPCmzfXCEEANi2vPnmm2eccUZzc/MnPvGJuXPn7r333uXbBwwYcPTRRx999NEnnHDC6aef/tZbb5166qlPPPFETY0P8QM95M0HEU2ePLlQKFx88cXr1q274oorDjnkkIEDBw4cOPAzn/nMLbfcstG7vP322xdeeOGnPvWpoUOH1tXVHXTQQdOnT3/sscdKpVKn5RNPPHHKKafsvvvuffv2HTFixKRJk26++eZNXaK84447JkyYMHjw4MGDBx922GHf/e53Gxoaujn5kiVLLrjggoMPPnjw4MF1dXVjx469+OKLV65cual97uNv1F133fX5z39+l1126du376677nrEEUd885vffO655zpuGhoaHnjggbPOOuuwww4bMWJEv3799thjj2nTps2ZM6frA7a//h9++OHFF188ZsyYurq6nXba6dhjj509e3Z588EHH3z7298+4IADBgwYMGLEiGnTpi1cuHBTj7Nq1aoLL7zwoIMOqqurGzZs2OTJkx9++OGs32PuCwtQUZdddtmaNWv69Onzi1/8or0GOzrllFMuvfTSlNKTTz55//33t9++zz77FAqFW2+9tetdTj755I5fsHfvvfcWCoUHH3wwpXT99dd3/ML7F154obzJfTO7dZ8d6CWb+hQ72I58+9vfTt1+ymgnxx57bErpy1/+8v7779/1f4rzzz+/0/7222/v37//Rv8Pmj9/fsfl+eefv9HZZz/72VWrVnVctra2fuUrX+m6/OQnP/mFL3whpfQP//APnY4xa9asjR5jt912e+mllzqNe/D4G3XmmWdu6q3H8uXL22eXX375pmYXX3zxRl//008/vevfcgqFwp133llfX7/zzjt3+qXhw4e//vrrXR/ntNNO22OPPbo+72WXXdbpeadPn55S+vu///steWEBKm3FihXl74U2ffr0bmbr1q0bNmxYSmnSpEntN5bfrv7nf/5n1/20adNSSt/4xjfK//rzn/98U2+3n3/++fIm983s1n12oHe4Qkhcd95553vvvffjH/946dKl69evnz9/fvk939VXX/273/2uffbwww+feeaZjY2NhxxyyL333rt8+fL169cvXLhw1qxZkydP7vhZOtddd93VV1+dUjrxxBPnz5/f2Ni4ZMmSyy+/vLa29re//W2nPLvmmmvuuOOOlNKZZ5756quvbtiw4Y033jjnnHNefvnlRx55pOtp586dO3369MbGxokTJz722GNr1qxpaGiYM2fOoYce+v7770+dOnXt2rVb8vgb9fjjj992220ppQsuuOCVV15Zt27d6tWrFyxY8JOf/OTII48sFArty4EDB5566qn333//q6++umbNmlWrVj333HMzZsxIKc2cOXOjz/jf//3fq1evvvHGGz/44IM1a9bMmTNn3333LZVK55133kknnTRgwIB77rnnww8/XL169d133z18+PDVq1f/8z//c9fHmTVr1rvvvnvZZZctWbKksbHxxRdfPP7441NKV1555S9/+cs/+3vMfWEBKu3Xv/51+fNKTjnllG5mdXV1J5xwQkrpqaee2rBhQ+6znHzyyaVS6aSTTkodOq1s/PjxHZdb+GZ2C58dqLhejE+olPIVwm50uhpWDr+U0ty5czve/vHHH48cOTKldPnll5dvaW5u3nfffVNK48ePX7t2bTdnaGho2GmnnVJKX/ziF1tbWzv+0jXXXFN+uscff7x9PHz48JTSV77ylU6Pc84553Q9c2tr64EHHphSmjBhQktLS8f9ypUrR4wYkVL693//946HyXr8TSlf9/vc5z73Z5eb8o//+I8ppeOPP77jje2v/7x58zre3v75pYMHD168eHHHX/qP//iPlFKxWFy/fn3Xx/nXf/3XjuOWlpbPfe5zKaWxY8d2vL3rFcLcFxagF3z3u98tv3F7++23u19eddVV5eXvf//78i2bf42ubKNJ1i73zezWfXagd7hCSFxHH330pEmTOt4yePDgY445JqX02muvlW95/PHHFy9enFK69tprBw4c2M2jzZkzZ9WqVSmlH/7wh52+uP/cc88tv4+cNWtW+ZZHH3109erVhULhe9/7XqfHufLKK/v06dPpxieffLL8FXTXX399bW1tx1/aaaedyp+net9997XfmPv4m1L+RMot+R6t5b8BPPPMM11/qfx9ETrecswxx5TPdvrpp++zzz4df+m4444rn2TRokWdHmfnnXc+77zzOt5SW1tb/uvUK6+80vFib1e5LyxAL2j/Aubyxyi70f7Z9RX9mucteTMLbPsEITuObr6G8Cc/+UnX/eGHH971xj333DOl9PHHH5f/9amnnkop7bLLLn/5l3/Z/bOXvwh+9913P/jggzv9Uk1NzZQpU9o3KaX6+vqU0oEHHrjXXnt1Go8cObLrwX7729+mlHbeeedPfepTXZ/6iCOOSCktWLCg/Zbcx9+U8geAf/Ob35x66qmPP/5495+S9NZbb1144YXjx48fPnx4bW1t+XsDlJNv5cqVXauy6zFqamp23XXXlFLX3+bo0aPL/9D+n6bdxIkTuybuhAkT6urqUofXfKNyX1iAbUqpyzc2q4QteTMLbPsEIXFt9PuIlL+Ov62trfyv77//fkqp/Fmj3Vu2bFn6U092VQ6z8qb9Hzb6NfobfZClS5emlJYvX14sFovFYm1tbW1tbU1NTU1NTaFQ+OIXv5hSWrduXXuw5T7+phx11FEXXXRRSunuu+8+5phjhg4detRRR33nO9/p+vHgBx98cMyYMf/2b/9WX1//4Ycftr+A7RobGzvdstHXv3yZrusvtV++6/rIG/1t1tTUjBo1KnV4zTcq94UF6AXlz1dPKa1YsaL7Zfug/S6VsCVvZoFtnyCEP6/jd0/ZkmWnX938h21tbW3/h9bW1ra2tra2tvLFz46z5ubmnj1+N2bOnPnss89+7WtfO+CAAzZs2PDss8/OnDnz0EMPPfvss9tPtXLlyvL33Rk3btw999yzePHihoaG8gmfffbZLT9D97bkt9mzFxagosaOHVv+hxdffLH7ZXnQr1+//fbbr3Ln2SrvTYBtliCE7pQ//PmHP/zhzy7LX8ixZMmSjf5q+fb2L/bYZZddUkpvv/32RsfvvPNOp1vKn0j56U9/uvuvCR40aFDPHr97n/nMZ2688caFCxeuWLHigQce+NKXvpRS+tnPfnbttdeWBw8//PBHH300bNiwuXPnnnLKKfvss8+AAQPKf4HohY8cb/S32dbW9t5776U/vRSbkvvCAvSCiRMnlj8topufzZBSWr9+/f/+7/+mlCZMmND+iRXlL2Jv/2hXp33PzrP5b2Yr8exApQlC6M6ECRNSSsuWLXv66ae7X5a/TfbSpUtffvnlTr9U+tP3z2z/Vtrjxo1LKS1cuLDre9kVK1bMnz+/041/9Vd/lVJasGDBZvZV7uNvphEjRpx00km/+MUvpk6dmlIq/0Uk/envCgceeOCQIUM63WWjP5h+65o3b17XK3hPPfVUQ0ND6vCab1TuCwvQC0aMGPG3f/u3KaW77777//7v/zY1+9GPfrR69eqU0te//vX2GwcPHpw29oG/tra2jX5FdLk8u/9axM1/M1uJZwcqTRBCdyZNmvSJT3wipfStb32r/J5vU6ZMmVL+sRPf+c53Or1vu/766996662U0mmnndY+HjZsWKlUuvTSSzs9zuWXX97U1NT1GPvtt19zc/O3vvWtrl9Et9HDZD1+rvI3vms/ydChQ1NKCxcu7PQj+15++eVbbrllC5/rz1q+fHn7D/Yoa21tveyyy1JKY8aMOeSQQ7q5b+4LC9A7rrjiikGDBjU3N0+bNm2jF+juv//+8vf5/OxnP1v+xI2yMWPGpJQeeuihTvubb7753Xff7fo45Q/kdf/Fipv/ZrYSzw5UmiCE7tTW1l533XWFQuGFF1446qij7rvvvpUrVzY2Nr7xxht33XXXlClT2j92O2DAgCuuuCKl9NBDD02bNu2ll15qamp65513rrzyyvJ36z7++OPL37SzPL7kkktSSrfffvuMGTMWLlzY1NT05ptvfv3rX7/hhhu6HqNYLN50003FYvHuu++eOHHigw8+uGzZspaWlmXLli1YsOC2226bNm1ax58wkfv4m3Luueeefvrps2bN+t3vfrdq1aoNGzYsXrx45syZt956a0qp/K1Ty/9QU1Pz4YcfTp06tb6+vrGxcdmyZTfffPPEiRO7r+it5eKLL7788svfeeedpqamBQsWTJ069fHHH08pff/73+/+jrkvLEDv2H///W+99dY+ffosWrTo0EMP/cEPfvDaa681NjZ+9NFHv/nNb2bMmDFt2rTm5ua99trrrrvu6vizjk4++eSUUn19/YwZM15//fWmpqY//OEPl1566Te+8Y2NPlE55x599NF58+Z181mdm/lmtkLPDlTW5vywQtjG/dkfTJ9Sev7559v35Z+0e9FFF3V9qPI31TzuuOM63njbbbf169dvow87f/78jsvyT67rasKECatWreq4bG1tPf3007sux44d+4UvfCFt7AfH33///V0/J7Ndp99ODx6/q/JPct+oKVOmbNiwodPr1kmxWGy/fc2aNZvz+m/qhxq3/0XhySef7PQ4p5566u6779712S+55JKN/nY6/mD6HrywAL3mscce29T3i04pTZo06b333ut0l7a2tuOPP77reMKECeWP4nX6KfBLly7t+gaw/T1m7pvZrfvsQO9whRD+vDPOOOO1114777zzxo4dO3DgwEGDBo0ZM2b69Olz5sw57LDDOi6vuuqquXPnTps2bdSoUX369Bk+fPjEiRN/+tOfzps3b/jw4R2XNTU1d95556233nrkkUcOHDiwrq7u4IMPvuyyy5577rnyp552NXXq1EWLFl1xxRVHHnnk8OHD+/TpM2rUqMMPP/yrX/3q/fff/y//8i9b+PhdXXfddbNmzZo+ffohhxwyfPjwYrG46667fv7zn7/zzjsfeeSRvn37ti9nzpx5xx13HHXUUXV1df37999nn33OOuusl156qfzR4orae++9FyxYcP755x9wwAEDBgwYMmTIpEmTHnrooc2/spf1wgL0mmOPPfaNN9646aabTjjhhD322KNfv36DBw/ef//9Z8yYMXv27Llz5+62226d7lIoFO67774f/OAHn/zkJ/v37z9o0KDx48dfddVVTzzxRPkL/DoZPXr0vHnzTj755FGjRpV/8FJXm/9mthLPDlRaoeQLeYHt0+TJk3/1q19ddNFFM2fOrPZZAHZA3sxCBK4QAgAABCUIAQAAghKEAAAAQQlCAACAoHxTGQAAgKBcIQQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABFWs9gF2HCtXrqyvr6/2KQB2EOPGjRsxYkS1T7HVeB8BsBXtYO8jqksQbjX19fXHHXdctU8BsIOYPXv2lClTqn2Krcb7CICtaAd7H1FdPmUUAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFDFah8AAADouUKhkLUvlUoVOgnbI1cIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAiqWO0DAACwuWpra7P2ra2tFToJ245SqVTtI7Adc4UQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKCK1T4AAMAOYtiwYVn7UqmU+xQ9uEuWmpq8qwX9+vXL2tfW1mbte6C5uTlrv27duqx9U1NT1j6l1NbWlrXP/a9c6T8V7NhcIQQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgqGK1DwBVduCBB2bt58yZk7Xffffds/Y98Oijj2btp0yZUqGTlF166aUVffweOO+887L2I0eOrNBJ2v34xz/O2l9wwQUVOglUTv/+/Su6HzZsWNY+pTR06NCsfV1dXdZ+yJAhWfvc8/TgKQYNGpS1HzhwYNa+X79+Wfva2tqsfUqpUChk7Zubm7P2K1euzNq/+eabWfuU0osvvpi1X758eda+paUlaw8duUIIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIKhitQ8AVTZjxoys/ejRo7P2pVIpa98Df/3Xf521r/SRvve97+XepRdepSy9cJ6/+7u/y9rfdNNNWfuFCxdm7WFz9O/fv6L7kSNHZu333XffrH1Kae+9967oU+y5555Z+9122y1rn1IaOnRo1r5fv35Z+2Ix7y+HuW8wm5ubs/YppdbW1oruP/7446z9q6++mrVPKQ0aNChr/8ILL2Tt33zzzaw9dOQKIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBFat9ANjK9ttvv6z99OnTK3QS6MaaNWuy9m+//XaFTkJYffv2zb1LoVDI2vfp0ydrP3DgwKz9XnvtlbVPKR1++OFZ+4MOOihrP3r06Kx97m85pdTU1JS137BhQ9a+sbExa9/Q0JC1zz1Pyv+DNHz48Kz9gAEDsvYffvhh1j6ltPPOO2ftBw8enPsU0GOuEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQVLHaB4Ct7Oyzz87ajxo1qkIn6TX19fVZ+/feey9rf8IJJ2Tt2Rw/+tGPsvYNDQ0VOglhNTU15d6lf//+lThJb2pra8va5/6v9+6772btP/roo6x9Smnp0qVZ+3feeSdrv2LFiqx9Y2Nj1n7w4MFZ+5TSmDFjsvaf/vSns/a5RyoWs//+3NLSUtF9TU3eNZ7c/xFSSoVCIWtfKpVyn4JqcYUQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKCK1T4AbGXTp0+v9hG21FtvvZW1P/7447P2q1evztofccQRWftnnnkma78NWrVqVe5dzj333Kz9o48+mvsUsHXV1tbm3qW1tTVr39LSUtHHX7duXdY+pbRmzZqs/R//+Mes/cqVK7P2b7/9dtY+pfT6669n7RcvXpy1b2pqytoPGjQoa7/nnntm7VNKo0ePztrn/tmuqcm7QLJ27dqsfUrpgw8+yNrn/tlua2vL2vdAqVSq9FNQLa4QAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAiqWO0DAJ3dcMMNWfsVK1ZU6CRl77//fkUffxv00EMP5d7l7rvvrsRJoHJaW1tz71IqlbL2zc3NWfumpqas/fr167P2KaU333wza79q1aqs/fLly7P2H330UdY+pbRs2bKs/Zo1a3KfoqKKxey/fO67775Z+2HDhmXtGxoasvZ//OMfs/Yp/z3p4sWLc58CeswVQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCKlb7ALCVLVq0KGs/cuTIrH3fvn2z9k899VTWPqU0a9as3Ltkqa2tzdpfcsklFTpJr1mzZk3W/uqrr67QSWC71tbWlrVvbm7O2jc2Nmbt33///ax9SumDDz7I2q9evTprn/tbaGlpydqn/P8KxWLeX/YKhULWvq6uLmt/8MEHZ+1TSgcddFDWvk+fPln79957L2u/ePHirH3K/4MHvckVQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCKlb7ALCVHXPMMVn7E088MWs/dOjQrP2cOXOy9iml999/P3dHjE4AACAASURBVPcuWc4666ys/dlnn12Zg/Sec845J2v/+9//vkIngVDa2tqy9mvXrs3aL1myJGuf8o/U3Nycta+pyftQe21tbdY+pVQoFCr6FH379s3a/8Vf/EXWftKkSVn7lP+ed82aNVn7N954I2u/ePHirH1Kaf369Vn73P/KpVIpaw8duUIIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFDFah8AquyXv/xltY+wpYYNG5a1P/fccyt0kl7z61//Oms/e/bsCp0E6EapVMrar127Nmu/fv36rH0PFAqFrH2xmPc3q5qa7A/N576quYYPH561nzRpUtZ+zJgxWfuU/6q+8847Wfv58+dn7ZcuXZq1Tyk1NDRk7Wtra7P2LS0tWXvoyBVCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIqVvsAwJb60pe+lLU/+OCDK3SSHnv66aez9ieeeGLWfu3atVl7YKtoa2ur6L61tTVrn1IqFApZ+5qavA+dl0qliu57oH///ln7MWPGZO2PPvrorP1OO+2UtU8pLV26NGv/3HPPZe3r6+uz9suWLcvap5TWrVuXtW9pacl9CugxVwgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABFWs9gGAzsaMGZO1v/nmmyt0kp6pqcn+SNM111yTtV+7dm3uUwC9r1QqbdeP34OnaG1trdBJ2vXp0ydrP3r06Kz95MmTs/b77rtv1r4H/9VefvnlrP3DDz+ctV+0aFHWfv369Vl72Ma5QggAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUMVqHwDo7Kijjsral0qlCp2kZ55//vncu/zP//xPJU4CbF+2tbdmKaXW1taKPn6hUMi9y5AhQ7L2hxxySNb+2GOPzdr369cva7948eKsfUrpnnvuydq/8sorWfv169dn7WEH4wohAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEVq30A2MGNHDky9y7nnHNOJU7SayZPnpx7lw0bNlTiJAA7nqFDh2btv/rVr2btd9lll6x97hvwu+66K2ufUnryySez9uvWrct9CojMFUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAgipW+wCwg7v22mtz73LEEUdU4iS95uOPP672EYBtQqlUqvYRtnU77bRT7l3+5m/+Jms/fvz4rH1bW1vW/qWXXsra33vvvVn75N0KVJgrhAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCKlb7ALCDGzlyZLWPsKVuuOGGah8BYPuQ+zb/0EMPzX2KGTNmZO1ravI++r9q1aqs/Y033pi1f/fdd7P2KaXm5ubcuwCbzxVCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIqVvsAsJ0ZOnRo1n7IkCEVOkmPtba2Zu1feOGFCp0EYBs3atSorP1uu+2WtT/ttNOy9imlnXfeOWvf2NiYtX/mmWey9k8//XTWPvd9EFBprhACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEFSx2geA7cz48eMruu8FP/3pT7P2t99+e4VOAtDLamtrs/YDBw7M2h9xxBFZ+3HjxmXtU0qlUilrv3z58qz9z3/+86z9Rx99lLXPPT9Qaa4QAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABBUsdoHgO3MySefXO0jbKlXX3212kcA2IhiMe+vJX369Ml9iv79+2fthwwZkrU/9NBDs/Z9+/bN2qeUGhoasvYvvfRS1n7+/PlZ++bm5qx9qVTK2gOV5gohAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKCK1T4AbGfGjh1b7SN01tbWlrVvamqq0EkAtkRLS0vWvn///rlPUVtbm7Wvq6vLfYosy5cvz71L7tvwRx55JGu/atWqrH2pVMraA9saVwgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACKpY7QPAdubMM8/M2s+ePTv3Kfbbb7+s/a233pq1v+WWW7L2ANumtWvX5t5l3bp1Wftly5Zl7X/1q19l7RctWpS1TymtXr06a//ss89m7Zubm7P2wPbOFUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAgipW+wCwnXnrrbey9gceeGBlDgJAtlKplLV//fXXK7oHqDpXCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIqlAqlap9hh3EypUr6+vrq30KgB3EuHHjRowYUe1TbDXeRwBsRTvY+4jqEoQAAABB+ZRRAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIKhitQ+w41i5cmV9fX21TwGwgxg3btyIESOqfYqtxvsIgK1oB3sfUV2CcKupr68/7rjjqn0KgB3E7Nmzp0yZUu1TbDXeRwBsRTvY+4jq8imjAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAARVrPYBAACCKhQKlX6KUqmUta/0kXLPA1SaK4QAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABFWs9gEAAHpJoVDI2vfp0ydrX1tbm7XPPU8P7tLa2pq1b2lpydq3tbVl7XugVCpV9PF78F9hW1Ppl4gdmyuEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAARVrPYBAAB6Sd++fbP2hUKhovuamuwPzZdKpdy7VPTxK73vBb1wpNw/GNCbXCEEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEFSx2gcAAOglNTWV/VB4oVDI2vfgPKVSKWvf0tJS0cdnc+S+qrl/kHL3PeAPxg7MFUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAgP9n125jLK/vOu7/zsyZu70ZdtldWCjsjQVaNA0oqQWqgq1Snmka0WCMprXxLqlJExVjHxgTtdEYNajExgRbkvaJKTYoJjUFQ62iIqZlk0YkUOgWtsvO7uzM7M6cmTlzzvWgSa+mVy/qZ+TsYff7ej3+/M//N3POMvOePxQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFNUd9wGAS83b3va29JKdO3dG+xMnTkT7l156KdoDF4Xdu3enl0xMZH8Kn5qaivZzc3PRfnZ2Ntq31nq9XrRfWFiI9p1OJ9oPh8ORvv42Lkl/pqRfQrpv+bs2GAyi/TaOBN/gCSEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARXXHfQDgW3W72T/MK664Itrfc8890f79739/tL/++uujfWttdnY22i8sLET7e++9N9o/9thj0R74tjqdTrS/7LLLov309HS0b63t3bs32h8+fHikrz8YDKJ9a+3UqVPRfteuXdH+3Llz0b7f70f7ycnJaN9a27NnT7RPf6acP38+2p8+fTrat9aGw2G039jYiPbpuwDfzBNCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIrqjvsAcJHZv39/tL/99tvTW/z6r/96tH/nO9+Z3uJi9/nPfz7aP/bYYyM6CVy8Op3OqG+xe/fuaL9jx45of+DAgWjfWrv++uuj/Zvf/OZov2vXrmh//vz5aL+NW8zPz0f71dXVaD8cDqP9zMxMtN/GJYPBINovLCxE+16vF+1ba5ubm9G+3+9H+/Sfc/qucWnzhBAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUd1xHwBeyw/+4A+ml/zAD/xAtP/Jn/zJaH/llVdG+4MHD0b7N6DTp09H+62trfQWDz74YLT//Oc/n94C+L+bnZ2N9t1u9mtG+vpXXXVVtG+t3XjjjdH+2muvjfbpfwDPnz8f7Vtr586di/ZLS0vRvtfrRfuZmZlov3PnzmjfWtu1a1e0n56ejvaDwSDaLy8vR/uWv2udTie9BWybJ4QAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFNUd9wGoZe/evdH+oYceSm9x+PDh9JI3mv/4j/+I9o8++mi0P3bsWLR/5JFHov1gMIj2wFjMzMyklwyHw2g/MZH93XnHjh3Rfhv/wT9y5Ei037lzZ7R/8cUXo/3//M//RPvW2nPPPRftl5aWon36Ls/NzUX7bfyM2LdvX7Q/cOBAtE8/qAsLC9G+tTY9PR3tO51OegvYNk8IAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAACiqO+4DUMtb3vKWaH/48OERneQbvvzlL0f7j33sY9H+E5/4RLRvrb388svRfn19Pb0FwMbGRnrJzMxMtJ+eno72+/bti/bpz5TW2t69e6P9yspKtH/++eejffozqLW2sLAQ7dOfERMT2dOCycnJaL+1tRXtW2tzc3PR/oorroj2u3btivavvPJKtG+tnThxItqn78LU1FS07/f70b61NhgM0ku4WHhCCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBR3XEfgFruvvvucR/hW911113R/vnnnx/RSQAupLm5ufSSTqcT7Xfu3BntDx8+HO2vueaaaN9am5qaivanTp2K9sePH4/2J0+ejPattfPnz0f7wWAQ7aenp6N9+i3dvXt3tG+tXXHFFdE+/WCk39LLL7882rf8u9rtjvZX9OFwOOpLtnELxsUTQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFdcd9AGr5rd/6rVHf4p//+Z+j/Ze//OURneSCueGGG6L95OTkiE7yddv4lvZ6vVGcBHgNa2tr6SW7d++O9rt27Yr2V111VbSfm5uL9q214XAY7ZeWlqL9qVOnov3Kykq0b61tbGxE+/RduPzyy6P9/v37o/2VV14Z7VtrR44cifYHDx6M9um7tmPHjmjfWpufn4/2i4uL0b7f70f7bfwmkN6Ci4gnhAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAU1R33AajlxRdfjPbXXXddeot+vx/tB4NBeovIH/zBH6SX3H777dH+1ltvjfaTk5PRPvXFL34xveT555+P9g8//HC0/+QnPxntoYJt/NdvYiL7O/Ls7Gy03717d7SfmpqK9i3/GdHtZr8p7dq1K9rv27cv2rfWZmZmov38/Hy0T9+Fyy+/PNofPXo02rfWjhw5Eu337NkT7ZeXl6P93NxctG/5uzbqn9TwzTwhBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKCo7rgPQC2f+tSnov19992X3uLOO++M9jfccEO0f+CBB6L9u971rmh/CbjppptGfcl73/veaP+xj30s2v/Lv/xLtG+t/dqv/Vq0/+IXvxjt+/1+tIfvqNPpjPqSUe+73fjXmNnZ2Wh/+PDhaH/LLbdE+0OHDkX7bdixY0e0n5+fj/Z79+6N9umP3dbavn37ov3U1FS0n5ubi/bpp6jln9V0PzGRPeMZDAbRvuVH2tzcTG/BuHhCCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBR3XEfgFo+/elPR/v77rsvvUWn04n2jzzySLS/4YYbov02PPfcc9H+93//96P9sWPHon1qYiL+S9P73ve+aH/PPfdE+/3790f7O+64I9q31p566qlo/+CDD0b73/md34n2x48fj/YUNBwOR32Lfr8f7VdWVqJ9r9eL9q21ffv2RfujR49G+x07dkT7M2fORPvW2mAwiPbT09PRfm5ubqT7+fn5aN9a29zcjPbplzw5ORntL4Bt/CQdtfT3Ky4ib7hPGwAAABeGIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUd1xH4Ba/v3f/z3aP/bYY+kt3v3ud0f7G264Ib1FZH19Pb3kZ37mZ6L9U089ld7ijeY///M/o/2HPvShaH/vvfdG+5/92Z+N9q21H/7hH47273//+0f6+um+tfaVr3wlvYRqNjc3o/3y8nK0Tz+EL7zwQrRvrU1MZH8K37VrV7Tfv39/tN+5c2e0b61tbGxE+8FgEO37/X60X1tbi/bpp6jl79rs7Gy07/V60T59C7Zha2sr2qfv8nA4jPbbu4SLhSeEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIrqjvsA8Foefvjh9JJ3v/vdozjJtv35n/95eslTTz01ipNcSjY2NqL9xz/+8Wj/iU98Itq31n7+538+2j/wwAPR/ujRo9H+t3/7t6N9a+0Xf/EXo32/309vwcWu1+tF+7Nnz0b7Z599Ntpv40P4yiuvRPtrrrkm2s/NzUX7bXwJa2tr0f78+fMj3aefivn5+WjfWut0OiO9xcrKSrRPv+TW2urqarRP3+XBYBDth8NhtG/+m39J84QQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKK64z4AvJaHHnooveQv/uIvRnGSbfvbv/3bcR+BWL/fTy/56Ec/Gu1vuOGGaP+hD30o2r/vfe+L9q21Rx99NNo//PDD6S242G1tbUX75eXlEZ3k686dO5de8sILL0T7vXv3Rvtdu3ZF+243/k1sfX092m9sbET7tbW1aD8YDKL9oUOHon1r7Zprron26ZeQ7peWlqJ9a21xcTHap+9y+i4Mh8Nov71LuFh4QggAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUd1xHwBey+bmZnrJU089Fe3f/va3p7eI3Hrrrekl//qv/zqKk/CG8pu/+ZvR/rbbbov22/jg3X333dH+4YcfTm/BxW44HEb7jY2NaL+wsBDtz507F+1ba51OJ9qnX3L6+pOTk9G+5Ueampoa6X52djba79y5M9q31nq9XrQ/f/58tF9aWor2i4uL0b7lX8JgMIj26adiG79fcQnzhBAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAorrjPgC8lo2NjfSSD3zgA9H+ySefjPY7duyI9jt37oz2FLG5uRntn3322Wh/6623RvvW2pEjR9JL4LVtbW2N9PVXV1fTSzqdTrQfDofpLUb9+umXMDU1NdJ9ahs/2c+fPx/tT58+He2Xl5ej/TY+eOm/hfRdTn+mwDfzhBAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUd1xHwBeZ8eOHYv2Dz30ULT/0R/90Wi/uLgY7eHb+uxnPxvtf+7nfi69xY/8yI+kl1DNcDiM9p1OZ0Qn+brBYJBeMuojpd+iC2BzczPap1/CxsZGtO/1etG+tXbmzJlov2fPnmjf7/ej/TakH7zJyckRnQT+vzwhBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKCo7rgPAGP2K7/yK+M+Anxnb37zm8d9BIgNh8No3+l0RnSSb3gDHimVfgnpvt/vR/u1tbVof/r06WjfWltYWIj2l19+ebTf2tqK9jt27Ij2rbXZ2dlon35X4f/CE0IAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAiuqO+wAAfGeHDh2K9seOHUtv8eyzz6aXwOtrOBxG+06nM6KTfEN6pEtA+iVvbm5G+5WVlWjfWjt58mS0P3DgQLTv9/vRfnZ2Ntq3/LM6GAyi/cRE9ownfX0ubZ4QjvHzkQAAIABJREFUAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFBUd9wHAOA7+8AHPjDuI8AbznA4HPcRaJ1OJ9qvrKykt3jxxRejfbeb/X47OTkZ7U+cOBHtW2urq6vRfmtrK9oPBoNoD9/ME0IAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARXXHfQAAAN4ohsNhtO/3+9F+MBhE+9baSy+9FO3Pnj0b7aempqL96upqtN/GJefPn09vAdvmCSEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARXXHfQAAAC5Ww+FwpPvWWq/Xi/b9fj/adzqdkb5+a21jYyO9BC4YTwgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKKo77gMAAHCxGg6H0X5rayu9RXrJ5uZmeguozBNCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIrqDIfDcZ/hEnH69Omnn3563KcAuETccsst+/btG/cpXjd+RgC8ji6xnxHjJQgBAACK8r+MAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEV1x32AS8fp06effvrpcZ8C4BJxyy237Nu3b9yneN34GQHwOrrEfkaMlyB83Tz99NPvec97xn0KgEvEZz7zmbvuumvcp3jd+BkB8Dq6xH5GjJf/ZRQAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKK64z4AAAD8/+p0OiPdXwDD4XCke/i/8IQQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKK64z4AAMB2dDqd9JLhcDiKk3zD5ORktN/GlzBq6ZFGvb8Atra2xn2EbzUYDKJ9+sEe9T8ELi6eEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoqjvuAwAAl6ZOpzPS/eTkZLTfxiUTE9mfztPX38aX0O1mv7yN+kjpuzYcDqN9a21rayva9/v9aJ8eaTAYRPvW2sbGRrRPv4T0SOm3tG3rjeNi4QkhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEV1x30AAODi0Ol0ov3k5GS0n56ejvYzMzPRvrU2NzcX7S+77LJov3///mh/1VVXRfvW2hVXXBHt0y9hYiJ7WtDr9aL92bNno31r7eTJk9H+zJkz0X5lZSXar62tRfvW2urq6khvsbGxEe3X19ejfWut3+9H+8FgkN6CcfGEEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiuuM+AIzZkSNHov299947moP8v+65555of/PNN4/oJF/3uc99Ltrfdddd6S02NjbSS4ALb2Ii+zvyzMxMtN+xY0e037NnT7RvrR08eDDaHzp0KNpfd9110f7o0aPRvrV24MCBaD83NxftJycno/1gMIj2S0tL0b619pWvfCXaP//889H+5ZdfjvavvvpqtG+tLS4uRvv031qq3++nl6RvdLpnjDwhBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKCo7rgPAGP20EMPRft3vvOdIzrJtg2Hw5G+/g/90A9F+4WFhfQWH/nIR6L9yspKeouL3RNPPBHtjx07NqKTcMnodDrpJRMT2d+Rp6amov2OHTui/Z49e6J9a+2KK66I9ocOHYr2V199dbRPv+TWWq/Xi/arq6vRfnJyMtrPzMxE+2188Hbt2hXtL7vssmi/uLgY7bvd+Pfn9Cf11tbWG2rf8i8hfaNH/csMr8ETQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFdcd9AGp561vfGu3379+f3uKOO+6I9t///d8f7RcWFqL9P/zDP0T7bXjHO94R7d/ylreM6CRft3PnzvSS3/3d3x3FSS4lv/qrvxrtjx07NqKTwP9ep9MZ6X5iIv679tTUVHpJZGVlJdovLS2ltzh79my0T4/U7Wa/HO7duzfaz87ORvvWWq/Xi/anTp2K9mfOnIn2i4uL0b61try8HO3X1tai/cbGRrTf2tqK9q214XA40j1j5AkhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEV1x30Aavn7v//7aL9v3770FrOzs9H+7Nmz0f6nf/qno/3jjz8e7bfhyiuvjPY/9VM/Fe1/4Rd+IdrfeOON0f7S8KUvfSnanzx5Mtp/8pOfjPbwHQ2Hw/SSzc3NaN/v90f6+ul+G5ecO3dupK+/srIS7Vtrr7zySrRfXl6O9umP0YMHD0b7+fn5aN9aW19fj/avvvpqtE//g3z69Olo3/IPUq/Xi/bpv7VOpxPt27b+i8HFwhNCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIrqjvsA1PI3f/M30f43fuM3RnSSb3jmmWei/eOPPz6ik2zbyZMno/39998f7dN37Zprron2rbX3vOc90f748ePR/tZbb432Dz74YLRvrX31q1+N9pubm9F+cXEx2sMbQa/Xi/Zzc3PRfn19Pdq31lZXV6P92tpatE//aZ89ezbat9ZWVlaifXqk2dnZaD8cDqP98vJytG+tnTt3LtqfPn062qfvQvoWtPxdGAwG0b7T6UT7fr8f7VtrExMeI12yvLUAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFNUd9wGo5eWXXx73Eb7V/Pz8SPdXXnlltG+tLS4uRvuFhYX0FpETJ06MdN9ae+qpp9JLIh//+MdH+vrAt7W1tRXtNzY2on2/34/2LT9Seovp6elov2vXrmjfWpuYyP6a3+l0ov3OnTujfbeb/TJ59uzZaN9aW15ejvZLS0vR/ty5c9H+AnzwUuvr6yN9/dbaYDAY9S0YF08IAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABTVHfcBYMze/va3R/tHH3002h89ejTat9ZOnDgR7Y8fP57eIvK5z30u2t9///3pLQaDQXoJ8MY3HA6jfb/fj/abm5vRvrW2tbUV7TudTrSfn58f6b61Njs7G+273eyXvcnJyWi/tLQU7dO3oLV26tSpaN/r9aJ9+sFLPxWttY2NjWjvxyIXkieEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABTVHfcBqOWzn/1stD979mx6iz179qSXRG6//fZof+bMmfQWb33rW6P9ddddF+3n5+ej/Y//+I9H+62trWjfWnvggQdGfQvgjW9zczPar6+vp7fo9XrRfjgcRvtdu3ZF+8OHD0f71tqBAwei/ezsbLRP/wO7sLAQ7aempqJ9a21tbS3ap0daXl6O9oPBINpfGjqdTrRP/+0wRp4QAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFBUd9wHoJb//u//jvZra2vpLfbs2ZNeEvmzP/uzaH///fent/iu7/quaH/mzJlof9ttt0X7P/qjP4r2f/qnfxrtW2ubm5vR/i//8i/TWwBvfFtbW9H+3Llz6S3Onj070n16pI2NjWjf8v9gdruj/WVvfn4+2h89ejS9RfolDAaDaH/+/Plo3+v1on1rbWpqKtqvr6+ntxi14XA47iMwKp4QAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFBUd9wHgIvMI488Eu1feOGF9BbbuCTyX//1XyPdP/7449G+tfbLv/zL0f5Tn/pUtD916lS0B14XExOj/btzr9dLLzl58mS0n5qaGum+3+9H+9bayy+/nF4S6XazXw53794d7Q8cOBDtW2vXXXddtE/fhTNnzkT7tbW1aL8Nw+Ew2m9sbIz09bm0eUIIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFHdcR8AXsuHP/zh9JK//uu/jvaPPfZYtH/88cej/SXgySefjPbvete70lt85jOfifYf/ehHo/173/veaA98W51O5w31+sPhML1Fr9eL9qdOnYr2ExPZn9oXFxejfcu/S6urq9E+/a4ePHgw2r/tbW+L9q21m266KdofOnRopK9/4sSJaN9a29zcjPZbW1vRvt/vj/T1ubR5QggAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgqO64DwCvZTgcjvqSO+64Y6T7J554ItpfAp588sn0kuXl5Wi/jQ8G8C06nc6obzExkf3deXJyckQn+Yb0q97c3Iz2S0tL0b7X60X71trGxka0X11djfZbW1vR/uTJk9E+PU9r7fLLL4/2119/fbS/9tpro/3VV18d7Vtri4uL0X59fT3ad7vZr/SDwSDat/yfc/pBYow8IQQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgqO64DwBj9uSTT0b7f/u3fxvRSfjfu/vuu6P9zTffHO2/8IUvRHsootPpRPupqamR7ofDYbRvrXW72W8+ExPZn84Hg0G0X19fj/bbuGR1dTXab21tjXR/6tSpaN9aW1hYiPZHjhyJ9nNzc9F+79690b61NjMzE+0nJyfTW4z69dPPNhcRTwgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKKo77gPAazl+/Hh6ye/93u9F+7/6q7+K9uvr69Ge/40/+ZM/ifZ/+Id/GO1vuummaP+FL3wh2sPFqNPppJdMTk5G++np6Wg/MzMT7S/Al5AeaWpqKtpvw9raWrQfDocjOsnXTUxkTxdmZ2fTW6TvQr/fj/aDwSDab0P6wUs/2+m7fAG+ZC4inhACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUFR33AeA1/JP//RPF+ASxu7UqVMjff0PfvCD0f7v/u7v0lucOXMmvQReX51OZ6T71trERPZ35HQ/NTUV7WdmZqL9Ni6ZnZ2N9tPT09F+a2sr2rf8jUu/q+nr79u3L9rfeOON0b61dvXVV0f7ycnJaL+0tBTtV1ZWon1rrd/vj3Q/HA6jPXwzTwgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFNUd9wEA2rXXXjvS1//e7/3eaH/zzTent3j88cfTS+D1NRwOx32Eb5UeaXJyMtrPzs5G+9ba7t27R7rfs2dPtJ+YiP80n35XNzY2ov309HS0P3ToULT/nu/5nmjfWrvqqqui/dmzZ6P9yZMnR/r6rbXV1dVov76+Hu0Hg0G034YLcAvGxRNCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIrqjvsAcJGZm5uL9h/+8IfTW3z3d393eknkH//xH6P9E088MaKTfMMHP/jBkb7+M888E+2PHTs2opPAG8dwOEwvGQwG0X5zczPab21tRfttfAmzs7PR/sorr4z2b3rTm6L9wYMHo31rbc+ePdE+/ZJ379490n23G//yeebMmWh//PjxaP/8889H+5MnT0b71trKykq07/f76S1g2zwhBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKCo7rgPABeZnTt3RvvbbrstvcWdd96ZXhL5sR/7sWg/MZH95WgwGET7C+DUqVMj3cPFaDgcppek/7o3Nzejfa/Xi/ZTU1PRvrV27ty5aL+6uhrt0+/qnj17on1r7dprr432u3fvTm8RSb9FL730UnqL5557Lto/88wz0f5LX/pStP/a174W7Vv+Xer3++ktYNs8IQQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgqO64DwAXmYWFhWj/Ez/xE+ktfumXfina33fffdF+eno62s/NzUX7C6Df70f7T3/60yM6CVy8hsNheslgMIj26T/Vc+fOjfT1W2sbGxvRvtfrRfuzZ89G+xMnTkT71trBgwej/e7du6P95uZmtE+/hJMnT0b71tpXv/rVaP/yyy9H+1dffTXawyXGE0IAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARXXHfQC4xC0uLqaXfOQjH4n2f/zHfxztv+/7vi/a33nnndH+He94R7Rvrb3pTW+K9um36OGHH472wLc1HA6j/dbWVrQfDAbRfn19Pdq31paWlqL91772/7RrbyFy3/X/xz+zO7tJdrt24zZpTkTTpBqpihC06I1eWW9UECqCinciCCqCBWkRiyLeiRak9NY7QTygYPVCkIoXukgSjTFtShrZmh62aZoD2cPM93chBP/lT+G9zWQ2+3o8rl8z389kdnfmOZMLpf2ZM2dK+16vV9pv4CbVf9XV1dXSHrjd+YYQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABC9cd9AODNWllZKe3//Oc/j3QPsDFd1436Er1er7QfDAYj3QOMnW8IAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAgVH/cBwAAuEW6rhv3EQA2F98QAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAqF7XdeM+wxaxvLy8uLg47lMAbBHHjh1bWFgY9yluGq8RADfRFnuNGC9BCAAAEMp/GQUAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUP1xH2DrWF5eXlxcHPcpALaIY8eOLSwsjPsUN43XCICbaIu9RoyXILxpFhcXH3jggXGfAmCLePLJJz/60Y+O+xQ3zeLi4sc+9rFxnwJgi/jtb3+7lV4jxst/GQUAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUP1xHwDYar7+9a9Xb/L973+/tP/2t79d2n/ve98r7QH+q9frjfT+JybKH81PTk6W9lNTU6X9jh07Svu3vOUtpf3c3Fxp3+oPYXV1tbS/dOlSaf/KK6+U9q21q1evlvbD4bB6Cdgw3xACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAECo/rgPAGx2H/rQh0r7Bx98sHqJfr/2t+jgwYPVSwC01nq93kj3ExO1j9qr+9ba9PR0aT87O1va79u3r7Q/cuRIab9///7SvrU2MzNT2l+7dq20f+6550r7M2fOlPattaWlpdL+8uXLpf1gMCjt4X/5hhAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFD9cR+HE2h0AAAXD0lEQVQA2Ow+/OEPl/b333//iE5yw8mTJ0d9CWDz6/V6o77JBi5RMjFR/mh+enq6tL/rrrtK+8OHD5f27373u0v73bt3l/attZWVldL+3//+d2l/9erV0v769eulfas/0dVneW1trbQfDAalfWut67rqTbhd+IYQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABC9cd9AOBW+9znPlfaP/zwwyM6yQ2DwaC0P3/+/IhOAtxEvV5vU+03oHqJfr/2zmrbtm2lfWttfn6+tD9w4EBpf++995b2u3fvLu2r/0SttQsXLpT2zzzzTGn/3HPPlfaXLl0q7Vtrq6urpX3XdaV99Qd1cnKytG/1V+rqQ2CMfEMIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAACh+uM+AHCrbd++vbSfmZkZ0UluGAwGpf3S0tKITgK8gV6vV9pPTGy6z52rR+r3a++UpqamSvs77rijtG+t7d27t7Q/fPhwaX/33XeX9sPhsLQ/e/Zsad9aO3HiRGl/5syZ0v7ixYul/crKSmnfWltfXy/tu67bVPtW/92pvrIzRpvuLzUAAAC3hiAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABC9cd9AODN+tKXvlTaf+c73xnRSf7r+PHj1Zt88pOfLO3Pnz9fvQTw5vV6vU11/xs4z8RE7aPw6iW2bdtW2i8sLJT2rbUjR46U9kePHi3t5+bmSvv//Oc/pf2//vWv0r61dvbs2dL+4sWLpf3q6mpp33VdaX8LVH9Qh8PhqC/BbcQ3hAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhOqP+wDA6913332l/WOPPVbaT05OlvbD4bC0/+53v1vat9bOnz9fvQlw63VdV9r3er0RnWTD91+9Sb9fe6e0Y8eO0v5tb3tbad9ae//731/a33PPPaX9pUuXSvvnn3++tF9aWirtW2uXL18u7QeDQWlffZnbgOorb/UhVFV/l9nafEMIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAACh+uM+AGxxk5OT1Zt861vfGvUlSn7yk5+U9j/72c9GdBJgvIbDYWnf6/VGut+AiYnaR+H9fu2d0s6dO0v7Y8eOlfattfvuu6+0r75GnDp1qrR/+umnS/vl5eXSvrW2urpa2nddV73Eprr/DVyiut/AO4fqrz+3Ed8QAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAqP64DwBb3Lve9a7qTR588MFRnOSGc+fOlfY///nPR3MQYIvrum6k+16vV9q31iYmah+FT09Pl/b33ntvaf++972vtG+t3XnnnaX92bNnS/u//vWvpX31NeXy5culfWttOByW9tUfjMnJydK+ep6N3aSk+pDX1taql6j+7nAb8dQCAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAECo/rgPALeZqamp0v6Xv/zliE5yQ6/XK+0///nPl/Z/+tOfSnuAjan+NduAiYnaR+Hz8/Ol/Xve857S/u677y7tW2tXr14t7Y8fP17aP/3006V99TyTk5OlfWut3x/t+9Wu60r7lZWV6iU2cJOStbW1kd5/a204HI76EoyLbwgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAjVH/cB4Dbz6KOPlvaHDh0a0UluOHHiRGl/8uTJEZ0E4M3o9Xql/eTkZPUSU1NTpf2+fftK+3e84x2l/fT0dGnfWjt37lxp/49//KO0v3LlSmk/MzNT2s/Ozpb2rf6vVH2Wh8Nhab+8vFzat9YuXLhQ2g8Gg9K++rvTdV1pz9bmG0IAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAjVH/cB4Dbzla98ZdxHeL0f/OAHpf1rr702opNs2BNPPFHav/e97y3tH3/88dL+97//fWnfWltaWqreBHidiYna59T9fvltzOzsbGl/5MiR0n5hYaG0v3LlSmnfWjt16lRpv7y8XNpXH8KePXtK+127dpX2rbW5ubnSftu2baV99Vl45plnSvvW2rVr10r7ixcvlvZra2ul/WAwKO1ba71er7Tvuq56CcbFN4QAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABCqP+4DwE129OjR0v7hhx8u7WdmZkr7DXjyySdL+7/85S+l/Wc+85nSft++faX9l7/85dK+tXbo0KHqTUo+8IEPlPa//vWvq5f4xCc+Ub0J3HZ6vd5I739iovY59dTUVPUSu3btKu3vueee0r7fr72zeuGFF0r71tqFCxdK+507d5b2e/bsKe2rf8Dn5+dL+9ba9PR0aV99FlZWVkZ6/63+RFePtLa2VtoPh8PSnq3NN4QAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABCqP+4DwE12//33l/af/exnR3SS//rGN75RvclPf/rT0v7RRx8t7b/whS+U9r1er7TfAmZnZ8d9BNgKJiZqnzv3+7W3JRv4Vd27d29pv2fPntK++pCvXr1a2rf6o37nO99Z2h88eLC0r55nbW2ttG+tra6ulvY7duwo7efm5kr7AwcOlPattfn5+dL+xRdfLO2rP0gbeBa6rqvehNuFbwgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACBUf9wHgDeyY8eO6k0eeuihUZzkhuFwWNofOHCgeonf/e53pf3Ro0dL+67rSvuqy5cvV28yNzdX2vd6vdK++pBfe+210h74/6r+qk5PT5f21T8drbXdu3eP9BL9fu2d1czMTGnfWnv7299e2ldfSavPwssvv1zav/TSS6V9a+2OO+4o7e+8887SfnZ2trTfvn17ad9aGwwGpf36+nppPzFR+46n+rvZRv/mgTHyDSEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKH64z4AvJGDBw9Wb3LXXXeN4iQ3TEzUPkb56le/OqKT3NB13Ujv//Tp06X9F7/4xeolvva1r5X2n/rUp6qXKPnb3/420vuH21Sv1xvpfmpqqrSfmZkp7Vtrs7Ozpf3k5GRp3+/X3lnNz8+X9q219fX10v769eul/UsvvVTaLy0tlfbV87TWdu/eXdpX/1WrP6gvv/xyad9aW15eLu1XV1dL++pPRfUhs7X5hhAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEL1x30AeCMPPfRQ9Sa7du0axUk2s5MnT5b2P/zhD0v7ubm50v6RRx4p7VtrR44cqd6k5A9/+ENp/+Mf/3hEJ4EoExOb7nPnwWBQ2q+srJT21Ydc/QPbWltbWyvtqw9hOByW9jt37izt3/rWt5b2rbXDhw+X9jMzM6X9uXPnSvvjx4+X9q21ixcvlvbVZ636g13dt9a6rqvehNvFpvtLDQAAwK0hCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFD9cR8A3sihQ4fGfYQxWF5eLu0fe+yx0v7jH/94af/AAw+U9tu3by/tN+CPf/xjaf/pT3+6tK8+BRCi1+uV9l3XlfZra2ul/ZUrV0r71tqrr7460n31n2hubq60b/W/sdu2bSvtd+3aVdpPTk6W9vPz86V9q/8gnT9/vrR/6qmnSvtTp06V9q3+szoYDEa6Hw6HpX2r/2xzG/ENIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhOqP+wDA6y0sLJT2TzzxxIhOcss8/vjjpf03v/nN0v7SpUulPXBTDIfD0v769eul/fLycmnfWnv22WdL+/3795f2e/fuLe2PHDlS2rfWZmZmSvvZ2dnSfn19vbQfDAal/ZUrV0r71trp06dL+6eeeqq0X1xcLO1ffPHF0r61trKyUtpXn4Xq7xr8L98QAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQqj/uA8Ab+dGPflS9yUc+8pERHCTa3//+99L+N7/5TfUSjzzySGk/GAyqlwDevK7rSvvhcFjar66ulvbr6+ulfWvt9OnTpf3Kykppf/Xq1dL+gx/8YGnfWtu/f39pv3379tL++vXrpf3S0lJpf+LEidJ+Azd59tlnS/tXXnmltK/+E7X6y1Z1X/3dhP/lG0IAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAjVH/cB4I384he/qN5kYsLHHAAj0XVdad/r9UZ6/+vr66V9a+3y5cul/YkTJ0r7f/7zn6X9r371q9K+tbZnz57SfmZmprSvPmvXrl0r7V999dXSvtWfteqR1tbWSvvhcFjat9YGg0FpX/1dqKo+y2xt3joDAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAECo/rgPAABsTV3Xbar9LbjEcDgs7Z9//vnSvrX2wgsvlPYTE7VP//v92pvD6v3fgmdtMBhsqvtv9R+M6h7eDN8QAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAqP64DwAAbE1d15X2w+FwpPe/Ab1eb6T3v4GHMBgMRnqJ6n5ycrK0vwWqD2F1dXWk9w+bnG8IAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAI1R/3AQAAWmut67otcIlRqz6E6n59fb20B253viEEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQ/XEfAADgFun1euM+wv+j67otcIk0ExPlL1Q8C2xmviEEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQ/XEfAAAgVK/XG/cRKOu6btxHgJvJN4QAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABCq13XduM+wRSwvLy8uLo77FABbxLFjxxYWFsZ9ipvGawTATbTFXiPGSxACAACE8l9GAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACPV/Me3teBQFAPQAAAAASUVORK5CYII=", + "text/plain": [ + "Scene (1200px, 2420px):\n", + " 159 Plots:\n", + " ├ MakieCore.Combined{Makie.poly, Tuple{Vector{Vector{GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Combined{Makie.poly, Tuple{Vector{Vector{GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Combined{Makie.poly, Tuple{Vector{Vector{GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Combined{Makie.poly, Tuple{Vector{Vector{GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Combined{Makie.poly, Tuple{Vector{Vector{GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Combined{Makie.poly, Tuple{Vector{Vector{GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Combined{Makie.poly, Tuple{Vector{Vector{GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Combined{Makie.poly, Tuple{Vector{Vector{GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " ├ MakieCore.Text{Tuple{String}}\n", + " └ MakieCore.Text{Tuple{String}}\n", + " 8 Child Scenes:\n", + " ├ Scene (500px, 500px)\n", + " ├ Scene (500px, 500px)\n", + " ├ Scene (500px, 500px)\n", + " ├ Scene (500px, 500px)\n", + " ├ Scene (500px, 500px)\n", + " ├ Scene (500px, 500px)\n", + " ├ Scene (500px, 500px)\n", + " └ Scene (500px, 500px)" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "xs = makebatch(method, data, rand(1:nobs(data), 4)) |> gpu\n", + "ypreds, _ = model(xs) |> cpu\n", + "showoutputbatch(method, cpu(xs), ypreds)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Julia 1.7.0-rc3", + "language": "julia", + "name": "julia-1.7" + }, + "language_info": { + "file_extension": ".jl", + "mimetype": "application/julia", + "name": "julia", + "version": "1.7.0-rc3" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 922ec9e6579602816de25e837764c725d3f1fe5e Mon Sep 17 00:00:00 2001 From: lorenzoh Date: Mon, 20 Dec 2021 15:31:04 +0100 Subject: [PATCH 03/23] Comment VAE notebook --- notebooks/assets/vae.png | Bin 0 -> 25411 bytes notebooks/vae.ipynb | 307 +++++++++++++++++++++++++++++++++------ 2 files changed, 260 insertions(+), 47 deletions(-) create mode 100644 notebooks/assets/vae.png diff --git a/notebooks/assets/vae.png b/notebooks/assets/vae.png new file mode 100644 index 0000000000000000000000000000000000000000..e66e2115e257620c0da32ada7cb7c791c4ffac05 GIT binary patch literal 25411 zcmbrmbySpH+dfP)^pJwoFr4K;Z@?7P@VNJw~!3Nl(qNT}}MdJlpE{@?r< z*8y+HZd&qENae#+o8UvNwZ5W_sw&cba1BAiK*m5qy)A+S$|Do~bBzqH?;@f9xsQbO z5SjAdYZGLa|CSH8Lqhwfj173d{j&lu@bo|LsM*N>tuY(rKc!LKvr+$ZjkI_BSX99# zIe5czRxofwLLxA~eIXZ#7uq2qq2So*=)3Eyst8#+IUvlfoGh#n-VV;UPa%nT3xP`q zYj<;)w}ZW-n~=9C{7(rXaD9859S-|b#Qmu#TwhfKChg>E4dX*_AUNP+*f1DO#MR11 zNJ~cUpU1%`QTStbcV{7Xb}uh4gcmo$$<>yfQ&3Qlor8;=i|YX>@xaZ;(cRqpfukG! z-%9>dkBqgOrK_E@yPcCG?6zKW3nveEQ8@gzp@0AW_S4m4!zjO2FE|?rKY!UW9=k+>+b5}KYIkJi%Uwfw%+@0seZg>Z@H?@jE6!|q$D5^WI7mHNC31K9YuIG z7n(?S`{BRe)WTt|@Av;(415E#g)n3qIZjnBl{fC)-U{Wo9Sk~|(%a6uqHSW{G z^IumN$8KR&*hK#}upEt4YIc6G6i(y&c%neHM7QK&%k%4C0Ss0X(W@|N293XcadWZt zwPa3KSRz-@d9k%E7_-JI(8brwZ21xlv zWzKiIEZOvVeKL1nAr@mY`{(oJsverBTC>6ULL0TSj@w_}PrMf1>n*G_YYR#?@!kA5 zY2pXQ)AIYqZ-3?S!e>!mw+Nes&(FMO?3nk<)k6Ox^}s2*gAM|5l?(wwEXEAi;S52O zYgWDT!Tgb(7_l0M+1l@+;wmW|OY7-Q>x2WiZm{4pGYj^`f=ACQ*R2-Y3pRkV2m6d@zq%AyZnwjb6?;8D^jQn zS`xddnD5Dko{3Lr%l2f&*^}8)18yc?7W$ciySz`ly#zxo@TrAP3ep{y$qff(qsVrf zj)ym=iM$yjSC7WzX{ouaAelo_Xl^g5J*R5*CVFUJ4EFQ{*&MC*iC>>+A4mtL?A45j zRjQEj%r$u6Xg@k|p~gY}y9%Kw=+cWn!0Z4Km_GZdyMFRsyU;g_d4t36bjL^kK4*RS zd)@u{W@gninc*H9znw4I>UhTrV?Pw(^&%Im3BNB!UW|W)FpWhf7spJN8g%@K7^iYs z=U?z&5sfluCn26t=}h(5`f|9prStF7FM^__d`_)P{hZo4QMiwRcX#m`@0}G!h2ba1 z_fH_niv~1;*C!LSz8g7OqK2iZK-30ND(QT9k@RmLl|A~{|Lr5&GyV5d2vKlmKHp3ZM=~mU70-8XQJ6Grevh z8E*Q0^J_o3uvYfio~Y0Iczr1OfuWq)0NAfw%+!%%7*TX5>z3Woiu^NE9+Sowo@436 zs@!>Fa%x-IQvcny&{k`UovA9D!K5*3ly{5?TvAFY5^7M(+;GiSgxIg0nhX;1GpTq5 zgx9PCzZAZC5l+{j!|y8psuX>;`1PUmAEd7vr;#zK%%bwYl^Oc|@1A%=5g3YdCr!xZ zd!5T>n~C!(uZP9&n`<|k9mW?;xr2{&c7ox_7ws?l;u+b}&qo)0eq^3V7<7K=M8?bQ z(?k5I_Wv2!Px?nK1m?;uHtv6Ra~Zx}6%}QVOQtc<)gm*htp}3k=cuw?8u^-mSHS8G ziA|(GP142Xc127%eAz>nl2|pc?f>vyOd&Z>tqg@p%}fnM?X?zXMlyw|b>hbUMM$g?x6<@tqVfi5%M7tS7s!fgP!ld;l*<|%6 zNcT?eaYl<>ZqV0AW^q|#6TS}Og_&r@(1;FVq(XKnyoj*ZC#(KhN`ZCst%C{~ZmaAn zuLq+|D#a}@IJu=*QmG;zkcXyfu8#j&l|hI~+&^;vTC8iRyPhXU_r<8$_oE)EOp0OG zbvSj{E2do4|4})ecahN&rZi?mB3j3v;#3V0s?Uh@CNa*>y%YugJGcY-7uk8QN}pLR zqn99B*08}EYdyurpgp(jAc-*!Ng?sCxd^n7L~=sYURZh+VE%5Bt%*?OSq~y=XJ}*Y zRqy-Wcawy-Vv7@IM6k995g~q1^4TA3^EQHHU5dqcfIYBgMygEj;6Y7wE@p%%$~CiF7%m5<-EmR|Q=H$75&-(f05tJ@BxSt}i55i{k_ z%6M&#qbK#T*8`&#_c8J{=Vfuj?HYj{LKa^sfdxXXA~#W}es<7G`BD9?Si$+;R8^iA z|2MFPP0k|0)*|WHZ1C8I@C}Tl>#pdTK3n5`ua+r9=9&cA#k9Gg+d3Gw|xC zljWKb(uO)4C7ieZM&Pn+z=ie92fG49x`>bKyLxa_ZXS?R7+A>vExYN(F(o8)k-IX~qHj?rpyFYl=J~V2*%Pm=FOdeQYLSZ8Tn?AAh~|;^ z@m}Q({kunT?D9b0Wk>xmSg(02hvP2k~Jxn&~W` zD3=TVp@VD8y^|l`pw#bfra-(hpJE=VLb3s4a=z=M2KUV%bR4s!RR*%e6zkGo#))@O zLgfQIN~bmGtQBJNzBhP0T}KgMY`Ztzly~rTg3>ZDpx2tM;og3tRW$hz@F|49 zca`|(p;ar(Izf3UOMD;3cdz^Fr0??Cll!$;)D1ZY3viS9wRC?nB{Y0~JjCzzWB45w zOC4CNbCu>D(L-%3;d5X4oaP7TjJ@VaJ1{ykDBQOGc%A_tSvrI>h~!7`DEYhtvgxV> zehB#HqMhCD7Ar{!Vs$d-_~%h$s~b8f5ft1uQ@Ke<^)^F83MWSQU6K{#Y<#5c^Q<1Z z{rG7ZXauieP!MyHw8u(x8qZhk=I7GQ9_Mn69O?jGF~T*KkOfR?q%4w~102&FxhqzU zjgA0Q4wr9+Y}!&Pz_QMR%l9hLM6tb`X8Sp zM!5}FP5Qr~voAp|TxD+{ytNk+cAzn~2NsGWBX!3mI_5%0 z!$)^qnC$kO&yqg5`h2GBg<~W`<*@^eh~DtzJ@UW4{HcI&ATaf>>@85t6{*}dpo4m$ zktc!uWXbS`;f)%bcJKoWeWqbSX`YN~`52nJ?}8UM>M|e^MoGl2t75|nAw(G`$t_pt zTb5zMdGaycg=iY&T@eL}a0I=O5oHU<_ckr=`q;$xYbpd9p2UVhf}AfkJ#E>rP9xPa zHt4?d1`qTnvf2*5j{=&AgcD_o%^cyx`oi!H2Bq?%VG}<^LRF!aW%uF(YMekLI+$xJ zD+94HKK+*$`9$j}`z^o4O02qhqta0k(T?=mSfmfc)@_D@Cw+41aOS)p3$2dhB2?`S zh};*W`D5agaS?_epD<>6KTb*1gmls%8$EE?>Dr&Qw+vFfbtYalKl^oR&44vOUIpzH zM7eBt#kb4Fu^NP*Q)~j?zWaP6Y1p#oHE(x^cMs=g^#DTp%7&hrkY{X(>F&qUSNy>P z9INr(o5PDl{OY&}#`;0*GDqydDSC+vfn3eIg+@)@Rg*N!ad_k$tt9f~=6~F{qj0b% zZz@%N`=YIXyd!GkayYj{llfp%$AN$IAQ1#{PdaGn%?6SlB)op~=$H9wrQE63^6UDS zw$YX`d8A9ZY$*_KY#MQv(FyDcbl|bSL`JS%v=CsddiZGr+V(K#qZzn%*ULDDaS0*F zQQ*nw!9w-y7VaIP8SazDUPI~pG023GOfM;CWy9j)w26?a(gjiBM5-kGTAL^kjO3@^ zN;y$caS<+~_xcuzP_SJ-9c&g5e^TtCDzhqn`~B_PM<#5L%Hht9h@5!3FVU6352RI> zqF|Fpxpl}<(4bw4O$@=x{>4y?Mo4l5LWNl^VDM{+9=+_hy@SZ>mtPL_>02Ad>XnHI z+n?f;`<<^b`8kTrBMnKe3RB%cWW z>DlSDZAs9N%8mmS=620o4Q^MvcxC!zhnWw35drUqRmr2(s8CUR8BfzaL`uts!A?^mTT<0zFo`B`Y`}gw zgV_oRm(R54P)cHz%j8;j^`mvC2xH>+_Dq+g<`?!$!;gPGzCYbgozy3Q!}`ENgQKof zt@SQMxHFn47`dW3pvHMg>YP8dK9m&MLV1dzho%3=$lDRp-K5pP0&O}dHJZj8EXS(U zt(a1e|M^e*@~4)@D7yUknADy4V5CzX2Jrj#lvml8^?ME58s`Ncr1R$ek5!3>bA);- zGeatg6Y|RLA9vuhiY3tN)BBGFK@BSMelis$*oxXQ835gs<#WvQcoZp|W+IYZzY@vS zXQ!&CeTK1KrITI$RWufgjk_a#I=7IQhb^AZK^x|5Z<1A>8)gK)#qI9dX<&09wXqlF zP8=na&aoHpJQC?%!_>kO`r7j1x7;fU4bcWA8g7pwIl=?dxhadYr|s|2qz!o@aJ z7=R?u2*2dSh_urv2qY3alTquhAyC!o0j_e-NDfLt-SdwaTv-PnD^Kx_17GBc`Md@Lq2z>7KE-<%apom;QU7!2 z!T9CZB(v0IM)l*e81DiVI8D8m(0k#RE;jd8(_qd(67XTGM{qJGKc^s!F+@F=PAvb+y&5%;yxpXeVsVzFLlAx1r61;j>+lPpBN0Nwi2>xl=twlR;R{J?kqI z$vfEB>f8v_KPBYzuxX3O6Z*WEY0q+x%9R?!J6@O{8bNO+>BRVu(@_{hf4yH(6=OlWsO3+SQQ5`ZgrJ|X35@iDgdy6(`%2S74p|U`2t1Wu~G=alk zX3{M54UIX|J>E8t!c;=XjSi}fHcH0pwjA)Exjmaw3go9H;V^Wv}lp@`ts$4~S8IOPQ5mN{qKo zIeeOf{$A7^Fd0;Q>L)c5WlJ!tJg#r$-9znVL}fUn6O#M3KW|3vejl=C%JLNSs{w?8 z3Ry0PSWN&KI+aBzo)wa(Svqv5;=q^B;8&*it4X^m_(7rgbzfT8Yk>lXE8u-_)P)#m zy;zQ~%(PUY%zc6r7(yQ~kwdrG{VoJGcbyp-Y($0=*X5b1MlYv{u*L}4s$e8}Mk!Wp z2V4Y}@iahfrg$Yu$xx(_(CkSpWGsw`U{WwTV91&B^Af(Q80oVe9puVJS0>!4UhkpM)-^>9k1T%1@#j(0rvvN zAg}H!(&8Er@$g|!M=wf0Rc)y<4?X8%YHdIzijFu@cOF%7w_r6 z-NY9M6BAOfw~*F*k>3=la{WKcSpNrs>SY9Gt6oOi8Y}QSJb+)lBMdJ%J@QAbey&A+VZK5c)UnRAQ>n);hRp^ z6dsW_Y+8TxhDgx*&R+loTTWuNI!IWe;tmQ0A?!)tQaEkaVG`nRR2Tq_;6kack9OTJ za{M2dHi80lk8_(tSMOi=3W}l$D&p#I&TGK>6Hxx`r40~vcWH-?@qdstP>T!&m^YTQ z*IWM?%WXWC0vsKR8hmY!|9~k#3qh^XNM5qsR>_DG2=K#lU^syHcIAPY&gXD>*h4#Q zA|i^c^iCYp*JJNH!;=#%bR)nNRY?PRcp4XA z%ab5my@~>;Fo^1}*IHlmJIxza@rETxb}2CGq9PYoscftR5OJ(Pm69*R9-xr`F;Vu- z$DM352lIX~!zD66ztR&Ue)b*22bqSoJwJ8Kq52>c`tn|z_WAje=kClLzyYp*ov{!4 z{UigDhKGXXmERzp7>S1hyqVPNeV(V^*c1rheOg~XgNQkT;6ho-XO}vAMQ%Ei^N`Ga zjb-S;H0KNRPnG;czMjI1?D3kbc?`X40A)_E3bT?XQ^20Lc00BiN~0JHd0O4&Q>c~+ zKwGLo?cx=ntTTXf5hHp!6^KcrUyRvz)k`0f;qlGj!+oT z=~4p7T3LrC>JxqYTvE%wt~t172w$0OorMa#r~6b!5gpx z0Ntsi@oa9DRM>Q(Pv);Y`tX?U!rEO(ftL^^RMO0^Ahx~fo{*CyYOl^qov=u5p^sXs zu2B+#eyIyxO+LqF3EBlJOZy+}CaRx)%87=#U42pSpm(DLym3+^x25*b z;~yiYm1cq0p{!Ou^m{VKU_Jp%vaj=Gy4pHvu(jH%k7yv^fQXPuM_BoJW2hP}{OQcW z!p-TY7r%ck04Tm`sPIjW1>rfsYDd62B&3(qz&lxC>x;L4VK7`(WYMVQL=YL^JB_LY zU=f)@*&qpu=Ud)S0P_E9|MM?IZZd)R71Y{AQ6JVS{n!@E;4Y@e_O}Q3+)69*uEQTc zKk8%Vwiyh7AMi(=(pEoaOK7=;$zR%-hoAeP<+Pp?S)XM~1_#}lhEYG@FjR0BTJ^9UCWB3)Cfux#k+Z_Gdn_>QoP!+^a(U1OOpX?Y^&61_ktK3C!x< zk6Hu6zEL6+;M{D`(&(q~87kfvMfucD1|a-nN<(Z zRiznUX}Cg@%|t0?<7cDDh4B>_S;@k4K+4E$xigZONoCP&cjlufbsK46{!&M{oAr-O zFSQDcTFe82$JfN_B7vz&z8K{K{Pa6)+%}_Kz3|0`K@SV7lH!*@OU4?4h3^(ss1(J5 z;=P)^^PE0$v|PwMy##T?m_l&6JxTB*`1*8~z}Cu38PfpTI&!_J11GQ|{K?NoT&^gU zx~J06@@ua#9@__)_S1R4eURANh@gM1Fk(sgW&_{nAU~M+kky@nI{bH?hehYplC9fD zrEeRB6uf*%h2pFfv@3bf#Ye7t{VTx`wU*)(Lq@gJ!e=zpBd(1PIuJAQ0CPsH5lWg) z6JjZX0U6epOrpw~z40+ZTSdz3%_Ncqcg%W|kmtMDWH%q?Ln~h^n-0hp#}`Kf^(QC~ z)ZdTT;#LSwoifo7X?s z8M!JXaEE(D#BIi|F!7DS|*- zJ?p^i`o7|}ArUx-@5y7kPsb4zFM*E&3?o4xb*>)cSA|=AQc_~D+WS)L0paa8A-A@K zO0Ay}?XUF*j|20F0_-Rhb}#DIANX_UcJu;ZdOMc#jsCrJYl!K9PlAgaUk#oDW&TO> zflB9>mz=c_fA2+Lq1>i9lO7mOP51*RuJaw6Xg{e}U&mfW;(a@!XvMWMc!#)DE>g4Mm zuG=auab8|pmc*oX2kYvi!z|gU0q!e3$`FYYFn`e|Z;Ryq&Z@>;w9&{H05GI|cA}Y# zFm|k4!pQL6vGZo=PJZiu)CYpLU#+MF{+FADnZjOTE79D(XL}Dqz3q@5n$T$offb@0 zp6vS9*#z6A$e?B&=tZnD_rdY|iY&RnLs;4f&ZstS^aSc`J|~>!5%94fe3H6ED6+v| zTx0I94%5|)L;y>jVNGW5l{EnAvwjisc|)+MVJg0Zba!IGv5dY=o{2@3`q=!ge! z0{}9&5>kd$i`g~>U{L_t6F2Es^_af6=Iu-tlE{J1>DpW0Ki!2efsFq8*ITCvt*Pl^ zJzcL<-(~mX+=3Wf~#x;U6vY`uya0eO1!q_c$=zh^!^9TD0t1-iY;ajqc}Spts~ zg)|ZfMK-s>3y2iTiv}Qj1>J1WYfibv=Ck>{ zQ;@65woJ;f=nIcJ6#S9V!P#CJJ=9??xboA%gzY$(RR4(2)Hsl$QaV%6jcM!)mR&tS zos6B^(9ffz3?7=rY<$I%0V*Xhvz!U^Sp$xOCbLSTCbB80Ci+uHYRYT|@~o;+;2J9G z0=gB_2%RYaV%;z)RDZjd#puZ|;?W+06)}o0e&zh7dR1^e#gYtRq(B3KdR39gj10E{ z$?novX;YMoTyEsqGrRTDg#)l7U>6$`m)sd79qKhM9@i$Xv@Rqi9{J z>XamW)Bow}a^k)8>o8s!Vea25yvqLK0w;qFCz@jbNO&&HSma~oTUTJKr(H*BLNww7 z7X{Y)UuoB@*(z@@T<>{#?nwC%^I%#$A-K7mY*{Zq9&_=ji=e`%ml}|xE`?J0?qFL@ z&UcxfHQcfmra$3^UKuX#bs$I0c~zRUdSwmK&~59F`tEi?iDMr?0=!t&3(+y?)XOxg zrU^CJ{mvC!?dWT6{zO%^@PoI%FW!PwV+S?btWqtsv*jF+G+<1c4NV7tBNCXG?fvZM zSH0;+&VZ%j^!b9RT`GmPx&xL5^bpw<5vv4Ru9aMOq$PBmS0)}Zm0h{9d}z_F1(hCI z3>FaPfAZz=mm*jaB%_yeK&#K*HrimDKYx{hb_DzKge%6=OyO6kKSR%K_HGl6yBRw7;G8 z1UQt3K$>U}k+$jzf`tHq276~Enh+Esg&2vXuA)CICt< zXnpeIja9sIEzc@B{#}1Gp#j+-VJY3#xyEchlpCMf{Z?u;M$%Mae*%>x47Y~O83KL%3s7Y`IH{0V>s(ft)H3QooMDv!piziiEQE`_VFS)Zk7~&w zi{(fl^)ObWdI(k+U0Ke9590x(Ax}N1rz`Hh4A&N7W>qF|4;!{-YH%_t8iyQxP4K7NDMnf6xcB_O&9dhfcMiU;)fH1smuxMWm648u&Uh0hrWFcpx zuEM5!XX1Y-D+IB`Aj8O_vl+|I34;W>!UM=cmm*9x&RrvFDXEsdHCDMqeEaVK?HmLR zD5%WAiR8P*7ejgP+J?L!x2{&%pZ&!C1^Q7yP4IjhHimm7NWerg(Z&iv6?k{G*p;pO zu5te3Q#~(x%;;ayE03q%$IBxKCk<$(^3UF6&X|Z%ICY)DMQB#zXo*>|ag@ndK}^DL zc=7xCj9!aTSNhgKVu+R= zvczW6h(6=Fb77aK)>Ur{=vgdnE@$h>H?+SGX+e_HD;*nQ+`tpQN_3b%DO6O31ru#! zp8o#r$Hw7+IKTJ)tMz;5;(&wm`|5}I*e^>C2_&W>hMb7Sg()Pn!2s3!3`F=(FmN$x z{W%7SvZJ%*Yc$v1>nhd|EJ$yED>I;{+g*>sLuB+HH?)hF2o@crB=5fZ{o}##2tgjz zw-@@=#FV(RBuR_*_Rim3ddxG1U6B*=08a|n&EtdAIfDhm>mImBFk#XHM@(`VM%URJ zTV7p%Je{Zzi7I&OcYB`0&%gGRo#@3K(9DeF4X*(r)qX6{jW)rl944tkH$lsGoOs8? z8<~G_afQWmvdplDG(W#>n<`Je^4UGnWH$`Y2wA`egk{dtyNJKT(TM0EiF8tb%-qPOd%i+k+49TmJsRo^Lz{?I7fd?+-|;rIMk`pYGCDrBZC2XaxPn3IoB9wd znE>QhKPm6gx4C(7Klk_@Z{)gPQt??T(RsVhICZ62V+Kyeh&pYx#%?H^}ArdaIfHye& zyDf!8__UqT?qd8U56dj*9CQd?y_%3*8JYtb=(Lz?;fu(c$323`AzV2!8Y(3iNtwXC z5@X~RD@;(nA7g~nw@rIh+*!?Am4HON=qDZU!nf@jo- zT!&zFQ~$%RVX4TdqTWD4!-`y;2u$Iw^kC|TU(-_}j5Z?o<7yW>8I?})I>2}3%CgweQgBri5oD{*D}9Y={Vk%r#2tht z5^PWjqDW1sSQqf(=t&r<@PCPfY0AWR>*xn^pTn?&-g6MOFICoBU4>e6k#YB02(=I5 zP0~Sa1k(pOg0Qgtb{F%cqvd)Tr4ebSd~<#JT<(C&nH3}reMn5rlGql~=gsk?XxVTm zcOL!9o#pYphhegok5lGbcl4Cfo}J6Q_*r6V&WA*2*4H*K>rXcwH9qG}C*h9MG~GX4 z29;0BgFvz7x{#HPI-_uX6|N1sswPouH=~uUDyN#8UawJ+;Y245tN52Z$_OIo*F+p> z%cL)+Fn;1dvq1_i;d*?8LKUGeSCcK@EVUWI*Vl1Bwhhs%k26Yv=4#o=%tRH&`|uS? z(}&m!j&C&UuPD~OGK%UK50YrjTv>bPvouyPTzQCILRys%Cd~yM+62fct57UDgAKTR zbM<9}6j=4`EQLD-@XM%#J=u~rq7%qXEu8xxSZ{-utex!Tifx0C&tOc7Ex)h#DU6xb zvvAke2y1_sckYkBi(0F?!foe6p~r8vxM{~>w^~xwhg;NW6U$(*^iwn&R|&B^K!L5EFwhj1*k!%Z49H(H-RbX>I$@NVw~PqSVvNAsdr@Mz(M;xAqSo5-W|5Ts1G5A)UtCrdO$u%|LId8r#1H8p7%WuxAl(E; zpMF9u6V717~q$VC04Ha0oXkE_>Kc@~;85|oN# z9pD;)*N~@HcjP1J>Dj)Iu?;k1&z?tag>}_7b~z9Bj8txyzgcPxDyrcX9I399PmFAR zuhYSn$M4dmDXQ7Wr2}J*(J?z)%fBgy7KqC|5`o%Fc}#Xtkr0J&)DX;*IG%Pc-J1%D zrLPyw6=Z=J)AQjfRd%k~@Sbin$HK7P?9`nHVS@Musr${zbjR-7a}(;~)F?dx z$}}uYcWIwE_ungo>egj+6rGK86iT{oX6Iw22v)-eOr$=G2XTn+o8i4ul4`o86it$O z1-Rl#dU1#ww{o>wLug>lbpVPCPTe%kZ^{zrq%n7BEnHL;QL&ur!W?k#+3{=hYJV?} zxHz{jAxfUW_rAx#t54z0{0TMz&|~*pN61qS_43!pa|89oh5dc6B`kh%Dj;g4N!A)= zC!R^-=-E81Ww%{%bvj^sPmhfy)jQo|uzJNo4%h{m4g%}dk}^^^4d3bOg?bebcuL=rQxYrN!VVWt_}72Y zLoSXzNkQ`63P%bfT>3m(ofNN%Z0Y+CF1OY?o5@Yh+ELjj1EhSSHBoPZVbf~__r+kL z{`{-*^LD8hPjd;AY5Q{La+x4(74_?P#c;(`livoz>O0p5$(bh!Z907KL5DKe3b`PI z#!GUDsyJ6{$;$SR6V|;;lArwq!*z8v4WJ74;xCPCF79K$7?`F*la~@xV?c=rubC{2 zU2yK5#tcm2?Zl#2;*Ecbue5(Cb?SOKJC_)}I)0fZNF4~vJ&PxSc^^Qha1!r@X5V+Q zb`9X8sx~L)KNfhYl(0-E-oDlSS*YLE?GaVLPFop9vOc~f%&efTT0l;uFAJ0D&GNIy z_H&jd!|iWv=#uw(Oye6w7H9*JRe587+9*0ESkNc0%7=y9C8HAGk3mo5?b&Q3G@!%_ z^*y4l8!SZ|7CBZK@b4s)pYlK(IprPQPO+#PWW#b+s`RbKcDWRjnS6+H#~>N)xPxn# z>_zZueBw*v@EVM15!2USi9gJF&?hI+It7E2#Ukm_eEe*%|Cy`f(~@x|DOlAti>tvn zvnJgn_kG*%u?h-bKzK?h91`upy2?)#ZzL4Xfg4dayd>{BBuP3yK*4HurzTeUtlmZ^ zxdR(&ywZb0fb}w@RG+Pv&KQb?Ejfi|%oFx9QqV+*Jez)Ol|Lvs#zfAgM`|~B;)@g>(P|tw^DjHk&11?KZX-qeqC%r@=jVVr#sK>j;zXJk%#GJK$ zRW3)`2UZ)Tt;Bs#M$c8E{-;OIrGgXIQ=_V^Lx(!Rx35e;EI)fmm9^fO?53|vnus3Q zc}o8n8Tz%6gTC_bfnIfu`|x}sL)a8=C>k9@y%n=ciMSh@?t`xi z<$^3z^?Osk6gdQ#q$w%a6D@=BH6(fZ9`y?BG`Wok?i!uPi1mu& zP_nWYp$#e%x|USJiHHnqQ z;4k4H)ec_X0x%Fbk}4f~1<4_22jmj3fd2*~!HeHW~l$*DMw!fE?>F|9YLcW@n0%h-H!-Ec~j z;97*HM1@pWwnc&_!Ybclo6Q*QNM3h&Z#fgbHOnP5_q690Qwg4?<%?HCs*N9rj$9yJ zePc>;q|l44k|8zmmOv~@;3Y2}%J5>B2z4HpO%ucbkS6`#w&BZ2iD})~N8i)o@)s1C zOUD*Ik5J5+BMlSbVb54@h_IS9_QGrv4Q&e-K>D7$8l8$&gL@rXYmuPL_l?TC$yH*> z-@IJKE8ray?`ns$UPS-Xp1PPNn!R#AO#6sCO( z5IJ(CK^mdQ+KhuZ-65M~uu&y@I(x~WT+;PL&V=Z}N*PzIuM#@=E%`~#T=^E+NhX#U z=z0huE@pf6kD6W05UQ_O(wFYJ_j}iFNI(Ihy9=2%&H9aN&w{1ss0+INk$<5EH zW_kr0NF`56Dx!4tQ{; z@oXk;nfxyDNa*+_56KduIpijzOMJczWx8r;VU%-c`FYi>e&SM2e{0UMGM+qSA0ZL{ zPPmBD&y}bZ{e_ZACZlcWf`yznfT_yv${{*pR?~v-H~s22LKagNvn(Qf%hD1Ry_V0d zT-|n(9aLKvWF&RLWpj*uHR*(eMRD&Oi5{hkf;}*rp7-aI4vRO}ol~&z(fEUfXmN#o z>kF|m=ufZDl%#BLO#-;a?~xgZQoBiI5$(U(W_gdrF6$3pO=csiEX49GPmFY)>3rpn za7RQQ#c|~_7P?LJ$iG>y^>op7g`1<0S9v5fW?cu|u^!28l!9qND!(s#wXsWW=Z>F7 zT~D=@S#$}$EA#ZdIXN6%lZ>^ru_ceGkJ1!_4OR~A=u1qKn-t%!**C)cmUBT{`_UBD zMx9Xn&w)l*FC#+Y7OD-+tbAS?3QGu+^T)kRfW1jdU(TkEReotH8Ad}qOBmA+)yPbw zs=Z5>h(5ionbp`k2&qtb7Lnh9;0@&|WmEq+aLPvL?+rE7_mn?SGV^KkRFZknwf)_y zkI%~Ne&Fo;<^f+2B<=XgdCF936l0{x)=w!nwwDc}XUHKrt?Mnbyznu`| zjIHx%kYCol5vrM@XUo-AN@{5}!}56`a-NGu1Hwda?!6q9pcAIHXDjV&SY&Ph8-GMV zgco*^xuW$7OyKg-bt6a*5?LI}g~(eM2`WErXG$}|T4`n@i+{d^wn4YbJ1psMa;}}{ z6sloC+&>&|GZn6uV2W=awvC`WfqSz#wFmbcKl1W9@(uVTa)~pDbMRb#@L{pnG)wD( zmME!MO_lSBBpq#)hK&AL!4I$bvpTz3{9aMBo~8vdff8i-;q?uN(IGQpPGNOxpbCO8We~g{}%E`KwPJOz_YeQ-+|NK$I01g z8?O<`dXBS)na5Eg$i~IJ_w&^=YHW3}%Egt$SYaX!JWa-Jj?<1cOw=(Mo4%to4~`WW z>6#mE`7G2cj<~|P+>&hTG>nr;Go?jfWTlD9nvOBn=KC;b1OcmpEd@nW)*AU3TeV^e zJn-$5XXJr|vp^DBrTVX-yc^H`u2Yf`X5W{Nq0!X}0dKfpLz^=MWNZxk^de$XnhC%1 zS;Y-ow@DcBOYF@xetJe10`VMUHTp^33*qZr4pD#UI^{uMuqrQIgYKah|4q-`dD@?Lu1FO9%4IG}R1^5OF!dy+@k2dm|Q= zF9>t6y_58QdTjhUxv;yOdxnMN5-x%;RI7Trp{`ci0~%OkF51{oH|JYBAL^;k^jy#x zM`zV2kaKY}>`7vmPAEIa$fprn^wc%@wF6-rs4sL&fp=oS+{%H ze#cM&S}XV~HjpEuAonB<`kY~sYE`nw4$S=Z?ph1V$6#FO*S1`IgTRO+j@FvWWM`$2 z$SkDuZICu*UfKtdGeWu|uX9GC+Cg`H$r1QZ9e ztB5#MAkACh>4l;0BGJ(`q5biX-3`51qaYt4d_+Njc9bE_AUv{qRW`4)68U%WiPQ#t zP!i^fg+>QB5XVhQ7os*26LOT{HteDkKT|!pXt_ARO%u)Y#qZ_fE=1#%@T2W>`-rU2 zB9W8{l5MRhOwKVEPqd@!r$aZMjg(nN7W%Frb(c4|#N1t-aY!83a^0{0?YN`rE#3V) zujly6n$QoC9zjkp*p6Ik%urFLqDv^#u#f1uJ4$J6TE!->K395}L*=L>YIi~#U9n$4 zL-$zyOPcw~h(Z7&iprkG0ynPNW;e9){)&?oM%Cvd!}^GWD}s$Hd3kUH1(qFWX$C5(8HpyLB_>Tm`gMyGs94s8 zIPy$qZfj`wQ}7y+?uSQSXy+aEZJ_`l3)J}>IV;FI82I#TU z8H^(3aQZ6sE6%Ke<+CM6mrCj%Q>#^yv@~&7j8zc1VJyK##TzM-EZ?nrjn*%!iaOls z^F~?q-K}C)ZU|+_rWABlnyKFJvA9|1j>eb+>?^lQv4{Z`5zHL*{DVTQlnA|@STL96 zy#d#9Tq=QB%W9X*jFcER!fA0-_QI3`v)LZfb4KC!u|2(vdK&nx^gef_khO!{2(RT7 zLz5{^jYP>q_Xczg`;A^puyDI``f}_w??~xOaK>qH^~Q2607i8KR*Oup_gbPJJnMs6 zATpN_U1f*?{7t89&vt1U;Zsg^n~?q#m73rWTr0-6M}%(K3Q&nCJoXUzd6yn)49Ft} zqiy|IOZ9dpeU=CXo90gE9SmI_4QF|tO|*|yNcf$Z)}k?uGDxS9BDbLGG8Lv6Fk^wNTqZwbg?2CP^%qe z()qtTqoOFYA%+E3aJy+M4iuai7QgD==AR*fkKq9v@#e`N8+2oGC|(gUIePY|ea?T< zp;RBL$T&vH77)DF`GMLsQin?cLD3n!W`;*lFC%lesmSt{Gyu+~Nbp5+5)|8iS%}t) zHfHl8G?{rVwxL+r(-jv5L-jss=dEJ5ogReBj8aCsv6($Ncn43GjEOkuFqO&@`xzAvkD?pM34w?(Ru%(Is)R5qcj6VY%gk+0I zUG}Pxf45=Vo2;NMW+XegW9+WEj(PJe(d?z+Kr<2&?B4Bf0nk!$Ri}4fEhPW`Nn4}L z{L|e+lRf6|QSmhyBX7ZNJTgQJ%K4QN&f{P;rGUVB~(`AJK<{8rPvJq=1qmF$NEy zWkm2&OaMx`(Bo0qqZ@E`#$tY0?v8zOs5&EI#2^#aYlT)ASBg2~^JGB0Zhaids&9x- z$%orf09fiF4pkiAWDdX7*s9xef+M>9Wi5}G~n*iKu`w%JZr z6cCXrGCY=6v9_5WWWOxiBs);kstyifSpN=44`RP>MChh#aWY^YY>(m(Ri{n|^2b*BEeMHsCJ2Oy=_OOVS%N z4h{ISe7ME__2K_8i{oyY#p82%{eMV4KpwYC-vr3NyQD?O@b6jxw*Z0aE})Mnc*Wdj zzLnaaTaZBSTV`>^jh4e-lGY!h6x0rIVCd&Wi!=Tyg$!x|ZTDI?T`GV4&$(%^$RzS_ z$z<+lMXrBz*59@w?ttm||KCYHwLb)xb}Z)afW8|=_Tav)rmxYzK={y)wou zZvV5-wQf%%J?TA>h=xu9Rr}&{ley%eUFY_PG5~A35S+?u<624s1D3f-$M|-fJ;deAQU`D+dh@AV3_9k0v_E5FW?x=kJnD?t-%e z05an=Zx6ZUYbgOP8Nq(RIPav9Hed?_A{IEn{2Y*!!yiip+|N#uz3xXw#UQT-3H<+8 z#d*h5`TzglIZio*&OxMP93!M`5oNFJJ<7-?dz2l@CVQV`OR}OUva-quQBjmxGD3(* zzvor&_vicmb-Vd5m+N|6uj}<(kH?)Qqgc0V{!O_NzoPi>1pgJ7ynAbO^?~u|&szjx zxt?QcfU?giel!S6eqAW@9yKrm44B%Y)UICW{RoQufKfJGZOVD=$kz4b9gQYpps#0vEpD{c3Fdv%$!eVJK z5YVMWBqng@{M}Xx6v%$`iR9H|#ej}@?QU7eE|^7Wxl1@l9vhO+5V79=a{@5QPiV8PuJo05<(f4d2E-VX6wXf!1 zCUdG)cZn28fS4eIY+=5<%*M#{u`@YJ?&Bm#%Wj{LBxK3|UJ+U(VO-q+g|S8sa?Q23 zJdL&NUMo*N!x{9z^pVq8?Ly^0_sj(M9CbbMzL7S@GQNmSK4SF^NVlB;oT0aWryA>R z5PCB?H~6Pxqnu?SJaDU!8DZ&mFYy*>Yp6nUoW>dBYJ{CZ{n7o?F$vFE7~V4PQpPX)2CsNw(=&$JPUqRHkG?k zTW_XKKF2xSFRptL@I|Fk!EHXADk-IjCz%%6_{0&3HjL0#Zzxgg%>ZE1vL$?YBkucNpwr*zm*(jv46_%gSv174+1(a|*7_R?DkFp|#m2M(QRBo)Fec&0Px_${HB zauhVPn9rn!C@??axP9LIf^$x4v5o5^w~xVJN6MI3k9ED34zQbur$c*ZjYew5>38Zg z!a=tl4_`(z-f$;chJ=l&d6wg?>BhXnu*Rp))Y#Bl3WIYsq~c6M%)??QEO4lHQ4c@a z;RTww*C)JWc0LCEII)q`!l;*J(3faAO_WrWZW_GJ)@B|%tTpDD(j8wRcb?ori=y!K zdC-S}hv5nZB03t=;W@c+?wSA7WG_rqy-bEljknc6uTdy0(EmIm^wyYn+Kpbk2i}u> zmw$2SEyp~*Ktm0@9GqmQ3?GEx4+@vE)Eb=Q`Q`6g)oOviW7>|NtWov;X-mzQpmbco zoz3la2~}G}7@t(Y+!QD1!JcDLNvk`b@BaSL^4n}rNYYBD;ZSW+?H3_ldB77W*I zuV@40g$J6I!AK#gI~mvi$``-D^>ZVXlx*01FLca@2jB$!zvv(8$=0g$xhRjvK65%44svA?iIA3K#Wv=Lzx%eWe{g?J zb#JqG_1v`?4(%Zm`#Xw@d3?UBdf4VYovW6N*@iX=gR=8&*2@JqzlcAr>d7vfhnQQf z0VjDqd6Ez|RE0gnI7;6x+^22aP|VJ8lzp9he^bFkS=!YojHAR-V=%Uz9i}~5#hJFg z9jSU)I$Ai6I!AJxVsYH2m^>bZAb-I$;27MD3{c%7Q9L9&dpv^kV1D5$Rd-KvNA}5e z1H%J|pz#p=nqR0vIH=(gy)dk&x{GxO)C`f=4EF;KsB1k}On5K;FAyTpehQ*J*B+_% z_x}X(L^Zf`ujrDO|3`SbCDRV^T78==Zyx&65WoNC3K>YQMx&ATANe#`$v_LbdRV6a zuMg_a{{}%T?nxhKHm}-${y4d^jK>+;ebl|3p+Byn7cO&;LD}?y{@(DvNk$QJt=rL+ z*w7f|kY#(2uQYymbMrKzli%x5*+p73@?@djaJR1ohmNpmuVEipwCsaMnQFz|m{->B zTr?5{Uy0Ae!HHC(Y|(jEOZ1=QWm*x$=)Sv#r(=cD;D|hN&fB5-KpL;XI0))rn^+4R zw%I7XQR*!;USqBNI9Mr|F;JoV?>e$#JY(Slc3snVlw&=MWwrQVn_-q{Vd~*rK2dS0 z9(JG@WU6;WlgTdb0M^P4Y$yHMqH21M2D~nlO=Hp0E!)$F)?wXBBK2ebhKAG-xFC7! z7tcOC2ysbIgKpSP{k;yzRrIvZ`^@$vfqtA+BZL)nQ8gfG0^{N^Ko#oxmD)t@U**PI z1#Ns80zG)G9Ban4>i+IDSy+|JY?g(I@hVSv_2#glQS7XX<$q!iJ*IhYy%*A*H5*WX z_O1tFa%G+UATnQH9v|KKq;iVk%;jy6R%vF7yig_7>eH&)gJzU11GG<{Tet7^*wS}+ zt6^u!QEzUg_C<9xq?Q`pGe)$z3TF!6u;16B#`(gUSEOzytBKE<2Q2$66lXwJUhRWi;{~tG#)WL zOrI$FnsyAuJ)&`TqQg3BQ4@5~y3#CJWMFF_hXe>b1GLi2<#b$`Aa7;b^Mv3sRqh8V z;KuTM20gVzS+m?B8{t65KLWlooF$#mC~AhLrA_yId7m6>36k^H&FC-pVL|165|XLz zwD9+k&Ktrz5c(YZVw6iG+tP0gGO5oTx1Dxjz8?fFJM3w?W_&5p^CAGsnty&hB^BH* zU5r$LsVpcZwl2cUE0?Xjt>SoZWbw*k75PoO+Yadn? zrzJiF1Q?kp1y?Ooj*#Ip{J zr%eEWRcFtwbV7JtoM`+3f@i%ap0=ls)W6m#RC)$3gZVnTc>^G7$p*?5skhrltdUXZ zm8`>Qf}{EK*Sk2^g_Q(#1S*_uME_f8$_!ClIQvI_hM_3tl}-OVk{^^XydpBnwJEjATO ztu55Zmwi~CDjJWDQ;AcH(_Bin9i8v)dDn^U67O>8$}}|f$~Y=&VW4pg9pjZ%$DD9A zz^L-_PSYY~4v)RtzM9rCcPMb{Dd^;CELdYXaiE{CM6s687m_?n+)?ffdrcr}mkby7&T^6ZxOaCJWR|PZhqq zoLt)w8sjT{-(Rkktu^5%2q9}6yNcQVQ6xBIw?G?>Jq2PbA50KU-u_A`i~oQDd+ZH^xqm+%J- zD+>MKI54Oj)E0rO;>-Ck_XCt9N6A&sh&5&?mmUcV0YXwE{6uLUiXk~RiE#Slq6KZA zrpd)069p0q|vx-ItrJ)bej6PxnG_P#M=oB^v$H}=*VZYe)}b02}W z*~*426Bv&&GIz%@)_Tk+Rbk&zlCnRa0i$IP;dNvAYcxcj1r!1+DgZBWHT1!i)D1ri z^ln(XL4hP#@dMm`{4P~*S+HW&^3UedFRjkY>bPEy?(1WoZh(oGfEo7e)U zM~z_0T=84Nwbd|-@ysWn*z~>bV|oQwnE-qcM^j4dJ3!;1o@COiRO9AB5<;5B`9>| z9^x9X`D|R#cW7)AC{^PKsp?$6AX*?3<)O=0IDZH7dc?I+ z%46hLAOpxTcI&bF`qQ{hF}c^+TThC2XY;KE?GzDpJ57W}v}$;ewH+k<>tjz92 zD3I8Z4oB4CWn8dWbD$``#}w zsWx%Ve2-*Jkf`m?rm)(G4gP+3-ipV)5v*xkrG@Z9ol#q26gr;&(y7FII)O2@vZRMeiXf;6SWOLj~Q3CjC~n4z#k z>0&B4jERJ-NI{3*rzL$8r#%;BPv#)+%}ze#1eUSLm}=&SOZxe?4<{tAiKtz)aeTN$ zPDn|ywi0^=1vEw{(u78LA23xNEf+Xyi_jRGl3i|05?~4o5lms!0rZ1aEiFz;>!b3{ zl=i2*%B6-;N=InktUi!(p>%s&X~vgba#4vM3bUST8v49l5hG^?{GMtN+5*7D(%PP} z&9JW8XK^=wv|ZE*avx_4g^>+d%iTfw!&KiaUd`OFxSZ{f(FX)vz$c7WG5z6!wi*uj z5Th?BJ`A_)b8gki_cxXyXgH!ZLN5cBH2WmRGTk@m0sgYX!vL{J@%xFM;lmC1=F4-CPn2~)LdKbNjVvMCN))GSuHl%~I% zVQ|Dm!J(To>QoPFt{Ug2Vcr)Kl>31)+j&e9JeMBP zPKSBkN($R_fKUym&uIZPchTz0NE3qT%Ing@!^#hqk&41e_^ zb~Oo9T-!`_477xV3DrM|B96%n8VrY|(sSv93m`rV^~9-DiHLUf|WD#pzrMXv&*bi_q5a*f&m8^Rr09U3J4qHTEU+i;`B0cZ# z7zT%uI*?_nlDQYgaP*_$fXgDV zu*SMSt(^Sc$bEeadTKy((!v%P(A3O{UA?+ax<4-KAG|TsL8~+J)Z4~e5h@o?LhE$8 zCA*Rctk!Z`ge>)%Xjagt>^SGi2?D|+JcsZIInqnNoF~^mSjutQyy@Ll()`C`9Eo>D zkQ@_EeI;6yLW060zG$xWih;N#m9knjnO?((xqk{;aSXH;oxbDQ#>VR2nAMY_4c`SS zWZFzS_+Pj7M_7zR$TC!jns`_IXW~aV+Q*c;%g^mLu$D}py{s;6Srz{e05lX_=NJ}s zgS){$%Z#+@xD_1ildbLpXQ24{S#t!#cmu&OD*U3;sCS``-C{G_)Q0SOn#fS|mF??9 z;m^NacNs1W@ddrC0I@Bh_lx@DpIghvyYuJ?8I`A&WsnCgD24t@b^p<5|AydS{#EJD zAKG6Y){IWjgtX^BEK(dEQ@zu+OSXeeIW{%iWIE-c;!as2%=Kv4_2|?QgC0G|7hK>c z5qpQmFv3z~53Li}=zh2|*w+_VM<0%smb)B1;qzCh@b?sFxv^K%@8SIQ3Vz(|(akIC zKn=!wSbHR?z4f?;BuBbyr5$;)QjX>4tX}#axo^?Z9P4FX?`npyNVd10W`=H$x4n~%Wm#WFQ+)vP-)gk$mej3Jxvyi7@46&&+EZqT{Wa& z`oeFeWK+)t6xC*VDh^#XIp*1(`;?yud9vO6#Cni;MT>siwP7< z91pYc7L)nZPUkl$OJ>ke8`@L+g9;(e(oLso>@AVW6Tf?xJ^{qX*x?BA&mP{{r!Zmu zk$>xPj)SQ7B3%9iMi9Uto8uapbcDJb))FYqCMVj7a1rrXxX|okYS2!Umn%`ox`BV1 zc)T<&prI;|?Vjh~rw2|a?$ekt7k*s$ps!fEY6xQaePZbVdqVrquWzTuQ?RCTjC^D~ z%NwGdY?(fu%HKU$F8CVjshigwX^mq z3@g%&2-&V16OZfUNLT)gd)DT~45qy&#%?hI4c8g0;w}Trz`}qDL{p=L1bj-CbONnHfqj97(S~g zx#H6nJO6GIZ0Ind)m#16wVXTAgPOMrEy>lp0aQ^xmUYu}V4$fS-qLn$z0;@@;62(H zIJd;3XOq8?ruvTykmLp)wKW#SXjmb>mWnOxj^42`( z&{farPP%HUk%&7_yqIadwKDO3ehr`-;Z-O5MuoonZkNVL+S71mz+0*BZ!js$V{seI zlTE(>zBk(2lYy~CfJ^q1sLACRc(PLnY}}E?a=pL%18LrcVFSU%EmljLe?m>8Dgq$C zeYbo4{-Ac-^rBGI^7J!7FH?EA$8G{Bov7fcYR>{iFMrc2n#@?G3Hm?_=I>RW^c&8= z&SX<4{t;yP$T2o2_}5Q-^XUKghv>1@Y2;6xqqtJ`!E@boj+W00oO5Rv`~DE5$LblA z?uCaVZecnV2!6?=DqXM+rurHm6 zh+*|VBwENGMbR*@6H0$ka`Mt+=e6;$E1XOuz4fpe^S3%=sC}Ng)J7K3*4RSFrAe74 zAY0p0^@cnPxoRS_-{>wK%z~~wT!ZnKez{@)f1d!(6U)~rHP+qPlY5X$_D-JT$gXNy zr{L{h8l&+!3kA08B&2VU4@z2fV5!5N-8SIAV9~|1w*QCV__t{Y0hkG6_bS(gF9o}t z%g@i-4{#h7f-I(gmV~CHIGQWtAP)7)BrlJu`#Ko|OJGoS(MTlC{Gtx?zpsp3mgI5k zR=~aUNw#mw^vK+(HtxdET1Dl%+NgR`uWnm$-~Todatw~)wr{+l6zUhU66hm=f2xX_ K3YBt}VgC dataiter,),\n", ")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, we can visualize how well inputs are reconstructed:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import CairoMakie # run if in an interactive environment like a notebook\n", + "showoutputs(learner, method, n = 3)" + ] + }, { "cell_type": "code", "execution_count": 43, @@ -645,6 +846,18 @@ "ypreds, _ = model(xs) |> cpu\n", "showoutputbatch(method, cpu(xs), ypreds)" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "Next, why not play around a little bit:\n", + "\n", + "- What happens if you increase or decrease the size of the latent space, `Dlatent`?\n", + "- How do the results change if you adjust the model size (`Dhidden`) or use a different model architecture like convolutional nets?" + ] } ], "metadata": { From fab7a2c91e20ff7e195cbcbc7c2ebfd4d51553bb Mon Sep 17 00:00:00 2001 From: lorenzoh Date: Wed, 22 Dec 2021 13:55:46 +0100 Subject: [PATCH 04/23] Make `fitonecycle!` more flexible --- src/training/onecycle.jl | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/training/onecycle.jl b/src/training/onecycle.jl index 190a348e14..171cf86311 100644 --- a/src/training/onecycle.jl +++ b/src/training/onecycle.jl @@ -15,12 +15,14 @@ to `lrmax` and then goes down to `lrmax/div_final` over the remaining duration. """ function fitonecycle!( learner::Learner, nepochs::Int, maxlr=0.1; - phases = (TrainingPhase(), ValidationPhase()), - dataiters=(learner.data.training, learner.data.validation), + phases = ( + TrainingPhase() => learner.data.training, + ValidationPhase() => learner.data.validation + ), wd=0., kwargs...) - nsteps = length(learner.data.training) + nsteps = length(phases[1][2]) scheduler = Scheduler(LearningRate => onecycle( nepochs * nsteps, maxlr; @@ -30,7 +32,7 @@ function fitonecycle!( withfields(learner, optimizer=wdoptim) do withcallbacks(learner, scheduler) do for _ in 1:nepochs - for (phase, data) in zip(phases, dataiters) + for (phase, data) in phases epoch!(learner, phase, data) end end From cbe85def0ba75bf60d0ad0aaedae3a0c08514fbe Mon Sep 17 00:00:00 2001 From: lorenzoh Date: Wed, 22 Dec 2021 14:22:56 +0100 Subject: [PATCH 05/23] Improve Block printing --- src/Vision/encodings/imagepreprocessing.jl | 2 ++ src/encodings/onehot.jl | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/Vision/encodings/imagepreprocessing.jl b/src/Vision/encodings/imagepreprocessing.jl index 3d7de1cfe9..a4972f3372 100644 --- a/src/Vision/encodings/imagepreprocessing.jl +++ b/src/Vision/encodings/imagepreprocessing.jl @@ -22,6 +22,8 @@ function checkblock(block::ImageTensor{N}, a::AbstractArray{T,M}) where {M,N,T} return (N + 1 == M) && (size(a, M) == block.nchannels) end +Base.summary(io::IO, ::ImageTensor{N}) where N = print(io, "ImageTensor{$N}") + """ ImagePreprocessing([; kwargs...]) <: Encoding diff --git a/src/encodings/onehot.jl b/src/encodings/onehot.jl index 661c81c58b..c1fdc4d387 100644 --- a/src/encodings/onehot.jl +++ b/src/encodings/onehot.jl @@ -16,6 +16,8 @@ end const OneHotLabel{T} = OneHotTensor{0, T} +Base.summary(io::IO, ::OneHotLabel{T}) where T = print(io, "OneHotLabel{$T}") + function checkblock(block::OneHotTensor{N}, a::AbstractArray{T, M}) where {M, N, T} return N + 1 == M && last(size(a)) == length(block.classes) end From ecd0171e2705193c40b2846109cecf944e059548 Mon Sep 17 00:00:00 2001 From: lorenzoh Date: Wed, 22 Dec 2021 14:23:40 +0100 Subject: [PATCH 06/23] Replace `BlockMethod` with `SupervisedMethod`, make `BlockMethod` struct --- src/FastAI.jl | 2 +- src/Tabular/learningmethods/classification.jl | 2 +- src/Tabular/learningmethods/regression.jl | 2 +- src/Vision/learningmethods/classification.jl | 4 +- .../learningmethods/keypointregression.jl | 2 +- src/Vision/learningmethods/segmentation.jl | 4 +- src/datablock/describe.jl | 63 +++++--- src/datablock/method.jl | 152 ++++++++++++++++-- src/interpretation/learner.jl | 2 - 9 files changed, 191 insertions(+), 42 deletions(-) diff --git a/src/FastAI.jl b/src/FastAI.jl index b08bc904eb..696d6c710d 100644 --- a/src/FastAI.jl +++ b/src/FastAI.jl @@ -166,7 +166,7 @@ export TabularPreprocessing, SupervisedMethod, - AbstractBlockMethod, + BlockMethod, describemethod, checkblock, makebatch, diff --git a/src/Tabular/learningmethods/classification.jl b/src/Tabular/learningmethods/classification.jl index 6869ffe0ed..c927b2d85a 100644 --- a/src/Tabular/learningmethods/classification.jl +++ b/src/Tabular/learningmethods/classification.jl @@ -5,7 +5,7 @@ function TabularClassificationSingle( tabledata, targetdata = data tabledata isa TableDataset || error("`data` needs to be a tuple of a `TableDataset` and targets") - return BlockMethod( + return SupervisedMethod( blocks, ( setup(TabularPreprocessing, blocks[1], tabledata), diff --git a/src/Tabular/learningmethods/regression.jl b/src/Tabular/learningmethods/regression.jl index 664a602787..a4d11746a7 100644 --- a/src/Tabular/learningmethods/regression.jl +++ b/src/Tabular/learningmethods/regression.jl @@ -4,7 +4,7 @@ function TabularRegression( data) tabledata, targetdata = data tabledata isa TableDataset || error("`data` needs to be a tuple of a `TableDataset` and targets") - return BlockMethod( + return SupervisedMethod( blocks, (setup(TabularPreprocessing, blocks[1], tabledata),), ŷblock=blocks[2], diff --git a/src/Vision/learningmethods/classification.jl b/src/Vision/learningmethods/classification.jl index 5ad53333a8..dc1183675a 100644 --- a/src/Vision/learningmethods/classification.jl +++ b/src/Vision/learningmethods/classification.jl @@ -8,7 +8,7 @@ function ImageClassificationSingle( C=RGB{N0f8}, computestats=false, ) where N - return BlockMethod( + return SupervisedMethod( blocks, ( ProjectiveTransforms(size; augmentations=aug_projections), @@ -57,7 +57,7 @@ function ImageClassificationMulti( C=RGB{N0f8}, computestats=false, ) where N - return BlockMethod( + return SupervisedMethod( blocks, ( ProjectiveTransforms(size; augmentations=aug_projections), diff --git a/src/Vision/learningmethods/keypointregression.jl b/src/Vision/learningmethods/keypointregression.jl index aac08674c7..ec04db162c 100644 --- a/src/Vision/learningmethods/keypointregression.jl +++ b/src/Vision/learningmethods/keypointregression.jl @@ -8,7 +8,7 @@ function ImageKeypointRegression( C=RGB{N0f8}, computestats=false, ) where N - return BlockMethod( + return SupervisedMethod( blocks, ( ProjectiveTransforms(size; augmentations=aug_projections), diff --git a/src/Vision/learningmethods/segmentation.jl b/src/Vision/learningmethods/segmentation.jl index b5cbc33829..e36fdbfe70 100644 --- a/src/Vision/learningmethods/segmentation.jl +++ b/src/Vision/learningmethods/segmentation.jl @@ -8,7 +8,7 @@ function ImageSegmentation( C=RGB{N0f8}, computestats=false, ) where N - return BlockMethod( + return SupervisedMethod( blocks, ( ProjectiveTransforms(size; augmentations=aug_projections), @@ -60,7 +60,7 @@ registerlearningmethod!(FASTAI_METHOD_REGISTRY, ImageSegmentation, (Image, Mask) end end @testset "3D" begin - method = BlockMethod( + method = SupervisedMethod( (Image{3}(), Mask{3}(1:4)), ( ProjectiveTransforms((16, 16, 16), inferencefactor=8), diff --git a/src/datablock/describe.jl b/src/datablock/describe.jl index 6e7f59bf56..28863edffe 100644 --- a/src/datablock/describe.jl +++ b/src/datablock/describe.jl @@ -28,9 +28,10 @@ tuplemap(f, args::Vararg{Tuple}) = map((as...) -> tuplemap(f, as...), args...) function blockcolumn(encodings, block; decode = false) blocks, changed = decode ? listdecodeblocks(encodings, block) : listencodeblocks(encodings, block) + n = length(blocks) blockscol = [ - tuplemap((b, c) -> c ? "**`$(typeof(b))`**" : "`$(typeof(b))`", bs, ch) for - (bs, ch) in zip(blocks, changed) + tuplemap((b, c) -> _blockcell(b, c, i), bs, ch) for + (i, bs, ch) in zip(1:n, blocks, changed) ] if block isa Tuple blockscol = [join(row, ", ") for row in blockscol] @@ -38,6 +39,16 @@ function blockcolumn(encodings, block; decode = false) return reshape(blockscol, :, 1) end +function _blockcell(block, haschanged, i) + if haschanged + return "**`$(summary(block))`**" + elseif i == 1 + return "`$(summary(block))`" + else + return "" + end +end + encodingscolumn(encodings) = reshape(["", ["`$(typeof(enc).name.name)`" for enc in encodings]...], :, 1) @@ -73,38 +84,54 @@ end function describemethod(method::SupervisedMethod) blocks = getblocks(method) - input, target, x, ŷ = blocks.input, blocks.target, blocks.x + input, target, x, ŷ = blocks.input, blocks.target, blocks.x, blocks.ŷ encoding = describeencodings( getencodings(method), getblocks(method).sample, - blocknames = ["`$(summary(input))`", "`$(summary(input))`"], + blocknames = ["`blocks.input`", "`blocks.target`"], inname = "`(input, target)`", outname = "`(x, y)`", ) - decoding = describeencodings( + s = """ + **`SupervisedMethod` summary** + + Learning method for the supervised task with input `$(summary(input))` and + target `$(summary(target))`. Compatible with `model`s that take in + `$(summary(x))` and output `$(summary(ŷ))`. + + Encoding a sample (`encode(method, context, sample)`) is done through + the following encodings: + + $encoding + """ + + return Markdown.parse(s) +end + +function describemethod(method::BlockMethod) + blocks = getblocks(method) + + encoding = describeencodings( getencodings(method), - (ŷ,), - blocknames = ["`getblocks(method).ŷ`"], - inname = "`ŷ`", - outname = "`target_pred`", - decode = true, + (blocks.sample,), + blocknames = ["sample"], + inname = "`sample`", + outname = "`encodedsample`", ) s = """ - #### `LearningMethod` summary - - - Task: `$(summary(input)) -> $(summary(target))` - - Model blocks: `$(summary(x)) -> $(summary(ŷ))` + **`BlockMethod` summary** - Encoding a sample (`encode(method, context, sample)`) + Learning method with blocks - $encoding + $(join(["- $k: $(summary(v))" for (k, v) in zip(keys(blocks), values(blocks))], '\n')) - Decoding a model output (`decode(method, context, ŷ)`) + Encoding a sample (`encode(method, context, sample)`) is done through + the following encodings: - $decoding + $encoding """ return Markdown.parse(s) diff --git a/src/datablock/method.jl b/src/datablock/method.jl index 4ca0676872..9f0794238c 100644 --- a/src/datablock/method.jl +++ b/src/datablock/method.jl @@ -1,5 +1,88 @@ +""" + abstract type AbstractBlockMethod <: LearningMethod + +Abstract supertype for learning methods that derive their +functionality from [`Block`](#)s and [`Encoding`](#)s. + +These learning methods require you only to specify blocks and +encodings by defining which blocks of data show up at which +stage of the pipeline. Generally, a subtype will have a field +`blocks` of type `NamedTuple` that contains this information +and a field `encodings` of encodings that are applied to samples. +They can be accessed with `getblocks` and `getencodings` +respectively. For example, [`SupervisedMethod`](#) represents a +learning task where each sample consists of an input and a target. + +{cell=main} +```julia +method = SupervisedMethod( + (Image{2}(), Label(["cat", "dog"])), + (ImagePreprocessing(), OneHot(),) +) +getblocks(method) +``` + +To implement a new `AbstractBlockMethod` either + +- use the helper [`BlockMethod`](#) (simpler) +- or subtype [`AbstractBlockMethod`](#) (allows customization through + dispatch) + +## Blocks and interfaces + +To support different learning method interfaces, a `AbstractBlockMethod`'s +blocks need to contain different blocks. Below we list first block names +with descriptions, and afterwards relevant interface functions and which +blocks are required to use them. + +### Blocks + +Each name corresponds to a key of the named tuple +`blocks = getblocks(method)`). A block is referred to with `blocks.\$name` +and an instance of data from a block is referred to as `\$name`. + +- `blocks.sample`: The most important block, representing one full + observation of unprocessed data. Data containers used with a learning + method should have compatible observations, i.e. + `checkblock(blocks.sample, getobs(data, i))`. +- `blocks.x`: Data that will be fed into the model, i.e. (neglecting batching) + `model(x)` should work +- `blocks.ŷ`: Data that is output by the model, i.e. (neglecting batching) + `checkblock(blocks.ŷ, model(x))` +- `blocks.y`: Data that is compared to the model output using a loss function, + i.e. `lossfn(ŷ, y)` +- `blocks.encodedsample`: An encoded version of `blocks.sample`. Will usually + correspond to `encodedblockfilled(getencodings(method), blocks.sample)`. + +### Interfaces/functionality and required blocks: + +Core: +- [`encode`](#)`(method, ctx, sample)` requires `sample`. Also enables use of + [`methoddataset`](#), [`methoddataloaders`](#) +- [`decode`](#)`(method, ctx, encodedsample)` requires `encodedsample` +- [`decodeŷ`](#)`(method, ctx, ŷ)` requires `ŷ` +- [`decodey`](#)`(method, ctx, y)` requires `y` + +Training: +- [`methodmodel`](#)`(method)` requires `x`, `ŷ` +- [`methodlossfn`](#)`(method)` requires `y`, `ŷ` + +Visualization: +- [`showsample`](#), [`showsamples`](#) require `sample` +- [`showencodedsample`](#), [`showencodedsamples`](#), [`showbatch`](#) + require `encodedsample` +- [`showsample`](#), [`showsamples`](#) require `sample` +- [`showoutput`](#), [`showoutputs`](#), [`showoutputbatch`](#) require + `ŷ`, `encodedsample` + +Testing: +- [`mockmodel`](#)`(method)` requires `x`, `ŷ` +- [`mocksample`](#)`(method)` requires `sample` + + +""" abstract type AbstractBlockMethod <: LearningMethod end getblocks(method::AbstractBlockMethod) = method.blocks @@ -52,25 +135,71 @@ end mocksample(method::AbstractBlockMethod) = mockblock(method, :sample) mockblock(method::AbstractBlockMethod, name::Symbol) = mockblock(getblocks(method)[name]) -mockmodel(method::AbstractBlockMethod) = mockmodel(getblocks(method).x, getblocks(method).ŷ) +mockmodel(method::AbstractBlockMethod) = + mockmodel(getblocks(method).x, getblocks(method).ŷ) + +""" + mockmodel(xblock, ŷblock) + mockmodel(method::AbstractBlockMethod) +Create a fake model that maps batches of block `xblock` to batches of block +`ŷblock`. Useful for testing. +""" function mockmodel(xblock, ŷblock) return function mockmodel_block(xs) out = mockblock(ŷblock) - DataLoaders.collate([out]) + bs = DataLoaders._batchsize(xs, DataLoaders.BatchDimLast()) + return DataLoaders.collate([out for _ in 1:bs]) end end +# ## Block method +# +# `BlockMethod` is a helper to create anonymous block methods. + +""" + BlockMethod(blocks, encodings) + +Create an [`AbstractBlockMethod`](#) directly, passing in a named tuple `blocks` +and `encodings`. See [`SupervisedMethod`](#) for supervised training tasks. +""" +struct BlockMethod{B<:NamedTuple,E} <: AbstractBlockMethod + blocks::B + encodings::E +end + +Base.show(io::IO, method::BlockMethod) = print(io, + "BlockMethod(blocks=", keys(getblocks(method)), ")") + # ## Supervised learning method """ - SupervisedMethod((inputblock, targetblock), encodings) <: LearningMethod + SupervisedMethod((inputblock, targetblock), encodings) + +A [`AbstractBlockMethod`](#) learning method for the supervised +task of learning to predict a `target` given an `input`. `encodings` +are applied to samples before being input to the model. Model outputs +are decoded using those same encodings to get a target prediction. -Learning method for the supervised task of learning to predict a `target` -given an `input`. `encodings` are applied to samples before being input to -the model. Model outputs are decoded using those same encodings to get -a target prediction. +In addition, to the blocks defined by [`AbstractBlockMethod`](#), +`getblocks(::SupervisedMethod)` defines the following blocks: + +By default the model output is assumed to be an encoded target, but the +`ŷblock` keyword argument to overwrite this. + +- `blocks.input`: An unencoded input and the first element in the tuple + `sample = (input, target)` +- `blocks.target`: An unencoded target and the second element in the tuple + `sample = (input, target)` +- `blocks.pred`: A prediction. Usually the same as `blocks.target` but may + differ if a custom `ŷblock` is specified. + +A `SupervisedMethod` also enables some additional functionality: + +- [`encodeinput`](#) +- [`encodetarget`](#) +- [`showprediction`](#), [`showpredictions`](#) """ struct SupervisedMethod{B<:NamedTuple,E} <: AbstractBlockMethod blocks::B @@ -78,7 +207,7 @@ struct SupervisedMethod{B<:NamedTuple,E} <: AbstractBlockMethod end -function SupervisedMethod(blocks::Tuple{Any, Any}, encodings; ŷblock = nothing) +function SupervisedMethod(blocks::Tuple{Any,Any}, encodings; ŷblock = nothing) sample = input, target = blocks x, y = encodedsample = encodedblockfilled(encodings, sample) ŷ = isnothing(ŷblock) ? y : ŷblock @@ -102,9 +231,4 @@ end # ## Deprecations -BlockMethod(args...; kwargs...) = SupervisedMethod(args...; kwargs...) -Base.@deprecate BlockMethod(blocks, encodings; kwargs...) SupervisedMethod( - blocks, - encodings; - kwargs..., -) +Base.@deprecate BlockMethod(blocks::Tuple{Any, Any}, encodings; kwargs...) SupervisedMethod(blocks, encodings; kwargs...) diff --git a/src/interpretation/learner.jl b/src/interpretation/learner.jl index 19332456ac..c7b6212a4c 100644 --- a/src/interpretation/learner.jl +++ b/src/interpretation/learner.jl @@ -1,5 +1,3 @@ -# High-level plotting functions for use with `BlockMethod` and a `Learner` - """ showoutputs(method, learner[; n = 4, context = Validation()]) From c8b5490f867cd502d00a9c382284fafdfcb00b5d Mon Sep 17 00:00:00 2001 From: lorenzoh Date: Wed, 22 Dec 2021 14:23:48 +0100 Subject: [PATCH 07/23] Update vae notebook --- notebooks/vae.ipynb | 495 +++++++++++++------------------------------- 1 file changed, 149 insertions(+), 346 deletions(-) diff --git a/notebooks/vae.ipynb b/notebooks/vae.ipynb index d5cf8c72c2..93b8feb4f4 100644 --- a/notebooks/vae.ipynb +++ b/notebooks/vae.ipynb @@ -22,20 +22,13 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 35, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING: using Makie.Label in module FastAI conflicts with an existing identifier.\n" - ] - } - ], + "outputs": [], "source": [ "using FastAI, StaticArrays, Colors\n", - "using FastAI: FluxTraining" + "using FastAI: FluxTraining, Image\n", + "import CairoMakie # run if in an interactive environment like a notebook" ] }, { @@ -54,83 +47,89 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfQAAAImCAIAAAAqqu9kAAAABmJLR0QA/wD/AP+gvaeTAAALkUlEQVR4nO3cP6iW5QOH8bc0I6pDYFObOUgUDp6glpbGFMEQjoM4NaRjuGRSQUO5BA1BEAUNbUUgCEoggn/iEBLUUISDg0WQVFbUIp7f0vYbvB/wfU/n6vOZvzzvzRkubs5w37W2tjYDoOXu9T4AAHeeuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0DQ5gX/3ssvv3z58uUF/yjA+lpeXn7zzTcX+YuLjvvly5c///zzBf8owH+Nf8sABIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEbV7vA9D3zDPPTNpfunRpfLxjx47x8Z49e8bHu3fvHh+fOnVqfDzVF198MT4+f/78/E7CBuLmDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBBnvzlH0tLS+Pjjz/+eHz87LPPTjrJ33//PT7esmXL+PiBBx6YdJJxU581nmTSH+Svv/4aHx8+fHh8/Mknn4yPWXdu7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4Q5Mlf/nHixInx8e7du+d3kvvuu298/O23346Pf/755/Hx77//Pj6e6u67J9yrnnvuufHxpL/eBx98MD7+/vvvx8ez2ezrr7+etOfOcnMHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwjynnvZ448/Pj7ev3//nI5x7dq1SftDhw6Nj69cuTI+/u2338bHf/755/h4qknvub/66qvj4+PHj4+Pl5aWxsevvfba+Hg2m73wwgvj419//XXSx7ktN3eAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCPLkb9mDDz44Pt66dev4eG1tbXx84sSJ8fFsNjt37tyk/UZ069at8fHrr78+Pt6yZcv4+OjRo+Pjffv2jY9ns9mHH344Pj516tSkj3Nbbu4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOEOTJ37J77713Tl/+6KOPxsfvvvvunI7B/zt27Nj4eGVlZXy8bdu2SSd5/vnnx8ee/L3j3NwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIMiTv2VvvPHGnL68uro6py+zSGfOnBkfv/jii5M+/vTTT088DneSmztAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBHnyd4N59NFHx8ePPPLI+PjGjRvj42+++WZ8zL/W2bNnx8dTn/xlfbm5AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEec99gzl48OD4eNLj759++un4+NKlS+NjYPHc3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gyJO/G8yBAwfGxzdu3Bgfv/POO9OPA/xLubkDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJAnf8u+++678fGFCxfmdxJgwdzcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSDIk7/r7P7775+0v+eee+Z0EqDEzR0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgjz5u85WVlYm7bdv3z4+vn79+sTj8N+yd+/e+X385s2b8/s4t+XmDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4Q5D13SFleXh4f79mzZ34nOXbs2Pw+zm25uQMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkCd/4d9u0iu+L7300vj4oYceGh9fvHhxfDybzc6cOTNpz53l5g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQZ78XWdXr16dtP/jjz/mcxAWatOmTePjo0ePjo9XVlbGxz/88MOcjjGbzW7evDlpz53l5g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQZ78XWdnz56dtJ/0RuvS0tL4+OGHHx4fX79+fXy8Qe3cuXN8fOTIkUkf37Vr1/j4ySefnPTxcQcPHhwfr66uzukYzIObO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEefK37LHHHhsfnz59enz8008/TT/OBvPUU0+Nj7du3Tq/k0x6YPnkyZPj4y+//HL6cdgY3NwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYK8577BvPLKK+Pj48ePj4937do1/Tj849atW5P2v/zyy/j47bffHh+/9dZbk05ClZs7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwR58neD+eyzz8bHq6ur4+PTp0+Pj5944onx8Qb1/vvvj4+/+uqrSR9/7733Jh4HpnFzBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgT/6W/fjjj+PjnTt3zu8kwIK5uQMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwRtXvDvLS8vL/gXAdbd4tN319ra2oJ/EoB5828ZgCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gKD/AU1d8UvOdiFWAAAAAElFTkSuQmCC", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfQAAAImCAIAAAAqqu9kAAAABmJLR0QA/wD/AP+gvaeTAAAK60lEQVR4nO3cv6vWZQPH8dsHx8RFEJoaIq1BCTe3WoTEwaYQMWwRRGzJQQoCQQ7ij0FwsiEhRMEEFykMxKBBUISaHMTJgw6J4+EQnmd5/oDre+g+R9/P6zV/+N7X9ObiHq4NKysrMwBa/rPeBwDg3yfuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOELRxjX/v5MmTDx8+XOMfBVhfu3btWlhYWMtfXOu4P3z48M6dO2v8owD/b/wtAxAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOELRxvQ/AW+mDDz4YH9+6dWvSx7dv3z4+vn379vj4xo0b4+PFxcXx8a+//jo+hjXg5g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQZ78ZTWOHDkyPt66deukj1+5cmV8fP/+/fHxiRMnxsfbtm0bH//+++/j49ls9sknn0zaw1Ru7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4Q5MlfVmPDhg3j43/++WfSx//444/x8eXLl8fH169fHx9/9dVX4+MzZ86Mj2ez2enTp8fH33777aSPw8zNHSBJ3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gyHvurMa5c+fGx++9996kj3/88cfTTjPs5cuX4+MLFy6Mj99///1JJ/nyyy/Hx95zZxXc3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gyJO/rMbi4uL4+PPPP5/fSebn9evX4+MnT57M7ySwCm7uAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQRvX+wDwhtqyZcv4+OjRo5M+/vfff088Dkzj5g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQZ78Ze527NgxaX/w4ME5nWTTpk3j43379o2P33333UknOXz48KQ9TOXmDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4Q5D135m7Sw+iz2eybb76Z00mWlpbGx0+fPh0fT33P/fjx4+PjK1euTPo4zNzcAZLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSDIk7/M3fnz5yftb9y4MaeTLC8vj4+fPXs2Pv7iiy8mneSHH34YH096H/jixYuTTkKVmztAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBHnyl7lbWlqatH/8+PGcTjI/165dm7TfvXv3+Pj06dPj47t3746P//rrr/Exbxc3d4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcI8uQv/AuWl5cn7b///vvx8aFDh8bHJ06cmNOXebu4uQMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkCd/YR28ePFifPzgwYPx8WeffTb9OAS5uQMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBHnPHVKWlpbW+wi8EdzcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSDIk7+wDrZs2TI+3rFjx/j46tWr049DkJs7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwR58hfWwUcffTQ+fuedd8bHv/322/TjEOTmDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBBnvwt27Nnz/j47t274+Pl5eXpxymb9CrvbDZbWFgYHz9//nx8fPPmzUknocrNHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCPPlbdvXq1fHxn3/+OT7ev3//+PjVq1fj4zfHpk2bxsenTp2a9PGdO3eOj/fu3Tvp4zBzcwdIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCPKee9lPP/00Pj527Nj4+MmTJ+PjS5cujY9ns9njx4/Hx5s3bx4ff/jhh+Pj7du3j48//fTT8fFsNjt79uz4+N69e5M+DjM3d4AkcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcI8uRv2ddffz0+fvTo0fj4wIED4+PvvvtufPzm+Pnnn8fHe/funfTxX375ZeJxYBo3d4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcI8uQv//Pjjz/OaQysPTd3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAoI1r/Hu7du1a418EWHdrn74NKysra/yTAMybv2UAgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgv4Lr1nNHyHm+i8AAAAASUVORK5CYII=", "text/plain": [ - "Scene (500px, 550px):\n", - " 18 Plots:\n", - " ├ MakieCore.Combined{Makie.poly, Tuple{Vector{Vector{GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " └ MakieCore.Text{Tuple{String}}\n", - " 1 Child Scene:\n", - " └ Scene (468px, 468px)" + "Figure()" ] }, + "execution_count": 13, "metadata": {}, - "output_type": "display_data" + "output_type": "execute_result" } ], "source": [ - "path = datasetpath(\"mnist_png\")\n", + "path = datasetpath(\"mnist_tiny\")\n", "data = Datasets.loadfolderdata(\n", " path,\n", " filterfn = isimagefile,\n", " loadfn = loadfile,\n", ")\n", - "showblock(Image{2}(), getobs(data, 1))" + "showblock(FastAI.Vision.Image{2}(), getobs(data, 1))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Next we need to define a learning method that will handle data encoding and decoding as well as visualization for us. So far, we've used [`SupervisedMethod`](#) a lot which assumes there is an input that is fed to the model and a corresponding target output. Since we want to do unsupervised learning, we'll instead use [`EmbeddingMethod`](#). `Image{2}()` is the kind of data that we're using and `ImagePreprocessing` makes sure to encode and decode these images so they can be used to train a model." + "Next we need to define a learning method that will handle data encoding and decoding as well as visualization for us. So far, we've used [`SupervisedMethod`](#) a lot which assumes there is an input that is fed to the model and a corresponding target output. Since we want to do unsupervised learning, we'll instead create a custom learning method using [`BlockMethod`](#). It defines what kind of data we'll have at each step in the data pipeline for example `x` is a model input and `ŷ` a model output. See [`AbstractBlockMethod`](#) for more info." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "EmbeddingMethod (generic function with 1 method)" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "using FastAI: AbstractBlockMethod, encodedblockfilled, decodedblockfilled\n", - "struct EmbeddingMethod{B<:NamedTuple,E} <: AbstractBlockMethod\n", - " blocks::B\n", - " encodings::E\n", - "end\n", + "using FastAI: encodedblockfilled, decodedblockfilled\n", "\n", "function EmbeddingMethod(block, encodings)\n", - " sample = input = target = block\n", + " sample = block\n", " encodedsample = x = y = ŷ = encodedblockfilled(encodings, sample)\n", - " pred = decodedblockfilled(encodings, ŷ)\n", - " blocks = (; sample, x, y, ŷ, encodedsample, pred, input, target)\n", - " EmbeddingMethod(blocks, encodings)\n", - "end\n", - "\n", - "\n", - "Base.show(io::IO, task::EmbeddingMethod) =\n", - " print(io, \"EmbeddingMethod($(summary(task.blocks.sample)))\")\n", - "\n", - "\n", + " blocks = (; sample, x, y, ŷ, encodedsample)\n", + " BlockMethod(blocks, encodings)\n", + "end\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "With this helper defined, we can create a learning method for our task: `Image{2}()` is the kind of data we want to learn with and `ImagePreprocessing` makes sure to encode and decode these images so they can be used to train a model." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "BlockMethod(blocks=(:sample, :x, :y, :ŷ, :encodedsample))" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ "method = EmbeddingMethod(\n", " Image{2}(),\n", " (ImagePreprocessing(means = SVector(0.0), stds = SVector(1.0), C = Gray{Float32}),),\n", @@ -146,39 +145,19 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfQAAAImCAIAAAAqqu9kAAAABmJLR0QA/wD/AP+gvaeTAAALkUlEQVR4nO3cP6iW5QOH8bc0I6pDYFObOUgUDp6glpbGFMEQjoM4NaRjuGRSQUO5BA1BEAUNbUUgCEoggn/iEBLUUISDg0WQVFbUIp7f0vYbvB/wfU/n6vOZvzzvzRkubs5w37W2tjYDoOXu9T4AAHeeuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0DQ5gX/3ssvv3z58uUF/yjA+lpeXn7zzTcX+YuLjvvly5c///zzBf8owH+Nf8sABIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEbV7vA9D3zDPPTNpfunRpfLxjx47x8Z49e8bHu3fvHh+fOnVqfDzVF198MT4+f/78/E7CBuLmDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBBnvzlH0tLS+Pjjz/+eHz87LPPTjrJ33//PT7esmXL+PiBBx6YdJJxU581nmTSH+Svv/4aHx8+fHh8/Mknn4yPWXdu7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4Q5Mlf/nHixInx8e7du+d3kvvuu298/O23346Pf/755/Hx77//Pj6e6u67J9yrnnvuufHxpL/eBx98MD7+/vvvx8ez2ezrr7+etOfOcnMHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwjynnvZ448/Pj7ev3//nI5x7dq1SftDhw6Nj69cuTI+/u2338bHf/755/h4qknvub/66qvj4+PHj4+Pl5aWxsevvfba+Hg2m73wwgvj419//XXSx7ktN3eAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCPLkb9mDDz44Pt66dev4eG1tbXx84sSJ8fFsNjt37tyk/UZ069at8fHrr78+Pt6yZcv4+OjRo+Pjffv2jY9ns9mHH344Pj516tSkj3Nbbu4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOEOTJ37J77713Tl/+6KOPxsfvvvvunI7B/zt27Nj4eGVlZXy8bdu2SSd5/vnnx8ee/L3j3NwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIMiTv2VvvPHGnL68uro6py+zSGfOnBkfv/jii5M+/vTTT088DneSmztAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBHnyd4N59NFHx8ePPPLI+PjGjRvj42+++WZ8zL/W2bNnx8dTn/xlfbm5AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEec99gzl48OD4eNLj759++un4+NKlS+NjYPHc3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gyJO/G8yBAwfGxzdu3Bgfv/POO9OPA/xLubkDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJAnf8u+++678fGFCxfmdxJgwdzcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSDIk7/r7P7775+0v+eee+Z0EqDEzR0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgjz5u85WVlYm7bdv3z4+vn79+sTj8N+yd+/e+X385s2b8/s4t+XmDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4Q5D13SFleXh4f79mzZ34nOXbs2Pw+zm25uQMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkCd/4d9u0iu+L7300vj4oYceGh9fvHhxfDybzc6cOTNpz53l5g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQZ78XWdXr16dtP/jjz/mcxAWatOmTePjo0ePjo9XVlbGxz/88MOcjjGbzW7evDlpz53l5g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQeIOECTuAEHiDhAk7gBB4g4QJO4AQZ78XWdnz56dtJ/0RuvS0tL4+OGHHx4fX79+fXy8Qe3cuXN8fOTIkUkf37Vr1/j4ySefnPTxcQcPHhwfr66uzukYzIObO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEefK37LHHHhsfnz59enz8008/TT/OBvPUU0+Nj7du3Tq/k0x6YPnkyZPj4y+//HL6cdgY3NwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYLEHSBI3AGCxB0gSNwBgsQdIEjcAYK8577BvPLKK+Pj48ePj4937do1/Tj849atW5P2v/zyy/j47bffHh+/9dZbk05ClZs7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwR58neD+eyzz8bHq6ur4+PTp0+Pj5944onx8Qb1/vvvj4+/+uqrSR9/7733Jh4HpnFzBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgT/6W/fjjj+PjnTt3zu8kwIK5uQMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwSJO0CQuAMEiTtAkLgDBIk7QJC4AwRtXvDvLS8vL/gXAdbd4tN319ra2oJ/EoB5828ZgCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gCBxBwgSd4AgcQcIEneAIHEHCBJ3gKD/AU1d8UvOdiFWAAAAAElFTkSuQmCC", "text/plain": [ - "Scene (500px, 550px):\n", - " 18 Plots:\n", - " ├ MakieCore.Combined{Makie.poly, Tuple{Vector{Vector{GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " └ MakieCore.Text{Tuple{String}}\n", - " 1 Child Scene:\n", - " └ Scene (468px, 468px)" + "Figure()" ] }, + "execution_count": 6, "metadata": {}, - "output_type": "display_data" + "output_type": "execute_result" } ], "source": [ @@ -195,11 +174,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "1-element Vector{Any}:\n", + " [0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0;;;; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0;;;; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0;;;; … ;;;; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0;;;; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0;;;; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "BATCHSIZE = 1024\n", + "BATCHSIZE = nobs(data)\n", "dataloader = DataLoader(methoddataset(shuffleobs(data), method, Training()), BATCHSIZE)\n", "dataiter = collect(dataloader)" ] @@ -213,9 +204,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(28, 28, 1, 1428)" + ] + } + ], "source": [ "for xs in dataiter\n", " print(size(xs))\n", @@ -260,7 +259,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 26, "metadata": {}, "outputs": [ { @@ -269,12 +268,12 @@ "βELBO (generic function with 1 method)" ] }, + "execution_count": 26, "metadata": {}, - "output_type": "display_data" + "output_type": "execute_result" } ], "source": [ - "\n", "struct VAE{E, D}\n", " encoder::E\n", " decoder::D\n", @@ -314,11 +313,10 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ - "\n", "SIZE = (28, 28, 1)\n", "Din = prod(SIZE)\n", "Dhidden = 512\n", @@ -331,7 +329,7 @@ " Parallel(\n", " tuple,\n", " Dense(Dhidden, Dlatent), # μ\n", - " Dense(Dhidden, Dlatent), # logσ²\n", + " Dense(Dhidden, Dlatent), # logσ²\n", " ),\n", " ) |> gpu\n", "\n", @@ -356,7 +354,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 37, "metadata": {}, "outputs": [], "source": [ @@ -372,7 +370,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 46, "metadata": {}, "outputs": [], "source": [ @@ -407,7 +405,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 48, "metadata": {}, "outputs": [], "source": [ @@ -417,7 +415,7 @@ " cb::ToDevice,\n", " learner,\n", ")\n", - " learner.step.x = cb.movedatafn(learner.step.x)\n", + " learner.step.xs = cb.movedatafn(learner.step.xs)\n", "end" ] }, @@ -437,19 +435,9 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 40, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "ProgressPrinter()" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "learner = Learner(model, (), ADAM(), βELBO, ToGPU())\n", "FluxTraining.removecallback!(learner, ProgressPrinter);" @@ -464,7 +452,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 49, "metadata": {}, "outputs": [ { @@ -474,158 +462,152 @@ "┌──────────────────┬───────┬─────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼─────────┤\n", - "│ VAETrainingPhase │ 1.0 │ 100.814 │\n", + "│ VAETrainingPhase │ 1.0 │ 181.964 │\n", "└──────────────────┴───────┴─────────┘\n", "┌──────────────────┬───────┬─────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼─────────┤\n", - "│ VAETrainingPhase │ 2.0 │ 54.5541 │\n", + "│ VAETrainingPhase │ 2.0 │ 178.809 │\n", "└──────────────────┴───────┴─────────┘\n", "┌──────────────────┬───────┬─────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼─────────┤\n", - "│ VAETrainingPhase │ 3.0 │ 49.6636 │\n", + "│ VAETrainingPhase │ 3.0 │ 167.562 │\n", "└──────────────────┴───────┴─────────┘\n", - "┌──────────────────┬───────┬────────┐\n", - "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", - "├──────────────────┼───────┼────────┤\n", - "│ VAETrainingPhase │ 4.0 │ 45.739 │\n", - "└──────────────────┴───────┴────────┘\n", "┌──────────────────┬───────┬─────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼─────────┤\n", - "│ VAETrainingPhase │ 5.0 │ 43.4472 │\n", + "│ VAETrainingPhase │ 4.0 │ 155.867 │\n", "└──────────────────┴───────┴─────────┘\n", "┌──────────────────┬───────┬─────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼─────────┤\n", - "│ VAETrainingPhase │ 6.0 │ 42.2355 │\n", + "│ VAETrainingPhase │ 5.0 │ 102.829 │\n", "└──────────────────┴───────┴─────────┘\n", "┌──────────────────┬───────┬─────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼─────────┤\n", - "│ VAETrainingPhase │ 7.0 │ 41.5158 │\n", + "│ VAETrainingPhase │ 6.0 │ 88.4556 │\n", "└──────────────────┴───────┴─────────┘\n", "┌──────────────────┬───────┬─────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼─────────┤\n", - "│ VAETrainingPhase │ 8.0 │ 40.8949 │\n", + "│ VAETrainingPhase │ 7.0 │ 75.3091 │\n", "└──────────────────┴───────┴─────────┘\n", "┌──────────────────┬───────┬─────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼─────────┤\n", - "│ VAETrainingPhase │ 9.0 │ 40.3357 │\n", + "│ VAETrainingPhase │ 8.0 │ 62.4067 │\n", "└──────────────────┴───────┴─────────┘\n", "┌──────────────────┬───────┬─────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼─────────┤\n", - "│ VAETrainingPhase │ 10.0 │ 39.9624 │\n", + "│ VAETrainingPhase │ 9.0 │ 57.2527 │\n", "└──────────────────┴───────┴─────────┘\n", "┌──────────────────┬───────┬─────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼─────────┤\n", - "│ VAETrainingPhase │ 11.0 │ 39.5658 │\n", + "│ VAETrainingPhase │ 10.0 │ 58.5014 │\n", "└──────────────────┴───────┴─────────┘\n", "┌──────────────────┬───────┬─────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼─────────┤\n", - "│ VAETrainingPhase │ 12.0 │ 39.2665 │\n", + "│ VAETrainingPhase │ 11.0 │ 53.1275 │\n", "└──────────────────┴───────┴─────────┘\n", "┌──────────────────┬───────┬─────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼─────────┤\n", - "│ VAETrainingPhase │ 13.0 │ 38.9916 │\n", + "│ VAETrainingPhase │ 12.0 │ 50.8608 │\n", "└──────────────────┴───────┴─────────┘\n", "┌──────────────────┬───────┬─────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼─────────┤\n", - "│ VAETrainingPhase │ 14.0 │ 38.7168 │\n", + "│ VAETrainingPhase │ 13.0 │ 50.1279 │\n", "└──────────────────┴───────┴─────────┘\n", "┌──────────────────┬───────┬─────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼─────────┤\n", - "│ VAETrainingPhase │ 15.0 │ 38.5706 │\n", + "│ VAETrainingPhase │ 14.0 │ 48.7308 │\n", "└──────────────────┴───────┴─────────┘\n", "┌──────────────────┬───────┬─────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼─────────┤\n", - "│ VAETrainingPhase │ 16.0 │ 38.3802 │\n", + "│ VAETrainingPhase │ 15.0 │ 47.3018 │\n", "└──────────────────┴───────┴─────────┘\n", "┌──────────────────┬───────┬─────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼─────────┤\n", - "│ VAETrainingPhase │ 17.0 │ 38.2334 │\n", + "│ VAETrainingPhase │ 16.0 │ 46.7513 │\n", "└──────────────────┴───────┴─────────┘\n", "┌──────────────────┬───────┬─────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼─────────┤\n", - "│ VAETrainingPhase │ 18.0 │ 38.0221 │\n", + "│ VAETrainingPhase │ 17.0 │ 46.2341 │\n", "└──────────────────┴───────┴─────────┘\n", + "┌──────────────────┬───────┬───────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼───────┤\n", + "│ VAETrainingPhase │ 18.0 │ 45.92 │\n", + "└──────────────────┴───────┴───────┘\n", "┌──────────────────┬───────┬─────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼─────────┤\n", - "│ VAETrainingPhase │ 19.0 │ 37.8736 │\n", + "│ VAETrainingPhase │ 19.0 │ 45.5328 │\n", "└──────────────────┴───────┴─────────┘\n", "┌──────────────────┬───────┬─────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼─────────┤\n", - "│ VAETrainingPhase │ 20.0 │ 37.7326 │\n", + "│ VAETrainingPhase │ 20.0 │ 45.2383 │\n", "└──────────────────┴───────┴─────────┘\n", "┌──────────────────┬───────┬─────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼─────────┤\n", - "│ VAETrainingPhase │ 21.0 │ 37.6447 │\n", + "│ VAETrainingPhase │ 21.0 │ 44.8402 │\n", "└──────────────────┴───────┴─────────┘\n", "┌──────────────────┬───────┬─────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼─────────┤\n", - "│ VAETrainingPhase │ 22.0 │ 37.5145 │\n", + "│ VAETrainingPhase │ 22.0 │ 44.5037 │\n", "└──────────────────┴───────┴─────────┘\n", "┌──────────────────┬───────┬─────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼─────────┤\n", - "│ VAETrainingPhase │ 23.0 │ 37.3824 │\n", - "└──────────────────┴───────┴─────────┘\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ + "│ VAETrainingPhase │ 23.0 │ 44.2519 │\n", + "└──────────────────┴───────┴─────────┘\n", "┌──────────────────┬───────┬─────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼─────────┤\n", - "│ VAETrainingPhase │ 24.0 │ 37.2937 │\n", + "│ VAETrainingPhase │ 24.0 │ 43.9507 │\n", "└──────────────────┴───────┴─────────┘\n", "┌──────────────────┬───────┬─────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼─────────┤\n", - "│ VAETrainingPhase │ 25.0 │ 37.2094 │\n", + "│ VAETrainingPhase │ 25.0 │ 43.9711 │\n", "└──────────────────┴───────┴─────────┘\n", "┌──────────────────┬───────┬────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼────────┤\n", - "│ VAETrainingPhase │ 26.0 │ 37.139 │\n", + "│ VAETrainingPhase │ 26.0 │ 43.592 │\n", + "└──────────────────┴───────┴────────┘\n", + "┌──────────────────┬───────┬────────┐\n", + "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", + "├──────────────────┼───────┼────────┤\n", + "│ VAETrainingPhase │ 27.0 │ 43.496 │\n", "└──────────────────┴───────┴────────┘\n", "┌──────────────────┬───────┬─────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼─────────┤\n", - "│ VAETrainingPhase │ 27.0 │ 37.0836 │\n", + "│ VAETrainingPhase │ 28.0 │ 43.5969 │\n", "└──────────────────┴───────┴─────────┘\n", "┌──────────────────┬───────┬─────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼─────────┤\n", - "│ VAETrainingPhase │ 28.0 │ 37.0444 │\n", + "│ VAETrainingPhase │ 29.0 │ 43.5515 │\n", "└──────────────────┴───────┴─────────┘\n", - "┌──────────────────┬───────┬────────┐\n", - "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", - "├──────────────────┼───────┼────────┤\n", - "│ VAETrainingPhase │ 29.0 │ 37.021 │\n", - "└──────────────────┴───────┴────────┘\n", "┌──────────────────┬───────┬─────────┐\n", "│\u001b[1m Phase \u001b[0m│\u001b[1m Epoch \u001b[0m│\u001b[1m Loss \u001b[0m│\n", "├──────────────────┼───────┼─────────┤\n", - "│ VAETrainingPhase │ 30.0 │ 36.9836 │\n", + "│ VAETrainingPhase │ 30.0 │ 43.5932 │\n", "└──────────────────┴───────┴─────────┘\n" ] } @@ -634,7 +616,7 @@ "fitonecycle!(\n", " learner,\n", " 30,\n", - " 0.01,\n", + " 0.01;\n", " phases = (VAETrainingPhase() => dataiter,),\n", ")" ] @@ -648,203 +630,25 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import CairoMakie # run if in an interactive environment like a notebook\n", - "showoutputs(learner, method, n = 3)" - ] - }, - { - "cell_type": "code", - "execution_count": 43, + "execution_count": 55, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABLAAAAl0CAIAAACP7Rn6AAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nOzdeZDX9X3H8c9v98e13IIHeMd6QDyikGhJR4JSTKJVJmirkqikzjQmMVXTqhmtjiaZkDZVYz1iNFOvFjXGI7Y6iEGJifcKNvFAMRgUD04VWJa9fv3jN9nZ2YUNn4Xf/oD34/GX/nj9fr8PP3GX5373KJRKpQQAAEA8NdU+AAAAANUhCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQghG3Ol7/85UKh8LWvfW07ffxtVvk3fvbZZ1f7IAAA2wpByI7gn/7pnwrdKhaL1T4jAGTbsGHDLbfcctJJJ+21114DBgwYOnToQQcddPbZZz/22GNb5fGnTp1aKBS++c1vbpVH276eHSjzt2QAgG3RE088ceaZZy5ZsqT9lsbGxo8//njhwoU/+9nPJk+e/F//9V+77LJLFU8I7ABcIWTHMWLEiNImtLS0VPt0AJDhgQcemDJlypIlS4YOHfr973//lVdeaWhoWL169bx5884444xCofDYY48deeSR7777brVPCmzfXCEEANi2vPnmm2eccUZzc/MnPvGJuXPn7r333uXbBwwYcPTRRx999NEnnHDC6aef/tZbb5166qlPPPFETY0P8QM95M0HEU2ePLlQKFx88cXr1q274oorDjnkkIEDBw4cOPAzn/nMLbfcstG7vP322xdeeOGnPvWpoUOH1tXVHXTQQdOnT3/sscdKpVKn5RNPPHHKKafsvvvuffv2HTFixKRJk26++eZNXaK84447JkyYMHjw4MGDBx922GHf/e53Gxoaujn5kiVLLrjggoMPPnjw4MF1dXVjx469+OKLV65cual97uNv1F133fX5z39+l1126du376677nrEEUd885vffO655zpuGhoaHnjggbPOOuuwww4bMWJEv3799thjj2nTps2ZM6frA7a//h9++OHFF188ZsyYurq6nXba6dhjj509e3Z588EHH3z7298+4IADBgwYMGLEiGnTpi1cuHBTj7Nq1aoLL7zwoIMOqqurGzZs2OTJkx9++OGs32PuCwtQUZdddtmaNWv69Onzi1/8or0GOzrllFMuvfTSlNKTTz55//33t9++zz77FAqFW2+9tetdTj755I5fsHfvvfcWCoUHH3wwpXT99dd3/ML7F154obzJfTO7dZ8d6CWb+hQ72I58+9vfTt1+ymgnxx57bErpy1/+8v7779/1f4rzzz+/0/7222/v37//Rv8Pmj9/fsfl+eefv9HZZz/72VWrVnVctra2fuUrX+m6/OQnP/mFL3whpfQP//APnY4xa9asjR5jt912e+mllzqNe/D4G3XmmWdu6q3H8uXL22eXX375pmYXX3zxRl//008/vevfcgqFwp133llfX7/zzjt3+qXhw4e//vrrXR/ntNNO22OPPbo+72WXXdbpeadPn55S+vu///steWEBKm3FihXl74U2ffr0bmbr1q0bNmxYSmnSpEntN5bfrv7nf/5n1/20adNSSt/4xjfK//rzn/98U2+3n3/++fIm983s1n12oHe4Qkhcd95553vvvffjH/946dKl69evnz9/fvk939VXX/273/2uffbwww+feeaZjY2NhxxyyL333rt8+fL169cvXLhw1qxZkydP7vhZOtddd93VV1+dUjrxxBPnz5/f2Ni4ZMmSyy+/vLa29re//W2nPLvmmmvuuOOOlNKZZ5756quvbtiw4Y033jjnnHNefvnlRx55pOtp586dO3369MbGxokTJz722GNr1qxpaGiYM2fOoYce+v7770+dOnXt2rVb8vgb9fjjj992220ppQsuuOCVV15Zt27d6tWrFyxY8JOf/OTII48sFArty4EDB5566qn333//q6++umbNmlWrVj333HMzZsxIKc2cOXOjz/jf//3fq1evvvHGGz/44IM1a9bMmTNn3333LZVK55133kknnTRgwIB77rnnww8/XL169d133z18+PDVq1f/8z//c9fHmTVr1rvvvnvZZZctWbKksbHxxRdfPP7441NKV1555S9/+cs/+3vMfWEBKu3Xv/51+fNKTjnllG5mdXV1J5xwQkrpqaee2rBhQ+6znHzyyaVS6aSTTkodOq1s/PjxHZdb+GZ2C58dqLhejE+olPIVwm50uhpWDr+U0ty5czve/vHHH48cOTKldPnll5dvaW5u3nfffVNK48ePX7t2bTdnaGho2GmnnVJKX/ziF1tbWzv+0jXXXFN+uscff7x9PHz48JTSV77ylU6Pc84553Q9c2tr64EHHphSmjBhQktLS8f9ypUrR4wYkVL693//946HyXr8TSlf9/vc5z73Z5eb8o//+I8ppeOPP77jje2v/7x58zre3v75pYMHD168eHHHX/qP//iPlFKxWFy/fn3Xx/nXf/3XjuOWlpbPfe5zKaWxY8d2vL3rFcLcFxagF3z3u98tv3F7++23u19eddVV5eXvf//78i2bf42ubKNJ1i73zezWfXagd7hCSFxHH330pEmTOt4yePDgY445JqX02muvlW95/PHHFy9enFK69tprBw4c2M2jzZkzZ9WqVSmlH/7wh52+uP/cc88tv4+cNWtW+ZZHH3109erVhULhe9/7XqfHufLKK/v06dPpxieffLL8FXTXX399bW1tx1/aaaedyp+net9997XfmPv4m1L+RMot+R6t5b8BPPPMM11/qfx9ETrecswxx5TPdvrpp++zzz4df+m4444rn2TRokWdHmfnnXc+77zzOt5SW1tb/uvUK6+80vFib1e5LyxAL2j/Aubyxyi70f7Z9RX9mucteTMLbPsEITuObr6G8Cc/+UnX/eGHH971xj333DOl9PHHH5f/9amnnkop7bLLLn/5l3/Z/bOXvwh+9913P/jggzv9Uk1NzZQpU9o3KaX6+vqU0oEHHrjXXnt1Go8cObLrwX7729+mlHbeeedPfepTXZ/6iCOOSCktWLCg/Zbcx9+U8geAf/Ob35x66qmPP/5495+S9NZbb1144YXjx48fPnx4bW1t+XsDlJNv5cqVXauy6zFqamp23XXXlFLX3+bo0aPL/9D+n6bdxIkTuybuhAkT6urqUofXfKNyX1iAbUqpyzc2q4QteTMLbPsEIXFt9PuIlL+Ov62trfyv77//fkqp/Fmj3Vu2bFn6U092VQ6z8qb9Hzb6NfobfZClS5emlJYvX14sFovFYm1tbW1tbU1NTU1NTaFQ+OIXv5hSWrduXXuw5T7+phx11FEXXXRRSunuu+8+5phjhg4detRRR33nO9/p+vHgBx98cMyYMf/2b/9WX1//4Ycftr+A7RobGzvdstHXv3yZrusvtV++6/rIG/1t1tTUjBo1KnV4zTcq94UF6AXlz1dPKa1YsaL7Zfug/S6VsCVvZoFtnyCEP6/jd0/ZkmWnX938h21tbW3/h9bW1ra2tra2tvLFz46z5ubmnj1+N2bOnPnss89+7WtfO+CAAzZs2PDss8/OnDnz0EMPPfvss9tPtXLlyvL33Rk3btw999yzePHihoaG8gmfffbZLT9D97bkt9mzFxagosaOHVv+hxdffLH7ZXnQr1+//fbbr3Ln2SrvTYBtliCE7pQ//PmHP/zhzy7LX8ixZMmSjf5q+fb2L/bYZZddUkpvv/32RsfvvPNOp1vKn0j56U9/uvuvCR40aFDPHr97n/nMZ2688caFCxeuWLHigQce+NKXvpRS+tnPfnbttdeWBw8//PBHH300bNiwuXPnnnLKKfvss8+AAQPKf4HohY8cb/S32dbW9t5776U/vRSbkvvCAvSCiRMnlj8topufzZBSWr9+/f/+7/+mlCZMmND+iRXlL2Jv/2hXp33PzrP5b2Yr8exApQlC6M6ECRNSSsuWLXv66ae7X5a/TfbSpUtffvnlTr9U+tP3z2z/Vtrjxo1LKS1cuLDre9kVK1bMnz+/041/9Vd/lVJasGDBZvZV7uNvphEjRpx00km/+MUvpk6dmlIq/0Uk/envCgceeOCQIUM63WWjP5h+65o3b17XK3hPPfVUQ0ND6vCab1TuCwvQC0aMGPG3f/u3KaW77777//7v/zY1+9GPfrR69eqU0te//vX2GwcPHpw29oG/tra2jX5FdLk8u/9axM1/M1uJZwcqTRBCdyZNmvSJT3wipfStb32r/J5vU6ZMmVL+sRPf+c53Or1vu/766996662U0mmnndY+HjZsWKlUuvTSSzs9zuWXX97U1NT1GPvtt19zc/O3vvWtrl9Et9HDZD1+rvI3vms/ydChQ1NKCxcu7PQj+15++eVbbrllC5/rz1q+fHn7D/Yoa21tveyyy1JKY8aMOeSQQ7q5b+4LC9A7rrjiikGDBjU3N0+bNm2jF+juv//+8vf5/OxnP1v+xI2yMWPGpJQeeuihTvubb7753Xff7fo45Q/kdf/Fipv/ZrYSzw5UmiCE7tTW1l533XWFQuGFF1446qij7rvvvpUrVzY2Nr7xxht33XXXlClT2j92O2DAgCuuuCKl9NBDD02bNu2ll15qamp65513rrzyyvJ36z7++OPL37SzPL7kkktSSrfffvuMGTMWLlzY1NT05ptvfv3rX7/hhhu6HqNYLN50003FYvHuu++eOHHigw8+uGzZspaWlmXLli1YsOC2226bNm1ax58wkfv4m3Luueeefvrps2bN+t3vfrdq1aoNGzYsXrx45syZt956a0qp/K1Ty/9QU1Pz4YcfTp06tb6+vrGxcdmyZTfffPPEiRO7r+it5eKLL7788svfeeedpqamBQsWTJ069fHHH08pff/73+/+jrkvLEDv2H///W+99dY+ffosWrTo0EMP/cEPfvDaa681NjZ+9NFHv/nNb2bMmDFt2rTm5ua99trrrrvu6vizjk4++eSUUn19/YwZM15//fWmpqY//OEPl1566Te+8Y2NPlE55x599NF58+Z181mdm/lmtkLPDlTW5vywQtjG/dkfTJ9Sev7559v35Z+0e9FFF3V9qPI31TzuuOM63njbbbf169dvow87f/78jsvyT67rasKECatWreq4bG1tPf3007sux44d+4UvfCFt7AfH33///V0/J7Ndp99ODx6/q/JPct+oKVOmbNiwodPr1kmxWGy/fc2aNZvz+m/qhxq3/0XhySef7PQ4p5566u6779712S+55JKN/nY6/mD6HrywAL3mscce29T3i04pTZo06b333ut0l7a2tuOPP77reMKECeWP4nX6KfBLly7t+gaw/T1m7pvZrfvsQO9whRD+vDPOOOO1114777zzxo4dO3DgwEGDBo0ZM2b69Olz5sw57LDDOi6vuuqquXPnTps2bdSoUX369Bk+fPjEiRN/+tOfzps3b/jw4R2XNTU1d95556233nrkkUcOHDiwrq7u4IMPvuyyy5577rnyp552NXXq1EWLFl1xxRVHHnnk8OHD+/TpM2rUqMMPP/yrX/3q/fff/y//8i9b+PhdXXfddbNmzZo+ffohhxwyfPjwYrG46667fv7zn7/zzjsfeeSRvn37ti9nzpx5xx13HHXUUXV1df37999nn33OOuusl156qfzR4orae++9FyxYcP755x9wwAEDBgwYMmTIpEmTHnrooc2/spf1wgL0mmOPPfaNN9646aabTjjhhD322KNfv36DBw/ef//9Z8yYMXv27Llz5+62226d7lIoFO67774f/OAHn/zkJ/v37z9o0KDx48dfddVVTzzxRPkL/DoZPXr0vHnzTj755FGjRpV/8FJXm/9mthLPDlRaoeQLeYHt0+TJk3/1q19ddNFFM2fOrPZZAHZA3sxCBK4QAgAABCUIAQAAghKEAAAAQQlCAACAoHxTGQAAgKBcIQQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABFWs9gF2HCtXrqyvr6/2KQB2EOPGjRsxYkS1T7HVeB8BsBXtYO8jqksQbjX19fXHHXdctU8BsIOYPXv2lClTqn2Krcb7CICtaAd7H1FdPmUUAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFDFah8AAADouUKhkLUvlUoVOgnbI1cIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAiqWO0DAACwuWpra7P2ra2tFToJ245SqVTtI7Adc4UQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKCK1T4AAMAOYtiwYVn7UqmU+xQ9uEuWmpq8qwX9+vXL2tfW1mbte6C5uTlrv27duqx9U1NT1j6l1NbWlrXP/a9c6T8V7NhcIQQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgqGK1DwBVduCBB2bt58yZk7Xffffds/Y98Oijj2btp0yZUqGTlF166aUVffweOO+887L2I0eOrNBJ2v34xz/O2l9wwQUVOglUTv/+/Su6HzZsWNY+pTR06NCsfV1dXdZ+yJAhWfvc8/TgKQYNGpS1HzhwYNa+X79+Wfva2tqsfUqpUChk7Zubm7P2K1euzNq/+eabWfuU0osvvpi1X758eda+paUlaw8duUIIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIKhitQ8AVTZjxoys/ejRo7P2pVIpa98Df/3Xf521r/SRvve97+XepRdepSy9cJ6/+7u/y9rfdNNNWfuFCxdm7WFz9O/fv6L7kSNHZu333XffrH1Kae+9967oU+y5555Z+9122y1rn1IaOnRo1r5fv35Z+2Ix7y+HuW8wm5ubs/YppdbW1oruP/7446z9q6++mrVPKQ0aNChr/8ILL2Tt33zzzaw9dOQKIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBFat9ANjK9ttvv6z99OnTK3QS6MaaNWuy9m+//XaFTkJYffv2zb1LoVDI2vfp0ydrP3DgwKz9XnvtlbVPKR1++OFZ+4MOOihrP3r06Kx97m85pdTU1JS137BhQ9a+sbExa9/Q0JC1zz1Pyv+DNHz48Kz9gAEDsvYffvhh1j6ltPPOO2ftBw8enPsU0GOuEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQVLHaB4Ct7Oyzz87ajxo1qkIn6TX19fVZ+/feey9rf8IJJ2Tt2Rw/+tGPsvYNDQ0VOglhNTU15d6lf//+lThJb2pra8va5/6v9+6772btP/roo6x9Smnp0qVZ+3feeSdrv2LFiqx9Y2Nj1n7w4MFZ+5TSmDFjsvaf/vSns/a5RyoWs//+3NLSUtF9TU3eNZ7c/xFSSoVCIWtfKpVyn4JqcYUQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKCK1T4AbGXTp0+v9hG21FtvvZW1P/7447P2q1evztofccQRWftnnnkma78NWrVqVe5dzj333Kz9o48+mvsUsHXV1tbm3qW1tTVr39LSUtHHX7duXdY+pbRmzZqs/R//+Mes/cqVK7P2b7/9dtY+pfT6669n7RcvXpy1b2pqytoPGjQoa7/nnntm7VNKo0ePztrn/tmuqcm7QLJ27dqsfUrpgw8+yNrn/tlua2vL2vdAqVSq9FNQLa4QAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAiqWO0DAJ3dcMMNWfsVK1ZU6CRl77//fkUffxv00EMP5d7l7rvvrsRJoHJaW1tz71IqlbL2zc3NWfumpqas/fr167P2KaU333wza79q1aqs/fLly7P2H330UdY+pbRs2bKs/Zo1a3KfoqKKxey/fO67775Z+2HDhmXtGxoasvZ//OMfs/Yp/z3p4sWLc58CeswVQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCKlb7ALCVLVq0KGs/cuTIrH3fvn2z9k899VTWPqU0a9as3Ltkqa2tzdpfcsklFTpJr1mzZk3W/uqrr67QSWC71tbWlrVvbm7O2jc2Nmbt33///ax9SumDDz7I2q9evTprn/tbaGlpydqn/P8KxWLeX/YKhULWvq6uLmt/8MEHZ+1TSgcddFDWvk+fPln79957L2u/ePHirH3K/4MHvckVQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCKlb7ALCVHXPMMVn7E088MWs/dOjQrP2cOXOy9iml999/P3dHjE4AACAASURBVPcuWc4666ys/dlnn12Zg/Sec845J2v/+9//vkIngVDa2tqy9mvXrs3aL1myJGuf8o/U3Nycta+pyftQe21tbdY+pVQoFCr6FH379s3a/8Vf/EXWftKkSVn7lP+ed82aNVn7N954I2u/ePHirH1Kaf369Vn73P/KpVIpaw8duUIIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFDFah8AquyXv/xltY+wpYYNG5a1P/fccyt0kl7z61//Oms/e/bsCp0E6EapVMrar127Nmu/fv36rH0PFAqFrH2xmPc3q5qa7A/N576quYYPH561nzRpUtZ+zJgxWfuU/6q+8847Wfv58+dn7ZcuXZq1Tyk1NDRk7Wtra7P2LS0tWXvoyBVCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIqVvsAwJb60pe+lLU/+OCDK3SSHnv66aez9ieeeGLWfu3atVl7YKtoa2ur6L61tTVrn1IqFApZ+5qavA+dl0qliu57oH///ln7MWPGZO2PPvrorP1OO+2UtU8pLV26NGv/3HPPZe3r6+uz9suWLcvap5TWrVuXtW9pacl9CugxVwgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABFWs9gGAzsaMGZO1v/nmmyt0kp6pqcn+SNM111yTtV+7dm3uUwC9r1QqbdeP34OnaG1trdBJ2vXp0ydrP3r06Kz95MmTs/b77rtv1r4H/9VefvnlrP3DDz+ctV+0aFHWfv369Vl72Ma5QggAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUMVqHwDo7Kijjsral0qlCp2kZ55//vncu/zP//xPJU4CbF+2tbdmKaXW1taKPn6hUMi9y5AhQ7L2hxxySNb+2GOPzdr369cva7948eKsfUrpnnvuydq/8sorWfv169dn7WEH4wohAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEVq30A2MGNHDky9y7nnHNOJU7SayZPnpx7lw0bNlTiJAA7nqFDh2btv/rVr2btd9lll6x97hvwu+66K2ufUnryySez9uvWrct9CojMFUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAgipW+wCwg7v22mtz73LEEUdU4iS95uOPP672EYBtQqlUqvYRtnU77bRT7l3+5m/+Jms/fvz4rH1bW1vW/qWXXsra33vvvVn75N0KVJgrhAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCKlb7ALCDGzlyZLWPsKVuuOGGah8BYPuQ+zb/0EMPzX2KGTNmZO1ravI++r9q1aqs/Y033pi1f/fdd7P2KaXm5ubcuwCbzxVCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIqVvsAsJ0ZOnRo1n7IkCEVOkmPtba2Zu1feOGFCp0EYBs3atSorP1uu+2WtT/ttNOy9imlnXfeOWvf2NiYtX/mmWey9k8//XTWPvd9EFBprhACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEFSx2geA7cz48eMruu8FP/3pT7P2t99+e4VOAtDLamtrs/YDBw7M2h9xxBFZ+3HjxmXtU0qlUilrv3z58qz9z3/+86z9Rx99lLXPPT9Qaa4QAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABBUsdoHgO3MySefXO0jbKlXX3212kcA2IhiMe+vJX369Ml9iv79+2fthwwZkrU/9NBDs/Z9+/bN2qeUGhoasvYvvfRS1n7+/PlZ++bm5qx9qVTK2gOV5gohAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKCK1T4AbGfGjh1b7SN01tbWlrVvamqq0EkAtkRLS0vWvn///rlPUVtbm7Wvq6vLfYosy5cvz71L7tvwRx55JGu/atWqrH2pVMraA9saVwgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACKpY7QPAdubMM8/M2s+ePTv3Kfbbb7+s/a233pq1v+WWW7L2ANumtWvX5t5l3bp1Wftly5Zl7X/1q19l7RctWpS1TymtXr06a//ss89m7Zubm7P2wPbOFUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAgipW+wCwnXnrrbey9gceeGBlDgJAtlKplLV//fXXK7oHqDpXCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIqlAqlap9hh3EypUr6+vrq30KgB3EuHHjRowYUe1TbDXeRwBsRTvY+4jqEoQAAABB+ZRRAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIKhitQ+w41i5cmV9fX21TwGwgxg3btyIESOqfYqtxvsIgK1oB3sfUV2CcKupr68/7rjjqn0KgB3E7Nmzp0yZUu1TbDXeRwBsRTvY+4jq8imjAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAARVrPYBAACCKhQKlX6KUqmUta/0kXLPA1SaK4QAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABFWs9gEAAHpJoVDI2vfp0ydrX1tbm7XPPU8P7tLa2pq1b2lpydq3tbVl7XugVCpV9PF78F9hW1Ppl4gdmyuEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAARVrPYBAAB6Sd++fbP2hUKhovuamuwPzZdKpdy7VPTxK73vBb1wpNw/GNCbXCEEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEFSx2gcAAOglNTWV/VB4oVDI2vfgPKVSKWvf0tJS0cdnc+S+qrl/kHL3PeAPxg7MFUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAgP9n125jLK/vOu7/zsyZu70ZdtldWCjsjQVaNA0oqQWqgq1Snmka0WCMprXxLqlJExVjHxgTtdEYNajExgRbkvaJKTYoJjUFQ62iIqZlk0YkUOgWtsvO7uzM7M6cmTlzzvWgSa+mVy/qZ+TsYff7ej3+/M//N3POMvOePxQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFNUd9wGAS83b3va29JKdO3dG+xMnTkT7l156KdoDF4Xdu3enl0xMZH8Kn5qaivZzc3PRfnZ2Ntq31nq9XrRfWFiI9p1OJ9oPh8ORvv42Lkl/pqRfQrpv+bs2GAyi/TaOBN/gCSEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARXXHfQDgW3W72T/MK664Itrfc8890f79739/tL/++uujfWttdnY22i8sLET7e++9N9o/9thj0R74tjqdTrS/7LLLov309HS0b63t3bs32h8+fHikrz8YDKJ9a+3UqVPRfteuXdH+3Llz0b7f70f7ycnJaN9a27NnT7RPf6acP38+2p8+fTrat9aGw2G039jYiPbpuwDfzBNCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIrqjvsAcJHZv39/tL/99tvTW/z6r/96tH/nO9+Z3uJi9/nPfz7aP/bYYyM6CVy8Op3OqG+xe/fuaL9jx45of+DAgWjfWrv++uuj/Zvf/OZov2vXrmh//vz5aL+NW8zPz0f71dXVaD8cDqP9zMxMtN/GJYPBINovLCxE+16vF+1ba5ubm9G+3+9H+/Sfc/qucWnzhBAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUd1xHwBeyw/+4A+ml/zAD/xAtP/Jn/zJaH/llVdG+4MHD0b7N6DTp09H+62trfQWDz74YLT//Oc/n94C+L+bnZ2N9t1u9mtG+vpXXXVVtG+t3XjjjdH+2muvjfbpfwDPnz8f7Vtr586di/ZLS0vRvtfrRfuZmZlov3PnzmjfWtu1a1e0n56ejvaDwSDaLy8vR/uWv2udTie9BWybJ4QAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFNUd9wGoZe/evdH+oYceSm9x+PDh9JI3mv/4j/+I9o8++mi0P3bsWLR/5JFHov1gMIj2wFjMzMyklwyHw2g/MZH93XnHjh3Rfhv/wT9y5Ei037lzZ7R/8cUXo/3//M//RPvW2nPPPRftl5aWon36Ls/NzUX7bfyM2LdvX7Q/cOBAtE8/qAsLC9G+tTY9PR3tO51OegvYNk8IAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAACiqO+4DUMtb3vKWaH/48OERneQbvvzlL0f7j33sY9H+E5/4RLRvrb388svRfn19Pb0FwMbGRnrJzMxMtJ+eno72+/bti/bpz5TW2t69e6P9yspKtH/++eejffozqLW2sLAQ7dOfERMT2dOCycnJaL+1tRXtW2tzc3PR/oorroj2u3btivavvPJKtG+tnThxItqn78LU1FS07/f70b61NhgM0ku4WHhCCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBR3XEfgFruvvvucR/hW911113R/vnnnx/RSQAupLm5ufSSTqcT7Xfu3BntDx8+HO2vueaaaN9am5qaivanTp2K9sePH4/2J0+ejPattfPnz0f7wWAQ7aenp6N9+i3dvXt3tG+tXXHFFdE+/WCk39LLL7882rf8u9rtjvZX9OFwOOpLtnELxsUTQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFdcd9AGr5rd/6rVHf4p//+Z+j/Ze//OURneSCueGGG6L95OTkiE7yddv4lvZ6vVGcBHgNa2tr6SW7d++O9rt27Yr2V111VbSfm5uL9q214XAY7ZeWlqL9qVOnov3Kykq0b61tbGxE+/RduPzyy6P9/v37o/2VV14Z7VtrR44cifYHDx6M9um7tmPHjmjfWpufn4/2i4uL0b7f70f7bfwmkN6Ci4gnhAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAU1R33AajlxRdfjPbXXXddeot+vx/tB4NBeovIH/zBH6SX3H777dH+1ltvjfaTk5PRPvXFL34xveT555+P9g8//HC0/+QnPxntoYJt/NdvYiL7O/Ls7Gy03717d7SfmpqK9i3/GdHtZr8p7dq1K9rv27cv2rfWZmZmov38/Hy0T9+Fyy+/PNofPXo02rfWjhw5Eu337NkT7ZeXl6P93NxctG/5uzbqn9TwzTwhBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKCo7rgPQC2f+tSnov19992X3uLOO++M9jfccEO0f+CBB6L9u971rmh/CbjppptGfcl73/veaP+xj30s2v/Lv/xLtG+t/dqv/Vq0/+IXvxjt+/1+tIfvqNPpjPqSUe+73fjXmNnZ2Wh/+PDhaH/LLbdE+0OHDkX7bdixY0e0n5+fj/Z79+6N9umP3dbavn37ov3U1FS0n5ubi/bpp6jln9V0PzGRPeMZDAbRvuVH2tzcTG/BuHhCCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBR3XEfgFo+/elPR/v77rsvvUWn04n2jzzySLS/4YYbov02PPfcc9H+93//96P9sWPHon1qYiL+S9P73ve+aH/PPfdE+/3790f7O+64I9q31p566qlo/+CDD0b73/md34n2x48fj/YUNBwOR32Lfr8f7VdWVqJ9r9eL9q21ffv2RfujR49G+x07dkT7M2fORPvW2mAwiPbT09PRfm5ubqT7+fn5aN9a29zcjPbplzw5ORntL4Bt/CQdtfT3Ky4ib7hPGwAAABeGIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUd1xH4Ba/v3f/z3aP/bYY+kt3v3ud0f7G264Ib1FZH19Pb3kZ37mZ6L9U089ld7ijeY///M/o/2HPvShaH/vvfdG+5/92Z+N9q21H/7hH47273//+0f6+um+tfaVr3wlvYRqNjc3o/3y8nK0Tz+EL7zwQrRvrU1MZH8K37VrV7Tfv39/tN+5c2e0b61tbGxE+8FgEO37/X60X1tbi/bpp6jl79rs7Gy07/V60T59C7Zha2sr2qfv8nA4jPbbu4SLhSeEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIrqjvsA8Foefvjh9JJ3v/vdozjJtv35n/95eslTTz01ipNcSjY2NqL9xz/+8Wj/iU98Itq31n7+538+2j/wwAPR/ujRo9H+t3/7t6N9a+0Xf/EXo32/309vwcWu1+tF+7Nnz0b7Z599Ntpv40P4yiuvRPtrrrkm2s/NzUX7bXwJa2tr0f78+fMj3aefivn5+WjfWut0OiO9xcrKSrRPv+TW2urqarRP3+XBYBDth8NhtG/+m39J84QQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKK64z4AvJaHHnooveQv/uIvRnGSbfvbv/3bcR+BWL/fTy/56Ec/Gu1vuOGGaP+hD30o2r/vfe+L9q21Rx99NNo//PDD6S242G1tbUX75eXlEZ3k686dO5de8sILL0T7vXv3Rvtdu3ZF+243/k1sfX092m9sbET7tbW1aD8YDKL9oUOHon1r7Zprron26ZeQ7peWlqJ9a21xcTHap+9y+i4Mh8Nov71LuFh4QggAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUd1xHwBey+bmZnrJU089Fe3f/va3p7eI3Hrrrekl//qv/zqKk/CG8pu/+ZvR/rbbbov22/jg3X333dH+4YcfTm/BxW44HEb7jY2NaL+wsBDtz507F+1ba51OJ9qnX3L6+pOTk9G+5Ueampoa6X52djba79y5M9q31nq9XrQ/f/58tF9aWor2i4uL0b7lX8JgMIj26adiG79fcQnzhBAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAorrjPgC8lo2NjfSSD3zgA9H+ySefjPY7duyI9jt37oz2FLG5uRntn3322Wh/6623RvvW2pEjR9JL4LVtbW2N9PVXV1fTSzqdTrQfDofpLUb9+umXMDU1NdJ9ahs/2c+fPx/tT58+He2Xl5ej/TY+eOm/hfRdTn+mwDfzhBAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUd1xHwBeZ8eOHYv2Dz30ULT/0R/90Wi/uLgY7eHb+uxnPxvtf+7nfi69xY/8yI+kl1DNcDiM9p1OZ0Qn+brBYJBeMuojpd+iC2BzczPap1/CxsZGtO/1etG+tXbmzJlov2fPnmjf7/ej/TakH7zJyckRnQT+vzwhBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKCo7rgPAGP2K7/yK+M+Anxnb37zm8d9BIgNh8No3+l0RnSSb3gDHimVfgnpvt/vR/u1tbVof/r06WjfWltYWIj2l19+ebTf2tqK9jt27Ij2rbXZ2dlon35X4f/CE0IAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAiuqO+wAAfGeHDh2K9seOHUtv8eyzz6aXwOtrOBxG+06nM6KTfEN6pEtA+iVvbm5G+5WVlWjfWjt58mS0P3DgQLTv9/vRfnZ2Ntq3/LM6GAyi/cRE9ownfX0ubZ4QjvHzkQAAIABJREFUAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFBUd9wHAOA7+8AHPjDuI8AbznA4HPcRaJ1OJ9qvrKykt3jxxRejfbeb/X47OTkZ7U+cOBHtW2urq6vRfmtrK9oPBoNoD9/ME0IAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARXXHfQAAAN4ohsNhtO/3+9F+MBhE+9baSy+9FO3Pnj0b7aempqL96upqtN/GJefPn09vAdvmCSEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARXXHfQAAAC5Ww+FwpPvWWq/Xi/b9fj/adzqdkb5+a21jYyO9BC4YTwgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKKo77gMAAHCxGg6H0X5rayu9RXrJ5uZmeguozBNCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIrqDIfDcZ/hEnH69Omnn3563KcAuETccsst+/btG/cpXjd+RgC8ji6xnxHjJQgBAACK8r+MAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEV1x32AS8fp06effvrpcZ8C4BJxyy237Nu3b9yneN34GQHwOrrEfkaMlyB83Tz99NPvec97xn0KgEvEZz7zmbvuumvcp3jd+BkB8Dq6xH5GjJf/ZRQAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKK64z4AAAD8/+p0OiPdXwDD4XCke/i/8IQQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKK64z4AAMB2dDqd9JLhcDiKk3zD5ORktN/GlzBq6ZFGvb8Atra2xn2EbzUYDKJ9+sEe9T8ELi6eEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoqjvuAwAAl6ZOpzPS/eTkZLTfxiUTE9mfztPX38aX0O1mv7yN+kjpuzYcDqN9a21rayva9/v9aJ8eaTAYRPvW2sbGRrRPv4T0SOm3tG3rjeNi4QkhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEV1x30AAODi0Ol0ov3k5GS0n56ejvYzMzPRvrU2NzcX7S+77LJov3///mh/1VVXRfvW2hVXXBHt0y9hYiJ7WtDr9aL92bNno31r7eTJk9H+zJkz0X5lZSXar62tRfvW2urq6khvsbGxEe3X19ejfWut3+9H+8FgkN6CcfGEEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiuuM+AIzZkSNHov299947moP8v+65555of/PNN4/oJF/3uc99Ltrfdddd6S02NjbSS4ALb2Ii+zvyzMxMtN+xY0e037NnT7RvrR08eDDaHzp0KNpfd9110f7o0aPRvrV24MCBaD83NxftJycno/1gMIj2S0tL0b619pWvfCXaP//889H+5ZdfjvavvvpqtG+tLS4uRvv031qq3++nl6RvdLpnjDwhBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKCo7rgPAGP20EMPRft3vvOdIzrJtg2Hw5G+/g/90A9F+4WFhfQWH/nIR6L9yspKeouL3RNPPBHtjx07NqKTcMnodDrpJRMT2d+Rp6amov2OHTui/Z49e6J9a+2KK66I9ocOHYr2V199dbRPv+TWWq/Xi/arq6vRfnJyMtrPzMxE+2188Hbt2hXtL7vssmi/uLgY7bvd+Pfn9Cf11tbWG2rf8i8hfaNH/csMr8ETQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFdcd9AGp561vfGu3379+f3uKOO+6I9t///d8f7RcWFqL9P/zDP0T7bXjHO94R7d/ylreM6CRft3PnzvSS3/3d3x3FSS4lv/qrvxrtjx07NqKTwP9ep9MZ6X5iIv679tTUVHpJZGVlJdovLS2ltzh79my0T4/U7Wa/HO7duzfaz87ORvvWWq/Xi/anTp2K9mfOnIn2i4uL0b61try8HO3X1tai/cbGRrTf2tqK9q214XA40j1j5AkhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEV1x30Aavn7v//7aL9v3770FrOzs9H+7Nmz0f6nf/qno/3jjz8e7bfhyiuvjPY/9VM/Fe1/4Rd+IdrfeOON0f7S8KUvfSnanzx5Mtp/8pOfjPbwHQ2Hw/SSzc3NaN/v90f6+ul+G5ecO3dupK+/srIS7Vtrr7zySrRfXl6O9umP0YMHD0b7+fn5aN9aW19fj/avvvpqtE//g3z69Olo3/IPUq/Xi/bpv7VOpxPt27b+i8HFwhNCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIrqjvsA1PI3f/M30f43fuM3RnSSb3jmmWei/eOPPz6ik2zbyZMno/39998f7dN37Zprron2rbX3vOc90f748ePR/tZbb432Dz74YLRvrX31q1+N9pubm9F+cXEx2sMbQa/Xi/Zzc3PRfn19Pdq31lZXV6P92tpatE//aZ89ezbat9ZWVlaifXqk2dnZaD8cDqP98vJytG+tnTt3LtqfPn062qfvQvoWtPxdGAwG0b7T6UT7fr8f7VtrExMeI12yvLUAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFNUd9wGo5eWXXx73Eb7V/Pz8SPdXXnlltG+tLS4uRvuFhYX0FpETJ06MdN9ae+qpp9JLIh//+MdH+vrAt7W1tRXtNzY2on2/34/2LT9Seovp6elov2vXrmjfWpuYyP6a3+l0ov3OnTujfbeb/TJ59uzZaN9aW15ejvZLS0vR/ty5c9H+AnzwUuvr6yN9/dbaYDAY9S0YF08IAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABTVHfcBYMze/va3R/tHH3002h89ejTat9ZOnDgR7Y8fP57eIvK5z30u2t9///3pLQaDQXoJ8MY3HA6jfb/fj/abm5vRvrW2tbUV7TudTrSfn58f6b61Njs7G+273eyXvcnJyWi/tLQU7dO3oLV26tSpaN/r9aJ9+sFLPxWttY2NjWjvxyIXkieEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABTVHfcBqOWzn/1stD979mx6iz179qSXRG6//fZof+bMmfQWb33rW6P9ddddF+3n5+ej/Y//+I9H+62trWjfWnvggQdGfQvgjW9zczPar6+vp7fo9XrRfjgcRvtdu3ZF+8OHD0f71tqBAwei/ezsbLRP/wO7sLAQ7aempqJ9a21tbS3ap0daXl6O9oPBINpfGjqdTrRP/+0wRp4QAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFBUd9wHoJb//u//jvZra2vpLfbs2ZNeEvmzP/uzaH///fent/iu7/quaH/mzJlof9ttt0X7P/qjP4r2f/qnfxrtW2ubm5vR/i//8i/TWwBvfFtbW9H+3Llz6S3Onj070n16pI2NjWjf8v9gdruj/WVvfn4+2h89ejS9RfolDAaDaH/+/Plo3+v1on1rbWpqKtqvr6+ntxi14XA47iMwKp4QAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFBUd9wHgIvMI488Eu1feOGF9BbbuCTyX//1XyPdP/7449G+tfbLv/zL0f5Tn/pUtD916lS0B14XExOj/btzr9dLLzl58mS0n5qaGum+3+9H+9bayy+/nF4S6XazXw53794d7Q8cOBDtW2vXXXddtE/fhTNnzkT7tbW1aL8Nw+Ew2m9sbIz09bm0eUIIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFHdcR8AXsuHP/zh9JK//uu/jvaPPfZYtH/88cej/SXgySefjPbvete70lt85jOfifYf/ehHo/173/veaA98W51O5w31+sPhML1Fr9eL9qdOnYr2ExPZn9oXFxejfcu/S6urq9E+/a4ePHgw2r/tbW+L9q21m266KdofOnRopK9/4sSJaN9a29zcjPZbW1vRvt/vj/T1ubR5QggAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgqO64DwCvZTgcjvqSO+64Y6T7J554ItpfAp588sn0kuXl5Wi/jQ8G8C06nc6obzExkf3deXJyckQn+Yb0q97c3Iz2S0tL0b7X60X71trGxka0X11djfZbW1vR/uTJk9E+PU9r7fLLL4/2119/fbS/9tpro/3VV18d7Vtri4uL0X59fT3ad7vZr/SDwSDat/yfc/pBYow8IQQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgqO64DwBj9uSTT0b7f/u3fxvRSfjfu/vuu6P9zTffHO2/8IUvRHsootPpRPupqamR7ofDYbRvrXW72W8+ExPZn84Hg0G0X19fj/bbuGR1dTXab21tjXR/6tSpaN9aW1hYiPZHjhyJ9nNzc9F+79690b61NjMzE+0nJyfTW4z69dPPNhcRTwgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKKo77gPAazl+/Hh6ye/93u9F+7/6q7+K9uvr69Ge/40/+ZM/ifZ/+Id/GO1vuummaP+FL3wh2sPFqNPppJdMTk5G++np6Wg/MzMT7S/Al5AeaWpqKtpvw9raWrQfDocjOsnXTUxkTxdmZ2fTW6TvQr/fj/aDwSDab0P6wUs/2+m7fAG+ZC4inhACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUFR33AeA1/JP//RPF+ASxu7UqVMjff0PfvCD0f7v/u7v0lucOXMmvQReX51OZ6T71trERPZ35HQ/NTUV7WdmZqL9Ni6ZnZ2N9tPT09F+a2sr2rf8jUu/q+nr79u3L9rfeOON0b61dvXVV0f7ycnJaL+0tBTtV1ZWon1rrd/vj3Q/HA6jPXwzTwgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFNUd9wEA2rXXXjvS1//e7/3eaH/zzTent3j88cfTS+D1NRwOx32Eb5UeaXJyMtrPzs5G+9ba7t27R7rfs2dPtJ+YiP80n35XNzY2ov309HS0P3ToULT/nu/5nmjfWrvqqqui/dmzZ6P9yZMnR/r6rbXV1dVov76+Hu0Hg0G034YLcAvGxRNCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIrqjvsAcJGZm5uL9h/+8IfTW3z3d393eknkH//xH6P9E088MaKTfMMHP/jBkb7+M888E+2PHTs2opPAG8dwOEwvGQwG0X5zczPab21tRfttfAmzs7PR/sorr4z2b3rTm6L9wYMHo31rbc+ePdE+/ZJ379490n23G//yeebMmWh//PjxaP/8889H+5MnT0b71trKykq07/f76S1g2zwhBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKCo7rgPABeZnTt3RvvbbrstvcWdd96ZXhL5sR/7sWg/MZH95WgwGET7C+DUqVMj3cPFaDgcppek/7o3Nzejfa/Xi/ZTU1PRvrV27ty5aL+6uhrt0+/qnj17on1r7dprr432u3fvTm8RSb9FL730UnqL5557Lto/88wz0f5LX/pStP/a174W7Vv+Xer3++ktYNs8IQQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgqO64DwAXmYWFhWj/Ez/xE+ktfumXfina33fffdF+eno62s/NzUX7C6Df70f7T3/60yM6CVy8hsNheslgMIj26T/Vc+fOjfT1W2sbGxvRvtfrRfuzZ89G+xMnTkT71trBgwej/e7du6P95uZmtE+/hJMnT0b71tpXv/rVaP/yyy9H+1dffTXawyXGE0IAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARXXHfQC4xC0uLqaXfOQjH4n2f/zHfxztv+/7vi/a33nnndH+He94R7Rvrb3pTW+K9um36OGHH472wLc1HA6j/dbWVrQfDAbRfn19Pdq31paWlqL91772/7RrbyFy3/X/xz+zO7tJdrt24zZpTkTTpBqpihC06I1eWW9UECqCinciCCqCBWkRiyLeiRak9NY7QTygYPVCkIoXukgSjTFtShrZmh62aZoD2cPM93chBP/lT+G9zWQ2+3o8rl8z389kdnfmOZMLpf2ZM2dK+16vV9pv4CbVf9XV1dXSHrjd+YYQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABC9cd9AODNWllZKe3//Oc/j3QPsDFd1436Er1er7QfDAYj3QOMnW8IAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAgVH/cBwAAuEW6rhv3EQA2F98QAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAqF7XdeM+wxaxvLy8uLg47lMAbBHHjh1bWFgY9yluGq8RADfRFnuNGC9BCAAAEMp/GQUAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUP1xH2DrWF5eXlxcHPcpALaIY8eOLSwsjPsUN43XCICbaIu9RoyXILxpFhcXH3jggXGfAmCLePLJJz/60Y+O+xQ3zeLi4sc+9rFxnwJgi/jtb3+7lV4jxst/GQUAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUP1xHwDYar7+9a9Xb/L973+/tP/2t79d2n/ve98r7QH+q9frjfT+JybKH81PTk6W9lNTU6X9jh07Svu3vOUtpf3c3Fxp3+oPYXV1tbS/dOlSaf/KK6+U9q21q1evlvbD4bB6Cdgw3xACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAECo/rgPAGx2H/rQh0r7Bx98sHqJfr/2t+jgwYPVSwC01nq93kj3ExO1j9qr+9ba9PR0aT87O1va79u3r7Q/cuRIab9///7SvrU2MzNT2l+7dq20f+6550r7M2fOlPattaWlpdL+8uXLpf1gMCjt4X/5hhAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFD9cR+HE2h0AAAXD0lEQVQA2Ow+/OEPl/b333//iE5yw8mTJ0d9CWDz6/V6o77JBi5RMjFR/mh+enq6tL/rrrtK+8OHD5f27373u0v73bt3l/attZWVldL+3//+d2l/9erV0v769eulfas/0dVneW1trbQfDAalfWut67rqTbhd+IYQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABC9cd9AOBW+9znPlfaP/zwwyM6yQ2DwaC0P3/+/IhOAtxEvV5vU+03oHqJfr/2zmrbtm2lfWttfn6+tD9w4EBpf++995b2u3fvLu2r/0SttQsXLpT2zzzzTGn/3HPPlfaXLl0q7Vtrq6urpX3XdaV99Qd1cnKytG/1V+rqQ2CMfEMIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAACh+uM+AHCrbd++vbSfmZkZ0UluGAwGpf3S0tKITgK8gV6vV9pPTGy6z52rR+r3a++UpqamSvs77rijtG+t7d27t7Q/fPhwaX/33XeX9sPhsLQ/e/Zsad9aO3HiRGl/5syZ0v7ixYul/crKSmnfWltfXy/tu67bVPtW/92pvrIzRpvuLzUAAAC3hiAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABC9cd9AODN+tKXvlTaf+c73xnRSf7r+PHj1Zt88pOfLO3Pnz9fvQTw5vV6vU11/xs4z8RE7aPw6iW2bdtW2i8sLJT2rbUjR46U9kePHi3t5+bmSvv//Oc/pf2//vWv0r61dvbs2dL+4sWLpf3q6mpp33VdaX8LVH9Qh8PhqC/BbcQ3hAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhOqP+wDA6913332l/WOPPVbaT05OlvbD4bC0/+53v1vat9bOnz9fvQlw63VdV9r3er0RnWTD91+9Sb9fe6e0Y8eO0v5tb3tbad9ae//731/a33PPPaX9pUuXSvvnn3++tF9aWirtW2uXL18u7QeDQWlffZnbgOorb/UhVFV/l9nafEMIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAACh+uM+AGxxk5OT1Zt861vfGvUlSn7yk5+U9j/72c9GdBJgvIbDYWnf6/VGut+AiYnaR+H9fu2d0s6dO0v7Y8eOlfattfvuu6+0r75GnDp1qrR/+umnS/vl5eXSvrW2urpa2nddV73Eprr/DVyiut/AO4fqrz+3Ed8QAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAqP64DwBb3Lve9a7qTR588MFRnOSGc+fOlfY///nPR3MQYIvrum6k+16vV9q31iYmah+FT09Pl/b33ntvaf++972vtG+t3XnnnaX92bNnS/u//vWvpX31NeXy5culfWttOByW9tUfjMnJydK+ep6N3aSk+pDX1taql6j+7nAb8dQCAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAECo/rgPALeZqamp0v6Xv/zliE5yQ6/XK+0///nPl/Z/+tOfSnuAjan+NduAiYnaR+Hz8/Ol/Xve857S/u677y7tW2tXr14t7Y8fP17aP/3006V99TyTk5OlfWut3x/t+9Wu60r7lZWV6iU2cJOStbW1kd5/a204HI76EoyLbwgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAjVH/cB4Dbz6KOPlvaHDh0a0UluOHHiRGl/8uTJEZ0E4M3o9Xql/eTkZPUSU1NTpf2+fftK+3e84x2l/fT0dGnfWjt37lxp/49//KO0v3LlSmk/MzNT2s/Ozpb2rf6vVH2Wh8Nhab+8vFzat9YuXLhQ2g8Gg9K++rvTdV1pz9bmG0IAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAjVH/cB4Dbzla98ZdxHeL0f/OAHpf1rr702opNs2BNPPFHav/e97y3tH3/88dL+97//fWnfWltaWqreBHidiYna59T9fvltzOzsbGl/5MiR0n5hYaG0v3LlSmnfWjt16lRpv7y8XNpXH8KePXtK+127dpX2rbW5ubnSftu2baV99Vl45plnSvvW2rVr10r7ixcvlvZra2ul/WAwKO1ba71er7Tvuq56CcbFN4QAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABCqP+4DwE129OjR0v7hhx8u7WdmZkr7DXjyySdL+7/85S+l/Wc+85nSft++faX9l7/85dK+tXbo0KHqTUo+8IEPlPa//vWvq5f4xCc+Ub0J3HZ6vd5I739iovY59dTUVPUSu3btKu3vueee0r7fr72zeuGFF0r71tqFCxdK+507d5b2e/bsKe2rf8Dn5+dL+9ba9PR0aV99FlZWVkZ6/63+RFePtLa2VtoPh8PSnq3NN4QAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABCqP+4DwE12//33l/af/exnR3SS//rGN75RvclPf/rT0v7RRx8t7b/whS+U9r1er7TfAmZnZ8d9BNgKJiZqnzv3+7W3JRv4Vd27d29pv2fPntK++pCvXr1a2rf6o37nO99Z2h88eLC0r55nbW2ttG+tra6ulvY7duwo7efm5kr7AwcOlPattfn5+dL+xRdfLO2rP0gbeBa6rqvehNuFbwgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACBUf9wHgDeyY8eO6k0eeuihUZzkhuFwWNofOHCgeonf/e53pf3Ro0dL+67rSvuqy5cvV28yNzdX2vd6vdK++pBfe+210h74/6r+qk5PT5f21T8drbXdu3eP9BL9fu2d1czMTGnfWnv7299e2ldfSavPwssvv1zav/TSS6V9a+2OO+4o7e+8887SfnZ2trTfvn17ad9aGwwGpf36+nppPzFR+46n+rvZRv/mgTHyDSEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKH64z4AvJGDBw9Wb3LXXXeN4iQ3TEzUPkb56le/OqKT3NB13Ujv//Tp06X9F7/4xeolvva1r5X2n/rUp6qXKPnb3/420vuH21Sv1xvpfmpqqrSfmZkp7Vtrs7Ozpf3k5GRp3+/X3lnNz8+X9q219fX10v769eul/UsvvVTaLy0tlfbV87TWdu/eXdpX/1WrP6gvv/xyad9aW15eLu1XV1dL++pPRfUhs7X5hhAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEL1x30AeCMPPfRQ9Sa7du0axUk2s5MnT5b2P/zhD0v7ubm50v6RRx4p7VtrR44cqd6k5A9/+ENp/+Mf/3hEJ4EoExOb7nPnwWBQ2q+srJT21Ydc/QPbWltbWyvtqw9hOByW9jt37izt3/rWt5b2rbXDhw+X9jMzM6X9uXPnSvvjx4+X9q21ixcvlvbVZ636g13dt9a6rqvehNvFpvtLDQAAwK0hCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFD9cR8A3sihQ4fGfYQxWF5eLu0fe+yx0v7jH/94af/AAw+U9tu3by/tN+CPf/xjaf/pT3+6tK8+BRCi1+uV9l3XlfZra2ul/ZUrV0r71tqrr7460n31n2hubq60b/W/sdu2bSvtd+3aVdpPTk6W9vPz86V9q/8gnT9/vrR/6qmnSvtTp06V9q3+szoYDEa6Hw6HpX2r/2xzG/ENIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhOqP+wDA6y0sLJT2TzzxxIhOcss8/vjjpf03v/nN0v7SpUulPXBTDIfD0v769eul/fLycmnfWnv22WdL+/3795f2e/fuLe2PHDlS2rfWZmZmSvvZ2dnSfn19vbQfDAal/ZUrV0r71trp06dL+6eeeqq0X1xcLO1ffPHF0r61trKyUtpXn4Xq7xr8L98QAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQqj/uA8Ab+dGPflS9yUc+8pERHCTa3//+99L+N7/5TfUSjzzySGk/GAyqlwDevK7rSvvhcFjar66ulvbr6+ulfWvt9OnTpf3Kykppf/Xq1dL+gx/8YGnfWtu/f39pv3379tL++vXrpf3S0lJpf+LEidJ+Azd59tlnS/tXXnmltK/+E7X6y1Z1X/3dhP/lG0IAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAjVH/cB4I384he/qN5kYsLHHAAj0XVdad/r9UZ6/+vr66V9a+3y5cul/YkTJ0r7f/7zn6X9r371q9K+tbZnz57SfmZmprSvPmvXrl0r7V999dXSvtWfteqR1tbWSvvhcFjat9YGg0FpX/1dqKo+y2xt3joDAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAECo/rgPAABsTV3Xbar9LbjEcDgs7Z9//vnSvrX2wgsvlPYTE7VP//v92pvD6v3fgmdtMBhsqvtv9R+M6h7eDN8QAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAqP64DwAAbE1d15X2w+FwpPe/Ab1eb6T3v4GHMBgMRnqJ6n5ycrK0vwWqD2F1dXWk9w+bnG8IAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAI1R/3AQAAWmut67otcIlRqz6E6n59fb20B253viEEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQ/XEfAADgFun1euM+wv+j67otcIk0ExPlL1Q8C2xmviEEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQ/XEfAAAgVK/XG/cRKOu6btxHgJvJN4QAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABCq13XduM+wRSwvLy8uLo77FABbxLFjxxYWFsZ9ipvGawTATbTFXiPGSxACAACE8l9GAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACPV/Me3teBQFAPQAAAAASUVORK5CYII=", + "image/png": "iVBORw0KGgoAAAANSUhEUgAABLAAAAl0CAIAAACP7Rn6AAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nOzdeXSV5Z3A8ecmlyUBZIliUavU1gUUN5zCUI8IKrbVGTlF51ipW+s4dh1tpy6jo8cuR2aznVbHVu1Rq1O0i0s9o7UyKHVp1SLgjhuKO7LLkkCSO3/cMzmZBNAncHNDfp/PX3r53fd93je5N/ne9yYplEqlBAAAQDw11V4AAAAA1SEIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCCEHucLX/hCoVA4++yzt9Pt91jlAz/zzDOrvRAAgJ5CENIb/MM//ENhi4rFYrXXCADZmpqarrvuuuOPP3733Xevq6sbPHjwvvvue+aZZ86aNWubbH/q1KmFQuFrX/vaNtna9rV3oMx3yQAAPdEDDzxw2mmnLV68uO2WxsbG1atXL1y48Gc/+9lRRx31X//1X8OHD6/iCoFewBVCeo+GhobSZjQ3N1d7dQCQ4Y477pgyZcrixYsHDx78/e9//9lnn123bt2KFSvmzJlz6qmnFgqFWbNmjRs37q233qr2SoHtmyuEAAA9y8svv3zqqadu3Lhxzz33nD179h577FG+va6u7vDDDz/88MOPO+64k08++dVXXz3ppJMeeOCBmhov8QNd5OmDiI466qhCoXDBBResXbv2sssuGzNmzIABAwYMGPDJT37yuuuu2+RdXn/99fPOO++ggw4aPHhwfX39vvvuO3369FmzZpVKpQ6TDzzwwIknnrjrrrv27du3oaFh0qRJ11577eYuUd50000TJkwYNGjQoEGDDjzwwO9+97vr1q3bwsoXL178zW9+c//99x80aFB9ff3o0aMvuOCCZcuWbW4+d/ubdMstt3z6058ePnx43759d95550MOOeRrX/vaY4891n5m3bp1d9xxx+mnn37ggQc2NDT069dvt912mzZt2n333dd5g23nf+XKlRdccMGoUaPq6+uHDRt25JFH3nvvveWZd99991vf+tbee+9dV1fX0NAwbdq0hQsXbm47y5cvP++88/bdd9/6+vohQ4YcddRRd999d9Yx5p5YgIq65JJL3n///T59+vzmN79pq8H2TjzxxIsvvjil9OCDD95+++1tt48cObJQKNxwww2d73LCCSe0/4G9X//614VC4c4770wpXXXVVe1/8P7Pf/5zeSb3aXbb7h3oJpt7ix1sR771rW+lLb5ltIMjjzwypfSFL3xhr7326vygOPfcczvM//znP+/fv/8mH0Hz5s1rP3nuueducuxTn/rU8uXL20+2tLSccsopnSf322+/z3zmMymlv/u7v+uwjJkzZ25yGR/5yEcWLFjQYbgL29+k0047bXPPHu+9917b2KWXXrq5sQsuuGCT5//kk0/u/F1OoVC4+eab586du9NOO3X4p6FDh77wwgudt/P5z39+t91267zfSy65pMN+p0+fnlL60pe+tDUnFqDSli5dWv5daNOnT9/C2Nq1a4cMGZJSmjRpUtuN5efV66+/vvP8tGnTUkpf/epXy//7q1/9anPP248//nh5JvdpdtvuHegerhAS18033/z222//x3/8x5tvvrl+/fp58+aVv/L94Ac/eOqpp9rG7r777tNOO62xsXHMmDG//vWv33vvvfXr1y9cuHDmzJlHHXVU+3fpXHnllT/4wQ9SSn/91389b968xsbGxYsXX3rppbW1tQ8//HCHPPvhD3940003pZROO+205557rqmp6cUXX/zyl7/8zDPP3HPPPZ1XO3v27OnTpzc2Nk6cOHHWrFnvv//+unXr7rvvvgMOOOCdd96ZOnXqmjVrtmb7m3T//fffeOONKaVvfvObzz777Nq1a1esWDF//vyf/OQn48aNKxQKbZMDBgw46aSTbr/99ueee+79999fvnz5Y489dsYZZ6SUZsyYsck9/uIXv1ixYsXVV1/97rvvvv/++/fdd9/HPvaxUql0zjnnHH/88XV1db/85S9Xrly5YsWKW2+9dejQoStWrPj2t7/deTszZ8586623LrnkksWLFzc2Nj7xxBPHHntsSuk73/nOb3/72w88xtwTC1Bpf/jDH8rvKznxxBO3MFZfX3/cccellB555JGmpqbcvZxwwgmlUun4449P7Tqt7NBDD20/uZVPs1u5d6DiujE+oVLKVwi3oMPVsHL4pZRmz57d/vbVq1fvuOOOKaVLL720fMvGjRs/9rGPpZQOPfTQNWvWbGEN69atGzZsWErps5/9bEtLS/t/+uEPf1je3f333982PHTo0JTSKaec0mE7X/7ylzuvuaWlZZ999kkpTZgwobm5uf38smXLGhoaUkr//u//3n4xWdvfnPJ1vyOOOOIDJzfn7//+71NKxx57bPsb287/nDlz2t/e9v7SQYMGLVq0qP0//fjHP04pFYvF9evXd97Ov/zLv7Qfbm5uPuKII1JKo0ePbn975yuEuScWoBt897vfLT+5vf7661uevOKKK8qTTz/9dPmWD3+NrmyTSdYm92l22+4d6B6uEBLX4YcfPmnSpPa3DBo0aPLkySml559/vnzL/fffv2jRopTSj370owEDBmxha/fdd9/y5ctTSv/8z//c4Yf7v/71r5e/Rs6cObN8y+9///sVK1YUCoXvfe97Hbbzne98p0+fPh1ufPDBB8s/QXfVVVfV1ta2/6dhw4aV36d62223td2Yu/3NKb+Rcmt+R2v5O4A//elPnf+p/HsR2t8yefLk8tpOPvnkkSNHtv+nY445prySl156qcN2dtppp3POOaf9LbW1teVvp5599tn2F3s7yz2xAN2g7QeYy69RbkHbu+sr+jPPW/M0C/R8gpDeYws/Q/iTn/yk8/zBBx/c+caPfvSjKaXVq1eX//eRRx5JKQ0fPvwv//Ivt7z38g/B77rrrvvvv3+Hf6qpqZkyZUrbTEpp7ty5KaV99tln99137zC84447dl7Yww8/nFLaaaedDjrooM67PuSQQ1JK8+fPb7sld/ubU34B+KGHHjrppJPuv//+Lb8l6dVXXz3vvPMOPfTQoUOH1tbWln83QDn5li1b1rkqOy+jpqZm5513Til1Psxddtml/B9tH5o2EydO7Jy4EyZMqK+vT+3O+SblnliAHqXU6RebVcLWPM0CPZ8gJK5N/h6R8s/xt7a2lv/3nXfeSSmV3zW6ZUuWLEn/15OdlcOsPNP2H5v8Gf1NbuTNN99MKb333nvFYrFYLNbW1tbW1tbU1NTU1BQKhc9+9rMppbVr17YFW+72N2f8+PHnn39+SunWW2+dPHny4MGDx48ff+GFF3Z+PfjOO+8cNWrUv/7rv86dO3flypVtJ7BNY2Njh1s2ef7Ll+k6/1Pb5bvOW97kYdbU1IwYMSK1O+eblHtiAbpB+f3qKaWlS5duebJtoO0ulbA1T7NAzycI4YO1/+0pWzPZ4V8//GZbWlra/qOlpaW1tbW1tbV88bP92MaNG7u2/S2YMWPGo48+evbZZ++9995NTU2PPvrojBkzDjjggDPPPLNtVcuWLSv/3p2xY8f+8pe/XLRo0bp168orfPTRR7d+DVu2NYfZtRMLUFGjR48u/8cTTzyx5cnyQL9+/T7+8Y9Xbj3b5KsJ0GMJQtiS8sufr7zyygdOln+QY/HixZv81/LtbT/sMXz48JTS66+/vsnhN954o8Mt5TdS/sVf/MWWfyZ44MCBXdv+ln3yk5+8+uqrFy5cuHTp0jvuuONzn/tcSulnP/vZj370o/LA3XffvWrVqiFDhsyePfvEE08cOXJkXV1d+RuIbnjleJOH2dra+vbbb6f/OxWbk3tiAbrBxIkTy2+L2MLfZkgprV+//r//+79TShMmTGh7Y0X5h9jbXu3qMN+19Xz4p9lK7B2oNEEIWzJhwoSU0pIlS/74xz9uebL8a7LffPPNZ555psM/lf7v92e2/SrtsWPHppQWLlzY+avs0qVL582b1+HGww47LKU0f/78D9lXudv/kBoaGo4//vjf/OY3U6dOTSmVvxFJ//e9wj777LPDDjt0uMsm/zD9tjVnzpzOV/AeeeSRdevWpXbnfJNyTyxAN2hoaPibv/mblNKtt9765JNPbm7s3/7t31asWJFS+spXvtJ246BBg9KmXvhrbW3d5E9El8tzyz+L+OGfZiuxd6DSBCFsyaRJk/bcc8+U0je+8Y3yV77NmTJlSvnPTlx44YUdvrZdddVVr776akrp85//fNvwkCFDSqXSxRdf3GE7l1566YYNGzov4+Mf//jGjRu/8Y1vdP4huk0uJmv7ucq/+K5tJYMHD04pLVy4sMOf7HvmmWeuu+66rdzXB3rvvffa/rBHWUtLyyWXXJJSGjVq1JgxY7Zw39wTC9A9LrvssoEDB27cuHHatGmbvEB3++23l3/P56c+9anyGzfKRo0alVK66667Osxfe+21b731VuftlF/I2/IPK374p9lK7B2oNEEIW1JbW3vllVcWCoU///nP48ePv+2225YtW9bY2Pjiiy/ecsstU6ZMaXvttq6u7rLLLksp3XXXXdOmTVuwYMGGDRveeOON73znO+Xf1n3ssceWf2lnefiiiy5KKf385z8/44wzFi5cuGHDhpdffvkrX/nKf/7nf3ZeRrFY/OlPf1osFm+99daJEyfeeeedS5YsaW5uXrJkyfz582+88cZp06a1/wsTudvfnK9//esnn3zyzJkzn3rqqeXLlzc1NS1atGjGjBk33HBDSqn8q1PL/1FTU7Ny5cqpU6fOnTu3sbFxyZIl11577cSJE7dc0dvKBRdccOmll77xxhsbNmyYP3/+1KlT77///pTS97///S3fMffEAnSPvfba64YbbujTp89LL710wAEHXH755c8//3xjY+OqVaseeuihM844Y9q0aRs3btx9991vueWW9n/r6IQTTkgpzZ0794wzznjhhRc2bNjwyiuvXHzxxV/96lc3uaNyzv3+97+fM2fOFt7V+SGfZiu0d6CyPswfK4Qe7gP/MH1K6fHHH2+bL/+l3fPPP7/zpsq/VPOYY45pf+ONN97Yr1+/TW523rx57SfLf7muswkTJixfvrz9ZEtLy8knn9x5cvTo0Z/5zGfSpv5w/O233975PZltOhxOF7bfWfkvuW/SlClTmpqaOpy3DorFYtvt77///oc5/5v7o8Zt3yg8+OCDHbZz0kkn7brrrp33ftFFF23ycNr/YfounFiAbjNr1qzN/b7olNKkSZPefvvtDndpbW099thjOw9PmDCh/Cpeh78C/+abb3Z+Amz7ipn7NLtt9w50D1cI4YOdeuqpzz///DnnnDN69OgBAwYMHDhw1KhR06dPv++++w488MD2k1dcccXs2bOnTZs2YsSIPn36DB06dOLEiddcc82cOXOGDh3afrKmpubmm2++4YYbxo0bN2DAgPr6+v333/+SSy557LHHym897Wzq1KkvvfTSZZddNm7cuKFDh/bp02fEiBEHH3zwF7/4xdtvv/2f/umftnL7nV155ZUzZ86cPn36mDFjhg4dWiwWd955509/+tM333zzPffc07dv37bJGTNm3HTTTePHj6+vr+/fv//IkSNPP/30BQsWlF8trqg99thj/vz555577t57711XV7fDDjtMmjTprrvu+vBX9rJOLEC3OfLII1988cWf/vSnxx133G677davX79BgwbttddeZ5xxxr333jt79uyPfOQjHe5SKBRuu+22yy+/fL/99uvfv//AgQMPPfTQK6644oEHHij/gF8Hu+yyy5w5c0444YQRI0aU//BSZx/+abYSewcqrVDyg7zA9umoo476n//5n/PPP3/GjBnVXgtAL+RpFiJwhRAAACAoQQgAABCUIAQAAAhKEAIAAATll8oAAAAE5QohAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACCoYrUX0HssW7Zs7ty51V4FQC8xduzYhoaGaq9im/E1AmAb6mVfI6pLEG4zc+fOPeaYY6q9CoBe4t57750yZUq1V7HNdOFrRKFQyJovlUpZ80Cvkft00Q0q/YzUy75GVJe3jAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQVLHaCwCAEAqFQtZ8qVSq0ErKcteTKr+kXF04hFw97ZC7oNKfeD4K26MunNJu+EBTLa4QAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABBUsdoLAIAQSqVStZfw//S09XRBLziEblDps+SjUAmFQiFr3keBreEKIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBFau9AACAHqpQKGTNl0qlCq2k18g9panyZ7UHfpR74CdSD1wS24orhAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCKlZ7AQAQQqFQyJovlUoVWkm3yT3kXN1winJ3UelDjqkXfCL1tId/F05pD1wS24orhAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEVaz2AgAghFKplDVfKBQquv1u0AOXlHtWK7392tra3F20trbm3mV716dPn6z55ubmrPmWlpas+S7oaY+Fnrae1COXFIcrhAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEVaz2AgCATSiVStVewtYqFArVXkJHuUuqq6vLmt+wYUPWfLGY/Z1YS0tL1nxNTd6r/7lLyt1+N3xi5y6pX79+WfOrVq3Kmu+CXvDwZzviCiEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQRWrvQDo5fbYY4/cuwwZMiRr/ogjjsia32effbLmDzvssKz5MWPGZM2nlEqlUtb85ZdfnjV/0UUXZc1DT1AoFLLmcx9HvUBtbW3uXT760Y9mzQ8fPryi229oaMiaTyntuOOOWfOrV6/Omq+rq8ua79OnT9Z8U1NT1nxKqX///lnzGzZsyJpvbGzMmn/yySez5lNKf/jDH3LvkqUXPPxzn/HYhlwhBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABBUsdoLgCo7/fTTs+b/9m//Nmt+r732yppPKTU0NOTepUdpbW2t9C5Gjx5d6V1A1ZVKpWovoaNCoVDR7dfU5L1OfdZZZ+XuYrfddsua33HHHSu6/Z133jlrPqU0ZMiQrPk+ffpkzed+FFauXJk134XPovXr12fNL126NGt+zZo1WfMjR47Mmk8prVu3Lmv+8ccfz5qv9GMzVf4ZqQc+48XhCiEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQRWrvQDYxorFvM/q6dOnZ82PHz8+ax6gawqFQtZ8qVSq0Eq6Te4hr1mzJncXI0aMyJrfZ599suZ32GGHrPm6urqs+ZT/Za65uTlrfunSpVnzuR+1xsbGrPmU0sCBA7Pm99hjj6z5VatWZc134RCOPvrorPnHH388a74bHv4Bn5HicIUQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKCK1V4AbGM777xz1vzkyZMrtBKArVEqlSq6/UKhUNHtd0HuIec+4aeUmpqasuYXLVqUNT948OCs+ebm5qz5lNKaNWuy5levXp01v2TJkqz53FO6YsWKrPmU0sSJE7PmR40alTVfV1eXNV9fX581n1L63e9+l3uXnqbSz0hUkSuEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAARVrPYCYBtbvXp11vwTTzyRNX/IIYdkzXfB2rVrs+ZXrVqVNT9gwICs+cGDB2fNd0FLS0vW/DXXXFOhlQBV1NramjX/4x//OHcXV199ddb8mjVrsuZfe+21rPkXX3wxaz6l9Pjjj2fNv/vuu1nzjY2NWfODBg3Kmj/66KOz5lP+RyH3EGpra7Pmc09pSmnevHm5d8lSKBSy5kulUoVWwvbIFUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQRWrvQDYxt5///2s+eOOOy5rfty4cVnzXfDKK69kzT/99NNZ81/60pey5q+55pqs+S544403subvueeeCq0Eeo5CoZA1XyqVKrSSLu8i9xBybdiwIfcu3/72t7Pmv/jFL2bNP//881nzs2fPzppPXTrqiqqtrc2aX7ZsWe4uhg8fnjWfu6SWlpas+aampqz5lP/Y6WmPNXo3VwgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACKpY7QVAlb377rtZ87/97W8rtJIuKxbzHsiHHnpohVbSZRs2bKj2EiCcUqlU7SVsrS4cwqpVq7LmH3300az51157LWt+48aNWfMppUKhkDWfe5ZqavKuFgwePDhrfq+99sqaTyntsssuWfP9+/fPmv/Tn/6UNX/RRRdlzXeDHvhwrvQnKtuQK4QAAABBCUIAAICgBCEAAEBQghAAACAoQdr2Mv8AACAASURBVAgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABFWs9gKArXXwwQdnzZ911lkVWkmXXX/99dVeAvQ4pVKp2kvohZqbm7PmH3rooaz5wYMHZ8337ds3az6l1NTUlDVfW1ubNV9XV5c1f8QRR2TNjx8/Pms+pTRw4MCs+aeffjprfuHChVnzGzduzJqPyTPYdsQVQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCKlZ7AcDWqqnZ7l/Zeeqpp6q9BKi4QqFQ0e2XSqWKbr8LcpdU6VPUBa2trVnzK1asyJrv27dv1nxKqU+fPlnz9fX1WfN77rln1vx+++2XNT9mzJis+ZT/idTc3Jw1//DDD2fN535WpMo/Frrh4d8Dl8S2st1/HwkAAEDXCEIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABFWs9gKAjmpq8l6pufDCCyu0kq557LHHcu8ya9asSqwEQikUClnzpVKpB+4img0bNlT6Lhs3bsyaP+aYY7LmDzvssKz5YjH7m8/nn38+az73y9CLL76YNZ/7ZTrlP3Z62vaTh3Ov5gohAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKCK1V4A0NH++++fNf9Xf/VXFVpJ19x99925d9mwYUMlVgI9SqlUqvYS/p9CoZB7l9xDyN1FN5yiSi+pC2e10j7xiU9kze+yyy4VnV+3bl3WfEpp6dKlWfOPPPJI1vyaNWuy5nvaYzn1yCXl6oGPnThcIQQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgqGK1FwB0dMQRR1R7CVtlzpw51V4CRFQoFKq9hI5KpVLWfO4h5G6/C3rgWa2pyXs1/+KLL86aHzZsWNZ87nqeeuqprPmU0h//+Mes+eeeey5rvqWlpaLzfBjd8HBmc1whBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACCoYrUXAPR0r732Wtb8ggULKrQSCKVQKGTNl0qlCq2kTe6SKq2nracLunAIZ599dtb8smXLsuYPOuigrPmnnnoqa37x4sVZ8ymluXPnZs03NTVlzXfDYydXD1wSvZgrhAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEVaz2AqCX69+/f+5dzjrrrEqspMtmz56dNb9q1aoKrQRCKZVKFd1+oVCo6Pa7QRcOodJntU+fPlnzkydPzt3FsGHDsuZ32GGHrPl33303a37x4sVZ888991zWfErp5ZdfzprP/SgXi3nfDzc1NWXNd4NueDhX+rFDFblCCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACCoYrUXAL3cuHHjcu8yatSoSqyky+68885qLwH4YIVCIWu+VCpVehe5amryXqcuFrO/jWlpacmaz11SQ0NDRedTSvX19Vnzffv2zZp/5plnsuZfeOGFrPknn3wyaz6ltHHjxqz53ENeu3Zt1nwXdOHhVlE9bT1UlyuEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAARVrPYCoJc75ZRTqr2ErfW73/2u2ksAPlipVKr2ErZWNxxCbW1t1nz//v2z5j/xiU9kzdfX12fNp5TWr1+fNb9s2bKs+ZUrV2bNr1ixImv+nXfeyZpPKRUKhaz5devW5e5ie5f72Mk9pV3YBdsRVwgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACKpY7QVAL1dfX1/tJQBsG6VSqdpL+H9aWlpy71JXV5c1v99++2XNjxw5Mmt+0KBBWfMp/xAWLFiQNV8oFLLmly1bljXfr1+/rPmU0vr167PmK/2J2tMeCF3QAw8h9xOPbcgVQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCKlZ7AUB3+9WvfpU139zcXKGVANuXQqGQNV8qlSo639LSkjWfUtq4cWPW/P777581v379+qz5XXfdNWs+pbRkyZKs+REjRmTN19TkXS148skns+YXLVqUNZ/yPzF62vb5MHwUqsgVQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBFau9AGBrrV27Nmv+F7/4RdZ8qVTKmgd6q9xng0KhUKGVdFlTU1PW/PXXX581P3bs2Kz5BQsWZM2nlBYuXJg1X1tbmzW/cePGrPmWlpas+W74muLLFmRxhRAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoIrVXgD0cs8++2zuXVpbW7Pmp0+fnjV/1113Zc0D24VCoVDpXZRKpYrO5+qGQ25ubs6af/TRRyu0Ej683E+MSn+iph65pErrhocn24orhAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEVaz2AqCX+973vtcNdwF6vkKhkDVfKpUqtJLu2X436IGH0NM+yl1Q6UPI3X4XdtED9YJDyBXwkLdfrhACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEFShVCpVew29xLJly+bOnVvtVQD0EmPHjm1oaKj2KrYZXyMAtqFe9jWiugQhAABAUN4yCgAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAARVrPYCeo9ly5bNnTu32qsA6CXGjh3b0NBQ7VVsM8uXL58/f361VwHVVyqVcu9SKBQqsZI2uUuq9Hr4MA466KBhw4ZVexW9hCDcZubOnXvMMcdUexUAvcS99947ZcqUaq9im5k/f/7nPve5aq/i/+mG78tz51tbWyu6/S7cpaWlJWu+trY2a74bdOEDnSX3lDY3N+fuotJnNfejXFNT8XfYVfqx1gW5Z6nSh3DbbbdNnjw56y5sjreMAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABBUsdoLAIDer1QqtbS0ZN2lWMz7Gt3a2po1X1tbmzWfUmpubs69S5bcQ+jTp0/uLkqlUtZ8TU3eS+e5h1AoFLLmU/4h5O4id/u5h5x7Sruwi0rrho9artynly58FCr9WKj0KWILXCEEAAAIShACAAAEJQgBAACCEoQAAABBCUIAAICgBCEAAEBQghAAACAoQQgAABCUIAQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIKhitRcAAL1foVCoqcl7EbalpaVCi+ny9it9CMVi3rclra2tWfNdUFtbmzWfu6RCoZA1n1IqlUoV3UU3nNVclT7kLnwUclX6ECr9idoNch/+3fBRi8MVQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQlCAEAAAIShACAAAEJQgBAACCKlZ7AQAQQqFQyJqvqcl70ba1tTVrvgtyd5F7CLnzXdDS0lLR7Vf6FKWUamtrs+Yr/YmRewi5D4SUUqlUqvQuKr393EPIlfuJnftZlCr/uZ27/Uqf0lBcIQQAAAhKEAIAAAQlCAEAAIIShAAAAEEJQgAAgKAEIQAAQFCCEAAAIChBCAAAEJQgBAAACEoQAgAABCUIAQAAghKEAAAAQQlCAACAoAQhAABAUIIQAAAgKEEIAAAQVLHaC4DtTN++fbPm//Ef/7HSuzj66KOz5sf+L7v2GmvpXdf9/3ettfae2XPYPdKh0xaahpYWQ6cwFvBANaEUqKhErSARjYlAoiHyRFBLUg8hUUuVoIlGEx6YSBTSGjDhIEibqqk1KRKgSBHa6YkyM+1MD3OevdZ1P/Af/ne4obk/655rr935vl6PP2tdv7XXYfZ7r9m9O9pvQB/+8Iej/e/93u9F+3379kX71trx48fTm1BN3/fRfjabRfvxeBzt19bWov0cuq6L9ulDTvfrYGlpKdqPRvGf5tN/IzZt2hTth37Wjh07Fu3nkF4ifcjpe7nlT/TQr+3pdJreZOiHkH6CcQr5hhAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAoiaLPgA8x9x6663R/vrrr08v0XVdtO/7ftD9BvQrv/Irg+6/9KUvRfvW2qte9apof/z48fQSVLO0tBTt19bWov1oFP9ROP10mkyyXzOm02m037RpU7Sfw/bt26P9VVddFe3f+MY3RvvW2rZt26J9+hBWVlai/cmTJ6P9k08+Ge1bawcPHoz2//7v/x7tP/WpT0X79IXa8s/89L2Wvp3Tj4vW2mw2S28SmeOnyqniG0IAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAipos+gDwHPO1r30t2r/hDW9IL9F1XXoTTq0rr7wyvcl5550X7R9++OH0ElTT9/2g9z/HR016pLW1tWg/GmV/p15aWor2rbWdO3dG+xtuuCHan3/++dE+/ehorZ155pnRfjLJftmbzWbRfsuWLdF+dXU12rfWnve850X79IV67rnnRvuPfOQj0b7l74X0IaTPWvqqaK1Np9Nonz6E9O3sl6VTyDeEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQ1WfQB4DnmN3/zN6N93/fpJa699tpo/5nPfCba33PPPdF+HVx11VXR/rd/+7cHOsnc3vnOd0b7973vfQOdhI2p67rxeBzdZDabDXSY/9F13aD331qbTLJfM9Ijbd26Ndq31n7gB34g2h85ciTaP/zww9H+vvvui/attZWVlWh/8uTJaL9p06Zov7q6Gu3TN0LLn+j0IT/99NPR/hWveEW0b63dfffd0f7w4cPpJSJz/HKS3iR9+6efeHM8BL4f3xACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKGqy6APAae4973nPoo+w3s4444z0Jm9961uHOMl6+uhHP7roI7Ch9X0/nU6jm3Rdl14i2q+trUX71trS0tKgl9i8eXO0X1lZifattf/6r/+K9gcPHoz2x48fj/ZzPAtHjhyJ9pNJ9ste+iyfeeaZ0f7SSy+N9q21Cy+8MNqfOHEi2u/YsSPaP/roo9G+tba6uhrtjx07Fu3TF1L6cdFaG42yr5GG/sTjFPINIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFTRZ9AGCjW11djfbveMc70kv89E//dHqTQT344IPpTb7+9a8PcRIq6/t+0Pvvum7om4zH42ifPuTjx49H+9ba3r17o/2hQ4ei/cmTJ6P9HI4dOxbtt27dGu137NgR7Xfu3BntzzzzzGjf8n+GnnzyyWh///33R/v0VdHyZ202m0X70Sj7jmc6nUb7lr/90336EDiF/OgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKGqy6AMA/69WVlai/Rvf+MZo/9a3vjXa/9RP/VS0Xwd79uyJ9tdff316iWPHjqU3gVNrNMr+yDubzdJLpDdJ9+lDOHr0aLRvra2trUX7/fv3R/uu66L9pk2bon1rbXV1NdpfeOGF0f7SSy+N9ldccUW0P//886N9y18Y6Wd++gF+4MCBaN9am06n0T59IQ19/3PcZB2OxKniG0IAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAipos+gDAd/vBH/zBaP+e97wn2v/sz/5stD8N3H777dH+vvvuG+gkVNb3fbQfj8fRfjqdDnr/62A2m0X748ePp5fYvHlztE+PlP5UL7nkkmjfWnv5y18e7S+//PJof/HFF0f7lZWVaH/y5Mlo31rbs2dPtP/Wt74V7R966KFB77/lb8/04yKVvrBba6NR9jVS13XRfuiHzLPwDSEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAoiaLPgDw3d7//vdH+2uvvXagk5w23vKWt0T72267Lb3EJz/5yfQm8Oz6vo/24/E42s9ms2jfWhuNsr8jd10X7dOHcPLkyWjf8iOde+650f6yyy6L9j/8wz8c7VtrL3zhC6P9xRdfHO23bt0a7SeT7JfJRx55JNq31h5//PFof+DAgWifvhdWVlaifWvt2LFj0f748ePRPn1hp+/l+W4SmU6ng94/z8I3hAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUNVn0AYDv9trXvjba930/0ElOGysrK9H+d37nd9JLfO5zn4v2J06cSC/Bc914PI72a2tr0X40yv7IO8dHR9d1g17i5MmTg97/HFZXV6P9pZdeGu0vuuiiaN9a27lzZ7Tftm1btE9/qumn2RzP2pYtW6L9BRdcEO1ns1m037t3b7Rvw3/mT6fTaJ9+HM1xCZ5DfEMIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGTRR8A+G6bN2+O9jfddFO03717d7Tfu3dvtD927Fi0b629/e1vT28yqKuvvjq9ydLSUrQ/ceJEegme66bTabQfjbI/2s5ms2jfdV20Xwd930f78XicXiJ96+3fvz/apz/VAwcORPuWf8aurq6ml4ikP6L037jW2rZt26L9JZdckl4icu+996Y3OXToULRP3wvLy8vR/vjx49G+5a/t9BMp/cTjFPKjBwAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKCoyaIPAHy3EydORPsbb7xxoJPM59WvfnV6k1/91V8d4iRz+4u/+Iv0JocPHx7iJJxOuq4bdD8ej6P9HPq+H/T+p9NptE9/RK210Sj7U/jTTz8d7T/60Y9G+127dkX7lj/RKysr0X55eTnap7Zs2ZLe5MUvfnG0P/PMM6P9JZdcEu23b98e7VtrTzzxRLQ/efJktE9/c5hD+t6Z4+3JoviGEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiJos+ALDRbdq0KdrfcMMNA51k3Tz55JOLPgKnm67rJpPs39zZbDbofg5930f7rusGOsn/WFtbS28yHo+j/dDPwr/+679G+5Y/hOXl5Wi/ffv2aJ++sHfs2BHtW2s7d+4cdL958+Zof/XVV0f71tqDDz4Y7ZeWlqJ9+l5I38uttel0Gu3Tt/9o5GuqhfGjBwAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFDUZNEHANbbpk2bov0f//EfR/tf//Vfj/br4Pbbb4/2f/iHfzjQSSir7/uTJ09GNxmNsj/adl0X7WezWbRvrU0m2a8N6SXG43G07/s+2s8hvcSJEyeiffqstdbSF1L6U923b1+0X11djfZnnHFGtG/5s5D+VIfet/xZOH78eLRPf0RzvHfSR51+gs3xicSp4htCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIqaLPoAbCyXXXZZtD948GC0379/f7SvaTweR/vrrrsu2r/3ve+N9q9+9auj/Tp4/PHHo/3v//7vR/tjx45FexjC2tpatE8/OiaTwX8H6Lpu0PtPH3JrbTabRfu+76P90A+5tba8vBzt04eQvvDS+9+xY0e0n+MS27dvj/ZPPPFEtL/vvvuifRv+hbQOL7z0vZPuWSDfEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQ1GTRB2BAr3jFK9KbfPzjH4/2Bw4ciPaf//zno/273vWuaL8BXXfddelNfvd3fzfav/KVr0wvsdHs378/2r/5zW+O9nfeeWe0hyGMRtkfYdP9bDYbdN/yI6Umk+zXkq7r0kssLy9H++l0Gu37vo/2czyEzZs3R/tDhw5F+3POOSfav+AFL4j2l19+ebRvrb3oRS+K9uPxONo/9dRT0X7v3r3RvrW2trYW7dMX0hxv540mfdbmeO/w/fiGEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiJos+AIHdu3dH+3/8x39ML3HuuedG+/POOy/aX3TRRdH+gQceiPattde+9rXRfseOHeklIi996UvTm4zH42jf9316iUF95CMfSW9y0003Rfv7778/vQQsVtd1o1H2R9i1tbVoP5lk/6an999aW1paivYrKyvR/swzz4z2559/frRvrR04cCDaT6fTaJ9+gG/ZsiXateE0hQAAIABJREFUt/yJTl946b/sV155ZbR//vOfH+3nuMm+ffui/aOPPhrt9+/fH+1ba13XRfv0X/b0vTnH2z99bafvHRbIN4QAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAipos+gAEfuZnfiban3vuuQOdZG5bt26N9jfffPNAJ6ns6aefjvZve9vbov3nP//5aN9aO3LkSHoTeG7p+346nUY36bou2qf3PxrFfxSezWaDXmLXrl3Rfvfu3dG+tbZp06Zov7y8HO2PHTsW7ffv3x/tW2vbt29PbxJ5/vOfH+1XVlai/XnnnRftW2vPPPNMtH/44Yej/d/93d9F+7W1tWg/x03S91rf94Pe/xw3ST/B0h9R+pB5Fr4hBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKCoyaIPQOCWW26J9tdff316iSuvvDK9CafcyZMno/3NN98c7T/4wQ9G+8cffzzaA/+nvu9ns1l0k9Eo+6Ntev9d10X7ORw9ejTaP/DAA9H+qquuivattbPPPjvan3feedG+7/tof9FFF0X71try8nK0H4/H0X5tbS3ab968Odrv27cv2rfWvvKVr0T7W2+9NdofPHgw2qfvtdbaZJL9yp2+kNLfHNKPl9badDpNbxJZh08kvh/fEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQ1GTRByBw4MCBaP+qV70qvcSb3vSmaP/Sl7402r/5zW+O9pdcckm0b6197Wtfi/b/8A//kF4ictddd6U3ueOOO6L94cOH00sA66zruvF4POgl+r6P9l3XDX2J48ePR/uvfvWr0X40iv+u/WM/9mPRfvv27dH+vPPOi/bbtm2L9q21Z555JtqvrKxE+wcffDDaP/XUU9F+z5490b61duedd0b79Ec0nU4H3c9xk9lsFu3Tj5f0/ttcb7fIHJ9InCq+IQQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgqMmiD8CAjh8/nt7k7//+7wfdv+9974v2AKeNvu+jfdd10X48Hkf7OYxG2d+RZ7PZoPt777032rfWHnjggWi/uroa7U+ePBntt27dGu1b/lNKfxnYvn17tD9y5Ei0P3DgQLRvra2trUX79FlI32tzSN876cfF0B8vc1witQ7PAt+PbwgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFDVZ9AEAoIS+7wfdj0bZH3m7rov2LT/SHJeITKfT9CbPPPNMtD9y5Ei0Tx/ywYMHo31rbTweR/sTJ05E+3379kX79CHPZrNo3/IXXvojWltbi/bpeeYw9HttHR5C+om0Dkfi+/ENIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFTRZ9AADge+j7PtpPp9NoPx6Po31rreu69CaR2WwW7ed4COkl0n36rM0hfaKHfgijUfbtwhw/ovSFd/LkyWg/9ENu+bOWXiL9Ec3xEIZ+bQ/98cKz8A0hAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEVNFn0AACih67pB97PZLNpPp9No31obj8fRvu/7aJ8+hPQ8LT9Suh+Nsj+1r62tRfs2/JGGfiGl52/5eyE1mWS/D8/xrKWXGPpZSF8Vbfi359DPMs/CN4QAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFNX1fb/oM5wmnnjiiXvuuWfRpwA4Tezevfucc85Z9ClOmQMHDnzxi18c9BLr8A9613WD3n/6EEaj+O/aG+3XnjnOM/SzMLTT4CFvwIeQHmmO86zDJSJXXXXV2WefPegl6hCEAAAARfkvowAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRk0Uf4PTxxBNP3HPPPYs+BcBpYvfu3eecc86iT3HKPPHEE1/4whcWfQqA08TLX/7y0+nfiMUShKfMPffc87rXvW7RpwA4TXzmM5+57rrrFn2KU+YLX/jCG97whkWfYr11XRft+74f6CTfMfSR1uEhb8Cf6nPd0M/yOij4qvjUpz712te+dtGnOE34L6MAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFDVZ9AEAgO+h67po3/f9oPc/xyWG3o9Gg/9de+hnYQ5DP9Hr8BA2mvRHtA7vHc8a68k3hAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUNVn0AQCA76Hv++f0/c+h67povw4PIT3SOtz/Bnzihjb0Qx76WZ5D+pA34HsntQGfhTp8QwgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUZNFHwAA+B66rov2fd8PdJJ1kz7kORT8KW20h7wOz3JB6bM8x7Mw9Atpo71QS/ENIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiJos+AADwPfR9v+gjfLf0SF3XDXr/G9BolP2pPf0RzXGJpaWl9BKR2WwW7dfW1gY6yXdMp9OhL/FctwHfa3O8FzhVfEMIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGTRR+AWn78x3882t9xxx2DnON/s3Pnzmj/2c9+NtpfccUV0X4d3HnnndH+qaeeivY/+ZM/Ge1ba3/6p38a7T/wgQ9E+8ceeyzaw0bQdV207/t+oJN8R3qk0Sj7u3P6ENLzzGF5eTna/8iP/Ei0v/HGG6N9a21lZSXab9myJdqPx+NoP5vNov2xY8eifWtt//790f5v//Zvo/3HPvaxaJ8+5NbadDpNb1LNOnyC8f34hhAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAoiaLPgC1rK6uLvoI3+3CCy+M9pdffnm07/s+2q+Da665JtqnD2GOh/zud7872r/zne+M9n/1V38V7f/8z/882rfW7r///vQm8Ow24KdHajabRfuu66L9eDyO9q218847L9q///3vj/YXXHBBtD/jjDOifWvt7LPPjvbLy8vRPn3W0vs/evRotG+tbdmyJdq/7W1vi/ZXX311tP+t3/qtaN/yn2pqA35cpEcajXxNtTB+9AAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUNVn0AajlE5/4xKKPsN7+7d/+Lb3JN7/5zSFO8h2//Mu/POj9r4OVlZVo/xu/8RvR/od+6Iei/Xw3oZqu6wa9/77vB73/OYxG2d+d0x/Rli1bon1r7U1velO0f+qpp6L9448/Hu2PHj0a7VtrO3fujPZPP/10tJ9Msl8Ot23bFu3neCOcddZZ0f7gwYPR/stf/nK0f81rXhPtW2v//M//HO1PnDiRXmKjSd/+G/ATrA7fEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoarLoA8CCffnLX472L3nJS6L9t7/97WjfWnvqqafSm0T+6I/+KNr3fT/QSb7j5ptvjvY/8RM/MdBJ/scFF1yQ3uQFL3hBtH/ooYfSS1BN+tZL913XRfs5bpIeaTTK/k69c+fOaN9a+4//+I9o/9hjj0X7vXv3Rvsnn3wy2rfWjhw5Eu2Xlpai/fLycrRfXV2N9rt37472rbUdO3ZE+/QhpJ/53/jGN6J9a+2ss86K9vv27Yv26/AvdWoDHonvxzeEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQ1WfQBYMGOHj0a7e+7776BTrJuvva1rw16/+eff356k3POOWeIk8ztrLPOSm/y4he/ONo/9NBD6SV4ruv7ftD777pu0PufQ3qkdH/gwIFo31p7+OGHo/0jjzwS7Q8dOhTtR6P4T/PT6TTaTybZL3vbt2+P9i972cui/ZYtW6J9a+15z3tetD948GC0T/9lT++/tba2thbt04+L9L0z9MfRHDbgJ1gdviEEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoKjJog8ArLedO3dG+7/+67+O9i984QujfWvtiiuuSG8SOXz4cLT/5Cc/mV7is5/9bHoTOO31fR/tZ7NZtD969Gi0b62dPHky2h84cCDad10X7cfjcbRvrW3bti3ap5/JV155ZbTftWtXtL/wwgujfct/qo888ki0f/rpp6P9wYMHo31r7dixY9E+fcjpe2cDSj8uOIV8QwgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUZNFHwBOsYsvvjjar6ysDHOQ/8/q6mp6k3e/+93R/u677472b3/726P95ZdfHu3n8Mwzz0T7Rx99NNrfcsst0f7DH/5wtIchdF0X7fu+H/T+10H6EI4cOZJeYtu2bdF+Op1G+/QhXHbZZdG+tXbttddG+yuvvDLaX3HFFdF+06ZN0X5paSnat9b+8z//M9p/9atfjfZf//rXo/3jjz8e7VtrJ06ciPbpCyk1x/1vwE8MThXfEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoarLoA8CzWVlZSW9y2223Rftdu3all9hofv7nfz7ad10X7fu+j/af/OQno31r7ZZbbon2d9xxR3oJeM5J33pDv7XnuMTQ9z+bzdJLHDlyJNqff/750f7lL395tH/DG94Q7VtrL3nJS6L985///Gi/bdu2aJ8+a3v27In2rbVvfvOb0f6xxx6L9ul7YcuWLdG+tXbixIloP/TbH/53viEEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoKjJog8Az+b1r399epNdu3YNcRL+7/3TP/1TepN/+Zd/GeIksKF0XRft+74fdL8O0iPNZrOBTvId6ZG2bt0a7X/0R3802l9++eXRvrW2Y8eOaL99+/b0EpH0RzqdToe+xKWXXhrtN2/eHO3vvvvuaN+Gf22n9z8axd8JbcBPGE4V3xACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUNRk0QeAZ3PvvfemN3nkkUei/YUXXphegmf3wQ9+ML1J13XR/kMf+lB6CVi4vu831P2n77sNaI6HMJ1Oo/1DDz0U7Uej7E/tR44cifattT179kT77du3R/v0hfTAAw9E+yeffDLat9Y2b94c7Xft2hXtt2zZEu2/+MUvRvs21xMdSV94s9lsoJPwXOQbQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKmiz6APBsvv71r6c3eeELXxjtX//610f7T3/609F+A7rjjjui/TXXXDPMQf5/73jHO6L9xz72sWj/2GOPRXt4Luq6btFH+H/V9/3Ql0h/SkePHo32N954Y7R/5StfGe1ba9u2bYv2V1xxRbQ/fPhwtP/Wt74V7bds2RLtW2u7d++O9i960Yui/datW6P9Jz7xiWjfWnvqqaei/XQ6jfYb8L2THuk0+AR77vINIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFTRZ9AFiwT3/604s+wnq77rrrov3jjz8e7bdu3RrtW2tXXHFFtL/hhhui/Yc+9KFoD0Poum7Q++/7ftD7X59LROY4T3qT9Fk7duxYtL/zzjujfcuP9LnPfS7aX3DBBdE+fchnnXVWtG+tXX755dH+3HPPjfbbtm2L9uk/o621v/mbv4n2o1H2nc1sNov2cxj67b/RPl5K8Q0hAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKImiz4AsN5OnDgR7fu+H+gkc9u5c+eijwCxDfhWSnVdF+034EMe+iGk+/Q8c1haWor2Dz30ULRfXl6O9meccUa0b61dcMEF0X40yr7wGI/Hg+5b/kTPZrNB738d3psb8Eh8P74hBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRghAAAKCoyaIPABC79dZbF30EGFzf99G+67pB93MY+hJz3H/6U02lR5rjPOPxONpPJtkve2tra9F+aWkp2p999tnRvrU2nU6j/erqarTfu3dvtL/rrruifU1Dv9c4hXxDCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiBCEAAEBRk0UfAFhvF198cbQfj8fDHAR4Nl3XDXr/fd+nNxn6SKPR4H+nTi8xx08pMsePdMuWLdH+6NGj0X51dTXan3/++dH++uuvj/attV27dkX7tbW1aH/o0KFo/61vfSvat9Zms1l6k8jQL9R1MPTHC8/CN4QAAABFCUIAAICiBCEAAEBRghAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFDVZ9AGA9fYLv/AL0X5lZWWgk0ApXddF+77vN9T9z3GJpaWlaL9169Zof9FFF0X71trevXujffpTWl5ejvYXXHBBtG+tHT16NNqvra1F+8suuyzav+51r4v2L3rRi6J9a+2MM86I9g8//HC0v++++6L9oUOHov062IBv/zkuwaL4hhAAAKAoQQgAAFCUIAQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUZNFHwCezZ/92Z+lN/n4xz8e7e+6665of/jw4Wg/h/F4HO3f9a53Rftf/MVfjPbr4O6774723/jGNwY6CQyn7/vn9P3Poeu6aH/NNddE+1/6pV+K9q21paWlaL+2thbtjxw5Eu2/8pWvRPvW2mSS/fKW7l/2spdF+3PPPTfa79ixI9q31g4cOBDtDx48GO3/4A/+INpPp9No31qbzWbpTTaagp9gdfiGEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgKEEIAABQlCAEAAAoShACAAAUJQgBAACKEoQAAABFCUIAAICiJos+ADyb17zmNelNfu3Xfi3af+5zn4v2H/jAB6L9o48+Gu1bazfeeGO0f8tb3hLtR6PsL0Gz2Szaf/vb3472rbU/+ZM/ifYHDx5MLwEL1/d9tO+6bkPd/xyXWFtbi/Zf+tKXov14PI72rbXl5eVof9FFF0X76XQa7VdWVqJ9a211dTXan3XWWdE+fZbT8zzwwAPRvrX23//939H+pptuivaHDx+O9umPqOX/8qaXmONI8B2+IQQAAChKEAIAABQlCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAACgqMmiDwDP5rbbbktv8p73vCfaX3vttYPuN6Djx49H+9tvvz3a/9zP/Vy0b60dPnw4vQk854xG2R9h+76P9l3XRfs5pEdaW1uL9nv27In2733ve6N9a+2d73xntH/wwQej/c6dO6P95s2bo31r7dChQ9H+2LFj0f6hhx6K9ul5vv3tb0f71tpf/uVfRvv0Iacv7HS/DpcY+uOF05tvCAEAAIoShAAAAEUJQgAAgKIEIQAAQFGCEAAAoChBCAAAUJQgBAAAKEoQAgAAFCUIAQAAihKEAAAARQlCAACAogQhAABAUYIQAAD4X+3aXajldb3H8d9/7TWzR2fGaRzTYWxSS1OibHJnIEgPEEVRN4EkRF3UVaTQRRdlEuFFVwVdSRcSgUGGdCFIlkXUTYYxUxg54UNaIzk6bZ/Sedzr/z8XBw6cQ8fTd53577Vnf16v689a/996HN97SShBCAAAEEoQAgAAhJou+gDwem6//fbqTfq+L+2/9rWvVS8xtlOnTpX2d9xxR2n/29/+trT/1a9+VdoD/9IwDIs+wv9X13WlffUhV7/AH3vssdK+tXbbbbeV9vv27SvtX3311dJ+x44dpX1rbTablfbTae0/9i6++OLS/siRI6X90aNHS/tWf8jVN1L1jT2HsS+xCb5eWCC/EAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEGq66APAWfb1r3991D0AI+m6rrTv+756iZMnT5b2Tz/9dGlffQgvvPBCaT+H6rP01FNPlfbDMIy6n+Mm1VdhjiNVrcMlYG5+IQQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFDTRR8AADgLhmEo7buuG/sSY9//OjyEvu9Hvf85zPGoN5Q5nqLqQ65eYh3eeLCR+YUQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCOgZ5AAAeGElEQVSCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCTRd9AACA1lobhqG077pupJP8l7GP1Pd9aZ+p+ipUTSa1H0jmeNXGfq9uwM9O1QY8Ug6/EAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQKhuGIZFn2GTWF1dPXjw4KJPAbBJrKys7NmzZ9GnOGtWV1cPHTq06FMAbBLXXXfdZvo3YrEEIQAAQCj/yygAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhJou+gCbx+rq6sGDBxd9CoBNYmVlZc+ePYs+xVnzwgsvHDp0aNGnANgkrrvuugsvvHDRp9gkBOFZc/DgwY985COLPgXAJvGzn/3swx/+8KJPcdYcOnTo4x//+KiX6LqutO/7fuxLbALDMJT2k0nt/72q3v8msAke8gb8IKzDkarfGGMf6f777//Qhz406iVy+F9GAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAg1HTRBwCAzW8Yhr7vSzeZTGp/tB2GYdT7b61VH0JV9SEsLS2NfYmu60r76lNUvf82/kMY+/4Zw9ifzTb+G4kF8gshAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEmi76AACw+XVd13XdqJcYhmHU+2+tVR9C3/el/WRS+zt19f7nsLS0VNqvw5Gqr0J1X30jrcMbr3qJ6htpAz7kjfYqr4Pqq8ZZ5KkHAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQ00UfAAAidF236CP8N8MwjH2J6kMee99a6/u+tK8+S9X9HA+hepOxX+gN+KpVrcNncx0+biVzPOSx39sb7SmK4hdCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCTRd9AACIMAzDqPc/mdT+yNv3ffUS1YdQPVL1/sd+SuewtLRU2nddV73EdFr7j7ctW7aU9tVXbTablfZra2ulfau/0GfOnKleYqPZgO/t6nu1+hDm+CxwtviFEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQk0XfQCyXH311aX9Jz/5yeolPve5z5X2b3nLW6qX2Ggmk9pfdj7xiU+U9vfff39pD/xLXdeV9tWPdt/3pf0clpaWSvvqQx6GobSvnqfVj3TeeeeV9u94xztK+5tuuqm0b61t27attN+9e3dpf8EFF5T2p06dKu2PHTtW2rfWjh8/Xtr/+te/Lu0feOCB0n42m5X2rbW1tbXSvvpGrap+1ua7yYa6f16HXwgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACDUdNEH4Nx2zTXXlPYPPPBAab9///7Sfg7DMIx9ibH1fV/a33vvvaX997///dL+C1/4QmkPIarfNmPvu64r7ee4xNhHWlpaKu1ba3v37i3tP/OZz5T2b3zjG0v7K664orRvre3atau0n+NZKlleXi7tq+dvrZ0+fbq0n81mpX31VfvBD35Q2rf6v9TVfdVkUv5NaAMeibPFUw8AABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKGmiz4A57Yf//jHpf3+/ftL+2effba0b639/Oc/L+2PHTtW2v/5z38u7ZeXl0v71tp73/ve0v7aa68t7Q8cOFDaf/rTny7tv/Wtb5X2rbUnn3yyehM450wmtT/CDsNQ2nddN+q+1Y+0tLRU2lePtH379tK+tfb2t7+9tD9x4kRpv7q6Wto/8cQTpX1rbdeuXaX98ePHS/sdO3aU9tV/5qrvilZ/oatv1Op/Cbzzne8s7VtrjzzySGl/8uTJ0r76kNfB2N94nEV+IQQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAINR00Qfg3PaNb3yjtP/Upz5V2n/7298u7VtrDz30UPUmG813v/vd0v6GG24o7X/xi1+U9ueff35pP536YoF/oe/70r7rutJ+GIbSvnqe1trS0lJpP5vNSvvl5eXSfseOHaV9a+2pp54q7SeT2p/OX3zxxdL+xIkTpX1r7fTp06X9tm3bSvstW7aMev8rKyulfau/Marv7be+9a2l/TPPPFPat/p7tfoqz/FxHtvY33icRX4hBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUNNFH4Bz27333jvqnn/Hj370o9J+eXm5tP/JT35S2v/1r38t7YGF6Lpu7JtU98MwlPanTp0q7Vtrx44dK+1feeWV0v7MmTOl/RyvwsmTJ0v7nTt3lvYXXXRRab+yslLav+ENbyjtW2v79u0r7Z9//vnS/vHHHy/tjx8/Xtq31k6fPl3aVz8L1TdS3/el/RzmeG+zKH4hBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUNNFHwD4n/bv31/aX3rppaX9MAylfVXXdaPeP4SoflSrH705vgqqN6nu+74v7U+dOlXat9Zms1lp/49//KO0r74KW7duLe1bazt37izt9+3bV9ofOHCgtL/mmmtK++q/Wa217du3l/aHDx+uXqLk5Zdfrt6k+t6uvpGq9z+Hsb9h/MfDAvmFEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQk0XfQDY5M4///zqTR588MExTjK3j33sY6X9N7/5zeolbrvtttL+xIkT1UvAOWcyqf3Rtu/7Ue+/tTYMw6j76kOY46tgeXm5tJ/NZqV99Vl905veVNq31q6//vrS/j3veU9p/7a3va2037JlS2m/devW0r619vTTT5f2Tz75ZGl/+PDh0v7YsWOlfWttbW2ttB/7s1bds7n5hRAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFDTRR8ANrkzZ85Ub/K73/2utN++fXtpPwxDab93797S/tZbby3t5/CVr3yltD916tRIJ4HxVD+qXdeNev9zXKJqMqn9nXptbW3sS+zZs6e0v/rqq0v7G264obSf4xJXXnllaV/9N2V5ebm0f/bZZ0v71trf//730v7o0aOlffWNVH3IrbW+70v76pHG/my2+men+g0zxzcSZ4tfCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAINR00QeATe7MmTPVm3z2s58d4yRz+853vlPaf/7zn69e4tZbby3tJ5PaH7O+/OUvl/ZzvGrwf6q+b/u+L+27rivth2Eo7ee4RHU/m81K+zlUn9Xt27eX9ldffXVpf9VVV5X2rbVLL720tK8+hKpTp06V9nO8ytWHcOWVV1YvUbK6ulq9ydjv7bG/Lua4BOcQvxACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAECo6aIPAGx0X/rSl0r7O++8s3qJBx98sLT/4he/WNr/8pe/LO3vu+++0h7+HX3fb6j777pupJP8l2EYSvvqkeZ4CGtra6X9888/X9pXj/Tyyy+X9q21F198sbTfvXt3aV99CEePHi3t9+7dW9q31nbt2lXaX3PNNaX90tJSaf/II4+U9q21V199tXqTki1btpT21Q/CHKof/8nEz1QL46kHAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQ00UfANhsHnvssepNnnvuudJ+//79pf373//+0v6+++4r7eHf0XXdOX3/rbVhGEa9/77vS/vJpPx37eqz9Nprr5X2d999d2l/4MCB0r7VH/XWrVtHvf/ZbFbaX3bZZaV9a21lZaW0v+KKK0r7tbW10n779u2lfWttdXW1tK9+FqoPYQ7VN8bYXxecRX4hBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUNNFHwCg/eY3vyntV1ZWSvtbbrmltH/44YdL+9baPffcU70JaSaT2h9hh2HYUPvWWtd1o+6rR+r7vrRv9SNVL1HdP/TQQ6V9q7+Rtm7dOup+y5Ytpf1LL71U2rfW9u/fX9pffvnlpf2ll15a2t94442lfWvthz/8YWm/0b4u1uES1YfMWeSpBwAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAg1HTRBwBod911V2l/0003lfZ79+4t7W+//fbSvrV2zz33VG9CmtlsVtp3XTfSSea+/+pNhmHYUPe/Dqqv8jqYTGp//T9x4kRpv3PnztJ+bW2ttG+tnX/++aX9mTNnNtS+tba0tFTaV99I1c/COnx2qm+8vu9HOgn/J78QAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAqOmiDwDQ/vSnP5X2jz76aGm/d+/e0n7nzp2lPYyh7/vSvuu60n5paam034Amk/LftYdhGOMkc99/9VVr9ReueqTqG6/6Klx00UWlfWvt9OnTpf10Wvvv2+r9Hz58uLRv6/LGGFv1IYz9WeMs8gshAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEmi76AADtwgsvHHUPG0HXdaPuh2Eo7fu+L+1ba5PJuH9HXlpaKu2rT1GrP4TZbFbaV1+FOR7Ctm3bSvvXXnuttN+9e3dpf/nll5f27373u0v71tpVV11V2m/durW0f+mll0r75557rrRv47+Rqh/nOd54VdWHMPbXC6/DUw8AABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKGmiz4AsNlce+211Zvccccdpf2BAweqlyh55plnRr1/MnVdV9r3fV/aTya1P/IOw1Dat/pDOO+880r7nTt3lvaXXHJJad9ae+mll0r72WxW2i8tLZX202n5v8SWl5dL++qRrrjiitL+gx/8YGm/b9++0r61tmfPntL+yJEjpf3Ro0dL+9XV1dK+1T871Y9n9VWufr20+kPgHOIXQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQk0XfQBgve3evbu0/+hHP1ra33nnnaV9a23Hjh3Vm5T8/ve/L+1vvvnmkU5CsmEYRr3/vu9L+67rqpeoPoTqJd71rneV9u973/tK+9ba0tJSaT+Z1P50fvLkydL+6NGjpX1rbXl5ubS/5JJLSvvLLrustN+zZ09pv2vXrtK+tfbss8+W9s8991xp/73vfa+0n81mpX2rfzyrn7Wx738dbMAj5fALIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEEoQAgAAhJou+gCwYFu2bCntL7744pFOMrcPfOADpf0tt9xS2l9//fWl/RxOnTpV2v/hD38o7W+++ebS/siRI6U9/DuGYTin73+OS1Q/2k8//XRpP8e30759+0r7N7/5zaV913Wl/Ysvvljat9YuuOCC0n7nzp3VS5Ts2LGjtP/b3/5WvcTDDz9c2t99992l/SuvvFLaz2Fpaam0r37WZrNZaV99o7b6kaqXWIdvMP43fiEEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQ00UfABbsyiuvLO3/+Mc/jnSSuXVdV9oPwzDSSf7Ta6+9Vr3JV7/61dL+zjvvrF4CFq76Ua0a+6PdWuv7vrQ/c+ZMaf/444+X9vfee29p31q78cYbS/u//OUvpf2+fftK+wsuuKC0b62dOHGitF9bWyvtH3300dL+5MmTpX31VW6t/fSnPy3tq/8MzWaz0r76QZhD9eM8mdR+45nj62Lsb7Cx75/X4RdCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAINV30AYD19tBDD5X2999/f2l/1113lfattdXV1epN4JwzDENp33VdaT+Z1P7IWz3POlyi7/vS/oknnijtW2tHjhwp7bdt21banz59urTfuXNnad/qz1LV7t27S/t//vOfpf0cX/hnzpwp7WezWWlf/axV9+tgjo/zRrvEBnxWc/iFEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUNNFHwAW7PDhw6X9dOpTA6yHYRhK+67rSvvJpPxH4eqRxjabzao3OX78eGl/8uTJ0r76rJ4+fbq0n0P1WTp27FhpX33j9X1f2rf6G6/6KszxRqqqHmmjfdbmEPiQz11+IQQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFDTRR8AAPgXhmFY9BH+p67rRr3/vu9L+8mk/Hft6rNa3c9ms9J+bW2ttG/jvwrVh1w9zzq8sauvQtUcL0H1vV1VPdI6PIQN+A3G/8YvhAAAAKEEIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEGq66AMAQISu60a9/2EYRt231iaTc/7vyGM/S9WnaI5XYaMdaY6HsNFUn6LZbDb2Jar6vi/tN+BneexvSF7Hhns3AAAAsD4EIQAAQChBCAAAEEoQAgAAhBKEAAAAoQQhAABAKEEIAAAQShACAACEEoQAAAChBCEAAEAoQQgAABBKEAIAAIQShAAAAKEEIQAAQChBCAAAEKobhmHRZ9gkVldXDx48uOhTAGwSKysre/bsWfQpzpoXXnjh0KFDo15iHf5B77pu7EvAxjfHZ23sz071SJvgs3zdddddeOGFiz7FJiEIAQAAQvlfRgEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAglCAEAAEIJQgAAgFCCEAAAIJQgBAAACCUIAQAAQglCAACAUIIQAAAglCAEAAAIJQgBAABCCUIAAIBQghAAACCUIAQAAAj1H1MkG2wYFGBDAAAAAElFTkSuQmCC", "text/plain": [ - "Scene (1200px, 2420px):\n", - " 159 Plots:\n", - " ├ MakieCore.Combined{Makie.poly, Tuple{Vector{Vector{GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Combined{Makie.poly, Tuple{Vector{Vector{GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Combined{Makie.poly, Tuple{Vector{Vector{GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Combined{Makie.poly, Tuple{Vector{Vector{GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Combined{Makie.poly, Tuple{Vector{Vector{GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Combined{Makie.poly, Tuple{Vector{Vector{GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Combined{Makie.poly, Tuple{Vector{Vector{GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Combined{Makie.poly, Tuple{Vector{Vector{GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.LineSegments{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{Vector{Tuple{AbstractString, GeometryBasics.Point{2, Float32}}}}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Lines{Tuple{Vector{GeometryBasics.Point{2, Float32}}}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " ├ MakieCore.Text{Tuple{String}}\n", - " └ MakieCore.Text{Tuple{String}}\n", - " 8 Child Scenes:\n", - " ├ Scene (500px, 500px)\n", - " ├ Scene (500px, 500px)\n", - " ├ Scene (500px, 500px)\n", - " ├ Scene (500px, 500px)\n", - " ├ Scene (500px, 500px)\n", - " ├ Scene (500px, 500px)\n", - " ├ Scene (500px, 500px)\n", - " └ Scene (500px, 500px)" + "Figure()" ] }, + "execution_count": 55, "metadata": {}, - "output_type": "display_data" + "output_type": "execute_result" } ], "source": [ "xs = makebatch(method, data, rand(1:nobs(data), 4)) |> gpu\n", - "ypreds, _ = model(xs) |> cpu\n", - "showoutputbatch(method, cpu(xs), ypreds)" + "ypreds, _ = model(xs)\n", + "showoutputbatch(method, cpu(xs), cpu(ypreds))" ] }, { @@ -862,18 +666,17 @@ ], "metadata": { "kernelspec": { - "display_name": "Julia 1.7.0-rc3", + "display_name": "Julia (12 threads) 1.7.0", "language": "julia", - "name": "julia-1.7" + "name": "julia-(12-threads)-1.7" }, "language_info": { "file_extension": ".jl", "mimetype": "application/julia", "name": "julia", - "version": "1.7.0-rc3" - }, - "orig_nbformat": 4 + "version": "1.7.0" + } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } From f2ef1548be026a4af99e6d120cfd54871c3519d5 Mon Sep 17 00:00:00 2001 From: lorenzoh Date: Thu, 23 Dec 2021 19:04:55 +0100 Subject: [PATCH 08/23] Add block invariant interface --- Project.toml | 1 + src/FastAI.jl | 2 ++ src/datablock/block.jl | 56 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index d8c1f67f05..9035258852 100644 --- a/Project.toml +++ b/Project.toml @@ -22,6 +22,7 @@ Glob = "c27321d9-0574-5035-807b-f59d2c89b15c" ImageInTerminal = "d8c32880-2388-543b-8c61-d9f865259254" IndirectArrays = "9b13fd28-a010-5f03-acff-a1bbcff69959" InlineTest = "bd334432-b1e7-49c7-a2dc-dd9149e4ebd6" +Invariants = "115d0255-0791-41d2-b533-80bc4cbe6c10" JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" LearnBase = "7f8f8fb0-2700-5f03-b4bd-41f8cfc144b6" MLDataPattern = "9920b226-0b2a-5f5f-9153-9aa70a013f8b" diff --git a/src/FastAI.jl b/src/FastAI.jl index 696d6c710d..96b546eae7 100644 --- a/src/FastAI.jl +++ b/src/FastAI.jl @@ -23,6 +23,8 @@ import Flux.Optimise: apply!, Optimiser, WeightDecay using FluxTraining: Learner, handle using FluxTraining.Events using JLD2: jldsave, jldopen +using Invariants: BooleanInvariant, WithMessage, AnyInvariant, AllInvariant, + SequenceInvariant, check_error, check using Markdown using MLDataPattern using PrettyTables diff --git a/src/datablock/block.jl b/src/datablock/block.jl index e206246b4c..e6c1389fa6 100644 --- a/src/datablock/block.jl +++ b/src/datablock/block.jl @@ -129,4 +129,58 @@ typify(t::Tuple) = Tuple{map(typify, t)...} typify(block::FastAI.AbstractBlock) = typeof(block) -# Continous +# ## Invariants +# +# Invariants allow specifying properties that an instance of a data for a block +# should have in more detail and such that actionable error messages can be given. + +""" + invariant_checkblock(block; kwargs...) + invariant_checkblock(blocks; kwargs...) + +Create a `Invariants.Invariant` that can be used to check whether an +observation is a valid instance of `block`. This should always agree +with `checkblock` (i.e. `checkblock(block, obs)` implies that +`check(invariant_checkblock(block), obs)`). The invariant can however +be used to give much more detailed information about the problem and +be used to throw helpful error messages from functions that depend +on these properties. +""" +function invariant_checkblock end + + +# If `invariant_checkblock` is not implemented for a block, default to +# checking that `checkblock` returns `true`. + +function invariant_checkblock(block::AbstractBlock; obsname = "obs", blockname = "block") + return BooleanInvariant( + obs -> checkblock(block, obs), + "`$obsname` should be valid $(typeof(block))", + _ -> """Expected `$obsname` to be a valid instance of block $blockname + with above type, but `checkblock($blockname, $obsname)` returned `false`. + This probably means that `$obsname` is not a valid instance of the + block. Check `?$(typeof(block).name.name)` for more information on + the block and what data is valid. + """, + ) +end + +# For tuples of blocks, the invariant is composed of the individuals' blocks +# invariants, passing only if all the child invariants pass. + +function invariant_checkblock(blocks::Tuple; obsname = "obss", blockname = "blocks") + return AllInvariant( + [ + WithContext( + obss -> obss[i], + isblockinvariant( + blocks[i]; + obsname = "$obsname[$i]", + blockname = "$blockname[$i]", + ), + ) for (i, block) in enumerate(blocks) + ], + name = "`$obsname` should be valid `$blockname`", + description = "Each `$obsname[i]` is a valid instance of block `$blockname[i]`." + ) +end From f1432ee40da926374ec4c37ca3a04e8596ef80e1 Mon Sep 17 00:00:00 2001 From: lorenzoh Date: Thu, 23 Dec 2021 19:05:16 +0100 Subject: [PATCH 09/23] implement block invariants for basic blocks --- src/blocks/continuous.jl | 39 ++++++++++++++++++++ src/blocks/label.jl | 79 ++++++++++++++++++++++++++++++++++------ 2 files changed, 106 insertions(+), 12 deletions(-) diff --git a/src/blocks/continuous.jl b/src/blocks/continuous.jl index 3e345b48c5..7b0b08b169 100644 --- a/src/blocks/continuous.jl +++ b/src/blocks/continuous.jl @@ -20,3 +20,42 @@ function blocklossfn(outblock::Continuous, yblock::Continuous) outblock.size == yblock.size || error("Sizes of $outblock and $yblock differ!") return Flux.Losses.mse end + + +function invariant_checkblock(block::Continuous; blockname = "block", obsname = "obs") + return SequenceInvariant( + [ + BooleanInvariant( + obs -> obs isa AbstractVector, + name = "`$obsname` should be an `AbstractVector`", + messagefn = obs -> """`$obsname` should be an `AbstractVector`, instead + got type `$(typeof(obs))`. + """ + ), + BooleanInvariant( + obs -> length(obs) == block.size, + name = "length(`$obsname`) should be $(block.size)", + messagefn = obs -> """`$obsname` should have $(block.size) features, instead + found a vector with $(length(obs)) features. + """ + ), + BooleanInvariant( + obs -> eltype(obs) <: Number, + name = "`eltype($obsname)` should be a subtype of `Number`", + messagefn = obs -> """Found a non-numerical element type $(eltype(obs))""" + ), + ], + "`$obsname` should be a valid `$(summary(block))`", + "", + ) +end + + +@testset "Continuous [block]" begin + inv = invariant_checkblock(Continuous(5)) + + @test check(inv, zeros(5)) + @test !(check(inv, "hi")) + @test !(check(inv, ["hi"])) + @test !(check(inv, [5])) +end diff --git a/src/blocks/label.jl b/src/blocks/label.jl index 315a7b948b..6b4eb57838 100644 --- a/src/blocks/label.jl +++ b/src/blocks/label.jl @@ -31,11 +31,23 @@ struct Label{T} <: Block classes::AbstractVector{T} end -checkblock(label::Label{T}, obs::T) where T = obs ∈ label.classes -mockblock(label::Label{T}) where T = rand(label.classes)::T +checkblock(label::Label{T}, obs::T) where {T} = obs ∈ label.classes +mockblock(label::Label{T}) where {T} = rand(label.classes)::T setup(::Type{Label}, data) = Label(unique(eachobs(data))) +Base.summary(io::IO, ::Label{T}) where {T} = print(io, "Label{", T, "}") + +function invariant_checkblock(block::Label; blockname = "block", obsname = "obs") + return BooleanInvariant( + obs -> obs ∈ block.classes, + name = "`$obsname` should be a valid `$(summary(block))`", + messagefn = obs -> """`$obsname` should be one of the valid classes, i.e. + `$obsname ∈ $blockname.classes`. Instead got unknown class `$(sprint(show, obs))`. + Valid classes are: + `$(sprint(show, block.classes, context=:limit => true))`""", + ) +end """ LabelMulti(classes) @@ -65,34 +77,77 @@ struct LabelMulti{T} <: Block classes::AbstractVector{T} end -function checkblock(label::LabelMulti{T}, v::AbstractVector{T}) where T +function checkblock(label::LabelMulti{T}, v::AbstractVector{T}) where {T} return all(map(x -> x ∈ label.classes, v)) end mockblock(label::LabelMulti) = - unique([rand(label.classes) for _ in 1:rand(1:length(label.classes))]) + unique([rand(label.classes) for _ = 1:rand(1:length(label.classes))]) setup(::Type{LabelMulti}, data) = LabelMulti(unique(eachobs(data))) -InlineTest.@testset "Label [block]" begin +Base.summary(io::IO, ::LabelMulti{T}) where {T} = print(io, "LabelMulti{", T, "}") + +function invariant_checkblock(block::LabelMulti; blockname = "block", obsname = "obs") + return SequenceInvariant( + [ + BooleanInvariant( + obs -> obs isa AbstractVector, + name = "`$obsname` should be an `AbstractVector`", + messagefn = obs -> """`$obsname` should be an `AbstractVector`, instead + got type `$(typeof(obs))`. + """ + ), + BooleanInvariant( + obs -> all([y ∈ block.classes for y in obs]), + name = "Elements in `$obsname` should be valid classes", + messagefn = obs -> """`$obsname` should contain only valid classes, + i.e. `∀ y ∈ $obsname: y ∈ $blockname.classes`. + Instead got unknown classes `$( + sprint(show, [y for y in obs if !(y ∈ block.classes)]))`. + + Valid classes are: + `$(sprint(show, block.classes, context=:limit => true))`""", + ), + ], + "`$obsname` should be a valid `$(summary(block))`", + "", + ) +end + + +# ## Tests + +@testset "Label [block]" begin block = Label(["cat", "dog"]) - InlineTest.@test FastAI.checkblock(block, "cat") - InlineTest.@test !(FastAI.checkblock(block, "horsey")) + @test FastAI.checkblock(block, "cat") + @test !(FastAI.checkblock(block, "horsey")) targets = ["cat", "dog", "dog", "dog", "cat", "dog"] block = setup(Label, targets) - InlineTest.@test block.classes == ["cat", "dog"] + @test block.classes == ["cat", "dog"] + + inv = invariant_checkblock(Label([1, 2, 3])) + @test check(inv, 1) + @test !(check(inv, 0)) end -InlineTest.@testset "LabelMulti [block]" begin +@testset "LabelMulti [block]" begin block = LabelMulti(["cat", "dog"]) - InlineTest.@test FastAI.checkblock(block, ["cat"]) - InlineTest.@test !(FastAI.checkblock(block, ["horsey", "cat"])) + @test FastAI.checkblock(block, ["cat"]) + @test !(FastAI.checkblock(block, ["horsey", "cat"])) targets = ["cat", "dog", "dog", "dog", "cat", "dog"] block = setup(LabelMulti, targets) - InlineTest.@test block.classes == ["cat", "dog"] + @test block.classes == ["cat", "dog"] + + inv = invariant_checkblock(block) + @test check(inv, ["cat", "dog"]) + @test check(inv, []) + @test !(check(inv, "cat")) + @test !(check(inv, ["mouse"])) + @test !(check(inv, ["mouse", "cat"])) end From 55b1814d9c766e972c7a22682af9d0cb424142e0 Mon Sep 17 00:00:00 2001 From: lorenzoh Date: Mon, 3 Jan 2022 14:45:47 +0100 Subject: [PATCH 10/23] more invariants --- src/FastAI.jl | 3 +- src/Vision/blocks/bounded.jl | 41 ++++++++++++++++++++---- src/Vision/blocks/keypoints.jl | 26 +++++++++++----- src/blocks/many.jl | 2 +- src/datablock/block.jl | 33 +++++++++++++------- src/datablock/wrappers.jl | 13 +++++--- src/invariants.jl | 57 ++++++++++++++++++++++++++++++++++ src/learner.jl | 1 - 8 files changed, 145 insertions(+), 31 deletions(-) create mode 100644 src/invariants.jl diff --git a/src/FastAI.jl b/src/FastAI.jl index 96b546eae7..28061fbbf1 100644 --- a/src/FastAI.jl +++ b/src/FastAI.jl @@ -24,7 +24,7 @@ using FluxTraining: Learner, handle using FluxTraining.Events using JLD2: jldsave, jldopen using Invariants: BooleanInvariant, WithMessage, AnyInvariant, AllInvariant, - SequenceInvariant, check_error, check + SequenceInvariant, WithContext, check_error, check using Markdown using MLDataPattern using PrettyTables @@ -101,6 +101,7 @@ include("fasterai/learningmethods.jl") include("fasterai/defaults.jl") +include("invariants.jl") # Domain-specific include("Vision/Vision.jl") diff --git a/src/Vision/blocks/bounded.jl b/src/Vision/blocks/bounded.jl index a170961b0e..4fa1873ad3 100644 --- a/src/Vision/blocks/bounded.jl +++ b/src/Vision/blocks/bounded.jl @@ -1,3 +1,18 @@ +const DimSize = Union{Int, Colon} + +function checksize(targetsz::Tuple, sz::Tuple) + length(targetsz) == length(sz) || return false + return all(map(_checksizedim, targetsz, sz)) +end + +#checksize(targetsz::Tuple{N, <:DimSize}, sz::NTuple{M, Int}) where {N, M} = false + +_checksizedim(l1::Int, l2::Int) = l1 == l2 +_checksizedim(l1::Colon, l2::Int) = true + +mockarray(T, sz) = rand(T, map(l -> l isa Colon ? rand(8:16) : l, sz)) + + """ Bounded(block, size) <: WrapperBlock @@ -28,24 +43,38 @@ will update the bounds: block = Image{2}() Bounded(Bounded(block, (16, 16)), (8, 8)) == Bounded(block, (8, 8)) ``` - - - """ struct Bounded{N, B<:AbstractBlock} <: WrapperBlock block::B - size::NTuple{N, Int} + size::NTuple{N, DimSize} end -function Bounded(bounded::Bounded{M}, size::NTuple{N, Int}) where {N, M} +function Bounded(bounded::Bounded{M}, size::NTuple{N, DimSize}) where {N, M} N == M || error("Cannot rewrap a `Bounded` with different dimensionalities $N and $M") Bounded(wrapped(bounded), size) end -InlineTest.@testset "Bounded [block, wrapper]" begin +function checkblock(bounded::Bounded{N}, a::AbstractArray{N}) where N + return checksize(bounded.size, size(a)) && checkblock(parent(bounded), a) +end + + +@testset "Bounded [block, wrapper]" begin @test_nowarn Bounded(Image{2}(), (16, 16)) bounded = Bounded(Image{2}(), (16, 16)) + @test checkblock(bounded, rand(16, 16)) + + # composition @test Bounded(bounded, (16, 16)) == bounded end + +@testset "checksize" begin + @test checksize((10, 1), (10, 1)) + @test !checksize((100, 1), (10, 1)) + @test checksize((:, :, :), (1, 2, 3)) + @test !checksize((:, :, :), (1, 2)) + @test checksize((10, :, 1), (10, 20, 1)) + @test !checksize((10, :, 2), (10, 20, 1)) +end diff --git a/src/Vision/blocks/keypoints.jl b/src/Vision/blocks/keypoints.jl index 360ca91043..691bd2d12e 100644 --- a/src/Vision/blocks/keypoints.jl +++ b/src/Vision/blocks/keypoints.jl @@ -6,19 +6,19 @@ A block representing an array of size `sz` filled with keypoints of type `SVector{N}`. """ struct Keypoints{N,M} <: Block - sz::NTuple{M,Int} + sz::NTuple{M,DimSize} end Keypoints{N}(n::Int) where {N} = Keypoints{N,1}((n,)) -Keypoints{N}(t::NTuple{M,Int}) where {N,M} = Keypoints{N,M}(t) +Keypoints{N}(sz::Tuple) where {N} = Keypoints{N,length(sz)}(sz) function checkblock( - ::Keypoints{N,M}, - ::AbstractArray{<:Union{SVector{N,T},Nothing},M}, -) where {M,N,T} - return true + b::Keypoints{N,M}, + ks::AbstractArray{<:Union{<:SVector{N},Nothing},M}, +) where {M,N} + return checksize(b.sz, size(ks)) end -mockblock(block::Keypoints{N}) where {N} = rand(SVector{N,Float32}, block.sz) +mockblock(block::Keypoints{N}) where {N} = mockarray(SVector{N,Float32}, block.sz) # ## Visualization @@ -35,3 +35,15 @@ function showblock!(io, ::ShowText, block::Bounded{2, <:Keypoints{2}}, obs) xlim=(0, w), ylim=(0, h), marker=:cross) print(io, plot) end + + +@testset "Keypoints [block]" begin + block = Keypoints{2}((10, 10)) + @test checkblock(block, rand(SVector{2}, 10, 10)) + + block = Keypoints{2}((10, :)) + @test checkblock(block, rand(SVector{2}, 10, 10)) + + ks = map(k -> (rand() > .5) ? k : nothing, rand(SVector{2}, 10, 10)) + @test checkblock(block, ks) +end diff --git a/src/blocks/many.jl b/src/blocks/many.jl index e3bba7fa9b..44e5ec51ec 100644 --- a/src/blocks/many.jl +++ b/src/blocks/many.jl @@ -17,7 +17,7 @@ end FastAI.checkblock(many::Many, obss) = all(checkblock(wrapped(many), obs) for obs in obss) -FastAI.mockblock(many::Many) = [mockblock(wrapped(many)), mockblock(wrapped(many))] +FastAI.mockblock(many::Many) = [mockblock(wrapped(many)) for _ in 1:rand(1:3)] function FastAI.encode(enc::Encoding, ctx, many::Many, obss) return map(obss) do obs diff --git a/src/datablock/block.jl b/src/datablock/block.jl index e6c1389fa6..e41f6dda5c 100644 --- a/src/datablock/block.jl +++ b/src/datablock/block.jl @@ -169,18 +169,29 @@ end # invariants, passing only if all the child invariants pass. function invariant_checkblock(blocks::Tuple; obsname = "obss", blockname = "blocks") - return AllInvariant( + return SequenceInvariant( [ - WithContext( - obss -> obss[i], - isblockinvariant( - blocks[i]; - obsname = "$obsname[$i]", - blockname = "$blockname[$i]", - ), - ) for (i, block) in enumerate(blocks) + BooleanInvariant( + obss -> (obss isa Tuple && length(obss) == length(blocks)), + "$obsname should be a `Tuple` with $(length(blocks)) elements.", + obss -> """Instead, got a `$(sprint(show, typeof(obss)))`"""), + AllInvariant( + [ + WithContext( + obss -> obss[i], + invariant_checkblock( + blocks[i]; + obsname = "$obsname[$i]", + blockname = "$blockname[$i]", + ), + ) for (i, block) in enumerate(blocks) + ], + name = "`$obsname` should be valid `$blockname`", + description = "" + ) ], - name = "`$obsname` should be valid `$blockname`", - description = "Each `$obsname[i]` is a valid instance of block `$blockname[i]`." + "For a tuple of blocks, an instance should be a tuple of valid instances", + "", ) + end diff --git a/src/datablock/wrappers.jl b/src/datablock/wrappers.jl index 03611f03f8..87ee5f3ec8 100644 --- a/src/datablock/wrappers.jl +++ b/src/datablock/wrappers.jl @@ -2,13 +2,18 @@ abstract type WrapperBlock <: AbstractBlock end -wrapped(w::WrapperBlock) = w.block +Base.parent(w::WrapperBlock) = w.block +Base.parent(b::Block) = b +wrapped(w::WrapperBlock) = wrapped(parent(w)) wrapped(b::Block) = b function setwrapped(w::WrapperBlock, b) + # TODO: make recursive return Setfield.@set w.block = b end -mockblock(w::WrapperBlock) = mockblock(wrapped(w)) -checkblock(w::WrapperBlock, obs) = checkblock(wrapped(w), obs) +mockblock(w::WrapperBlock) = mockblock(parent(w)) +checkblock(w::WrapperBlock, obs) = checkblock(parent(w), obs) + +# TODO: add way to specify how wrapper blocks compose # If not overwritten, encodings are applied to the wrapped block """ @@ -61,7 +66,7 @@ end function encodedblock(enc::Encoding, wrapper::WrapperBlock, ::PropagateSameBlock) inner = encodedblock(enc, wrapped(wrapper)) - inner == wrapped(block) && return setwrapped(wrapper, inner) + inner == wrapped(wrapper) && return setwrapped(wrapper, inner) return inner end diff --git a/src/invariants.jl b/src/invariants.jl new file mode 100644 index 0000000000..6a302bec49 --- /dev/null +++ b/src/invariants.jl @@ -0,0 +1,57 @@ + + +""" + invariant_datacontainer(block) + +Create an `Invariants.Invariant` that checks that `data` is a data container +with observations that are valid instances of block `block`. See also +[`invariant_checkblock`](#). +""" +function invariant_datacontainer(block) + return SequenceInvariant([ + WithContext(data -> (data,), invariant_hasmethod(nobs, (data = Any,))), + WithContext(data -> (data, 1), invariant_hasmethod(getobs, (data = Any, idx = Int))), + WithContext(data -> getobs(data, 1), invariant_checkblock(block)), + ], "`data` should be a data container with block observations $(summary(block))", "") +end + + +function invariant_hasmethod(f, argtypes::NamedTuple) + methodcall = "`$f(" + for (i, (name, T)) in enumerate(zip(keys(argtypes), values(argtypes))) + if i != 1 + methodcall *= ", " + end + methodcall *= string(name) + if T !== Any + methodcall *= "::$T" + end + end + methodcall *= ")`" + return SequenceInvariant( + [ + BooleanInvariant( + args -> Base.hasmethod(f, typeof(args)), + "Method $methodcall should exist", + args -> "Expected method $methodcall to exist but it does not", + ), + BooleanInvariant( + args -> _hasmethod(f, args), + "Method $methodcall should not error", + args -> "Expected method $methodcall not to error but it did.", + ), + ], + "$methodcall should be a valid method call", + "", + ) + +end + +function _hasmethod(f, args) + try + f(args...) + return true + catch e + return false + end +end diff --git a/src/learner.jl b/src/learner.jl index a1459337a5..d4e4c4cf03 100644 --- a/src/learner.jl +++ b/src/learner.jl @@ -50,7 +50,6 @@ function methodlearner( backbone=nothing, model=nothing, callbacks=[], - pctgval=0.2, batchsize=16, optimizer=ADAM(), lossfn=methodlossfn(method), From 2054ce105700c8c922e10c2523cdd6654e8dcbc7 Mon Sep 17 00:00:00 2001 From: lorenzoh Date: Mon, 3 Jan 2022 15:51:49 +0100 Subject: [PATCH 11/23] Make UNet closer to fastai --- src/Vision/models/unet.jl | 149 ++++++++++++++++++++++++++------------ 1 file changed, 101 insertions(+), 48 deletions(-) diff --git a/src/Vision/models/unet.jl b/src/Vision/models/unet.jl index 3d0356ff03..0086b7e3a2 100644 --- a/src/Vision/models/unet.jl +++ b/src/Vision/models/unet.jl @@ -33,63 +33,119 @@ unet = UNetDynamic(backbone, (256, 256, 3, 1); fdownscalk_out = 10) Flux.outputsize(unet, (256, 256, 3, 1)) == (256, 256, 10, 1) ``` """ -function UNetDynamic(backbone, inputsize, final; kwargs...) +function UNetDynamic( + backbone, + inputsize, + k_out::Int; + final = UNetFinalBlock, + fdownscale = 0, + kwargs..., +) backbonelayers = collect(iterlayers(backbone)) - unet = unet_from_layers(backbonelayers, inputsize; kwargs...) + unet = unetlayers( + backbonelayers, + inputsize; + m_middle = UNetMiddleBlock, + skip_upscale = fdownscale, + kwargs..., + ) outsz = Flux.outputsize(unet, inputsize) - return Chain(unet, final(outsz)) + return Chain(unet, final(outsz[end-1], k_out)) end -function UNetDynamic(backbone, inputsize, k_out::Int; kwargs...) - final = insz -> convxlayer(insz[end-1], k_out; ks = 1) - return UNetDynamic(backbone, inputsize, final; kwargs...) + +function catchannels(x1, x2) + ndims(x1) == ndims(x2) || error("Expected inputs with same number of dimensions!") + cat(x1, x2; dims = ndims(x1) - 1) end -function unet_from_layers( - backbonelayers, - insz; - fdownscale = 0, - upsample = upsample_block_small, - agg = (mx, x) -> cat(mx, x; dims = length(insz)-1), - kwargs...) - layers = [] - channeldim = length(insz) - 1 - - - for (i, layer) in enumerate(backbonelayers) - outsz = Flux.outputsize(layer, insz) - if (insz[1] ÷ 2 == outsz[1]) - if fdownscale == 0 - child_unet = unet_from_layers( - backbonelayers[i+1:end], - outsz; - upsample = upsample, - fdownscale = fdownscale, - agg = agg) - outsz = Flux.outputsize(child_unet, outsz) - - upsample_k_out = insz[channeldim] - up = upsample(outsz, upsample_k_out; kwargs...) - - push!(layers, SkipConnection(Chain(layer, child_unet, up), agg)) - - break - else - fdownscale -= 1 - end - end - push!(layers, layer) - insz = outsz +function unetlayers( + layers, + sz; + k_out = nothing, + skip_upscale = 0, + m_middle = _ -> (identity,), +) + isempty(layers) && return m_middle(sz[end-1]) + + layer, layers = layers[1], layers[2:end] + outsz = Flux.outputsize(layer, sz) + does_downscale = sz[1] ÷ 2 == outsz[1] + + if !does_downscale + # If `layer` does not scale down the spatial dimensions, append + # it to a Chain + return Chain(layer, unetlayers(layers, outsz; k_out, skip_upscale)...) + elseif does_downscale && skip_upscale > 0 + # If `layer` does scale down the spatial dimensions, but we don't + # to upsample this one, recurse with modified arguments + return Chain( + layer, + unetlayers(layers, outsz; skip_upscale = skip_upscale - 1, k_out)..., + ) + else + # `layer` scales down the spatial dimensions and we add an upsampling block + # and a skip connection that scales the dimensions back up + childunet = Chain(unetlayers(layers, outsz; skip_upscale)...) + outsz = Flux.outputsize(childunet, outsz) + + k_in = sz[end-1] + k_mid = outsz[end-1] + k_out = isnothing(k_out) ? k_in : k_out + return FastAI.Vision.Models.UNetBlock( + Chain(layer, childunet), + k_in, # Input channels to upsampling layer + k_mid, + k_out, + ) end - - unet = length(layers) == 1 ? only(layers) : Chain(layers...) - return unet end iterlayers(m::Chain) = Iterators.flatten(iterlayers(l) for l in m.layers) iterlayers(m) = (m,) + +""" + UNetBlock(m, k_in) + +Given convolutional module `m` that halves the spatial dimensions +and outputs `k_in` filters, create a module that upsamples the +spatial dimensions and then aggregates features via a skip connection. +""" +function UNetBlock(m_child, k_in, k_mid, k_out = 2k_in) + return Chain( + upsample = SkipConnection( + Chain( + child = m_child, # Downsampling and processing + upsample = PixelShuffleICNR(k_mid, k_mid), # Upsampling + ), + Parallel(catchannels, identity, BatchNorm(k_in)), + ), + act = xs -> relu.(xs), + combine = UNetCombineLayer(k_in + k_mid, k_out), # Data from both branches is combined + ) +end + + +function PixelShuffleICNR(k_in, k_out; r = 2) + return Chain(convxlayer(k_in, k_out * (r^2), ks = 1), Flux.PixelShuffle(r)) +end + + +function UNetCombineLayer(k_in, k_out) + return Chain(convxlayer(k_in, k_out), convxlayer(k_out, k_out)) +end + +function UNetMiddleBlock(k) + return Chain(convxlayer(k, 2k), convxlayer(2k, k)) +end + + +function UNetFinalBlock(k_in, k_out) + return Chain(ResBlock(1, k_in, k_in), convxlayer(k_in, k_out, ks = 1)) +end + """ upsample_block_small(insize, k_out) @@ -97,10 +153,7 @@ An upsampling block that increases the spatial dimensions of the input by 2 using pixel-shuffle upsampling. """ function upsample_block_small(insize, k_out; ks = 3, kwargs...) - return Chain( - Flux.PixelShuffle(2), - convxlayer(insize[end-1] ÷ 4, k_out; kwargs...) - ) + return Chain(Flux.PixelShuffle(2), convxlayer(insize[end-1] ÷ 4, k_out; kwargs...)) end function conv_final(insize, k_out; ks = 1, kwargs...) From 2b3eae5519eb626aa9998d28a54ecc58e2f2be3f Mon Sep 17 00:00:00 2001 From: lorenzoh Date: Sat, 22 Jan 2022 14:30:17 +0100 Subject: [PATCH 12/23] Forward `usedefaultcallbacks` kwarg through `methodlearner` --- src/learner.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/learner.jl b/src/learner.jl index d4e4c4cf03..e1dd77a801 100644 --- a/src/learner.jl +++ b/src/learner.jl @@ -53,13 +53,14 @@ function methodlearner( batchsize=16, optimizer=ADAM(), lossfn=methodlossfn(method), + usedefaultcallbacks=true, kwargs..., ) if isnothing(model) model = isnothing(backbone) ? methodmodel(method) : methodmodel(method, backbone) end dls = methoddataloaders(traindata, validdata, method, batchsize; kwargs...) - return Learner(model, dls, optimizer, lossfn, callbacks...) + return Learner(model, dls, optimizer, lossfn, callbacks...; usedefaultcallbacks) end function methodlearner(method, data; pctgval=0.2, kwargs...) From 29a20822219ec488b1e2e9932864f7c87efdf35c Mon Sep 17 00:00:00 2001 From: lorenzoh Date: Sat, 22 Jan 2022 14:30:34 +0100 Subject: [PATCH 13/23] Add title to `showencodedsample` --- src/interpretation/method.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/interpretation/method.jl b/src/interpretation/method.jl index c38035e2ba..e320be4b56 100644 --- a/src/interpretation/method.jl +++ b/src/interpretation/method.jl @@ -55,7 +55,7 @@ function showencodedsample(backend::ShowBackend, method::AbstractBlockMethod, en showblockinterpretable( backend, getencodings(method), - getblocks(method).encodedsample, + "Encoded sample" => getblocks(method).encodedsample, encsample, ) end From 939e8b395ee128067b38d2291fcc2e2c5b0f332e Mon Sep 17 00:00:00 2001 From: lorenzoh Date: Sun, 29 May 2022 08:43:10 +0200 Subject: [PATCH 14/23] Switch LearnBase + MLDataPattern + DataLoaders -> MLUtils --- Project.toml | 10 +- src/FastAI.jl | 9 +- src/Tabular/Tabular.jl | 1 + src/Textual/recipes.jl | 2 +- src/Vision/Vision.jl | 1 + src/Vision/encodings/imagepreprocessing.jl | 6 +- src/Vision/recipes.jl | 8 +- src/datablock/task.jl | 8 +- src/datasets/Datasets.jl | 33 +--- src/datasets/batching.jl | 29 +++ src/datasets/containers.jl | 57 ++---- src/datasets/load.jl | 6 +- src/datasets/recipe.jl | 4 +- src/datasets/registry.jl | 4 - src/datasets/transformations.jl | 195 --------------------- src/interpretation/backend.jl | 2 +- src/interpretation/task.jl | 14 +- src/learner.jl | 11 +- src/tasks/check.jl | 2 +- src/tasks/predict.jl | 4 +- src/tasks/taskdata.jl | 17 +- src/training/utils.jl | 2 +- 22 files changed, 94 insertions(+), 331 deletions(-) create mode 100644 src/datasets/batching.jl delete mode 100644 src/datasets/registry.jl delete mode 100644 src/datasets/transformations.jl diff --git a/Project.toml b/Project.toml index b2ad342f8d..814b19d27e 100644 --- a/Project.toml +++ b/Project.toml @@ -4,7 +4,6 @@ authors = ["Lorenz Ohly", "Julia Community"] version = "0.4.3" [deps] -Animations = "27a7e980-b3e6-11e9-2bcd-0b925532e340" BSON = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0" CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" ColorVectorSpace = "c3611d14-8923-5661-9e6a-0046d554d3a4" @@ -12,7 +11,6 @@ Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" DataAugmentation = "88a5189c-e7ff-4f85-ac6b-e6158070f02e" DataDeps = "124859b0-ceae-595e-8997-d05f6a7a8dfe" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" -DataLoaders = "2e981812-ef13-4a9c-bfa0-ab13047b12a9" FeatureRegistries = "c6aefb4f-3ac3-4095-8805-528476b02c02" FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" FilePathsBase = "48062228-2e41-5def-b9a4-89aafe57970f" @@ -25,8 +23,8 @@ ImageInTerminal = "d8c32880-2388-543b-8c61-d9f865259254" IndirectArrays = "9b13fd28-a010-5f03-acff-a1bbcff69959" InlineTest = "bd334432-b1e7-49c7-a2dc-dd9149e4ebd6" JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" -LearnBase = "7f8f8fb0-2700-5f03-b4bd-41f8cfc144b6" -MLDataPattern = "9920b226-0b2a-5f5f-9153-9aa70a013f8b" +MLDatasets = "eb30cadb-4394-5ae3-aed4-317e484a6458" +MLUtils = "f1d291b0-491e-4a28-83b9-f70985020b54" Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" MosaicViews = "e94cdb99-869f-56ef-bcf0-1ae2bcbe0389" Parameters = "d96e819e-fc66-5662-9728-84c9c7592b0a" @@ -45,7 +43,6 @@ UnicodePlots = "b8865327-cd53-5732-bb35-84acbb429228" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [compat] -Animations = "0.4" BSON = "0.3" CSV = "0.8, 0.9, 0.10" ColorVectorSpace = "0.9" @@ -53,7 +50,6 @@ Colors = "0.12" DataAugmentation = "0.2.4" DataDeps = "0.7" DataFrames = "1" -DataLoaders = "0.1" FeatureRegistries = "0.1" FileIO = "1.7" FilePathsBase = "0.9" @@ -66,8 +62,6 @@ ImageInTerminal = "0.4" IndirectArrays = "0.5, 1" InlineTest = "0.2" JLD2 = "0.4" -LearnBase = "0.3, 0.4, 0.6" -MLDataPattern = "0.5" MosaicViews = "0.2, 0.3" Parameters = "0.12" PrettyTables = "1.2" diff --git a/src/FastAI.jl b/src/FastAI.jl index d7f370e1db..bf55fa7ddb 100644 --- a/src/FastAI.jl +++ b/src/FastAI.jl @@ -4,14 +4,12 @@ module FastAI using Base: NamedTuple using Reexport @reexport using FluxTraining -@reexport using DataLoaders +import MLUtils +using MLUtils: getobs, numobs, splitobs, eachobs, DataLoader using Flux -using Animations import DataAugmentation import DataAugmentation: getbounds, Bounds - -import LearnBase using FilePathsBase using Flux using Flux.Optimise @@ -20,7 +18,6 @@ using FluxTraining: Learner, handle using FluxTraining.Events using JLD2: jldsave, jldopen using Markdown -using MLDataPattern using PrettyTables using Requires using StaticArrays @@ -160,8 +157,6 @@ export taskdataset, taskdataloaders, tasklossfn, - getobs, - nobs, encodesample, predict, predictbatch, diff --git a/src/Tabular/Tabular.jl b/src/Tabular/Tabular.jl index 944e40a99a..a59729b2c6 100644 --- a/src/Tabular/Tabular.jl +++ b/src/Tabular/Tabular.jl @@ -24,6 +24,7 @@ import ..FastAI: import DataAugmentation import DataFrames: DataFrame +import MLUtils: MLUtils, eachobs, getobs, numobs import Flux import Flux: Embedding, Chain, Dropout, Dense, Parallel, BatchNorm import PrettyTables diff --git a/src/Textual/recipes.jl b/src/Textual/recipes.jl index 1817da7870..8dbc0bc5f0 100644 --- a/src/Textual/recipes.jl +++ b/src/Textual/recipes.jl @@ -20,7 +20,7 @@ function Datasets.loadrecipe(recipe::TextFolders, path) loadfn=(loadfile, recipe.labelfn), splitfn=recipe.split ? grandparentname : nothing) - (recipe.split ? length(data) > 0 : nobs(data) > 0) || error("No text files found in $path") + (recipe.split ? length(data) > 0 : numobs(data) > 0) || error("No text files found in $path") labels = recipe.split ? first(values(data))[2] : data[2] blocks = (Paragraph(), Label(unique(eachobs(labels)))) diff --git a/src/Vision/Vision.jl b/src/Vision/Vision.jl index 13cc2149c7..6286811878 100644 --- a/src/Vision/Vision.jl +++ b/src/Vision/Vision.jl @@ -38,6 +38,7 @@ using ..FastAI: Context, Training, Validation, Inference, Datasets import Flux +import MLUtils: getobs, numobs, mapobs, eachobs import FastAI.Datasets # for tests diff --git a/src/Vision/encodings/imagepreprocessing.jl b/src/Vision/encodings/imagepreprocessing.jl index b2c968558c..f41cc39e0e 100644 --- a/src/Vision/encodings/imagepreprocessing.jl +++ b/src/Vision/encodings/imagepreprocessing.jl @@ -131,15 +131,15 @@ function imagedatasetstats( C; progress=true) means, stds = imagestats(getobs(data, 1), C) - loaderfn = d -> eachobsparallel(d, buffered=false, useprimary=true) + loaderfn = d -> eachobs(d, parallel=true, buffer=false) - p = Progress(nobs(data), enabled=progress) + p = Progress(numobs(data), enabled=progress) for (means_, stds_) in mapobs(img -> imagestats(img, C), data) |> loaderfn means .+= means_ stds .+= stds_ next!(p) end - return means ./ nobs(data), stds ./ nobs(data) + return means ./ numobs(data), stds ./ numobs(data) end diff --git a/src/Vision/recipes.jl b/src/Vision/recipes.jl index b934aa3c48..65d5c18235 100644 --- a/src/Vision/recipes.jl +++ b/src/Vision/recipes.jl @@ -28,7 +28,7 @@ function Datasets.loadrecipe(recipe::ImageFolders, path) loadfn=(loadfile, recipe.labelfn), splitfn=recipe.split ? grandparentname : nothing) - (recipe.split ? length(data) > 0 : nobs(data) > 0) || error("No image files found in $path") + (recipe.split ? length(data) > 0 : numobs(data) > 0) || error("No image files found in $path") labels = recipe.split ? first(values(data))[2] : data[2] blocks = Image{2}(), Label(unique(eachobs(labels))) @@ -68,8 +68,8 @@ function Datasets.loadrecipe(recipe::ImageSegmentationFolders, path) images = loadfolderdata(imagepath, filterfn=isimagefile, loadfn=loadfile) masks = loadfolderdata(maskpath, filterfn=isimagefile, loadfn=f -> loadmask(f, classes)) - nobs(images) == nobs(masks) || error("Expected the same number of images and masks, but found $(nobs(images)) images and $(nobs(masks)) masks") - nobs(images) > 0 || error("No images or masks found in folders $imagepath and $maskpath") + numobs(images) == numobs(masks) || error("Expected the same number of images and masks, but found $(numobs(images)) images and $(numobs(masks)) masks") + numobs(images) > 0 || error("No images or masks found in folders $imagepath and $maskpath") blocks = Image{2}(), Mask{2}(classes) return (images, masks), blocks @@ -104,7 +104,7 @@ function Datasets.loadrecipe(recipe::ImageTableMultiLabel, path) data = (images, labels) blocks = Image{2}(), LabelMulti(unique(Iterators.flatten(labels))) if recipe.split - idxs = 1:nobs(data) + idxs = 1:numobs(data) splits = df[:, recipe.splitcol] data = Dict( "train" => datasubset(data, idxs[splits]), diff --git a/src/datablock/task.jl b/src/datablock/task.jl index 3789aec3c0..d986217c1d 100644 --- a/src/datablock/task.jl +++ b/src/datablock/task.jl @@ -47,7 +47,7 @@ and an instance of data from a block is referred to as `\$name`. - `blocks.sample`: The most important block, representing one full observation of unprocessed data. Data containers used with a learning task should have compatible observations, i.e. - `checkblock(blocks.sample, getobs(data, i))`. + `checkblock(blocks.sample, data[i])`. - `blocks.x`: Data that will be fed into the model, i.e. (neglecting batching) `model(x)` should work - `blocks.ŷ`: Data that is output by the model, i.e. (neglecting batching) @@ -145,11 +145,9 @@ mockmodel(task::AbstractBlockTask) = Create a fake model that maps batches of block `xblock` to batches of block `ŷblock`. Useful for testing. """ -function mockmodel(xblock, ŷblock) +function mockmodel(_, ŷblock) return function mockmodel_block(xs) - out = mockblock(ŷblock) - bs = DataLoaders._batchsize(xs, DataLoaders.BatchDimLast()) - return DataLoaders.collate([out for _ in 1:bs]) + return MLUtils.batch([mockblock(ŷblock) for _ in 1:Datasets.batchsize(xs)]) end end diff --git a/src/datasets/Datasets.jl b/src/datasets/Datasets.jl index 7b96857a29..ed6027654d 100644 --- a/src/datasets/Datasets.jl +++ b/src/datasets/Datasets.jl @@ -1,21 +1,11 @@ -""" - module Datasets - -Commonly used datasets and utilities for creating data containers. - -In the future, contents will be integrated into packages: - -- FastAI datasets and data containers will be moved into MLDatasets.jl -- data container transformations will be moved to MLDataPattern.jl - -This submodule will then reexport the same definitions. -""" module Datasets - using ..FastAI using ..FastAI: typify +import MLUtils: MLUtils, getobs, numobs, filterobs, groupobs, mapobs +import MLDatasets: FileDataset +using MLUtils: mapobs, groupobs using DataDeps using Glob using FilePathsBase @@ -23,9 +13,6 @@ import DataAugmentation using FilePathsBase: filename import FileIO using IndirectArrays: IndirectArray -using MLDataPattern -using MLDataPattern: splitobs -import LearnBase using Colors using FixedPointNumbers using DataFrames @@ -41,7 +28,7 @@ function __init__() end include("containers.jl") -include("transformations.jl") +include("batching.jl") include("load.jl") include("recipe.jl") @@ -52,18 +39,7 @@ include("loaders.jl") export - # reexports from MLDataPattern - splitobs, - - # container transformations - mapobs, - filterobs, - groupobs, - joinobs, - eachobs, - # primitive containers - FileDataset, TableDataset, # utilities @@ -78,7 +54,6 @@ export grandparentname, # datasets - #DATASETS, loadfolderdata, datasetpath, diff --git a/src/datasets/batching.jl b/src/datasets/batching.jl new file mode 100644 index 0000000000..e5650c4f6f --- /dev/null +++ b/src/datasets/batching.jl @@ -0,0 +1,29 @@ + + +batchsize(batch::Union{Tuple,NamedTuple}) = batchsize(batch[1]) +batchsize(batch::Dict) = batchsize(batch[first(keys(batch))]) +batchsize(batch::AbstractArray{T, N}) where {T, N} = size(batch, N) + + +unbatch(batch) = collect(obsslices(batch)) + +obsslices(batch) = + (obsslice(batch, i) for i in 1:batchsize(batch)) + +function obsslice(batch::AbstractArray{T, N}, i) where {T, N} + return view(batch, [(:) for _ in 1:N-1]..., i) +end + +obsslice(batch::AbstractVector, i) = batch[i] + +function obsslice(batch::Tuple, i) + return Tuple(obsslice(batch[j], i) for j in 1:length(batch)) +end + +function obsslice(batch::NamedTuple, i) + return (; zip(keys(batch), obsslice(values(batch), i))...) +end + +function obsslice(batch::Dict, i) + return Dict(k => obsslice(v, i) for (k, v) in batch) +end diff --git a/src/datasets/containers.jl b/src/datasets/containers.jl index d39ccafc58..ae3f3cc413 100644 --- a/src/datasets/containers.jl +++ b/src/datasets/containers.jl @@ -1,30 +1,3 @@ -# FileDataset - -function FileDataset(dir, pattern = "*") - return rglob(pattern, string(dir)) -end - -pathparent(p::String) = splitdir(p)[1] -pathname(p::String) = splitdir(p)[2] - -# File utilities - -""" - rglob(filepattern, dir = pwd(), depth = 4) - -Recursive glob up to 6 layers deep. -""" -function rglob(filepattern = "*", dir = pwd(), depth = 4) - patterns = [ - "$filepattern", - "*/$filepattern", - "*/*/$filepattern", - "*/*/*/$filepattern", - "*/*/*/*/$filepattern", - "*/*/*/*/*/$filepattern", - ] - return vcat([glob(pattern, dir) for pattern in patterns[1:depth]]...) -end """ @@ -60,7 +33,7 @@ end TableDataset(table::T) where {T} = TableDataset{T}(table) TableDataset(path::AbstractPath) = TableDataset(DataFrame(CSV.File(path))) -function LearnBase.getobs(dataset::FastAI.Datasets.TableDataset, idx) +function Base.getindex(dataset::FastAI.Datasets.TableDataset, idx) if Tables.rowaccess(dataset.table) row, _ = Iterators.peel(Iterators.drop(Tables.rows(dataset.table), idx - 1)) return row @@ -75,7 +48,7 @@ function LearnBase.getobs(dataset::FastAI.Datasets.TableDataset, idx) end end -function LearnBase.nobs(dataset::TableDataset) +function Base.length(dataset::TableDataset) if Tables.columnaccess(dataset.table) return length(Tables.getcolumn(dataset.table, 1)) elseif Tables.rowaccess(dataset.table) @@ -87,11 +60,11 @@ function LearnBase.nobs(dataset::TableDataset) end end -LearnBase.getobs(dataset::TableDataset{<:DataFrame}, idx) = dataset.table[idx, :] -LearnBase.nobs(dataset::TableDataset{<:DataFrame}) = nrow(dataset.table) +Base.getindex(dataset::TableDataset{<:DataFrame}, idx) = dataset.table[idx, :] +Base.length(dataset::TableDataset{<:DataFrame}) = nrow(dataset.table) -LearnBase.getobs(dataset::TableDataset{<:CSV.File}, idx) = dataset.table[idx] -LearnBase.nobs(dataset::TableDataset{<:CSV.File}) = length(dataset.table) +Base.getindex(dataset::TableDataset{<:CSV.File}, idx) = dataset.table[idx] +Base.length(dataset::TableDataset{<:CSV.File}) = length(dataset.table) # ## Tests @@ -104,8 +77,8 @@ LearnBase.nobs(dataset::TableDataset{<:CSV.File}) = length(dataset.table) testtable = Tables.table([1 4.0 "7"; 2 5.0 "8"; 3 6.0 "9"]) td = TableDataset(testtable) - @test all(getobs(td, 1) .== [1, 4.0, "7"]) - @test nobs(td) == 3 + @test all(td[1] .== [1, 4.0, "7"]) + @test length(td) == 3 end @testset "TableDataset from columnaccess table" begin @@ -115,10 +88,10 @@ LearnBase.nobs(dataset::TableDataset{<:CSV.File}) = length(dataset.table) testtable = Tables.table([1 4.0 "7"; 2 5.0 "8"; 3 6.0 "9"]) td = TableDataset(testtable) - @test [data for data in getobs(td, 2)] == [2, 5.0, "8"] - @test nobs(td) == 3 + @test [data for data in td[2]] == [2, 5.0, "8"] + @test length(td) == 3 - @test getobs(td, 1) isa NamedTuple + @test td[1] isa NamedTuple end @testset "TableDataset from DataFrames" begin @@ -133,8 +106,8 @@ LearnBase.nobs(dataset::TableDataset{<:CSV.File}) = length(dataset.table) td = TableDataset(testtable) @test td isa TableDataset{<:DataFrame} - @test [data for data in getobs(td, 1)] == [1, "a", 10, "A", 100.0, "train"] - @test nobs(td) == 5 + @test [data for data in td[1]] == [1, "a", 10, "A", 100.0, "train"] + @test length(td) == 5 end @testset "TableDataset from CSV" begin @@ -144,8 +117,8 @@ LearnBase.nobs(dataset::TableDataset{<:CSV.File}) = length(dataset.table) testtable = CSV.File("test.csv") td = TableDataset(testtable) @test td isa TableDataset{<:CSV.File} - @test [data for data in getobs(td, 1)] == [1, "a", 10, "A", 100.0, "train"] - @test nobs(td) == 1 + @test [data for data in td[1]] == [1, "a", 10, "A", 100.0, "train"] + @test length(td) == 1 rm("test.csv") end end diff --git a/src/datasets/load.jl b/src/datasets/load.jl index ca6c16665d..6f3a36c1b7 100644 --- a/src/datasets/load.jl +++ b/src/datasets/load.jl @@ -22,8 +22,8 @@ function loadfolderdata( splitfn = nothing, filterfn = nothing, loadfn = nothing) - data = FileDataset(dir, pattern) - if filterfn !== nothing + data = FileDataset(identity, dir, pattern) + if filterfn !== nothing && !isempty(data) data = filterobs(filterfn, data) end if splitfn !== nothing @@ -39,6 +39,8 @@ function loadfolderdata( return data end +pathparent(p::String) = splitdir(p)[1] +pathname(p::String) = splitdir(p)[2] parentname(f) = f |> pathparent |> pathname grandparentname(f) = f |> pathparent |> pathparent |> pathname matches(re::Regex) = f -> matches(re, f) diff --git a/src/datasets/recipe.jl b/src/datasets/recipe.jl index 9479ba6900..47d15b1e79 100644 --- a/src/datasets/recipe.jl +++ b/src/datasets/recipe.jl @@ -30,9 +30,9 @@ data, blocks = loadrecipe(recipe, args...; kwargs...) the following must hold: -- `∀i ∈ [1..nobs(data)]: checkblock(blocks, getobs(data, i))`, i.e. +- `∀i ∈ 1:numobs(data): checkblock(blocks, data[i])`, i.e. `data` must be a data container of observations that are valid `blocks`. -- `nobs(data) ≥ 1`, i.e. there is at least one observation if the data was loaded +- `numobs(data) ≥ 1`, i.e. there is at least one observation if the data was loaded without error. """ abstract type DatasetRecipe end diff --git a/src/datasets/registry.jl b/src/datasets/registry.jl deleted file mode 100644 index 161cd96079..0000000000 --- a/src/datasets/registry.jl +++ /dev/null @@ -1,4 +0,0 @@ - - - -# ## Tests diff --git a/src/datasets/transformations.jl b/src/datasets/transformations.jl deleted file mode 100644 index cec32cbc65..0000000000 --- a/src/datasets/transformations.jl +++ /dev/null @@ -1,195 +0,0 @@ - -# mapobs - -struct MappedData{F,D} - f::F - data::D -end - -Base.show(io::IO, data::MappedData) = print(io, "mapobs($(data.f), $(summary(data.data)))") -Base.show(io::IO, data::MappedData{F,<:AbstractArray}) where {F} = - print(io, "mapobs($(data.f), $(ShowLimit(data.data, limit=80)))") -LearnBase.nobs(data::MappedData) = nobs(data.data) -LearnBase.getobs(data::MappedData, idx::Int) = data.f(getobs(data.data, idx)) -LearnBase.getobs(data::MappedData, idxs::AbstractVector) = data.f.(getobs(data.data, idxs)) - - -""" - mapobs(f, data) - -Lazily map `f` over the observations in a data container `data`. - -```julia -data = 1:10 -getobs(data, 8) == 8 -mdata = mapobs(-, data) -getobs(mdata, 8) == -8 -``` -""" -mapobs(f, data) = MappedData(f, data) -mapobs(f::typeof(identity), data) = data - - -""" - mapobs(fs, data) - -Lazily map each function in tuple `fs` over the observations in data container `data`. -Returns a tuple of transformed data containers. -""" -mapobs(fs::Tuple, data) = Tuple(mapobs(f, data) for f in fs) - - -struct NamedTupleData{TData,F} - data::TData - namedfs::NamedTuple{F} -end - -LearnBase.nobs(data::NamedTupleData) = nobs(getfield(data, :data)) - -function LearnBase.getobs(data::NamedTupleData{TData,F}, idx::Int) where {TData,F} - obs = getobs(getfield(data, :data), idx) - namedfs = getfield(data, :namedfs) - return NamedTuple{F}(f(obs) for f in namedfs) -end - -Base.getproperty(data::NamedTupleData, field::Symbol) = - mapobs(getproperty(getfield(data, :namedfs), field), getfield(data, :data)) - -Base.show(io::IO, data::NamedTupleData) = - print(io, "mapobs($(getfield(data, :namedfs)), $(getfield(data, :data)))") - -""" - mapobs(namedfs::NamedTuple, data) - -Map a `NamedTuple` of functions over `data`, turning it into a data container -of `NamedTuple`s. Field syntax can be used to select a column of the resulting -data container. - -```julia -data = 1:10 -nameddata = mapobs((x = sqrt, y = log), data) -getobs(nameddata, 10) == (x = sqrt(10), y = log(10)) -getobs(nameddata.x, 10) == sqrt(10) -``` -""" -function mapobs(namedfs::NamedTuple, data) - return NamedTupleData(data, namedfs) -end - -# filterobs - -""" - filterobs(f, data) - -Return a subset of data container `data` including all indices `i` for -which `f(getobs(data, i)) === true`. - -```julia -data = 1:10 -nobs(data) == 10 -fdata = filterobs(>(5), data) -nobs(fdata) == 5 -``` -""" -function filterobs(f, data; iterfn = _iterobs) - return datasubset(data, [i for (i, obs) in enumerate(iterfn(data)) if f(obs)]) -end - -_iterobs(data) = [getobs(data, i) for i = 1:nobs(data)] - - -# groupobs - -""" - groupobs(f, data) - -Split data container data `data` into different data containers, grouping -observations by `f(obs)`. - -```julia -data = -10:10 -datas = groupobs(>(0), data) -length(datas) == 2 -``` -""" -function groupobs(f, data) - groups = Dict{Any,Vector{Int}}() - for i = 1:nobs(data) - group = f(getobs(data, i)) - if !haskey(groups, group) - groups[group] = [i] - else - push!(groups[group], i) - end - end - return Dict(group => datasubset(data, idxs) for (group, idxs) in groups) -end - -# joinobs - -struct JoinedData{T,N} - datas::NTuple{N,T} - ns::NTuple{N,Int} -end - -JoinedData(datas) = JoinedData(datas, nobs.(datas)) - -LearnBase.nobs(data::JoinedData) = sum(data.ns) -function LearnBase.getobs(data::JoinedData, idx) - for (i, n) in enumerate(data.ns) - if idx <= n - return getobs(data.datas[i], idx) - else - idx -= n - end - end -end - -""" - joinobs(datas...) - -Concatenate data containers `datas`. - -```julia -data1, data2 = 1:10, 11:20 -jdata = joinobs(data1, data2) -getobs(jdata, 15) == 15 -``` -""" -joinobs(datas...) = JoinedData(datas) - - -# ## Tests - -@testset "Data container transformations" begin - @testset "mapobs" begin - data = 1:10 - mdata = mapobs(-, data) - @test getobs(mdata, 8) == -8 - - mdata2 = mapobs((-, x -> 2x), data) - @test getobs(mdata2, 8) == (-8, 16) - - nameddata = mapobs((x = sqrt, y = log), data) - @test getobs(nameddata, 10) == (x = sqrt(10), y = log(10)) - @test getobs(nameddata.x, 10) == sqrt(10) - end - - @testset "filterobs" begin - data = 1:10 - fdata = filterobs(>(5), data) - @test nobs(fdata) == 5 - end - - @testset "groupobs" begin - data = -10:10 - datas = groupobs(>(0), data) - @test length(datas) == 2 - end - - @testset "joinobs" begin - data1, data2 = 1:10, 11:20 - jdata = joinobs(data1, data2) - @test getobs(jdata, 15) == 15 - end -end diff --git a/src/interpretation/backend.jl b/src/interpretation/backend.jl index 7b22717c1a..c3b1b7d88d 100644 --- a/src/interpretation/backend.jl +++ b/src/interpretation/backend.jl @@ -92,7 +92,7 @@ Show a vector of observations `obss` of the same `block` type. ```julia data, blocks = loaddataset("imagenette2-160") -samples = [getobs(data, i) for i in range(1:4)] +samples = [data[i] for i in range(1:4)] showblocks(blocks, samples) ``` diff --git a/src/interpretation/task.jl b/src/interpretation/task.jl index c3972da41e..fd3c31d372 100644 --- a/src/interpretation/task.jl +++ b/src/interpretation/task.jl @@ -11,7 +11,7 @@ Show an unprocessed `sample` for `LearningTask` `task` to ```julia data, blocks = loaddataset("imagenette2-160", (Image, Label)) task = ImageClassificationSingle(data) -sample = getobs(data, 1) +sample = data[1] showsample(task, sample) # select backend automatically showsample(ShowText(), task, sample) ``` @@ -35,7 +35,7 @@ Show a vector of unprocessed `samples` for `LearningTask` `task` to ```julia data, blocks = loaddataset("imagenette2-160", (Image, Label)) task = ImageClassificationSingle(data) -samples = [getobs(data, i) for i in 1:4] +samples = [data[i] for i in 1:4] showsamples(task, samples) # select backend automatically showsamples(ShowText(), task, samples) ``` @@ -87,8 +87,7 @@ end Show a collated batch of encoded samples to `backend`. """ function showbatch(backend::ShowBackend, task::AbstractBlockTask, batch) - encsamples = collect(DataLoaders.obsslices(batch)) - showencodedsamples(backend, task, encsamples) + showencodedsamples(backend, task, Datasets.unbatch(batch)) end showbatch(task, batch) = showbatch(default_showbackend(), task, batch) @@ -206,8 +205,7 @@ Show collated batch of outputs to `backend`. If a collated batch of encoded samp have vectors of outputs and not collated batches. """ function showoutputbatch(backend::ShowBackend, task::AbstractBlockTask, outputbatch) - outputs = collect(DataLoaders.obsslices(outputbatch)) - return showoutputs(backend, task, outputs) + return showoutputs(backend, task, Datasets.unbatch(outputbatch)) end function showoutputbatch( backend::ShowBackend, @@ -215,9 +213,7 @@ function showoutputbatch( batch, outputbatch, ) - encsamples = collect(DataLoaders.obsslices(batch)) - outputs = collect(DataLoaders.obsslices(outputbatch)) - return showoutputs(backend, task, encsamples, outputs) + return showoutputs(backend, task, Datasets.unbatch(batch), Datasets.unbatch(outputbatch)) end showoutputbatch(task::AbstractBlockTask, args...) = diff --git a/src/learner.jl b/src/learner.jl index 508cb204d7..bc5c8a8f4f 100644 --- a/src/learner.jl +++ b/src/learner.jl @@ -64,7 +64,7 @@ function tasklearner( end function tasklearner(task, data; pctgval=0.2, kwargs...) - traindata, validdata = splitobs(shuffleobs(data), at=1 - pctgval) + traindata, validdata = splitobs(data, at=1 - pctgval) return tasklearner(task, traindata, validdata; kwargs...) end @@ -78,11 +78,10 @@ or validation data if `validation = true`. If `n` take only the first """ function getbatch(learner; context = Training(), n = nothing) - dl = context==Validation() ? learner.data.validation : learner.data.training - batch = first(learner.data.validation) - bs = DataLoaders._batchsize(batch, DataLoaders.BatchDimLast()) - b = isnothing(n) ? bs : min(n, bs) - batch = DataLoaders.collate([s for (s, _) in zip(DataLoaders.obsslices(batch), 1:b)]) + dl = context == Validation() ? learner.data.validation : learner.data.training + batch = first(dl) + b = min(isnothing(n) ? Inf : n, Datasets.batchsize(batch)) + batch = MLUtils.batch([s for (s, _) in zip(Datasets.unbatch(batch), 1:b)]) return batch end diff --git a/src/tasks/check.jl b/src/tasks/check.jl index 50e7cf68c4..fb3ace621b 100644 --- a/src/tasks/check.jl +++ b/src/tasks/check.jl @@ -41,7 +41,7 @@ end function _predictx(method, model, x, device = identity) if shouldbatch(method) - x = DataLoaders.collate([x]) + x = MLUtils.batch([x]) end ŷs = device(model)(device(x)) if shouldbatch(method) diff --git a/src/tasks/predict.jl b/src/tasks/predict.jl index c5f7a9beb2..7f3c3b888e 100644 --- a/src/tasks/predict.jl +++ b/src/tasks/predict.jl @@ -41,9 +41,9 @@ function predictbatch( context = Inference(), ) xs = device( - DataLoaders.collate([copy(encodeinput(task, context, input)) for input in inputs]), + MLUtils.batch([copy(encodeinput(task, context, input)) for input in inputs]), ) ŷs = undevice(model(xs)) - targets = [decodeypred(task, context, ŷ) for ŷ in DataLoaders.obsslices(ŷs)] + targets = [decodeypred(task, context, ŷ) for ŷ in Datasets.unbatch(ŷs)] return targets end diff --git a/src/tasks/taskdata.jl b/src/tasks/taskdata.jl index 5d93319c5d..f052a8bf80 100644 --- a/src/tasks/taskdata.jl +++ b/src/tasks/taskdata.jl @@ -9,7 +9,7 @@ Transform data container `data` of samples into a data container of encoded samples. Maps `encodesample(task, context, sample)` over the observations in -`data`. Also handles in-place `getobs!` through `encode!`. +`data`. Also handles in-place `MLUtils.getobs!` through `encodesample!`. """ struct TaskDataset{TData, TTask<:LearningTask, TContext<:Context} data::TData @@ -17,13 +17,13 @@ struct TaskDataset{TData, TTask<:LearningTask, TContext<:Context} context::TContext end -LearnBase.nobs(ds::TaskDataset) = nobs(ds.data) +Base.length(ds::TaskDataset) = numobs(ds.data) -function LearnBase.getobs(ds::TaskDataset, idx) +function Base.getindex(ds::TaskDataset, idx) return encodesample(ds.task, ds.context, getobs(ds.data, idx)) end -function LearnBase.getobs!(buf, ds::TaskDataset, idx) +function MLUtils.getobs!(buf, ds::TaskDataset, idx) return encodesample!(buf, ds.task, ds.context, getobs(ds.data, idx)) end @@ -88,10 +88,10 @@ function taskdataloaders( shuffle = true, validbsfactor = 2, kwargs...) - traindata = shuffle ? shuffleobs(traindata) : traindata return ( - DataLoader(taskdataset(traindata, task, Training()), batchsize; kwargs...), - DataLoader(taskdataset(validdata, task, Validation()), validbsfactor * batchsize; kwargs...), + DataLoader(taskdataset(traindata, task, Training()); batchsize, shuffle, kwargs...), + DataLoader(taskdataset(validdata, task, Validation()); + batchsize = validbsfactor * batchsize, kwargs...), ) end @@ -101,8 +101,7 @@ function taskdataloaders( task::LearningTask, batchsize = 16; pctgval = 0.2, - shuffle = true, kwargs...) traindata, validdata = splitobs(shuffleobs(data), at = 1-pctgval) - taskdataloaders(traindata, validdata, task, batchsize; shuffle = false, kwargs...) + taskdataloaders(traindata, validdata, task, batchsize; kwargs...) end diff --git a/src/training/utils.jl b/src/training/utils.jl index d5f6cdf02b..782d023957 100644 --- a/src/training/utils.jl +++ b/src/training/utils.jl @@ -100,5 +100,5 @@ in `context` which defaults to `Training`. """ function makebatch(task::LearningTask, data, idxs = 1:8; context = Training()) xys = [deepcopy(encodesample(task, context, getobs(data, i))) for i in idxs] - return DataLoaders.collate(xys) + return MLUtils.batch(xys) end From 1c7c5db0a8339feabc50e751f366ca24a5f14486 Mon Sep 17 00:00:00 2001 From: lorenzoh Date: Sun, 29 May 2022 10:01:29 +0200 Subject: [PATCH 15/23] Remove unneeded dependencies --- Project.toml | 4 ---- src/FastAI.jl | 9 +++------ src/Vision/models/Models.jl | 1 - src/datasets/Datasets.jl | 14 +++++--------- 4 files changed, 8 insertions(+), 20 deletions(-) diff --git a/Project.toml b/Project.toml index 814b19d27e..caa2b6ee35 100644 --- a/Project.toml +++ b/Project.toml @@ -4,7 +4,6 @@ authors = ["Lorenz Ohly", "Julia Community"] version = "0.4.3" [deps] -BSON = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0" CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" ColorVectorSpace = "c3611d14-8923-5661-9e6a-0046d554d3a4" Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" @@ -17,7 +16,6 @@ FilePathsBase = "48062228-2e41-5def-b9a4-89aafe57970f" FixedPointNumbers = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" FluxTraining = "7bf95e4d-ca32-48da-9824-f0dc5310474f" -Glob = "c27321d9-0574-5035-807b-f59d2c89b15c" ImageIO = "82e4d734-157c-48bb-816b-45c225c6df19" ImageInTerminal = "d8c32880-2388-543b-8c61-d9f865259254" IndirectArrays = "9b13fd28-a010-5f03-acff-a1bbcff69959" @@ -43,7 +41,6 @@ UnicodePlots = "b8865327-cd53-5732-bb35-84acbb429228" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [compat] -BSON = "0.3" CSV = "0.8, 0.9, 0.10" ColorVectorSpace = "0.9" Colors = "0.12" @@ -56,7 +53,6 @@ FilePathsBase = "0.9" FixedPointNumbers = "0.8" Flux = "0.12, 0.13" FluxTraining = "0.2, 0.3" -Glob = "1" ImageIO = "0.6" ImageInTerminal = "0.4" IndirectArrays = "0.5, 1" diff --git a/src/FastAI.jl b/src/FastAI.jl index bf55fa7ddb..3c00ab82f0 100644 --- a/src/FastAI.jl +++ b/src/FastAI.jl @@ -90,7 +90,7 @@ include("serialization.jl") # submodules include("datasets/Datasets.jl") -@reexport using .Datasets +using .Datasets include("Registries/Registries.jl") @@ -146,11 +146,8 @@ export Datasets, Models, datasetpath, - mapobs, - groupobs, - filterobs, - shuffleobs, - datasubset, + getobs, + numobs, # task API taskmodel, diff --git a/src/Vision/models/Models.jl b/src/Vision/models/Models.jl index c65408d640..4772169c35 100644 --- a/src/Vision/models/Models.jl +++ b/src/Vision/models/Models.jl @@ -4,7 +4,6 @@ module Models using Base: Bool, Symbol using ..FastAI -using BSON using Flux using Zygote using DataDeps diff --git a/src/datasets/Datasets.jl b/src/datasets/Datasets.jl index ed6027654d..0e1ed229e4 100644 --- a/src/datasets/Datasets.jl +++ b/src/datasets/Datasets.jl @@ -7,7 +7,6 @@ import MLUtils: MLUtils, getobs, numobs, filterobs, groupobs, mapobs import MLDatasets: FileDataset using MLUtils: mapobs, groupobs using DataDeps -using Glob using FilePathsBase import DataAugmentation using FilePathsBase: filename @@ -23,20 +22,17 @@ using InlineTest include("fastaidatasets.jl") -function __init__() - initdatadeps() -end - -include("containers.jl") include("batching.jl") - +include("containers.jl") include("load.jl") +include("loaders.jl") include("recipe.jl") include("deprecations.jl") -include("loaders.jl") - +function __init__() + initdatadeps() +end export # primitive containers From d30a4ce26f1565282ed923184e57b4263ac10a45 Mon Sep 17 00:00:00 2001 From: lorenzoh Date: Tue, 31 May 2022 12:11:27 +0200 Subject: [PATCH 16/23] Update to newer Invariants --- Project.toml | 1 + src/FastAI.jl | 3 +- src/Vision/Vision.jl | 2 + src/Vision/blocks/bounded.jl | 12 +++ src/Vision/blocks/image.jl | 61 ++++++++++++- src/Vision/blocks/mask.jl | 33 +++++++ src/blocks/label.jl | 97 ++++++++++++--------- src/datablock/block.jl | 64 +++++++------- src/datablock/wrappers.jl | 13 ++- src/invariants.jl | 163 +++++++++++++++++++++++++---------- 10 files changed, 320 insertions(+), 129 deletions(-) diff --git a/Project.toml b/Project.toml index b86ec931d9..5a824f5674 100644 --- a/Project.toml +++ b/Project.toml @@ -20,6 +20,7 @@ FixedPointNumbers = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" FluxTraining = "7bf95e4d-ca32-48da-9824-f0dc5310474f" Glob = "c27321d9-0574-5035-807b-f59d2c89b15c" +ImageCore = "a09fc81d-aa75-5fe9-8630-4744c3626534" ImageIO = "82e4d734-157c-48bb-816b-45c225c6df19" ImageInTerminal = "d8c32880-2388-543b-8c61-d9f865259254" IndirectArrays = "9b13fd28-a010-5f03-acff-a1bbcff69959" diff --git a/src/FastAI.jl b/src/FastAI.jl index 29f57255f1..b79a22d480 100644 --- a/src/FastAI.jl +++ b/src/FastAI.jl @@ -10,6 +10,7 @@ using Flux using Animations import DataAugmentation import DataAugmentation: getbounds, Bounds +import Invariants: Invariants, invariant, check, check_throw, md import LearnBase using FilePathsBase @@ -19,8 +20,6 @@ import Flux.Optimise: apply!, Optimiser, WeightDecay using FluxTraining: Learner, handle using FluxTraining.Events using JLD2: jldsave, jldopen -using Invariants: BooleanInvariant, WithMessage, AnyInvariant, AllInvariant, - SequenceInvariant, WithContext, check_error, check using Markdown using MLDataPattern using PrettyTables diff --git a/src/Vision/Vision.jl b/src/Vision/Vision.jl index 13cc2149c7..6df8a1ac1e 100644 --- a/src/Vision/Vision.jl +++ b/src/Vision/Vision.jl @@ -59,8 +59,10 @@ import DataAugmentation: apply, Identity, ToEltype, ImageToTensor, Normalize, BufferedThreadsafe, ScaleKeepAspect, PinOrigin, RandomCrop, CenterResizeCrop, AdjustBrightness, AdjustContrast, Maybe, FlipX, FlipY, WarpAffine, Rotate, Zoom, ResizePadDivisible, itemdata +import ImageCore: colorview import ImageInTerminal import IndirectArrays: IndirectArray +import Invariants: invariant, md import ProgressMeter: Progress, next! import Requires: @require import StaticArrays: SVector diff --git a/src/Vision/blocks/bounded.jl b/src/Vision/blocks/bounded.jl index f0d97902ec..ca8bda9f4e 100644 --- a/src/Vision/blocks/bounded.jl +++ b/src/Vision/blocks/bounded.jl @@ -58,6 +58,7 @@ struct Bounded{N, B<:AbstractBlock} <: WrapperBlock size::NTuple{N, DimSize} end +Base.nameof(b::Bounded) = "Bounded($(nameof(parent(b))))" function Bounded(bounded::Bounded{M}, size::NTuple{N, DimSize}) where {N, M} N == M || error("Cannot rewrap a `Bounded` with different dimensionalities $N and $M") @@ -69,6 +70,17 @@ function checkblock(bounded::Bounded{N}, a::AbstractArray{N}) where N return checksize(bounded.size, size(a)) && checkblock(parent(bounded), a) end + +function FastAI.invariant_checkblock(block::Bounded{N}; blockname = "block", obsname = "obs") where N + return invariant([ + FastAI.invariant_checkblock(parent(block)), + ], + FastAI.__inv_checkblock_title(block, blockname, obsname), + :seq + ) +end + + @testset "Bounded [block, wrapper]" begin @test_nowarn Bounded(Image{2}(), (16, 16)) bounded = Bounded(Image{2}(), (16, 16)) diff --git a/src/Vision/blocks/image.jl b/src/Vision/blocks/image.jl index 729ff0603c..791f53a93a 100644 --- a/src/Vision/blocks/image.jl +++ b/src/Vision/blocks/image.jl @@ -52,9 +52,68 @@ mockblock(::Image{N}) where {N} = rand(RGB{N0f8}, ntuple(_ -> 16, N)) setup(::Type{Image}, data) = Image{ndims(getobs(data, 1))}() +Base.nameof(::Image{N}) where N = "Image{$N}" # Visualization -function showblock!(io, ::ShowText, block::Image{2}, obs) +showblock!(io, ::ShowText, block::Image{2}, obs::AbstractMatrix{<:Colorant}) = ImageInTerminal.imshow(io, obs) +showblock!(io, ::ShowText, block::Image{2}, obs::AbstractMatrix{<:Real}) = + ImageInTerminal.imshow(io, colorview(Gray, obs)) + + +function FastAI.invariant_checkblock(block::Image{N}; blockname = "block", obsname = "obs") where N + return invariant([ + invariant("`$obsname` is an `AbstractArray`", + description = md("`$obsname` should be of type `AbstractArray`.")) do obs + if !(obs isa AbstractArray) + return "Instead, got invalid type `$(nameof(typeof(obs)))`." |> md + end + end, + invariant("`$obsname` is `$N`-dimensional") do obs + if ndims(obs) != N + return "Instead, got invalid dimensionality `$N`." |> md + end + end, + invariant("`$obsname` should have a color or numerical element type") do obs + if !((eltype(obs) <: Color) ||(eltype(obs) <: Real)) + return "Instead, got invalid element type `$(eltype(obs))`." |> md + end + end, + ], + FastAI.__inv_checkblock_title(block, blockname, obsname), + :seq + ) end + +#= + +function isblockinvariant(block::Image{N}; dataname = "data", blockname = "block") where {N} + return SequenceInvariant( + [ + BooleanInvariant( + obs -> obs isa AbstractArray, + name = "Image data is an array", + messagefn = obs -> """Expected `$dataname` to be a subtype of + `AbstractArray`, but instead got type `$(typeof(obs))`.""", + ), + BooleanInvariant( + obs -> ndims(obs) == N, + name = "Image data is `$N`-dimensional", + messagefn = obs -> """Expected `$dataname` to be an `$N`-dimensional array, + but instead got a `$(ndims(obs))`-dimensional array.""", + ), + BooleanInvariant( + obs -> eltype(obs) <: Color || eltype(obs) <: Number, + name = "Image data has a color or numerical type.", + messagefn = obs -> """Expected `$dataname` to have an element type that is a + color (`eltype($dataname) <: Color`) or a number (`eltype($dataname) + <: Color`), but instead found `eltype($dataname) == $(eltype(obs)).` + """ + ) + ], + "`$dataname` is a valid `$(typeof(block))`", + "" + ) +end +=# diff --git a/src/Vision/blocks/mask.jl b/src/Vision/blocks/mask.jl index 1208ea8ef9..a3c7f6d145 100644 --- a/src/Vision/blocks/mask.jl +++ b/src/Vision/blocks/mask.jl @@ -17,6 +17,39 @@ end mockblock(mask::Mask{N, T}) where {N, T} = rand(mask.classes, ntuple(_ -> 16, N))::AbstractArray{T, N} +Base.nameof(::Mask{N}) where N = "Mask{$N}" + +function FastAI.invariant_checkblock(block::Mask{N}; blockname = "block", obsname = "obs") where N + return invariant([ + invariant("`$obsname` is an `AbstractArray`", + description = md("`$obsname` should be of type `AbstractArray`.")) do obs + if !(obs isa AbstractArray) + return "Instead, got invalid type `$(nameof(typeof(obs)))`." |> md + end + end, + invariant("`$obsname` is `$N`-dimensional") do obs + if ndims(obs) != N + return "Instead, got invalid dimensionality `$N`." |> md + end + end, + invariant("All elements are valid labels") do obs + valid = ∈(block.classes).(obs) + if !(all(valid)) + unknown = unique(obs[valid .== false]) + return md("""`$obsname` should contain only valid labels, + i.e. `∀ y ∈ $obsname: y ∈ $blockname.classes`, but `$obsname` includes + unknown labels: `$(sprint(show, unknown))`. + + Valid classes are: + `$(sprint(show, block.classes, context=:limit => true))`""") + end + end, + ], + FastAI.__inv_checkblock_title(block, blockname, obsname), + :seq + ) +end + # Visualization diff --git a/src/blocks/label.jl b/src/blocks/label.jl index 6b4eb57838..85f02ec2fb 100644 --- a/src/blocks/label.jl +++ b/src/blocks/label.jl @@ -38,34 +38,44 @@ setup(::Type{Label}, data) = Label(unique(eachobs(data))) Base.summary(io::IO, ::Label{T}) where {T} = print(io, "Label{", T, "}") + function invariant_checkblock(block::Label; blockname = "block", obsname = "obs") - return BooleanInvariant( - obs -> obs ∈ block.classes, - name = "`$obsname` should be a valid `$(summary(block))`", - messagefn = obs -> """`$obsname` should be one of the valid classes, i.e. - `$obsname ∈ $blockname.classes`. Instead got unknown class `$(sprint(show, obs))`. - Valid classes are: - `$(sprint(show, block.classes, context=:limit => true))`""", - ) + return invariant( + __inv_checkblock_title(block, blockname, obsname), + description=md("""`$obsname` should be a valid label, i.e. one of + `$(sprint(show, block.classes, context=:limit => true))`."""), + ) do obs + if !(obs ∈ block.classes) + return md("Instead, got invalid value `$obs`.") + end + end end + """ - LabelMulti(classes) + LabelMulti(classes) <: Block setup(LabelMulti, data) -`Block` for a categorical label in a multi-class context. -`data` is valid for `Label(classes)` if `data ∈ classes`. +`Block` for a categorical label in a multi-class context where multiple +labels can be associated for an input. Each label must be in `classes`. +For example, for a block `LabelMulti([1, 2, 3])`, `[1, 2]` is a valid +observation, unlike `[0, 2]` (invalid label) or `1` (not a vector of +labels). + +Use [`is_block_obs`](#) to make sure you have valid observations. ## Examples +An observation can contain all or none of the listed classes: + ```julia -block = Label(["cat", "dog"]) # an observation can be either "cat" or "dog" -@test FastAI.checkblock(block, "cat") -@test !(FastAI.checkblock(block, "horsey")) +block = LabelMulti(["cat", "dog", "person"]) +@test FastAI.checkblock(block, ["cat", "person"]) +@test !(FastAI.checkblock(block, ["horsey"])) ``` -You can use `setup` to create a `Label` instance from a data container containing -possible classes: +You can use `setup` to create a `Label` instance from a data container +containing possible classes: ```julia targets = ["cat", "dog", "dog", "dog", "cat", "dog"] @@ -90,30 +100,31 @@ setup(::Type{LabelMulti}, data) = LabelMulti(unique(eachobs(data))) Base.summary(io::IO, ::LabelMulti{T}) where {T} = print(io, "LabelMulti{", T, "}") + + function invariant_checkblock(block::LabelMulti; blockname = "block", obsname = "obs") - return SequenceInvariant( - [ - BooleanInvariant( - obs -> obs isa AbstractVector, - name = "`$obsname` should be an `AbstractVector`", - messagefn = obs -> """`$obsname` should be an `AbstractVector`, instead - got type `$(typeof(obs))`. - """ - ), - BooleanInvariant( - obs -> all([y ∈ block.classes for y in obs]), - name = "Elements in `$obsname` should be valid classes", - messagefn = obs -> """`$obsname` should contain only valid classes, - i.e. `∀ y ∈ $obsname: y ∈ $blockname.classes`. - Instead got unknown classes `$( - sprint(show, [y for y in obs if !(y ∈ block.classes)]))`. + return invariant([ + invariant("`$obsname` is an `AbstractVector`", + description = md("`$obsname` should be of type `AbstractVector`.")) do obs + if !(obs isa AbstractVector) + return md("Instead, got invalid type `$(nameof(typeof(obs)))`.") + end + end, + invariant("All elements are valid labels") do obs + valid = ∈(block.classes).(obs) + if !(all(valid)) + unknown = unique(obs[valid .== false]) + return md("""`$obsname` should contain only valid labels, + i.e. `∀ y ∈ $obsname: y ∈ $blockname.classes`, but `$obsname` includes + unknown labels: `$(sprint(show, unknown))`. Valid classes are: - `$(sprint(show, block.classes, context=:limit => true))`""", - ), + `$(sprint(show, block.classes, context=:limit => true))`""") + end + end ], - "`$obsname` should be a valid `$(summary(block))`", - "", + __inv_checkblock_title(block, blockname, obsname), + :seq ) end @@ -130,8 +141,8 @@ end @test block.classes == ["cat", "dog"] inv = invariant_checkblock(Label([1, 2, 3])) - @test check(inv, 1) - @test !(check(inv, 0)) + @test check(Bool, inv, 1) + @test !(check(Bool, inv, 0)) end @@ -145,9 +156,9 @@ end @test block.classes == ["cat", "dog"] inv = invariant_checkblock(block) - @test check(inv, ["cat", "dog"]) - @test check(inv, []) - @test !(check(inv, "cat")) - @test !(check(inv, ["mouse"])) - @test !(check(inv, ["mouse", "cat"])) + @test check(Bool, inv, ["cat", "dog"]) + @test check(Bool, inv, []) + @test !(check(Bool, inv, "cat")) + @test !(check(Bool, inv, ["mouse"])) + @test !(check(Bool, inv, ["mouse", "cat"])) end diff --git a/src/datablock/block.jl b/src/datablock/block.jl index e41f6dda5c..005d38d209 100644 --- a/src/datablock/block.jl +++ b/src/datablock/block.jl @@ -128,6 +128,7 @@ typify(T::Type) = T typify(t::Tuple) = Tuple{map(typify, t)...} typify(block::FastAI.AbstractBlock) = typeof(block) +Base.nameof(::B) where {B<:AbstractBlock} = nameof(B) # ## Invariants # @@ -138,7 +139,7 @@ typify(block::FastAI.AbstractBlock) = typeof(block) invariant_checkblock(block; kwargs...) invariant_checkblock(blocks; kwargs...) -Create a `Invariants.Invariant` that can be used to check whether an +Create an `Invariants.Invariant` that can be used to check whether an observation is a valid instance of `block`. This should always agree with `checkblock` (i.e. `checkblock(block, obs)` implies that `check(invariant_checkblock(block), obs)`). The invariant can however @@ -152,46 +153,39 @@ function invariant_checkblock end # If `invariant_checkblock` is not implemented for a block, default to # checking that `checkblock` returns `true`. -function invariant_checkblock(block::AbstractBlock; obsname = "obs", blockname = "block") - return BooleanInvariant( - obs -> checkblock(block, obs), - "`$obsname` should be valid $(typeof(block))", - _ -> """Expected `$obsname` to be a valid instance of block $blockname - with above type, but `checkblock($blockname, $obsname)` returned `false`. - This probably means that `$obsname` is not a valid instance of the - block. Check `?$(typeof(block).name.name)` for more information on - the block and what data is valid. - """, - ) +function invariant_checkblock(block::B; obsname = "obs", blockname = "block") where {B<:AbstractBlock} + return invariant(__inv_checkblock_title(block, blockname, obsname),) do obs + if !checkblock(block, obs) + """Expected `$obsname` to be a block with above type, but + `checkblock($blockname, $obsname)` returned `false`. + This probably means that `$obsname` is not a valid instance of the + block. Check `?$(string(parentmodule(B))).$(nameof(B))` for more information on + the block and what data is valid. + """ |> md + end + end end +__inv_checkblock_title(b, bname, oname) = "`$oname` is a valid observation for `$(bname) <: $(nameof(b))`" # For tuples of blocks, the invariant is composed of the individuals' blocks # invariants, passing only if all the child invariants pass. -function invariant_checkblock(blocks::Tuple; obsname = "obss", blockname = "blocks") - return SequenceInvariant( +function invariant_checkblock(blocks::Tuple; dataname = "obss", blockname = "blocks") + return invariant( [ - BooleanInvariant( - obss -> (obss isa Tuple && length(obss) == length(blocks)), - "$obsname should be a `Tuple` with $(length(blocks)) elements.", - obss -> """Instead, got a `$(sprint(show, typeof(obss)))`"""), - AllInvariant( - [ - WithContext( - obss -> obss[i], - invariant_checkblock( - blocks[i]; - obsname = "$obsname[$i]", - blockname = "$blockname[$i]", - ), - ) for (i, block) in enumerate(blocks) - ], - name = "`$obsname` should be valid `$blockname`", - description = "" - ) + invariant( + invariant_checkblock( + blocks[i], + obsname = "$dataname[$i]", + blockname = "$blockname[$i]" + ), + inputfn = obss -> obss[i] + ) for (i, block) in enumerate(blocks) ], - "For a tuple of blocks, an instance should be a tuple of valid instances", - "", + "`$dataname` are valid instances of blocks `$blockname`", + description = md("""The given observations `obss` should be valid instances of the + blocks `$blockname`. Since `$blockname` is a tuple of blocks, each observation + `$dataname[i]` should be a valid instance of the block `$blockname[i]`. + See `?Block` for more background on blocks.""") ) - end diff --git a/src/datablock/wrappers.jl b/src/datablock/wrappers.jl index 87ee5f3ec8..855ddbb26b 100644 --- a/src/datablock/wrappers.jl +++ b/src/datablock/wrappers.jl @@ -1,5 +1,14 @@ # # Wrapper blocks +""" + abstract type WrapperBlock + +Supertype for blocks that "wrap" an existing block, inheriting its +functionality, allowing you to override just parts of its interface. + +For examples of `WrapperBlock`, see [`Bounded`](#) + +""" abstract type WrapperBlock <: AbstractBlock end Base.parent(w::WrapperBlock) = w.block @@ -13,9 +22,6 @@ end mockblock(w::WrapperBlock) = mockblock(parent(w)) checkblock(w::WrapperBlock, obs) = checkblock(parent(w), obs) -# TODO: add way to specify how wrapper blocks compose - -# If not overwritten, encodings are applied to the wrapped block """ abstract type PropagateWrapper @@ -50,6 +56,7 @@ struct PropagateAlways <: PropagateWrapper end struct PropagateSameBlock <: PropagateWrapper end struct PropagateNever <: PropagateWrapper end +# If not overwritten, encodings are applied to the wrapped block propagatewrapper(::WrapperBlock) = PropagateAlways() encodedblock(enc::Encoding, wrapper::WrapperBlock) = diff --git a/src/invariants.jl b/src/invariants.jl index 6a302bec49..42c189e394 100644 --- a/src/invariants.jl +++ b/src/invariants.jl @@ -1,57 +1,130 @@ +#= +This file implements functions that allow checking common interfaces in the package using +[Invariants.jl](https://github.com/lorenzoh/Invariants.jl). +=# + +#= +`is_block_obs(block, obs` chekcs that a value `obs` is a valid observation for a block +`block`. +=# """ - invariant_datacontainer(block) + is_block(block, obs) + +Check whether `obs` is a valid observation for `block` and give +detailed output. + +## Examples + +{cell=is_block, show=false resultshow=false} +```julia +using FastAI +``` + +Basic check with a valid observation: + +{cell=is_block} +```julia +FastAI.is_block(LabelMulti([1, 2, 3]), [1]) +``` + +An invalid observation will show an error, detailing why the observatio is not valid for +the block: + +{cell=is_block} +```julia +FastAI.is_block(LabelMulti([1, 2, 3]), [2, "invalid:("])) +``` + +As a tuple of blocks is also a valid block, we can check that too. For example, a sample for +a supervised learning task is usually a tuple `(input, label)`. Using `is_block_obs` to +check observations for tuples of blocks (or nested tuples) details which specific +observations are valid. + +{cell=is_block} +```julia +using FastAI.Vision: RGB +FastAI.is_block(( + Image{2}(), # input block + Label(["cat", "dog"]), # target block + ), ( + rand(RGB, 100, 100), # valid input + "mouse", # invalid label + )) +``` -Create an `Invariants.Invariant` that checks that `data` is a data container -with observations that are valid instances of block `block`. See also -[`invariant_checkblock`](#). +## Extending + +To extend this check to work on a new block type `B`, implement +[`invariant_checkblock`](#)`(::B)`. For help implementing invariants see the documentation +of [Invariants.jl](https://github.com/lorenzoh/Invariants.jl). + +If `invariant_checkblock` is not implemented for `B`, it will fall back to checking +[`checkblock`](#) which is correct, but doesn't yield helpful output. """ -function invariant_datacontainer(block) - return SequenceInvariant([ - WithContext(data -> (data,), invariant_hasmethod(nobs, (data = Any,))), - WithContext(data -> (data, 1), invariant_hasmethod(getobs, (data = Any, idx = Int))), - WithContext(data -> getobs(data, 1), invariant_checkblock(block)), - ], "`data` should be a data container with block observations $(summary(block))", "") +function is_block(block, obs; kwargs...) + inv = invariant_checkblock(block; kwargs...) + check(inv, obs) end +function is_block(::Type{Bool}, block, obs; kwargs...) + inv = invariant_checkblock(block; kwargs...) + check(Bool, inv, obs) +end -function invariant_hasmethod(f, argtypes::NamedTuple) - methodcall = "`$f(" - for (i, (name, T)) in enumerate(zip(keys(argtypes), values(argtypes))) - if i != 1 - methodcall *= ", " - end - methodcall *= string(name) - if T !== Any - methodcall *= "::$T" - end - end - methodcall *= ")`" - return SequenceInvariant( - [ - BooleanInvariant( - args -> Base.hasmethod(f, typeof(args)), - "Method $methodcall should exist", - args -> "Expected method $methodcall to exist but it does not", - ), - BooleanInvariant( - args -> _hasmethod(f, args), - "Method $methodcall should not error", - args -> "Expected method $methodcall not to error but it did.", - ), - ], - "$methodcall should be a valid method call", - "", - ) +#= +`is_data(data)` checks that `data` is a valid data container. +=# + +function is_data(data; kwargs...) + inv = invariant_datacontainer(; kwargs...) + return check(inv, data) end -function _hasmethod(f, args) - try - f(args...) - return true - catch e - return false - end + +function invariant_datacontainer(; symbol = :data) + invariant([ + __invariant_numobs(; symbol), + __invariant_getobs(; symbol), + ], "`$symbol` implements the data container interface.", + description="""A data container stores a number of observations and allows loading + individual observations. See + [the tutorial](/documents/docs/tutorials/data_containers.md) for more + information.""" |> md) end + +__invariant_getobs(; symbol = :data) = invariant([ + Invariants.hasmethod_invariant(Base.getindex, :data, :idx => 1) + Invariants.hasmethod_invariant(MLUtils.getobs, :data, :idx => 1) + ], + "`$symbol` implements the `getobs` interface", + :any; + description=Invariants.md(""" + `$symbol` must provide a way load an observation by implementing **either** + (1) `Base.getindex($symbol, idx::Int)` (preferred) or (2) `MLUtils.getobs($symbol, idx::Int)` + (if regular indexing is already used and has different semantics). + """), + inputfn=data -> (; data), +) + +__invariant_numobs(; symbol = :data) = invariant([ + Invariants.hasmethod_invariant(Base.length, :data) + Invariants.hasmethod_invariant(MLUtils.numobs, :data) + invariant("`$symbol` is not empty") do input + n = numobs(input.data) + if n <= 0 + return "Instead, got a data container with $n observations." + end + end + ], + "`$symbol` implements the `numobs` interface", + :any; + description=Invariants.md(""" + `$symbol` must provide a way get the number of observations it contains implementing either + `Base.length($symbol)` (preferred) or `MLUtils.numobs($symbol, idx::Int)` + It must also be non-empty, i.e. contain at least one observation. + """), + inputfn=data -> (; data), +) From 839618094c8235439173b8fd43c7ba6c52ba7be4 Mon Sep 17 00:00:00 2001 From: lorenzoh Date: Tue, 31 May 2022 12:25:46 +0200 Subject: [PATCH 17/23] Fix submodule import --- src/Tabular/Tabular.jl | 2 ++ src/Textual/Textual.jl | 2 ++ src/Vision/Vision.jl | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Tabular/Tabular.jl b/src/Tabular/Tabular.jl index a59729b2c6..a403f8cd84 100644 --- a/src/Tabular/Tabular.jl +++ b/src/Tabular/Tabular.jl @@ -18,8 +18,10 @@ using ..FastAI: testencoding # extending import ..FastAI: + Datasets, blockmodel, blockbackbone, blocklossfn, encode, decode, checkblock, encodedblock, decodedblock, showblock!, mockblock, setup +using ..FastAI.Datasets import DataAugmentation diff --git a/src/Textual/Textual.jl b/src/Textual/Textual.jl index 5ec089bcad..6d11e711f8 100644 --- a/src/Textual/Textual.jl +++ b/src/Textual/Textual.jl @@ -13,6 +13,8 @@ using ..FastAI: # other Context, Training, Validation +using ..FastAI.Datasets + import Requires: @require using InlineTest diff --git a/src/Vision/Vision.jl b/src/Vision/Vision.jl index 6286811878..3204a2a4bb 100644 --- a/src/Vision/Vision.jl +++ b/src/Vision/Vision.jl @@ -39,7 +39,7 @@ using ..FastAI: Datasets import Flux import MLUtils: getobs, numobs, mapobs, eachobs -import FastAI.Datasets +using ..FastAI.Datasets # for tests using ..FastAI: testencoding From 8384eba38c55f40d403e181510caef4f00c2508f Mon Sep 17 00:00:00 2001 From: lorenzoh Date: Tue, 31 May 2022 12:28:14 +0200 Subject: [PATCH 18/23] One more import --- src/Registries/Registries.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Registries/Registries.jl b/src/Registries/Registries.jl index 23b3bf434b..9b0668330d 100644 --- a/src/Registries/Registries.jl +++ b/src/Registries/Registries.jl @@ -1,6 +1,7 @@ module Registries using ..FastAI +using ..FastAI.Datasets using ..FastAI.Datasets: DatasetLoader, DataDepLoader, isavailable, loaddata, typify import Markdown From c77d1c7d858cfe6f6c31ed4edd8e64aa4c7c9409 Mon Sep 17 00:00:00 2001 From: lorenzoh Date: Tue, 31 May 2022 13:22:18 +0200 Subject: [PATCH 19/23] Add data container invariant --- src/invariants.jl | 81 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 59 insertions(+), 22 deletions(-) diff --git a/src/invariants.jl b/src/invariants.jl index 42c189e394..ae0d3476d8 100644 --- a/src/invariants.jl +++ b/src/invariants.jl @@ -78,53 +78,90 @@ end =# +""" + is_data(data) + +Check that `data` implements the data container interface and give detailed info on missing +functionality if not. + +""" function is_data(data; kwargs...) inv = invariant_datacontainer(; kwargs...) return check(inv, data) end -function invariant_datacontainer(; symbol = :data) +""" + is_data(data, block) + +Check that `data` implements the data container interface and its observations are valid +instances of `block`, giving detailed errors if not. +""" +function is_data(data, block; kwargs...) + inv = invariant_datacontainer_block(block; kwargs...) + return check(inv, data) +end + +is_data(::Type{Bool}, args...; kwargs...) = convert(Bool, is_data(args...; kwargs...)) + + +function invariant_datacontainer(; var = :data) invariant([ - __invariant_numobs(; symbol), - __invariant_getobs(; symbol), - ], "`$symbol` implements the data container interface.", - description="""A data container stores a number of observations and allows loading - individual observations. See - [the tutorial](/documents/docs/tutorials/data_containers.md) for more + __invariant_numobs(; var), + invariant("`$var` contains at least one observation") do data + n = numobs(data) + if n <= 0 + return "Instead, got a data container with $n observations." + end + end, + __invariant_getobs(; var), + ], + "`$var` implements the data container interface.", + :seq; + description="""A data container stores observations and allows (1) getting + the number of observation and (2) loading an observation. + See [the tutorial](/documents/docs/tutorials/data_containers.md) for more information.""" |> md) end -__invariant_getobs(; symbol = :data) = invariant([ +function invariant_datacontainer_block(block; + datavar = "data", blockvar = "data", obsvar = "obs") + return invariant([ + invariant_datacontainer(; var = datavar), + invariant( + invariant_checkblock(block; blockname = blockvar, obsname = obsvar); + inputfn = data -> getobs(data, 1) + ) + ], + "`$datavar` is a data container with valid observations for block `$(nameof(block))`", + :seq) + + +end + +__invariant_getobs(; var = :data) = invariant([ Invariants.hasmethod_invariant(Base.getindex, :data, :idx => 1) Invariants.hasmethod_invariant(MLUtils.getobs, :data, :idx => 1) ], - "`$symbol` implements the `getobs` interface", + "`$var` implements the `getobs` interface", :any; description=Invariants.md(""" - `$symbol` must provide a way load an observation by implementing **either** - (1) `Base.getindex($symbol, idx::Int)` (preferred) or (2) `MLUtils.getobs($symbol, idx::Int)` + `$var` must provide a way load an observation by implementing **either** + (1) `Base.getindex($var, idx::Int)` (preferred) or (2) `MLUtils.getobs($var, idx::Int)` (if regular indexing is already used and has different semantics). """), inputfn=data -> (; data), ) -__invariant_numobs(; symbol = :data) = invariant([ +__invariant_numobs(; var = :data) = invariant([ Invariants.hasmethod_invariant(Base.length, :data) Invariants.hasmethod_invariant(MLUtils.numobs, :data) - invariant("`$symbol` is not empty") do input - n = numobs(input.data) - if n <= 0 - return "Instead, got a data container with $n observations." - end - end ], - "`$symbol` implements the `numobs` interface", + "`$var` implements the `numobs` interface", :any; description=Invariants.md(""" - `$symbol` must provide a way get the number of observations it contains implementing either - `Base.length($symbol)` (preferred) or `MLUtils.numobs($symbol, idx::Int)` - It must also be non-empty, i.e. contain at least one observation. + `$var` must provide a way get the number of observations it contains implementing either + `Base.length($var)` (preferred) or `MLUtils.numobs($var, idx::Int)` """), inputfn=data -> (; data), ) From e9d0c45568177ee0a692fb1cbf18b02180287c80 Mon Sep 17 00:00:00 2001 From: lorenzoh Date: Thu, 2 Jun 2022 09:03:17 +0200 Subject: [PATCH 20/23] Add encoding interface invariant --- src/Vision/blocks/bounded.jl | 4 +-- src/Vision/blocks/image.jl | 26 +++++++-------- src/Vision/blocks/mask.jl | 14 ++++---- src/blocks/continuous.jl | 14 ++++---- src/blocks/label.jl | 20 +++++------ src/datablock/block.jl | 25 +++++++------- src/datablock/encoding.jl | 64 ++++++++++++++++++++++++++++++++++++ src/invariants.jl | 4 +-- 8 files changed, 118 insertions(+), 53 deletions(-) diff --git a/src/Vision/blocks/bounded.jl b/src/Vision/blocks/bounded.jl index ca8bda9f4e..5f0be0ca1b 100644 --- a/src/Vision/blocks/bounded.jl +++ b/src/Vision/blocks/bounded.jl @@ -71,11 +71,11 @@ function checkblock(bounded::Bounded{N}, a::AbstractArray{N}) where N end -function FastAI.invariant_checkblock(block::Bounded{N}; blockname = "block", obsname = "obs") where N +function FastAI.invariant_checkblock(block::Bounded{N}; blockvar = "block", obsvar = "obs") where N return invariant([ FastAI.invariant_checkblock(parent(block)), ], - FastAI.__inv_checkblock_title(block, blockname, obsname), + FastAI.__inv_checkblock_title(block, blockvar, obsvar), :seq ) end diff --git a/src/Vision/blocks/image.jl b/src/Vision/blocks/image.jl index 791f53a93a..6840ddc537 100644 --- a/src/Vision/blocks/image.jl +++ b/src/Vision/blocks/image.jl @@ -62,57 +62,57 @@ showblock!(io, ::ShowText, block::Image{2}, obs::AbstractMatrix{<:Real}) = ImageInTerminal.imshow(io, colorview(Gray, obs)) -function FastAI.invariant_checkblock(block::Image{N}; blockname = "block", obsname = "obs") where N +function FastAI.invariant_checkblock(block::Image{N}; blockvar = "block", obsvar = "obs") where N return invariant([ - invariant("`$obsname` is an `AbstractArray`", - description = md("`$obsname` should be of type `AbstractArray`.")) do obs + invariant("`$obsvar` is an `AbstractArray`", + description = md("`$obsvar` should be of type `AbstractArray`.")) do obs if !(obs isa AbstractArray) return "Instead, got invalid type `$(nameof(typeof(obs)))`." |> md end end, - invariant("`$obsname` is `$N`-dimensional") do obs + invariant("`$obsvar` is `$N`-dimensional") do obs if ndims(obs) != N return "Instead, got invalid dimensionality `$N`." |> md end end, - invariant("`$obsname` should have a color or numerical element type") do obs + invariant("`$obsvar` should have a color or numerical element type") do obs if !((eltype(obs) <: Color) ||(eltype(obs) <: Real)) return "Instead, got invalid element type `$(eltype(obs))`." |> md end end, ], - FastAI.__inv_checkblock_title(block, blockname, obsname), + FastAI.__inv_checkblock_title(block, blockvar, obsvar), :seq ) end #= -function isblockinvariant(block::Image{N}; dataname = "data", blockname = "block") where {N} +function isblockinvariant(block::Image{N}; obsvar = "data", blockvar = "block") where {N} return SequenceInvariant( [ BooleanInvariant( obs -> obs isa AbstractArray, name = "Image data is an array", - messagefn = obs -> """Expected `$dataname` to be a subtype of + messagefn = obs -> """Expected `$obsvar` to be a subtype of `AbstractArray`, but instead got type `$(typeof(obs))`.""", ), BooleanInvariant( obs -> ndims(obs) == N, name = "Image data is `$N`-dimensional", - messagefn = obs -> """Expected `$dataname` to be an `$N`-dimensional array, + messagefn = obs -> """Expected `$obsvar` to be an `$N`-dimensional array, but instead got a `$(ndims(obs))`-dimensional array.""", ), BooleanInvariant( obs -> eltype(obs) <: Color || eltype(obs) <: Number, name = "Image data has a color or numerical type.", - messagefn = obs -> """Expected `$dataname` to have an element type that is a - color (`eltype($dataname) <: Color`) or a number (`eltype($dataname) - <: Color`), but instead found `eltype($dataname) == $(eltype(obs)).` + messagefn = obs -> """Expected `$obsvar` to have an element type that is a + color (`eltype($obsvar) <: Color`) or a number (`eltype($obsvar) + <: Color`), but instead found `eltype($obsvar) == $(eltype(obs)).` """ ) ], - "`$dataname` is a valid `$(typeof(block))`", + "`$obsvar` is a valid `$(typeof(block))`", "" ) end diff --git a/src/Vision/blocks/mask.jl b/src/Vision/blocks/mask.jl index a3c7f6d145..4ab4086d7b 100644 --- a/src/Vision/blocks/mask.jl +++ b/src/Vision/blocks/mask.jl @@ -19,15 +19,15 @@ mockblock(mask::Mask{N, T}) where {N, T} = rand(mask.classes, ntuple(_ -> 16, N) Base.nameof(::Mask{N}) where N = "Mask{$N}" -function FastAI.invariant_checkblock(block::Mask{N}; blockname = "block", obsname = "obs") where N +function FastAI.invariant_checkblock(block::Mask{N}; blockvar = "block", obsvar = "obs") where N return invariant([ - invariant("`$obsname` is an `AbstractArray`", - description = md("`$obsname` should be of type `AbstractArray`.")) do obs + invariant("`$obsvar` is an `AbstractArray`", + description = md("`$obsvar` should be of type `AbstractArray`.")) do obs if !(obs isa AbstractArray) return "Instead, got invalid type `$(nameof(typeof(obs)))`." |> md end end, - invariant("`$obsname` is `$N`-dimensional") do obs + invariant("`$obsvar` is `$N`-dimensional") do obs if ndims(obs) != N return "Instead, got invalid dimensionality `$N`." |> md end @@ -36,8 +36,8 @@ function FastAI.invariant_checkblock(block::Mask{N}; blockname = "block", obsnam valid = ∈(block.classes).(obs) if !(all(valid)) unknown = unique(obs[valid .== false]) - return md("""`$obsname` should contain only valid labels, - i.e. `∀ y ∈ $obsname: y ∈ $blockname.classes`, but `$obsname` includes + return md("""`$obsvar` should contain only valid labels, + i.e. `∀ y ∈ $obsvar: y ∈ $blockvar.classes`, but `$obsvar` includes unknown labels: `$(sprint(show, unknown))`. Valid classes are: @@ -45,7 +45,7 @@ function FastAI.invariant_checkblock(block::Mask{N}; blockname = "block", obsnam end end, ], - FastAI.__inv_checkblock_title(block, blockname, obsname), + FastAI.__inv_checkblock_title(block, blockvar, obsvar), :seq ) end diff --git a/src/blocks/continuous.jl b/src/blocks/continuous.jl index 7b0b08b169..e5e84ecbe9 100644 --- a/src/blocks/continuous.jl +++ b/src/blocks/continuous.jl @@ -22,30 +22,30 @@ function blocklossfn(outblock::Continuous, yblock::Continuous) end -function invariant_checkblock(block::Continuous; blockname = "block", obsname = "obs") +function invariant_checkblock(block::Continuous; blockvar = "block", obsvar = "obs") return SequenceInvariant( [ BooleanInvariant( obs -> obs isa AbstractVector, - name = "`$obsname` should be an `AbstractVector`", - messagefn = obs -> """`$obsname` should be an `AbstractVector`, instead + name = "`$obsvar` should be an `AbstractVector`", + messagefn = obs -> """`$obsvar` should be an `AbstractVector`, instead got type `$(typeof(obs))`. """ ), BooleanInvariant( obs -> length(obs) == block.size, - name = "length(`$obsname`) should be $(block.size)", - messagefn = obs -> """`$obsname` should have $(block.size) features, instead + name = "length(`$obsvar`) should be $(block.size)", + messagefn = obs -> """`$obsvar` should have $(block.size) features, instead found a vector with $(length(obs)) features. """ ), BooleanInvariant( obs -> eltype(obs) <: Number, - name = "`eltype($obsname)` should be a subtype of `Number`", + name = "`eltype($obsvar)` should be a subtype of `Number`", messagefn = obs -> """Found a non-numerical element type $(eltype(obs))""" ), ], - "`$obsname` should be a valid `$(summary(block))`", + "`$obsvar` should be a valid `$(summary(block))`", "", ) end diff --git a/src/blocks/label.jl b/src/blocks/label.jl index 85f02ec2fb..32c85afda2 100644 --- a/src/blocks/label.jl +++ b/src/blocks/label.jl @@ -39,14 +39,14 @@ setup(::Type{Label}, data) = Label(unique(eachobs(data))) Base.summary(io::IO, ::Label{T}) where {T} = print(io, "Label{", T, "}") -function invariant_checkblock(block::Label; blockname = "block", obsname = "obs") +function invariant_checkblock(block::Label; blockvar = "block", obsvar = "obs") return invariant( - __inv_checkblock_title(block, blockname, obsname), - description=md("""`$obsname` should be a valid label, i.e. one of + __inv_checkblock_title(block, blockvar, obsvar), + description=md("""`$obsvar` should be a valid label, i.e. one of `$(sprint(show, block.classes, context=:limit => true))`."""), ) do obs if !(obs ∈ block.classes) - return md("Instead, got invalid value `$obs`.") + return md("Instead, got invalid value `$(sprint(show, obs))`.") end end end @@ -102,10 +102,10 @@ Base.summary(io::IO, ::LabelMulti{T}) where {T} = print(io, "LabelMulti{", T, "} -function invariant_checkblock(block::LabelMulti; blockname = "block", obsname = "obs") +function invariant_checkblock(block::LabelMulti; blockvar = "block", obsvar = "obs") return invariant([ - invariant("`$obsname` is an `AbstractVector`", - description = md("`$obsname` should be of type `AbstractVector`.")) do obs + invariant("`$obsvar` is an `AbstractVector`", + description = md("`$obsvar` should be of type `AbstractVector`.")) do obs if !(obs isa AbstractVector) return md("Instead, got invalid type `$(nameof(typeof(obs)))`.") end @@ -114,8 +114,8 @@ function invariant_checkblock(block::LabelMulti; blockname = "block", obsname = valid = ∈(block.classes).(obs) if !(all(valid)) unknown = unique(obs[valid .== false]) - return md("""`$obsname` should contain only valid labels, - i.e. `∀ y ∈ $obsname: y ∈ $blockname.classes`, but `$obsname` includes + return md("""`$obsvar` should contain only valid labels, + i.e. `∀ y ∈ $obsvar: y ∈ $blockvar.classes`, but `$obsvar` includes unknown labels: `$(sprint(show, unknown))`. Valid classes are: @@ -123,7 +123,7 @@ function invariant_checkblock(block::LabelMulti; blockname = "block", obsname = end end ], - __inv_checkblock_title(block, blockname, obsname), + __inv_checkblock_title(block, blockvar, obsvar), :seq ) end diff --git a/src/datablock/block.jl b/src/datablock/block.jl index 005d38d209..2e64140a6a 100644 --- a/src/datablock/block.jl +++ b/src/datablock/block.jl @@ -153,39 +153,40 @@ function invariant_checkblock end # If `invariant_checkblock` is not implemented for a block, default to # checking that `checkblock` returns `true`. -function invariant_checkblock(block::B; obsname = "obs", blockname = "block") where {B<:AbstractBlock} - return invariant(__inv_checkblock_title(block, blockname, obsname),) do obs +function invariant_checkblock(block::AbstractBlock; obsvar = "obs", blockvar = "block") + return invariant(__inv_checkblock_title(block, blockvar, obsvar),) do obs if !checkblock(block, obs) - """Expected `$obsname` to be a block with above type, but - `checkblock($blockname, $obsname)` returned `false`. - This probably means that `$obsname` is not a valid instance of the - block. Check `?$(string(parentmodule(B))).$(nameof(B))` for more information on + """Expected `$obsvar` to be a valid observation for block `$(nameof(block))`, + but `checkblock($blockvar, $obsvar)` returned `false`. + This probably means that `$obsvar` is not a valid instance of the + block. Check `?$(__typename_qualified(block))` for more information on the block and what data is valid. """ |> md end end end +__typename_qualified(::T) where T = "$(string(parentmodule(T))).$(nameof(T))" __inv_checkblock_title(b, bname, oname) = "`$oname` is a valid observation for `$(bname) <: $(nameof(b))`" # For tuples of blocks, the invariant is composed of the individuals' blocks # invariants, passing only if all the child invariants pass. -function invariant_checkblock(blocks::Tuple; dataname = "obss", blockname = "blocks") +function invariant_checkblock(blocks::Tuple; obsvar = "obss", blockvar = "blocks") return invariant( [ invariant( invariant_checkblock( blocks[i], - obsname = "$dataname[$i]", - blockname = "$blockname[$i]" + obsvar = "$obsvar[$i]", + blockvar = "$blockvar[$i]" ), inputfn = obss -> obss[i] ) for (i, block) in enumerate(blocks) ], - "`$dataname` are valid instances of blocks `$blockname`", + "`$obsvar` are valid instances of blocks `$blockvar`", description = md("""The given observations `obss` should be valid instances of the - blocks `$blockname`. Since `$blockname` is a tuple of blocks, each observation - `$dataname[i]` should be a valid instance of the block `$blockname[i]`. + blocks `$blockvar`. Since `$blockvar` is a tuple of blocks, each observation + `$obsvar[i]` should be a valid instance of the block `$blockvar[i]`. See `?Block` for more background on blocks.""") ) end diff --git a/src/datablock/encoding.jl b/src/datablock/encoding.jl index 4c0309c824..bae63c8db4 100644 --- a/src/datablock/encoding.jl +++ b/src/datablock/encoding.jl @@ -29,6 +29,9 @@ decode them [`decode`] """ abstract type Encoding end +invertible(::Encoding) = true + +Base.nameof(::E) where {E<:Encoding} = nameof(E) """ fillblock(inblocks, outblocks) @@ -261,3 +264,64 @@ function testencoding(encoding, block, obs = mockblock(block)) end end end + + +function invariant_encoding(encoding, block; context = Validation(), encvar = "encoding", blockvar = "block", obsvar = "obs") + B = __typename_qualified(block) + E = __typename_qualified(encoding) + return invariant([ + invariant_checkblock(block; blockvar, obsvar), + invariant("`$encvar` is implemented for `$blockvar`") do _ + if isnothing(encodedblock(encoding, block)) + return """Expected `encodedblock($encvar, $blockvar)` to return a block, + indicating that the encoding does transform observations for block + `$blockvar`. Instead, it returned `nothing` which indicates that the + encoding does not transform observations of block`$B`. + + If the encoding should modify the block, this may mean that a method + for `FastAI.encodedblock` is missing. To fix this, implement a method + and return a block from it: + + ```julia + FastAI.encodedblock(::$E, ::$B) + ``` + """ |> md + end + end, + invariant(invariant_checkblock(encodedblockfilled(encoding, block); + blockvar = "enc$blockvar", obsvar = "enc$obsvar"); + title = "Encoded `$obsvar` is a valid instance of encoded `$blockvar`", + inputfn = obs -> encode(encoding, context, block, obs), + description = """The encoded observation + `encobs = encode($encvar, $context, $blockvar, $obsvar)` + should be a valid observation for the encoded block + `enc$blockvar = encodedblock($encvar, $blockvar)`. + """ |> md), + invariant([ + invariant("`$encvar <: $E` is not invertible") do _ + if invertible(encoding) + return "The encoding *is* invertible." |> md + end + end, + invariant([ + + invariant("`$encvar <: $E` is not invertible") do _ + if invertible(encoding) + return "The encoding *is* invertible." |> md + end + end, + invariant("`$encvar <: $E` is not invertible") do _ + if invertible(encoding) + return "The encoding *is* invertible." |> md + end + end, + ]) + ], + "Unless `$encvar <: $E` is not invertible, decoding should return the original block", + :any + ) + ], + "Encoding `$(nameof(encoding))` is implemented for block `$(nameof(block))`", + :seq; + description = "Description" |> md) +end diff --git a/src/invariants.jl b/src/invariants.jl index ae0d3476d8..59f14c9dcc 100644 --- a/src/invariants.jl +++ b/src/invariants.jl @@ -116,7 +116,7 @@ function invariant_datacontainer(; var = :data) end, __invariant_getobs(; var), ], - "`$var` implements the data container interface.", + "`$var` implements the data container interface", :seq; description="""A data container stores observations and allows (1) getting the number of observation and (2) loading an observation. @@ -129,7 +129,7 @@ function invariant_datacontainer_block(block; return invariant([ invariant_datacontainer(; var = datavar), invariant( - invariant_checkblock(block; blockname = blockvar, obsname = obsvar); + invariant_checkblock(block; blockvar = blockvar, obsvar = obsvar); inputfn = data -> getobs(data, 1) ) ], From 96c3dd720a3b9c502bb20bda44bb75ba0ecacd24 Mon Sep 17 00:00:00 2001 From: lorenzoh Date: Tue, 5 Jul 2022 21:22:24 +0200 Subject: [PATCH 21/23] Updat einvariants and merge #240 --- FastTabular/Project.toml | 2 + .../src/encodings/tabularpreprocessing.jl | 7 ++ FastVision/Project.toml | 2 + FastVision/src/FastVision.jl | 3 +- FastVision/src/blocks/bounded.jl | 11 +- FastVision/src/blocks/image.jl | 11 +- FastVision/src/blocks/mask.jl | 11 +- .../src/encodings/imagepreprocessing.jl | 14 ++- .../src/encodings/keypointpreprocessing.jl | 10 +- Project.toml | 1 + src/blocks/continuous.jl | 47 ++++---- src/blocks/label.jl | 28 +++-- src/blocks/many.jl | 1 + src/datablock/block.jl | 28 +++-- src/datablock/encoding.jl | 112 ++++++++++++------ src/encodings/only.jl | 2 + src/interpretation/showinterpretable.jl | 3 +- src/invariants.jl | 83 ++++++++----- src/learner.jl | 3 +- 19 files changed, 243 insertions(+), 136 deletions(-) diff --git a/FastTabular/Project.toml b/FastTabular/Project.toml index b3047e9055..fa7a6c4045 100644 --- a/FastTabular/Project.toml +++ b/FastTabular/Project.toml @@ -11,6 +11,8 @@ FastAI = "5d0beca9-ade8-49ae-ad0b-a3cf890e669f" FilePathsBase = "48062228-2e41-5def-b9a4-89aafe57970f" Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" InlineTest = "bd334432-b1e7-49c7-a2dc-dd9149e4ebd6" +Invariants = "115d0255-0791-41d2-b533-80bc4cbe6c10" +InvariantsCore = "69cbffe8-09de-43b1-81db-93034495284f" MLUtils = "f1d291b0-491e-4a28-83b9-f70985020b54" PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" ShowCases = "605ecd9f-84a6-4c9e-81e2-4798472b76a3" diff --git a/FastTabular/src/encodings/tabularpreprocessing.jl b/FastTabular/src/encodings/tabularpreprocessing.jl index e3be1473bb..d4f868ce44 100644 --- a/FastTabular/src/encodings/tabularpreprocessing.jl +++ b/FastTabular/src/encodings/tabularpreprocessing.jl @@ -14,6 +14,13 @@ function EncodedTableRow(catcols, contcols, categorydict) EncodedTableRow{length(catcols), length(contcols)}(catcols, contcols, categorydict) end +function mockblock(block::EncodedTableRow) + b = TableRow(block.catcols, block.contcols, block.categorydict) + obs = mockblock(b) + enc = setup(TabularPreprocessing, b, TableDataset(map(x -> [x], obs))) + return encode(enc, Validation(), b, obs) +end + function checkblock(::EncodedTableRow{M, N}, x::Tuple{Vector, Vector}) where {M, N} length(x[1]) == M && length(x[2]) == N end diff --git a/FastVision/Project.toml b/FastVision/Project.toml index da9b3c8bf6..e371b7a487 100644 --- a/FastVision/Project.toml +++ b/FastVision/Project.toml @@ -15,6 +15,8 @@ ImageIO = "82e4d734-157c-48bb-816b-45c225c6df19" ImageInTerminal = "d8c32880-2388-543b-8c61-d9f865259254" IndirectArrays = "9b13fd28-a010-5f03-acff-a1bbcff69959" InlineTest = "bd334432-b1e7-49c7-a2dc-dd9149e4ebd6" +Invariants = "115d0255-0791-41d2-b533-80bc4cbe6c10" +InvariantsCore = "69cbffe8-09de-43b1-81db-93034495284f" MLUtils = "f1d291b0-491e-4a28-83b9-f70985020b54" MakieCore = "20f20a25-4f0e-4fdf-b5d1-57303727442b" ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca" diff --git a/FastVision/src/FastVision.jl b/FastVision/src/FastVision.jl index dfcd0033d2..221cc48d08 100644 --- a/FastVision/src/FastVision.jl +++ b/FastVision/src/FastVision.jl @@ -58,7 +58,8 @@ import DataAugmentation: apply, Identity, ToEltype, ImageToTensor, Normalize, AdjustBrightness, AdjustContrast, Maybe, FlipX, FlipY, WarpAffine, Rotate, Zoom, ResizePadDivisible, itemdata -import Invariants: invariant, md +import Invariants: md +import InvariantsCore: invariant, check import ImageInTerminal import IndirectArrays: IndirectArray import MakieCore diff --git a/FastVision/src/blocks/bounded.jl b/FastVision/src/blocks/bounded.jl index c1066ef02b..6ef71c1c4e 100644 --- a/FastVision/src/blocks/bounded.jl +++ b/FastVision/src/blocks/bounded.jl @@ -65,12 +65,13 @@ function checkblock(bounded::Bounded{N}, a::AbstractArray{N}) where {N} end -function FastAI.invariant_checkblock(block::Bounded{N}; blockvar = "block", obsvar = "obs") where N - return invariant([ - FastAI.invariant_checkblock(parent(block)), - ], +function FastAI.invariant_checkblock(block::Bounded{N}; blockvar = "block", obsvar = "obs", kwargs...) where N + return invariant( FastAI.__inv_checkblock_title(block, blockvar, obsvar), - :seq + [ + FastAI.invariant_checkblock(parent(block)), + ]; + kwargs... ) end diff --git a/FastVision/src/blocks/image.jl b/FastVision/src/blocks/image.jl index 27c2dc9394..bbbb78966a 100644 --- a/FastVision/src/blocks/image.jl +++ b/FastVision/src/blocks/image.jl @@ -61,8 +61,10 @@ showblock!(io, ::ShowText, block::Image{2}, obs::AbstractMatrix{<:Real}) = ImageInTerminal.imshow(io, colorview(Gray, obs)) -function FastAI.invariant_checkblock(block::Image{N}; blockvar = "block", obsvar = "obs") where N - return invariant([ +function FastAI.invariant_checkblock(block::Image{N}; blockvar = "block", obsvar = "obs", kwargs...) where N + return invariant( + FastAI.__inv_checkblock_title(block, blockvar, obsvar), + [ invariant("`$obsvar` is an `AbstractArray`", description = md("`$obsvar` should be of type `AbstractArray`.")) do obs if !(obs isa AbstractArray) @@ -79,9 +81,8 @@ function FastAI.invariant_checkblock(block::Image{N}; blockvar = "block", obsvar return "Instead, got invalid element type `$(eltype(obs))`." |> md end end, - ], - FastAI.__inv_checkblock_title(block, blockvar, obsvar), - :seq + ]; + kwargs... ) end diff --git a/FastVision/src/blocks/mask.jl b/FastVision/src/blocks/mask.jl index f5da16ad53..e3973264d4 100644 --- a/FastVision/src/blocks/mask.jl +++ b/FastVision/src/blocks/mask.jl @@ -17,8 +17,10 @@ end mockblock(mask::Mask{N, T}) where {N, T} = rand(mask.classes, ntuple(_ -> 16, N))::AbstractArray{T, N} -function FastAI.invariant_checkblock(block::Mask{N}; blockvar = "block", obsvar = "obs") where N - return invariant([ +function FastAI.invariant_checkblock(block::Mask{N}; blockvar = "block", obsvar = "obs", kwargs...) where N + return invariant( + FastAI.__inv_checkblock_title(block, blockvar, obsvar), + [ invariant("`$obsvar` is an `AbstractArray`", description = md("`$obsvar` should be of type `AbstractArray`.")) do obs if !(obs isa AbstractArray) @@ -42,9 +44,8 @@ function FastAI.invariant_checkblock(block::Mask{N}; blockvar = "block", obsvar `$(sprint(show, block.classes, context=:limit => true))`""") end end, - ], - FastAI.__inv_checkblock_title(block, blockvar, obsvar), - :seq + ]; + kwargs... ) end diff --git a/FastVision/src/encodings/imagepreprocessing.jl b/FastVision/src/encodings/imagepreprocessing.jl index 495c0dfbbe..910ed00317 100644 --- a/FastVision/src/encodings/imagepreprocessing.jl +++ b/FastVision/src/encodings/imagepreprocessing.jl @@ -21,7 +21,19 @@ function checkblock(block::ImageTensor{N}, a::AbstractArray{T, M}) where {M, N, return (N + 1 == M) && (size(a, M) == block.nchannels) end -Base.summary(io::IO, ::ImageTensor{N}) where {N} = print(io, "ImageTensor{$N}") +FastAI.blockname(io::IO, ::ImageTensor{N}) where {N} = "ImageTensor{$N}" + +function FastAI.mockblock(block::ImageTensor{N}) where {N} + return randn(Float32, ntuple(n -> n == N+1 ? block.nchannels : 16, N + 1)) +end + +function FastAI.invariant_checkblock(block::ImageTensor{N}; blockvar = "block", obsvar = "obs", kwargs...) where N + return invariant( + Invariants.hastype_invariant(AbstractArray{<:Number, N+1}), + title = FastAI.__inv_checkblock_title(block, blockvar, obsvar); + kwargs... + ) +end """ ImagePreprocessing([; kwargs...]) <: Encoding diff --git a/FastVision/src/encodings/keypointpreprocessing.jl b/FastVision/src/encodings/keypointpreprocessing.jl index 60965db8bd..269f984204 100644 --- a/FastVision/src/encodings/keypointpreprocessing.jl +++ b/FastVision/src/encodings/keypointpreprocessing.jl @@ -8,9 +8,13 @@ struct KeypointTensor{N, T, M} <: Block sz::NTuple{M, Int} end -mockblock(block::KeypointTensor{N}) where {N} = rand(SVector{N, Float32}, block.sz) -function checkblock(block::KeypointTensor{N, T}, obs::AbstractArray{T}) where {N, T} - return length(obs) == (prod(block.sz) * N) +function mockblock(block::KeypointTensor{N}) where {N} + enc = KeypointPreprocessing((16, 16)) + b = Keypoints{N}(block.sz) + return encode(enc, Validation(), b, mockblock(b)) +end +function checkblock(block::KeypointTensor{N, T, M}, obs::AbstractArray{U, M}) where {N, T, U, M} + return length(obs) == prod(block.sz) * N end """ diff --git a/Project.toml b/Project.toml index 226db96298..bfad93ee7d 100644 --- a/Project.toml +++ b/Project.toml @@ -12,6 +12,7 @@ Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" FluxTraining = "7bf95e4d-ca32-48da-9824-f0dc5310474f" InlineTest = "bd334432-b1e7-49c7-a2dc-dd9149e4ebd6" Invariants = "115d0255-0791-41d2-b533-80bc4cbe6c10" +InvariantsCore = "69cbffe8-09de-43b1-81db-93034495284f" JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" MLDatasets = "eb30cadb-4394-5ae3-aed4-317e484a6458" MLUtils = "f1d291b0-491e-4a28-83b9-f70985020b54" diff --git a/src/blocks/continuous.jl b/src/blocks/continuous.jl index 3f5b590dd2..001df9a356 100644 --- a/src/blocks/continuous.jl +++ b/src/blocks/continuous.jl @@ -21,31 +21,24 @@ function blocklossfn(outblock::Continuous, yblock::Continuous) end -function invariant_checkblock(block::Continuous; blockvar = "block", obsvar = "obs") - return SequenceInvariant( +function invariant_checkblock(block::Continuous; blockvar = "block", obsvar = "obs", kwargs...) + return invariant( + __inv_checkblock_title(block, blockvar, obsvar), [ - BooleanInvariant( - obs -> obs isa AbstractVector, - name = "`$obsvar` should be an `AbstractVector`", - messagefn = obs -> """`$obsvar` should be an `AbstractVector`, instead - got type `$(typeof(obs))`. - """ + Invariants.hastype_invariant(AbstractVector; var = obsvar), + invariant("length(`$obsvar`) should be $(block.size)") do obs + if !(length(obs) == block.size) + return """`$obsvar` should have `$(block.size)` features, instead + found a vector with `$(length(obs))` features.""" |> md + end + end, + Invariants.hastype_invariant( + Number, + title = "`eltype($obsvar)` should be a subtype of number", + inputfn = eltype, ), - BooleanInvariant( - obs -> length(obs) == block.size, - name = "length(`$obsvar`) should be $(block.size)", - messagefn = obs -> """`$obsvar` should have $(block.size) features, instead - found a vector with $(length(obs)) features. - """ - ), - BooleanInvariant( - obs -> eltype(obs) <: Number, - name = "`eltype($obsvar)` should be a subtype of `Number`", - messagefn = obs -> """Found a non-numerical element type $(eltype(obs))""" - ), - ], - "`$obsvar` should be a valid `$(summary(block))`", - "", + ]; + kwargs... ) end @@ -53,8 +46,8 @@ end @testset "Continuous [block]" begin inv = invariant_checkblock(Continuous(5)) - @test check(inv, zeros(5)) - @test !(check(inv, "hi")) - @test !(check(inv, ["hi"])) - @test !(check(inv, [5])) + @test check(Bool, inv, zeros(5)) + @test !check(Bool, inv, "hi") + @test !check(Bool, inv, ["hi"]) + @test !check(Bool, inv, [5]) end diff --git a/src/blocks/label.jl b/src/blocks/label.jl index 1310773051..fa15f2636a 100644 --- a/src/blocks/label.jl +++ b/src/blocks/label.jl @@ -36,16 +36,18 @@ mockblock(label::Label{T}) where {T} = rand(label.classes)::T setup(::Type{Label}, data) = Label(unique(eachobs(data))) -function invariant_checkblock(block::Label; blockvar = "block", obsvar = "obs") - return invariant( - __inv_checkblock_title(block, blockvar, obsvar), - description=md("""`$obsvar` should be a valid label, i.e. one of - `$(sprint(show, block.classes, context=:limit => true))`."""), +function invariant_checkblock(block::Label; blockvar = "block", obsvar = "obs", kwargs...) + inv = invariant( + __inv_checkblock_title(block, blockvar, obsvar) ) do obs if !(obs ∈ block.classes) - return md("Instead, got invalid value `$(sprint(show, obs))`.") + return "\n" * ("""`$obsvar` should be a valid label, i.e. one of + `$blockvar.classes = $(sprint(show, block.classes, context=:limit => true))`. + Instead, got invalid value `$(sprint(show, obs))`. + """ |> Invariants.md) end end + invariant(inv; kwargs...) end @@ -99,12 +101,14 @@ Base.summary(io::IO, ::LabelMulti{T}) where {T} = print(io, "LabelMulti{", T, "} -function invariant_checkblock(block::LabelMulti; blockvar = "block", obsvar = "obs") - return invariant([ +function invariant_checkblock(block::LabelMulti; blockvar = "block", obsvar = "obs", kwargs...) + return invariant( + __inv_checkblock_title(block, blockvar, obsvar), + [ invariant("`$obsvar` is an `AbstractVector`", description = md("`$obsvar` should be of type `AbstractVector`.")) do obs if !(obs isa AbstractVector) - return md("Instead, got invalid type `$(nameof(typeof(obs)))`.") + return md("Instead, got invalid type `$(typeof(obs))`.") end end, invariant("All elements are valid labels") do obs @@ -119,9 +123,7 @@ function invariant_checkblock(block::LabelMulti; blockvar = "block", obsvar = "o `$(sprint(show, block.classes, context=:limit => true))`""") end end - ], - __inv_checkblock_title(block, blockvar, obsvar), - :seq + ]; kwargs... ) end @@ -153,7 +155,7 @@ end @test block.classes == ["cat", "dog"] inv = invariant_checkblock(block) - @test check(Bool, inv, ["cat", "dog"]) + @test_nowarn check(Exception, inv, ["cat", "dog"]) @test check(Bool, inv, []) @test !(check(Bool, inv, "cat")) @test !(check(Bool, inv, ["mouse"])) diff --git a/src/blocks/many.jl b/src/blocks/many.jl index 20bd69cd24..e96176d49c 100644 --- a/src/blocks/many.jl +++ b/src/blocks/many.jl @@ -35,5 +35,6 @@ end @testset "Many [block]" begin enc = OneHot() block = Many(Label(1:10)) + @test encodedblock(enc, block) isa Many{<:OneHotTensor} FastAI.testencoding(enc, block) end diff --git a/src/datablock/block.jl b/src/datablock/block.jl index f0dde30c65..eaad5c40d0 100644 --- a/src/datablock/block.jl +++ b/src/datablock/block.jl @@ -147,10 +147,10 @@ function invariant_checkblock end # If `invariant_checkblock` is not implemented for a block, default to # checking that `checkblock` returns `true`. -function invariant_checkblock(block::AbstractBlock; obsvar = "obs", blockvar = "block") - return invariant(__inv_checkblock_title(block, blockvar, obsvar),) do obs +function invariant_checkblock(block::AbstractBlock; obsvar = "obs", blockvar = "block", kwargs...) + return invariant(__inv_checkblock_title(block, blockvar, obsvar); kwargs...) do obs if !checkblock(block, obs) - """Expected `$obsvar` to be a valid observation for block `$(nameof(block))`, + """Expected `$obsvar` to be a valid observation for block `$(blockname(block))`, but `checkblock($blockvar, $obsvar)` returned `false`. This probably means that `$obsvar` is not a valid instance of the block. Check `?$(__typename_qualified(block))` for more information on @@ -161,12 +161,13 @@ function invariant_checkblock(block::AbstractBlock; obsvar = "obs", blockvar = " end __typename_qualified(::T) where T = "$(string(parentmodule(T))).$(nameof(T))" -__inv_checkblock_title(b, bname, oname) = "`$oname` is a valid observation for `$(bname) <: $(nameof(b))`" +__inv_checkblock_title(b, bname, oname) = "`$oname` is a valid observation for `$(bname) <: $(blockname(b))`" # For tuples of blocks, the invariant is composed of the individuals' blocks # invariants, passing only if all the child invariants pass. -function invariant_checkblock(blocks::Tuple; obsvar = "obss", blockvar = "blocks") +function invariant_checkblock(blocks::Tuple; obsvar = "obss", blockvar = "blocks", kwargs...) return invariant( + "`$obsvar` are valid instances of blocks `$blockvar`", [ invariant( invariant_checkblock( @@ -177,13 +178,16 @@ function invariant_checkblock(blocks::Tuple; obsvar = "obss", blockvar = "blocks inputfn = obss -> obss[i] ) for (i, block) in enumerate(blocks) ], - "`$obsvar` are valid instances of blocks `$blockvar`", description = md("""The given observations `obss` should be valid instances of the blocks `$blockvar`. Since `$blockvar` is a tuple of blocks, each observation `$obsvar[i]` should be a valid instance of the block `$blockvar[i]`. - See `?Block` for more background on blocks.""") + See `?Block` for more background on blocks."""); + kwargs... ) end + +invariant_checkblock((title, block)::Pair; kwargs...) = invariant_checkblock(block; kwargs...) + """ blockname(block) @@ -192,3 +196,13 @@ and other diagrams. """ blockname(block::Block) = string(nameof(typeof(block))) blockname(blocks::Tuple) = "(" * join(map(blockname, blocks), ", ") * ")" + + +@testset "block invariants" begin + @testset "is_block" begin + @test is_block(Bool, Label(1:10), 1) + @test !is_block(Bool, Label(1:10), 0) + @test is_block(Bool, OneHotLabel{Float32}([1, 2]), [0, 1]) + @test is_block(Bool, (Label([1]), Label([2])), (1, 2)) + end +end diff --git a/src/datablock/encoding.jl b/src/datablock/encoding.jl index 79538b31b7..3221cfcedf 100644 --- a/src/datablock/encoding.jl +++ b/src/datablock/encoding.jl @@ -3,7 +3,18 @@ abstract type Encoding Transformation of `Block`s. Can encode some `Block`s ([`encode`]), and optionally -decode them [`decode`] +decode them [`decode`]. `Encoding`s describe data transformations that are applied +to `Block` data. Together `Encoding`s and `Block`s, are used to construct complex +data preprocessing pipelines for training loops. + +Encodings operate on two levels: + +- On the value level, an encoding transforms an observation. +- On the `Block` level, applying an encoding to a block tells you what the output + block is. For example, the [`OneHot`](#) encoding turns a [`Label`](#) block + into a [`OneHotLabel`](#) block. + + By introspecting the block-level transformation ## Interface @@ -29,9 +40,10 @@ decode them [`decode`] """ abstract type Encoding end -invertible(::Encoding) = true +invertible(enc::Encoding, block::AbstractBlock) = + !isnothing(decodedblock(enc, encodedblockfilled(enc, block))) -Base.nameof(::E) where {E<:Encoding} = nameof(E) +encodingname(::E) where E<:Encoding = nameof(E) """ fillblock(inblocks, outblocks) @@ -219,6 +231,11 @@ Performs some tests that the encoding interface is set up properly for and that the block is identical to `block` """ function testencoding(encoding, block, obs = mockblock(block)) + Test.@testset "Encoding `$(typeof(encoding))` for block `$block`" begin + inv = invariant_encoding(encoding, block) + @test_nowarn inv(Exception, obs) + end + return Test.@testset "Encoding `$(typeof(encoding))` for block `$block`" begin # Test that `obs` is a valid instance of `block` Test.@test checkblock(block, obs) @@ -251,20 +268,35 @@ end function invariant_encoding(encoding, block; context = Validation(), encvar = "encoding", blockvar = "block", obsvar = "obs") - B = __typename_qualified(block) - E = __typename_qualified(encoding) - return invariant([ - invariant_checkblock(block; blockvar, obsvar), - invariant("`$encvar` is implemented for `$blockvar`") do _ - if isnothing(encodedblock(encoding, block)) - return """Expected `encodedblock($encvar, $blockvar)` to return a block, + B = blockname(block) + E = encodingname(encoding) + + encobs(obs) = encode(encoding, context, block, obs) + encblock() = encodedblock(encoding, block) + encblockfilled() = encodedblockfilled(encoding, block) + decblock() = decodedblock(encoding, encblockfilled()) + decblockfilled() = decodedblockfilled(encoding, encblockfilled()) + decobs(obs) = decode(encoding, context, encblockfilled(), encobs(obs)) + + return invariant( + "Encoding `$E` is implemented for block `$B`", + [ + invariant_checkblock(block; blockvar, obsvar, description=""" + Before checking that the encoding is properly implemented for the block, + we need to check that the observation `$obsvar` is a valid instance of + `$blockvar <: $B.` + + """ |> md), + invariant("`$encvar <: $E` is implemented for `$blockvar <: $B`") do _ + if isnothing(encblock()) + return """Expected `encodedblock($encvar::$E, $blockvar::$B)` to return a block, indicating that the encoding does transform observations for block `$blockvar`. Instead, it returned `nothing` which indicates that the - encoding does not transform observations of block`$B`. + encoding does not transform observations of block `$B`. If the encoding should modify the block, this may mean that a method - for `FastAI.encodedblock` is missing. To fix this, implement a method - and return a block from it: + for `FastAI.encodedblock` is missing. To fix this, implement the following + method, returning a block from it: ```julia FastAI.encodedblock(::$E, ::$B) @@ -272,40 +304,50 @@ function invariant_encoding(encoding, block; context = Validation(), encvar = "e """ |> md end end, - invariant(invariant_checkblock(encodedblockfilled(encoding, block); + invariant(invariant_checkblock(encblockfilled(); blockvar = "enc$blockvar", obsvar = "enc$obsvar"); title = "Encoded `$obsvar` is a valid instance of encoded `$blockvar`", - inputfn = obs -> encode(encoding, context, block, obs), + inputfn = encobs, description = """The encoded observation `encobs = encode($encvar, $context, $blockvar, $obsvar)` should be a valid observation for the encoded block `enc$blockvar = encodedblock($encvar, $blockvar)`. """ |> md), - invariant([ + invariant( + "If `$encvar <: $E` is invertible, decoding is implemented", + [ invariant("`$encvar <: $E` is not invertible") do _ - if invertible(encoding) + if invertible(encoding, block) return "The encoding *is* invertible." |> md end end, - invariant([ - - invariant("`$encvar <: $E` is not invertible") do _ - if invertible(encoding) - return "The encoding *is* invertible." |> md - end - end, - invariant("`$encvar <: $E` is not invertible") do _ - if invertible(encoding) - return "The encoding *is* invertible." |> md - end - end, - ]) + invariant("Decoding is implemented") do obs + if isnothing(decblock()) + return """ + `decodedblock(encoding, encodedblock(encoding, block))` returned + `nothing`, indicating that the encoding `$encvar <: $E` does not implement + a decoding step. + + This can mean that either the encoding is not invertible, or `decodedblock` + was not implemented for block `$blockvar <: $B`. To fix this, implement EITHER + + - `decodedblock(::$E, ::$B)` and return a non-`nothing` block value from it; OR + - `invertible(::$E, ::$B) = false` if the encoding is not invertible. + """ |> md + end + end, + invariant_checkblock( + decblockfilled(), + inputfn = decobs, + title = "Decoded `encobs` is a valid instance of `$blockvar <: $B`", + description="""Decoding the encoded observation should return a valid observation.""") ], - "Unless `$encvar <: $E` is not invertible, decoding should return the original block", - :any + any, ) ], - "Encoding `$(nameof(encoding))` is implemented for block `$(nameof(block))`", - :seq; - description = "Description" |> md) + description = """ + This invariant checks that the encoding `$encvar <: $E` is properly implemented + for `$blockvar <: $B`. Type `?FastAI.Encoding` to get an overview of the + interface for `Encoding`s. + """ |> md) end diff --git a/src/encodings/only.jl b/src/encodings/only.jl index cb5b4699ac..15dae073aa 100644 --- a/src/encodings/only.jl +++ b/src/encodings/only.jl @@ -31,6 +31,8 @@ struct Only{E <: Encoding} <: StatefulEncoding encoding::E end +encodingname(only::Only) = "Only{$(encodingname(only.encoding))}" + function Only(name::Symbol, encoding::Encoding) return Only(Named{name}, encoding) end diff --git a/src/interpretation/showinterpretable.jl b/src/interpretation/showinterpretable.jl index f216307cf0..04400e78c8 100644 --- a/src/interpretation/showinterpretable.jl +++ b/src/interpretation/showinterpretable.jl @@ -21,6 +21,7 @@ showblockinterpretable(ShowText(), encodings, block, x) # will decode to an `Im """ function showblockinterpretable(backend::ShowBackend, encodings, block, obs) + invariant_checkblock(block)(Exception, obs) res = decodewhile(block -> !isshowable(backend, block), encodings, Validation(), @@ -53,7 +54,7 @@ end # Helpers function isshowable(backend::S, block::B) where {S <: ShowBackend, B <: AbstractBlock} - hasmethod(FastAI.showblock!, (Any, S, B, Any)) + hasmethod(FastAI.showblock!, (Any, S, B, typeof(mockblock(block)))) end """ diff --git a/src/invariants.jl b/src/invariants.jl index 59f14c9dcc..89d96a88f6 100644 --- a/src/invariants.jl +++ b/src/invariants.jl @@ -106,45 +106,47 @@ is_data(::Type{Bool}, args...; kwargs...) = convert(Bool, is_data(args...; kwarg function invariant_datacontainer(; var = :data) - invariant([ - __invariant_numobs(; var), - invariant("`$var` contains at least one observation") do data - n = numobs(data) - if n <= 0 - return "Instead, got a data container with $n observations." - end - end, - __invariant_getobs(; var), - ], - "`$var` implements the data container interface", - :seq; - description="""A data container stores observations and allows (1) getting - the number of observation and (2) loading an observation. - See [the tutorial](/documents/docs/tutorials/data_containers.md) for more - information.""" |> md) + invariant( + "`$var` implements the data container interface", + [ + __invariant_numobs(; var), + invariant("`$var` contains at least one observation") do data + n = numobs(data) + if n <= 0 + return "Instead, got a data container with $n observations." + end + end, + __invariant_getobs(; var), + ], + all; + description="""A data container stores observations and allows (1) getting + the number of observation and (2) loading an observation. + See [the tutorial](/documents/docs/tutorials/data_containers.md) for more + information.""" |> md) end function invariant_datacontainer_block(block; datavar = "data", blockvar = "data", obsvar = "obs") - return invariant([ - invariant_datacontainer(; var = datavar), - invariant( - invariant_checkblock(block; blockvar = blockvar, obsvar = obsvar); - inputfn = data -> getobs(data, 1) - ) + return invariant( + "`$datavar` is a data container with valid observations for block `$(blockname(block))`", + [ + + invariant_datacontainer(; var = datavar), + invariant( + invariant_checkblock(block; blockvar = blockvar, obsvar = obsvar); + inputfn = data -> getobs(data, 1) + ) ], - "`$datavar` is a data container with valid observations for block `$(nameof(block))`", - :seq) - - + all) end -__invariant_getobs(; var = :data) = invariant([ +__invariant_getobs(; var = :data) = invariant( + "`$var` implements the `getobs` interface", + [ Invariants.hasmethod_invariant(Base.getindex, :data, :idx => 1) Invariants.hasmethod_invariant(MLUtils.getobs, :data, :idx => 1) ], - "`$var` implements the `getobs` interface", - :any; + any; description=Invariants.md(""" `$var` must provide a way load an observation by implementing **either** (1) `Base.getindex($var, idx::Int)` (preferred) or (2) `MLUtils.getobs($var, idx::Int)` @@ -153,15 +155,32 @@ __invariant_getobs(; var = :data) = invariant([ inputfn=data -> (; data), ) -__invariant_numobs(; var = :data) = invariant([ +__invariant_numobs(; var = :data) = invariant( + "`$var` implements the `numobs` interface", + [ Invariants.hasmethod_invariant(Base.length, :data) Invariants.hasmethod_invariant(MLUtils.numobs, :data) ], - "`$var` implements the `numobs` interface", - :any; + any; description=Invariants.md(""" `$var` must provide a way get the number of observations it contains implementing either `Base.length($var)` (preferred) or `MLUtils.numobs($var, idx::Int)` """), inputfn=data -> (; data), ) + + +@testset "data container invariants" begin + @testset "is_data" begin + @test is_data(Bool, 1:10) + @test !is_data(Bool, nothing) + @test is_data(Bool, [1]) + @test !is_data(Bool, []) + + @test is_data(Bool, 1:10, Label(1:10)) + @test !is_data(Bool, 0:10, Label(1:10)) + @test is_data(Bool, [[0, 1]], OneHotLabel{Float32}([1, 2])) + + @test is_data(Bool, [(1, 2)], (Label([1]), Label([2]))) + end +end diff --git a/src/learner.jl b/src/learner.jl index 1f11a4c9cf..e1ae4e6eef 100644 --- a/src/learner.jl +++ b/src/learner.jl @@ -52,11 +52,12 @@ function tasklearner(task::LearningTask, batchsize = 16, optimizer = ADAM(), lossfn = tasklossfn(task), + usedefaultcallbacks = true, kwargs...) if isnothing(model) model = isnothing(backbone) ? taskmodel(task) : taskmodel(task, backbone) end - dls = methoddataloaders(traindata, validdata, method, batchsize; kwargs...) + dls = taskdataloaders(traindata, validdata, task, batchsize; kwargs...) return Learner(model, dls, optimizer, lossfn, callbacks...; usedefaultcallbacks) end From a230a78cd9af24e74ad1bf4f7c4379e572de0d4a Mon Sep 17 00:00:00 2001 From: lorenzoh Date: Tue, 13 Dec 2022 08:28:52 +0100 Subject: [PATCH 22/23] Fix invariants for FastVision.jl --- FastVision/Project.toml | 1 - FastVision/src/FastVision.jl | 3 +-- FastVision/src/tasks/segmentation.jl | 2 ++ src/datablock/encoding.jl | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/FastVision/Project.toml b/FastVision/Project.toml index 176288a8af..1f09868825 100644 --- a/FastVision/Project.toml +++ b/FastVision/Project.toml @@ -16,7 +16,6 @@ ImageInTerminal = "d8c32880-2388-543b-8c61-d9f865259254" IndirectArrays = "9b13fd28-a010-5f03-acff-a1bbcff69959" InlineTest = "bd334432-b1e7-49c7-a2dc-dd9149e4ebd6" Invariants = "115d0255-0791-41d2-b533-80bc4cbe6c10" -InvariantsCore = "69cbffe8-09de-43b1-81db-93034495284f" MLUtils = "f1d291b0-491e-4a28-83b9-f70985020b54" MakieCore = "20f20a25-4f0e-4fdf-b5d1-57303727442b" ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca" diff --git a/FastVision/src/FastVision.jl b/FastVision/src/FastVision.jl index 221cc48d08..2f3981e0e3 100644 --- a/FastVision/src/FastVision.jl +++ b/FastVision/src/FastVision.jl @@ -58,8 +58,7 @@ import DataAugmentation: apply, Identity, ToEltype, ImageToTensor, Normalize, AdjustBrightness, AdjustContrast, Maybe, FlipX, FlipY, WarpAffine, Rotate, Zoom, ResizePadDivisible, itemdata -import Invariants: md -import InvariantsCore: invariant, check +import Invariants: Invariants, md, invariant, check import ImageInTerminal import IndirectArrays: IndirectArray import MakieCore diff --git a/FastVision/src/tasks/segmentation.jl b/FastVision/src/tasks/segmentation.jl index f018e2e696..9a4e13eac8 100644 --- a/FastVision/src/tasks/segmentation.jl +++ b/FastVision/src/tasks/segmentation.jl @@ -72,6 +72,8 @@ _tasks["imagesegmentation"] = (id = "vision/imagesegmentation", @testset "taskdataloaders" begin data, blocks = load(datarecipes()["camvid_tiny"]) traindl, _ = taskdataloaders(data, ImageSegmentation(blocks)) + # Iterate once so that precompilation does not print when testing + for batch in traindl end @test_nowarn for batch in traindl end end diff --git a/src/datablock/encoding.jl b/src/datablock/encoding.jl index 3221cfcedf..e858ebd555 100644 --- a/src/datablock/encoding.jl +++ b/src/datablock/encoding.jl @@ -44,6 +44,7 @@ invertible(enc::Encoding, block::AbstractBlock) = !isnothing(decodedblock(enc, encodedblockfilled(enc, block))) encodingname(::E) where E<:Encoding = nameof(E) +encodingname(t::Tuple) = map(encodingname, t) """ fillblock(inblocks, outblocks) From c0eea08b63f1334f57901a6f9c834c0483ad4305 Mon Sep 17 00:00:00 2001 From: lorenzoh Date: Sat, 17 Dec 2022 11:44:07 +0100 Subject: [PATCH 23/23] Improve docstring --- src/invariants.jl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/invariants.jl b/src/invariants.jl index 89d96a88f6..eb9bb7784b 100644 --- a/src/invariants.jl +++ b/src/invariants.jl @@ -80,10 +80,13 @@ end """ is_data(data) + is_data(Bool, data)::Bool Check that `data` implements the data container interface and give detailed info on missing functionality if not. +Pass `Bool` as a first argument to return a `Bool`. + """ function is_data(data; kwargs...) inv = invariant_datacontainer(; kwargs...) @@ -93,9 +96,12 @@ end """ is_data(data, block) + is_data(Bool, data, block)::Bool Check that `data` implements the data container interface and its observations are valid instances of `block`, giving detailed errors if not. + +Pass `Bool` as a first argument to return a `Bool`. """ function is_data(data, block; kwargs...) inv = invariant_datacontainer_block(block; kwargs...)