Lecture 10: Scripts

Author: sadda

scripts/lecture_10/Project.toml

@@ -4,6 +4,7 @@ Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
 ImageInspector = "b0ce21f1-0238-464b-b95f-8a4068743199"
 MLDatasets = "eb30cadb-4394-5ae3-aed4-317e484a6458"
 Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
+RDatasets = "ce6b1742-4840-55fa-b093-852dadbb1d8b"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 
@@ -12,4 +13,5 @@ BSON = "= 0.2.6"
 Flux = "= 0.11.6"
 MLDatasets = "= 0.5.6"
 Plots = "= 1.10.3"
+RDatasets = "= 0.7.4"
 julia = "1.5"

scripts/lecture_10/script.jl

@@ -0,0 +1,150 @@
+using Base.Iterators: partition
+using Random
+using BSON
+using Statistics
+using MLDatasets
+using Flux
+using Flux: onehotbatch, onecold, crossentropy
+using Flux.Data: DataLoader
+using Plots
+using ImageInspector
+import RDatasets: dataset
+
+# Introduction to Flux
+
+include("utilities.jl")
+
+Random.seed!(666)
+
+iris = dataset("datasets", "iris")
+
+X = Matrix(iris[:, 1:4])
+y = iris.Species
+
+X_train, y_train, X_test, y_test, classes = prepare_data(X', y; dims=2)
+
+n_hidden = 5
+m = Chain(
+    Dense(size(X_train,1), n_hidden, relu),
+    Dense(n_hidden, size(y_train,1), identity),
+    softmax,
+)
+
+m(X_train)
+
+params(m[2])[2] .= [-1;0;1]
+
+L(x,y) = crossentropy(m(x), y)
+
+L(X_train, y_train)
+
+ps = params(m)
+grad = gradient(() -> L(X_train, y_train), ps)
+
+grad = gradient(() -> L(X_train, y_train), params(X_train))
+
+size(grad[X_train])
+
+opt = Descent(0.1)
+max_iter = 250
+
+acc_test = zeros(max_iter)
+for i in 1:max_iter
+    gs = gradient(() -> L(X_train, y_train), ps)
+    Flux.Optimise.update!(opt, ps, gs)
+    acc_test[i] = accuracy(X_test, y_test)
+end
+
+plot(acc_test, xlabel="Iteration", ylabel="Test accuracy", label="", ylim=(-0.01,1.01))
+
+# Loading data
+
+T = Float32
+X_train, y_train = MLDatasets.MNIST.traindata(T)
+X_test, y_test = MLDatasets.MNIST.testdata(T)
+
+# Exercise
+
+
+
+# Exercise
+
+
+
+# Loading data
+
+function load_data(dataset; T=Float32, onehot=false, classes=0:9)
+    X_train, y_train = dataset.traindata(T)
+    X_test, y_test = dataset.testdata(T)
+    
+    X_train = reshape_data(X_train)
+    X_test = reshape_data(X_test)
+
+    if onehot
+        y_train = onehotbatch(y_train, classes)
+        y_test = onehotbatch(y_test, classes)
+    end
+
+    return X_train, y_train, X_test, y_test
+end
+
+X_train, y_train, X_test, y_test = load_data(MLDatasets.MNIST; T=T, onehot=true)
+
+# Exercise
+
+
+
+# Exercise
+
+
+
+# Bonus
+
+batches = map(partition(randperm(size(y, 2)), batchsize)) do inds
+    return (X[:, :, :, inds], y[:, inds])
+end
+
+[(X[:, :, :, inds], y[:, inds]) for inds in partition(randperm(size(y, 2)), batchsize)]
+
+# Define model
+
+Random.seed!(666)
+m = Chain(
+    Conv((2,2), 1=>16, relu),
+    MaxPool((2,2)),
+    Conv((2,2), 16=>8, relu),
+    MaxPool((2,2)),
+    flatten,
+    Dense(288, size(y_train,1)),
+    softmax,
+)
+
+L(X, y) = crossentropy(m(X), y)
+
+# Train model
+
+function train_model!(m, L, X, y;
+        opt = Descent(0.1),
+        batchsize = 128,
+        n_epochs = 10,
+        file_name = "")
+
+    batches = DataLoader((X, y); batchsize, shuffle = true)
+
+    for _ in 1:n_epochs
+        Flux.train!(L, params(m), batches, opt)
+    end
+
+    !isempty(file_name) && BSON.bson(file_name, m=m)
+
+    return
+end
+
+# Exercise
+
+
+
+# Exercise
+
+
+

scripts/lecture_10/script_init.jl

@@ -12,37 +12,22 @@ using Flux: onehotbatch, onecold, crossentropy
 using Flux.Data: DataLoader
 using Plots
 using ImageInspector
+import RDatasets: dataset
 
-function reshape_data(X::AbstractArray{<:Real, 3})
-    s = size(X)
-    return reshape(X, s[1], s[2], 1, s[3])
-end
+plot(1:10)
 
-reshape_data(X::AbstractArray{<:Real, 4}) = X
-
-function load_data(dataset; T=Float32, onehot=false, classes=0:9)
-    X_train, y_train = dataset.traindata(T)
-    X_test, y_test = dataset.testdata(T)
-    
-    X_train = reshape_data(X_train)
-    X_test = reshape_data(X_test)
-
-    if onehot
-        y_train = onehotbatch(y_train, classes)
-        y_test = onehotbatch(y_test, classes)
-    end
-
-    return X_train, y_train, X_test, y_test
-end
+iris = dataset("datasets", "iris")
 
 T = Float32
-X_train, y_train, X_test, y_test = load_data(MLDatasets.MNIST; T=T, onehot=true);
-load_data(MLDatasets.CIFAR10; T=T, onehot=true);
+X_train, y_train = MLDatasets.MNIST.traindata(T)
+X_train = reshape(X_train, size(X_train,1), size(X_train,2), 1, size(X_train,3))
+y_train = onehotbatch(y_train, 0:9)
+MLDatasets.MNIST.testdata(T)
+MLDatasets.CIFAR10.traindata(T)
 
-inds = findall(y_train .== 0)[1:15]
-imageplot(1 .- X_train, inds; nrows=3, size=(800,480))
+imageplot(1 .- X_train, 1:3; nrows=1, size=(800,480))
 
-m = Chain(
+m_aux = Chain(
     Conv((2,2), 1=>16, relu),
     MaxPool((2,2)),
     Conv((2,2), 16=>8, relu),
@@ -52,27 +37,10 @@ m = Chain(
     softmax,
 )
 
-L(X, y) = crossentropy(m(X), y)
-
-function train_model!(m, L, X, y;
-        opt = Descent(0.1),
-        batchsize = 128,
-        n_epochs = 10,
-        file_name = "")
-
-    batches = DataLoader((X, y); batchsize, shuffle = true)
-
-    for _ in 1:n_epochs
-        Flux.train!(L, params(m), batches, opt)
-    end
-
-    !isempty(file_name) && BSON.bson(file_name, m=m)
-
-    return
-end
+L_aux(X, y) = crossentropy(m_aux(X), y)
 
-train_model!(m, L, X_train, y_train; n_epochs=1)
+batches_aux = DataLoader((X_train, y_train); batchsize=64, shuffle = true)
 
-accuracy(x, y) = mean(onecold(m(x)) .== onecold(y))
+gradient(() -> L_aux(X_train[:,:,:,1:10], y_train[:,1:10]), params(m_aux))
 
-accuracy(X_test, y_test)
+onecold(m_aux(X_train[:,:,:,1:10]))

scripts/lecture_10/script_sol.jl

@@ -8,6 +8,56 @@ using Flux: onehotbatch, onecold, crossentropy
 using Flux.Data: DataLoader
 using Plots
 using ImageInspector
+import RDatasets: dataset
+
+# Introduction to Flux
+
+include("utilities.jl")
+
+Random.seed!(666)
+
+iris = dataset("datasets", "iris")
+
+X = Matrix(iris[:, 1:4])
+y = iris.Species
+
+X_train, y_train, X_test, y_test, classes = prepare_data(X', y; dims=2)
+
+n_hidden = 5
+m = Chain(
+    Dense(size(X_train,1), n_hidden, relu),
+    Dense(n_hidden, size(y_train,1), identity),
+    softmax,
+)
+
+m(X_train)
+
+params(m[2])[2] .= [-1;0;1]
+
+L(x,y) = crossentropy(m(x), y)
+
+L(X_train, y_train)
+
+ps = params(m)
+grad = gradient(() -> L(X_train, y_train), ps)
+
+grad = gradient(() -> L(X_train, y_train), params(X_train))
+
+size(grad[X_train])
+
+opt = Descent(0.1)
+max_iter = 250
+
+acc_test = zeros(max_iter)
+for i in 1:max_iter
+    gs = gradient(() -> L(X_train, y_train), ps)
+    Flux.Optimise.update!(opt, ps, gs)
+    acc_test[i] = accuracy(X_test, y_test)
+end
+
+plot(acc_test, xlabel="Iteration", ylabel="Test accuracy", label="", ylim=(-0.01,1.01))
+
+# Loading data
 
 T = Float32
 X_train, y_train = MLDatasets.MNIST.traindata(T)
@@ -66,7 +116,7 @@ batches = DataLoader((X_train, y_train); batchsize, shuffle = true)
 
 # Bonus
 
-batches = map(partition(randperm(size(y, 2)), batchsize)) do inds
+map(partition(randperm(size(y, 2)), batchsize)) do inds
     return (X[:, :, :, inds], y[:, inds])
 end
 

scripts/lecture_10/utilities.jl

@@ -0,0 +1,49 @@
+using Statistics
+
+function split(X, y::AbstractVector; dims=1, ratio_train=0.8, kwargs...)
+    n = length(y)
+    size(X, dims) == n || throw(DimensionMismatch("..."))
+
+    n_train = round(Int, ratio_train*n)
+    i_rand = randperm(n)
+    i_train = i_rand[1:n_train]
+    i_test = i_rand[n_train+1:end]
+
+    return selectdim(X, dims, i_train), y[i_train], selectdim(X, dims, i_test), y[i_test]
+end
+
+function normalize(X_train, X_test; dims=1, kwargs...)
+    col_mean = mean(X_train; dims)
+    col_std = std(X_train; dims)
+
+    return (X_train .- col_mean) ./ col_std, (X_test .- col_mean) ./ col_std
+end
+
+function prepare_data(X, y; do_normal=true, do_onehot=true, kwargs...)
+    X_train, y_train, X_test, y_test = split(X, y; kwargs...)
+
+    if do_normal
+        X_train, X_test = normalize(X_train, X_test; kwargs...)
+    end
+
+    classes = unique(y)
+
+    if do_onehot
+        y_train = onehot(y_train, classes)
+        y_test = onehot(y_test, classes)
+    end
+
+    return X_train, y_train, X_test, y_test, classes
+end
+
+function onehot(y, classes)
+    y_onehot = falses(length(classes), length(y))
+    for (i, class) in enumerate(classes)
+        y_onehot[i, y .== class] .= 1
+    end
+    return y_onehot
+end
+
+# onecold(y, classes) = [classes[argmax(y_col)] for y_col in eachcol(y)]
+
+accuracy(x, y) = mean(onecold(m(x), classes) .== onecold(y, classes))