Lab4 - Neural Networks

Autoencoder.py

@@ -0,0 +1,73 @@
+import numpy as np
+from tensorflow.keras import layers, Model, optimizers
+from typing import Tuple, Optional
+
+
+class Autoencoder(object):
+    """
+    * An autoencoder model for compressing and reconstructing input data.
+    * Made for MNIST input digits' database.
+    """
+
+    def __init__(self, input_shape: Tuple[int, int, int], encoding_dim: int, optimizer: str = 'adam',
+                 loss: str = 'binary_crossentropy'):
+        """
+        * Initializes the Autoencoder with specified parameters.
+
+        * :param input_shape: Shape of the input data (height, width, channels).
+        * :param encoding_dim: Dimension of the encoded representation.
+        * :param optimizer: Optimizer to use during training.
+        * :param loss: Loss function to use during training.
+        """
+        self.input_shape: Tuple[int, int, int] = input_shape
+        self.encoding_dim: int = encoding_dim
+        self.optimizer: str = optimizer
+        self.loss: str = loss
+        self.encoder: Optional[Model] = None
+        self.decoder: Optional[Model] = None
+        self.autoencoder: Optional[Model] = None
+
+    def _build_model(self, activation_encoding, activation_decoding) -> None:
+        """
+        * Builds the encoder, decoder, and autoencoder models.
+        """
+        input_img = layers.Input(shape=self.input_shape)
+        x = layers.Flatten()(input_img)
+        encoded = layers.Dense(self.encoding_dim, activation=activation_encoding)(x)
+
+        x = layers.Dense(int(np.prod(self.input_shape)), activation=activation_decoding)(encoded)
+        decoded = layers.Reshape(self.input_shape)(x)
+
+        self.autoencoder = Model(input_img, decoded)
+        self.encoder = Model(input_img, encoded)
+        encoded_input = layers.Input(shape=(self.encoding_dim,))
+        decoder_layer = self.autoencoder.layers[-2]
+        self.decoder = Model(encoded_input, decoder_layer(encoded_input))
+        self.autoencoder.compile(optimizer=self.optimizer, loss=self.loss)
+
+    def fit(self, x_train: np.ndarray, epochs: int = 50, batch_size: int = 256, shuffle: bool = True,
+            validation_data: Optional[Tuple[np.ndarray, np.ndarray]] = None,
+            activation_encoding: str = 'relu', activation_decoding: str = 'sigmoid') -> None:
+        """
+        * Trains the autoencoder on the training data.
+
+        * :param x_train: Training data.
+        * :param epochs: Number of training epochs.
+        * :param batch_size: Size of each batch.
+        * :param shuffle: Whether to shuffle the data.
+        * :param validation_data: Tuple of validation data (x_val, y_val).
+        * :param activation_encoding: Activation function for the encoding layer.
+        * :param activation_decoding: Activation function for the decoding layer.
+        """
+        self._build_model(activation_encoding=activation_encoding, activation_decoding=activation_decoding)
+        self.autoencoder.fit(x_train, x_train, epochs=epochs, batch_size=batch_size, shuffle=shuffle,
+                             validation_data=validation_data)
+
+    def predict(self, x_test: np.ndarray) -> np.ndarray:
+        """
+        * Encodes and reconstructs the test data.
+
+        * :param x_test: Test data to reconstruct.
+        * :return: Reconstructed data.
+        """
+        return self.decoder.predict(self.encoder.predict(x_test))

README.md

@@ -1 +1,60 @@
-# MachineLearning
+# Lab 4 – Neural Networks
+This branch contains the code, experiments, and report for Lab 4: Neural Networks, part of the Machine Learning coursework in the Robotics Engineering degree.
+
+*🧠 Objectives*
+
+  - Understand the use of MATLAB's Deep Learning Toolbox
+
+  - Train and test feedforward neural networks on classification tasks
+
+  - Implement and evaluate an autoencoder for dimensionality reduction
+
+*📚 Benchmark Data Sources*
+
+Suggested datasets:
+
+  - UCI Machine Learning Repository: Iris, Wine, 20 Newsgroups, Breast Cancer Wisconsin, etc.
+
+  - MNIST: Handwritten digit images
+
+  - Kaggle Datasets: ML competition data and open datasets
+
+*🔧 Task 0 – Neural Networks in MATLAB*
+
+MATLAB's Deep Learning Toolbox (previously Neural Network Toolbox) is used for the entire lab. To familiarize with it:
+
+  - 🔗 Tutorial: Fit Data with a Neural Network
+
+*🔗 Task 1 – Feedforward Neural Networks (Multi-Layer Perceptrons)*
+
+  - 🔗 Tutorial: Classify Patterns with a Neural Network
+
+💡 Use the GUI or function-based interface to experiment with:
+
+  - Different datasets (e.g., Iris, Wine, MNIST)
+
+  - Varying architectures (number of hidden layers and neurons)
+
+📊 Deliverables:
+
+  - Confusion matrices (automatically generated by MATLAB)
+
+  - Accuracy tables summarizing experiments with different configurations
+
+*🔄 Task 2 – Autoencoder*
+
+Train an autoencoder using a multi-layer perceptron structure where:
+
+  - Input layer = Output layer size
+
+  - Hidden layer = Compressed representation (fewer neurons)
+
+🧪 Experimental Workflow (using MNIST or similar):
+
+  - Select 2 classes (e.g., digits "1" and "8") from MNIST
+
+  - Create a dataset using only these classes
+
+  - Train the autoencoder
+
+  - Extract compressed (encoded) representations

main.py

@@ -0,0 +1,144 @@
+import numpy as np
+from typing import Optional
+import matplotlib.pyplot as plt
+from Autoencoder import Autoencoder
+from tensorflow.keras.datasets import mnist
+from sklearn.model_selection import train_test_split
+
+
+def main() -> None:
+    (x_train, y_train), (x_test, y_test) = mnist.load_data()
+
+    # Filter the data to get only the digits 1 and 8
+    x_train_18 = x_train[np.isin(y_train, [1, 8])]
+    x_test_18 = x_test[np.isin(y_test, [1, 8])]
+    y_train_18 = y_train[np.isin(y_train, [1, 8])]
+    y_test_18 = y_test[np.isin(y_test, [1, 8])]
+
+    # Split the data into train and test
+    x_train_18, _, y_train_18, _ = train_test_split(x_train_18, y_train_18, test_size=0.2, random_state=42)
+
+    # Normalize the data
+    x_train_18 = x_train_18 / 255.0
+    x_test_18 = x_test_18 / 255.0
+
+    # Reshape the images for the autoencoder
+    x_train_18 = x_train_18.reshape((x_train_18.shape[0], 28, 28, 1))
+    x_test_18 = x_test_18.reshape((x_test_18.shape[0], 28, 28, 1))
+    # Train the autoencoder
+    autoencoder = Autoencoder(input_shape=(28, 28, 1), encoding_dim=2, optimizer='adam', loss='binary_crossentropy')
+    autoencoder.fit(x_train_18, epochs=100, batch_size=256, validation_data=(x_test_18, x_test_18),
+                    activation_encoding='relu', activation_decoding='sigmoid')
+    # Predicting on test data
+    decoded_imgs = autoencoder.predict(x_test_18)
+
+    # Display original and reconstructed images
+    n = 20
+    plt.figure(figsize=(10, 4))
+    for i in range(n):
+        # Original
+        ax = plt.subplot(2, n, i + 1)
+        plt.imshow(x_test_18[i].reshape(28, 28), cmap='gray')
+        ax.set_xticks([])
+        ax.set_yticks([])
+
+        # Reconstructed
+        ax = plt.subplot(2, n, i + 1 + n)
+        plt.imshow(decoded_imgs[i].reshape(28, 28), cmap='gray')
+        ax.set_xticks([])
+        ax.set_yticks([])
+    plt.show()
+
+    # Plot the clusters
+    plt.figure(figsize=(10, 10))
+    # Obtain the labels for the test data
+    encoded_imgs = autoencoder.encoder.predict(x_test_18)
+    plotcl(encoded_imgs, y_test_18, coord=slice(0, 2))
+
+    plotmodelhistory(autoencoder.autoencoder.history)
+
+    # Compute Variance Accounted For (VAF)
+    vaf_value = compute_vaf(x_test_18.flatten(), decoded_imgs.flatten())
+
+    # Plot VAF as bar chart
+    plt.figure()
+    x = np.unique(y_test_18)
+    vaf_values = [compute_vaf(x_test_18[y_test_18 == digit].flatten(), decoded_imgs[y_test_18 == digit].flatten()) for digit in x]
+    plt.bar(x, vaf_values, color='skyblue')
+    plt.xlabel('Digits')
+    plt.ylabel('VAF (%)')
+    plt.grid(axis='y', linestyle='--', linewidth=0.5, color='gray', alpha=0.5)
+    for i, vaf in enumerate(vaf_values):
+        plt.text(x[i], vaf, f'{vaf:.2f}%', ha='center', va='bottom')
+    plt.xticks(x)
+    plt.ylim(0, 100)
+    plt.show()
+
+
+def compute_vaf(true_data: np.ndarray, predicted_data: np.ndarray) -> float:
+    """
+    * Compute the Variance Accounted For (VAF) between true data and predicted data.
+
+    * :param true_data: Actual data.
+    * :param predicted_data: Predicted data.
+    * :return: VAF value.
+    """
+    vaf = 1 - np.var(true_data - predicted_data) / np.var(true_data)
+    return vaf * 100  # Convert to percentage
+
+
+# * Autoencoder Model loss
+def plotmodelhistory(history):
+    plt.plot(history.history['loss'])
+    plt.plot(history.history['val_loss'])
+    plt.ylabel('Loss')
+    plt.xlabel('Epoch')
+    plt.legend(['Train', 'Test'], loc='upper left')
+    plt.show()
+
+
+def plotcl(x: np.ndarray, Xlbl: np.ndarray, coord: Optional[slice] = None, psize: int = 2) -> None:
+    """
+    * Plot clusters in 1, 2 or 3 dimensions.
+
+    * :param x : Data to plot.
+    * :param Xlbl : Labels of the data.
+    * :param coord : Coordinates to plot (default is to plot the first 3 coordinates).
+    * :param psize : Size of the points (default is 2).
+    """
+    unique_labels = np.unique(Xlbl)
+    colors = [[1, 0, 0], [0, 0, 1], [0, 1, 0], [1, 0, .5], [.5, 0, 0], [0, .5, 0], [0, 0, .5],
+              [1, .5, 0], [.5, 1, 0], [.5, 0, 1], [0, 1, .5], [0, .5, 1], [.5, .5, 0], [.5, 0, .5],
+              [0, .5, .5]]
+    pointstyles = 'o+*xsd^v><ph'
+    if coord is None:
+        coord = slice(0, min(3, x.shape[1]))
+    x = x[:, coord]
+
+    plt.figure()
+
+    for label in unique_labels:
+        indices = np.where(Xlbl == label)
+        cc = colors[label % len(colors)]
+        ps = pointstyles[label % len(pointstyles)]
+        if x.shape[1] == 1:
+            plt.scatter(x[indices, 0], np.zeros_like(x[indices, 0]), c=[cc], marker=ps, s=psize, linewidth=int(np.log(psize + 1) + 1), label=f"Label {label}")
+            plt.xlabel('1st dimension of the encoder')
+        elif x.shape[1] == 2:
+            plt.scatter(x[indices, 0], x[indices, 1], c=[cc], marker=ps, s=psize, linewidth=int(np.log(psize + 1) + 1), label=f"Label {label}")
+            plt.xlabel('1st dimension of the encoder')
+            plt.ylabel('2nd dimension of the encoder')
+        else:
+            ax = plt.axes(projection='3d')
+            ax.scatter3D(x[indices, 0], x[indices, 1], x[indices, 2], c=[cc], marker=ps, s=psize,
+                         linewidth=int(np.log(psize + 1) + 1), label=f"Label {label}")
+            ax.set_xlabel('1st dimension of the encoder')
+            ax.set_ylabel('2nd dimension of the encoder')
+            ax.set_zlabel('3rd dimension of the encoder')
+    plt.ticklabel_format(style='sci', axis='both', scilimits=(0, 0))
+    plt.legend(loc='best')
+    plt.show()
+
+
+if __name__ == '__main__':
+    main()