gda_svm_cifar10.py

# -*- coding: utf-8 -*-
"""gda_svm_cifar10.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1vTZ42zJyCdTqGwovZL-3sc9QeY_ZwK83

# SVM and GDA for CIFAR-10
"""

import torch
import torchvision
import torchvision.transforms as transforms
import numpy as np
import matplotlib.pyplot as plt

import sklearn
from sklearn.metrics import accuracy_score, confusion_matrix, classification_report
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.svm import LinearSVC, SVC
from scipy.spatial.distance import cdist

"""# The CIFAR10 dataset
- Download and normalize the CIFAR10 dataset from torchvision
- Split the CIFAR10 data into train, validation and test set
- Set the batch size for processing these datasets
- Build the dataloaders for train, validation, and test set which will be used in the training loop
- Define the string class labels (targets are numeric 0-9)
"""

# mean and std for the RGB channels in CIFAR10
tmean = [0.49139968, 0.48215841, 0.44653091]
tstd = [0.24703223, 0.24348513, 0.26158784]

# transform the 32x32x3 images into a tensor after normalizing
# each channel using the parameters above
transform = transforms.Compose(
    [transforms.ToTensor(),
     transforms.Normalize(tmean, tstd)])

# download and transform the  trainset and testset for training
trainset = torchvision.datasets.CIFAR10(root='./data',train=True,download=True,transform=transform)
testset = torchvision.datasets.CIFAR10(root='./data',train=False,download=True,transform=transform)

#split trainset into a train and a val set (90-10 split)
lengths = [int(p * len(trainset)) for p in [0.9,0.1]]
tr,v = torch.utils.data.random_split(trainset,lengths)
train_sampler = torch.utils.data.SubsetRandomSampler(tr.indices)
val_sampler = torch.utils.data.SubsetRandomSampler(v.indices)

# set batch size and set up the data generators for train, val, test sets
batch_size = 128
trainloader = torch.utils.data.DataLoader(trainset,batch_size=batch_size,sampler=train_sampler)
valloader = torch.utils.data.DataLoader(trainset,batch_size=batch_size,sampler=val_sampler)
testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size)

print("Number of training batches = ",len(trainloader))
print("Number of validation batches = ",len(valloader))
print("Number of test batches = ",len(testloader))

# define the output classes
classes = ('plane', 'car', 'bird', 'cat',
           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')

"""# Visualize the training data"""

Xtr,ytr = next(iter(trainloader))
# make a 8x8 grid and display 64 images from the first batch of training data
rows,cols = 8,8
fig = plt.figure(figsize=(8,8),constrained_layout=True)

for i in range(0,rows*cols):
    fig.add_subplot(rows,cols,i+1)
    tmp = np.transpose(Xtr[i].numpy(),(1,2,0))
    plt.imshow(((tmp*tstd + tmean)*255).astype(np.uint8))
    plt.xticks([])
    plt.yticks([])
    plt.title(classes[ytr[i].numpy()])

"""# Accumulate the training, validation, and test sets"""

def extract_data(generator):
    X = torch.zeros((0,3,32,32))
    y = torch.zeros((0,))
    for (Xtr,ytr) in generator:
        X = torch.vstack([X,Xtr])
        y = torch.concat((y,ytr),axis=0)
    return X,y

Xtrain,ytrain = extract_data(trainloader)
print("Training data: ",Xtrain.shape,ytrain.shape)
Xval,yval = extract_data(valloader)
print("Validation data: ",Xval.shape,yval.shape)
Xtest,ytest = extract_data(testloader)
print("Test data: ",Xtest.shape,ytest.shape)

"""# GDA with same covariance for each class (5 points)

Set up an LDA model using sklearn's [LDA](https://scikit-learn.org/stable/modules/generated/sklearn.discriminant_analysis.LinearDiscriminantAnalysis.html) function (10 points)
    
- set up the LinearDiscriminantAnalysis classifier with 'lsqr' solver and store_covariance=True.
- fit this classifier on the training images (flattened into 50000 x 3072) and labels (50000).
- evaluate the trained model with the set aside test set using built in functions from sklearn.metrics: accuracy, confusion matrix, classification report (f1 scores). Remember to flatten the test images before running the .predict() function.
- visualize the means of the 10 classes, and interpret the images in relation to the classes they represent.
"""

# concatenate the train and validation sets for GDA
trainX = torch.vstack((Xtrain,Xval))
trainy = torch.cat([ytrain,yval])
print(trainX.shape,trainy.shape)

# your code here (LDA model with same covariance for every class)

# remember to reshape trainX as (50000,3072)
# set up LDA model and fit on (trainX,trainy)
# about 2 lines

# Reshape trainX as (50000, 3072)
trainX_flat = trainX.reshape((50000,3072))

# Create the LDA classifier with 'lsqr' solver and store_covariance=True
lda = LinearDiscriminantAnalysis(solver='lsqr', store_covariance=True)

# Fit the LDA model on the training data
lda.fit(trainX_flat, trainy)

"""# Test model on Xtest and ytest (4 points)
- print accuracy of model on (Xtest,ytest)
- print classification report
- print confusion_matrix
"""

#your code here (remember to reshape Xtest)
# about 4 lines


# Reshape Xtest as (number of samples, number of features)
Xtest_flat = Xtest.reshape((Xtest.shape[0], -1))

# Predict labels using the trained LDA model
lda_predictions = lda.predict(Xtest_flat)

# Calculate accuracy
accuracy = accuracy_score(ytest, lda_predictions)

# Print accuracy
print(f'Accuracy on Test Set: {accuracy:.2f}')

# Print classification report
classification_rep = classification_report(ytest, lda_predictions)
print('Classification Report:\n', classification_rep)

# Print confusion matrix
confusion = confusion_matrix(ytest, lda_predictions)
print('Confusion Matrix:\n', confusion)

"""# Visualize the means (1 point)
- comment on the visualizations in this cell. That is, interpret these means in terms of the classes they represent.
"""

# Run this cell; no code to write.

mean = lda.means_.reshape(10, 32, 32, 3) # 10 x 32 x 32 x 3
class_min, class_max = np.min(lda.means_), np.max(lda.means_)

for i in range(10):
    plt.subplot(2, 5, i + 1)
    tmp = np.transpose(lda.means_[i].reshape((3,32,32)),(1,2,0))
    tmp = (255.0 * (tmp - class_min) / (class_max - class_min)).astype('uint8')
    plt.imshow(tmp)
    plt.axis('off')
    plt.title(classes[i])

plt.show()

"""From above visualization we can say Visual inspection of the mean images allows us to observe patterns, shapes, or colors that are characteristic of each class. This can be helpful for understanding what aspects of the data are important for the LDA model in making class distinctions.Features that are consistent within a class are likely to be more pronounced in the mean images. For example, if a certain class is associated with a particular shape or color, those features might stand out in the mean image for that class. But for point to observe here is that while mean images can provide insights, they might not capture the full complexity of the data. For a more comprehensive analysis, it's beneficial to explore other aspects of the LDA model, such as decision boundaries, covariance matrices, and overall classification performance.

# Run SVM on CIFAR10 (40 points)

The sklearn implementation of SVM (SVC) is very slow, because it uses all points in the training set as landmarks. So to speed up the process, we will only use a subset of 5000 images for training.

- build a linear kernel SVC model with probability = False and regularization constant C drawn from a set of values C_vals. For each C, train a linear kernel SVM on the 5000 image training set (sXtrain,sytrain) and measure accuracy on the validation set (Xval,yval). Plot C value vs val set accuracy on a semi-log plot. What is the best value of C for this training set? Build a linear model with that value of C and report accuracy, classification report, and confusion matrix on the full test set.
- build an RBF kernel SVC model with probability = False and regularization constant C drawn from a set of values C_vals. For each C, train an RBF kernel SVM on the 5000 image training set (sXtrain,sytrain) and measure accuracy on the validation set (Xval,yval). Plot C value vs val set accuracy on a semi-log plot. What is the best value of C for this training set? Build an RBF model with that value of C and report accuracy, classification report, and confusion matrix on the full test set.
- Comment on performance differences between the two models, if any.
"""

# subselect 5000 examples from the CIFAR-10 dataset
N = 5000
sXtrain = trainX[:N]
sytrain = trainy[:N]

"""# Linear SVM kernel
- choosing regularization C using a validation set
"""

C_vals = np.array([0.0001,0.001,0.01,0.1,1,10])

# your code here for fitting and selecting C (about 6 lines)


# Linear kernel SVM
linear_accuracies = []

for C in C_vals:
    # Train linear SVM
    linear_svm = SVC(kernel='linear', C=C, probability=False)
    linear_svm.fit(sXtrain.reshape((N, -1)), sytrain)

    # Predict on validation set
    y_val_pred = linear_svm.predict(Xval.reshape((Xval.shape[0], -1)))

    # Measure accuracy
    accuracy = accuracy_score(yval, y_val_pred)
    linear_accuracies.append(accuracy)

# plot C_vals and val set accuracy in a semilog plot
# 1 line of code
# your code here

plt.semilogx(C_vals, linear_accuracies, marker='o')
plt.title('Linear SVM - C Value vs Validation Set Accuracy')
plt.xlabel('C Value (Regularization Constant)')
plt.ylabel('Validation Set Accuracy')
plt.show()

# build model with best value of C
# print test accuracy, confusion matrix and classification report on test set
# about 5 lines of code


best_linear_C = C_vals[np.argmax(linear_accuracies)]
print(f'Best C value for Linear SVM: {best_linear_C}')

# Build a linear model with the best C value
best_linear_svm = SVC(kernel='linear', C=best_linear_C, probability=False)
best_linear_svm.fit(sXtrain.view(N, -1).numpy(), sytrain)

# Evaluate the linear model on the full test set
flat_Xtest = Xtest.view(Xtest.shape[0], -1).numpy()
ytest_np=ytest.view(ytest.shape[0], -1).numpy()
y_test_pred_linear = best_linear_svm.predict(flat_Xtest)
accuracy_linear = accuracy_score(ytest_np, y_test_pred_linear)
print(f'Linear SVM Test Accuracy: {accuracy_linear:.4f}')

# Display classification report and confusion matrix
print('Classification Report:')
print(classification_report(ytest_np, y_test_pred_linear))
print('Confusion Matrix:')
print(confusion_matrix(ytest_np, y_test_pred_linear))
best_linear_C = C_vals[np.argmax(linear_accuracies)]

"""# RBF SVM kernel
- choosing regularization C using a validation set
"""

C_vals = np.array([0.0001,0.001,0.01,0.1,1,10,100])

# your code here for finding good value of C for RBF kernel
# about 6 lines
# RBF kernel SVM
rbf_accuracies = []

for C in C_vals:
    # Train RBF SVM
    rbf_svm = SVC(kernel='rbf', C=C, probability=False)
    rbf_svm.fit(sXtrain.reshape((N, -1)), sytrain)

    # Predict on validation set
    y_val_pred_rbf = rbf_svm.predict(Xval.reshape((Xval.shape[0], -1)))

    # Measure accuracy
    accuracy_rbf = accuracy_score(yval, y_val_pred_rbf)
    rbf_accuracies.append(accuracy_rbf)

# your code here
# build best RBF model with best value of C
# report test set accuracy, confusion matrix and classification report on test set
# about 6 lines of code

plt.semilogx(C_vals, rbf_accuracies, marker='o')
plt.title('RBF SVM - C Value vs Validation Set Accuracy')
plt.xlabel('C Value (Regularization Constant)')
plt.ylabel('Validation Set Accuracy')
plt.show()

# your code here
# build best RBF model with best value of C
# report test set accuracy, confusion matrix and classification report on test set
# about 6 lines of code

best_rbf_C = C_vals[np.argmax(rbf_accuracies)]
print(f'Best C value for RBF SVM: {best_rbf_C}')

# Build an RBF model with the best C value
best_rbf_svm = SVC(kernel='rbf', C=best_rbf_C, probability=False)
best_rbf_svm.fit(sXtrain.view(N, -1).numpy(), sytrain)


# Evaluate the RBF model on the full test set
flat_Xtest_rbf = Xtest.view(Xtest.shape[0], -1).numpy()
ytest_np_rbf=ytest.view(ytest.shape[0], -1).numpy()
y_test_pred_rbf = best_rbf_svm.predict(flat_Xtest_rbf)

accuracy_rbf = accuracy_score(ytest_np_rbf, y_test_pred_rbf)
print(f'RBF SVM Test Accuracy: {accuracy_rbf:.4f}')

# Display classification report and confusion matrix
print('Classification Report:')
print(classification_report(ytest_np_rbf, y_test_pred_rbf))
print('Confusion Matrix:')
print(confusion_matrix(ytest_np_rbf, y_test_pred_rbf))

"""# Comment on linear SVM vs RBF kernel SVM

For the RBF kernel SVM, the best C value is 1.0.
For the linear SVM, the best C value is 0.001.

The test accuracy for the RBF kernel SVM is 0.4374.
The test accuracy for the linear SVM is 0.3777.

The RBF kernel SVM has a higher test accuracy compared to the linear SVM in this specific scenario. The RBF kernel is more flexible and able to capture complex, non-linear relationships in the data. This flexibility allows it to perform better, especially when the underlying patterns are not strictly linear.

The linear SVM, on the other hand, assumes a linear decision boundary. If the true decision boundary is non-linear, a linear SVM may struggle to capture the complexity of the data, leading to lower accuracy.

It's important to note that the choice between linear and non-linear kernels often involves a trade-off between model complexity and interpretability. Linear models are simpler and easier to interpret but may not perform well on complex datasets. Non-linear models, such as those using RBF kernels, can capture more intricate patterns but might be harder to interpret.

The specific choice of the best C value is also significant. It influences the trade-off between having a smooth decision boundary and correctly classifying training points. In the given results, the best C values indicate the optimal balance for each kernel.

In summary, the RBF kernel SVM outperforms the linear SVM in terms of test accuracy for the provided dataset. However, it's crucial to consider the specific characteristics of the data and the problem when choosing between linear and non-linear models. Additionally, hyperparameter tuning (including C values) is essential for achieving the best performance.
"""