From 1de3dfecf43f4af2d6a96754b0be3bf728b8515b Mon Sep 17 00:00:00 2001 From: mamoanwar97 Date: Wed, 20 Jan 2021 21:49:49 +0200 Subject: [PATCH] cnn --- MNIST_test.py | 33 +++++---- activation_functions.py | 10 +-- alexnet.py | 73 ++++++++++++++++++++ cnn.py | 119 +++++++++++++++++++++++++-------- dataset.py | 1 - lenet.py | 31 +++++---- loss.py | 12 +--- testing_pooling.py | 145 ++++++++++++++++++++++++++++++++++++++++ train1.csv | 9 --- 9 files changed, 352 insertions(+), 81 deletions(-) create mode 100644 alexnet.py create mode 100644 testing_pooling.py delete mode 100644 train1.csv diff --git a/MNIST_test.py b/MNIST_test.py index 4ba827a..bf594ba 100644 --- a/MNIST_test.py +++ b/MNIST_test.py @@ -1,4 +1,4 @@ -from dataset import Dataset, Data_Loader +from dataset import MNIST_dataset, Dataset, Data_Loader from model import Model from Linear import Dense from optim import GradientDecent, MomentumGD, Adam, StepLR @@ -14,20 +14,20 @@ # MNIST Dataset batch_size = 32 -dataset = Dataset("train.csv") +dataset = MNIST_dataset("train.csv") dataloader = Data_Loader(dataset, batch_size) -image = dataset[0][0]/255 -image = image.reshape(1,1,28,28) -print(image) -ob1 = conv(1, 4, 3) -mp = MaxPool2D(kernel_size=(2,2)) -ob1.forward(image) -#ob1.backward(image) +# image = dataset[0][0]/255 +# image = image.reshape(1,1,28,28) +# print(image) +# # ob1 = conv(1, 4, 3) +# # mp = MaxPool2D(kernel_size=(2,2)) +# # ob1.forward(image) +# # #ob1.backward(image) -ob1.forward(image) -mp.forward(image) -exit() -#ob1.backward() +# # ob1.forward(image) +# # mp.forward(image) +# # exit() +# # #ob1.backward() @@ -47,7 +47,7 @@ optimizer = Adam(model.parameters(), learning_rate = 0.01) lr_schedular = StepLR(optimizer, step_size = 1, gamma=0.1) -model = load_weights(path) +# model = load_weights(path) epochs = 1 for epoch in range(epochs): @@ -56,13 +56,12 @@ # if i == 1700: # break image = image/255 - image = image.reshape(28,28) i = i + 1 print("Iteration no.", i) predicted = model(image) loss = model.loss(predicted, label) - # model.backward() - # optimizer.step() + model.backward() + optimizer.step() # print("loss= ", loss) # time.sleep(0.1) print("===========") diff --git a/activation_functions.py b/activation_functions.py index 5734347..cf97d36 100644 --- a/activation_functions.py +++ b/activation_functions.py @@ -1,7 +1,7 @@ import numpy as np def sigmoid(x): - return 1 / (1 + np.exp(-x)) + return 1 / (1 + np.exp(-np.abs(x))) def sigmoid_derivative(x): return sigmoid(x) * (1-sigmoid(x)) @@ -19,12 +19,14 @@ def leaky_relu_derivative(x,alpha): return 1*(x > 0) + alpha*(x <= 0) def softMax(x): - exp_x = np.exp(x) - value = exp_x / np.sum(exp_x, axis=0, keepdims=True) + max_1 = np.max(x, axis=0, keepdims=True) + max_1 = np.subtract(x, max_1) + exp_x = np.exp(max_1) + value = np.divide(exp_x, np.sum(exp_x, axis=0, keepdims=True)) return value def tanh (x): return np.tanh(x) def tanh_derivative (x): - return 1-(np.tanh(x) ** 2) \ No newline at end of file + return 1-(np.tanh(x) ** 2) diff --git a/alexnet.py b/alexnet.py new file mode 100644 index 0000000..05f6561 --- /dev/null +++ b/alexnet.py @@ -0,0 +1,73 @@ +from dataset import MNIST_dataset, Dataset, Data_Loader +from model import Model +from Linear import Dense +from optim import GradientDecent, MomentumGD, Adam, StepLR +from activations import ReLU, Sigmoid +from loss import CrossEntropyLoss +from utils import save_weights, load_weights +from cnn import * +from PIL import Image +import cv2 +import time +import numpy as np + +# MNIST Dataset +batch_size = 1 +dataset = MNIST_dataset("train.csv") +dataloader = Data_Loader(dataset, batch_size) + + + +alexnet = Model() + +alexnet.add(conv(1, 96, 11, padding=1, stride=4)) +alexnet.add(ReLU()) +alexnet.add(MaxPool2D(kernel_size=3)) +alexnet.add(conv(96, 256, 5, padding=2)) +alexnet.add(Sigmoid()) +alexnet.add(MaxPool2D(kernel_size=3)) +alexnet.add(conv(256, 384, 3, padding=1)) +alexnet.add(Sigmoid()) +alexnet.add(conv(384, 384, 3, padding=1)) +alexnet.add(ReLU()) +alexnet.add(conv(384, 256, 3, padding=1)) +alexnet.add(ReLU()) +alexnet.add(MaxPool2D(kernel_size=3)) +alexnet.add(Flatten()) +alexnet.add(Dense(6400, 4096)) +alexnet.add(ReLU()) +alexnet.add(Dense(4096, 4096)) +alexnet.add(ReLU()) +alexnet.add(Dense(4096, 10)) + +alexnet.set_loss(CrossEntropyLoss()) + +optimizer = GradientDecent(alexnet.parameters(), learning_rate=0.01) + +epochs = 10 + +for epoch in range(epochs): + i = 0 + for img, label in dataloader: + # if i == 1700: + # break + img = img.reshape(28, 28) + img = np.asarray(img, dtype='int8') + + # cv_gray = cv2.CreateMat(28, 28, cv2.CV_32FC3) + image = Image.fromarray(img) + image = image.resize(size=(224, 224)) + image = np.asarray(image) + + print(image.shape) + image = image/255 + image = image.reshape(batch_size, 1, 224, 224) + i = i + 1 + print("Iteration no.", i) + predicted = alexnet(image) + loss = alexnet.loss(predicted, label) + alexnet.backward() + optimizer.step() + print("loss= ", loss) + #time.sleep(0.1) + print("===========") diff --git a/cnn.py b/cnn.py index ae86071..e3172df 100644 --- a/cnn.py +++ b/cnn.py @@ -1,7 +1,8 @@ from abstract_classes import * import numpy as np -from math import sqrt +from math import sqrt, isnan from itertools import product + ###calculate_local_grads in conv @@ -69,28 +70,29 @@ def forward(self, X): h_offset, w_offset = h*self.stride, w*self.stride rec_field = X[n, :, h_offset:h_offset + KH, w_offset:w_offset + KW] - #print(rec_field.shape) - #print((self.weight['W'][c_w]).shape) + # print(rec_field.shape) + # print((self.weight['W'][c_w]).shape) Y[n, c_w, h, w] = np.sum(self.weight['W'][c_w]*rec_field) + self.weight['b'][c_w] + assert(not isnan(np.max(Y))) + return Y def backward(self, dY): # calculating the global gradient to be propagated backwards # TODO: this is actually transpose convolution, move this to a util function - #print(dY.shape) - # print("conv backward", self.in_channels) # print(dY.shape) - + # print("\nConv backward started") X = self.cache['X'] dX = np.zeros_like(X) N, C, H, W = dX.shape KH, KW = self.kernel_size - for n in range(N): + for c_w in range(self.out_channels): + for h, w in product(range(dY.shape[2]), range(dY.shape[3])): h_offset, w_offset = h * self.stride, w * self.stride - #print("line 91",dY[n, c_w, h, w]) + # print("line 91",dY[n, c_w, h, w]) dX[n, :, h_offset:h_offset + KH, w_offset:w_offset + KW] += self.weight['W'][c_w] * dY[n, c_w, h, w] # calculating the global gradient wrt the conv filter weights @@ -107,55 +109,118 @@ def backward(self, dY): # caching the global gradients of the parameters self.weights_global_grads = {'w': dW, 'b': db} - if self.padding == 0: + # print("Conv backward Ended\n") + assert(not isnan(np.max(dX))) + if self.padding==0: return dX else: return dX[:, :, self.padding:-self.padding, self.padding:-self.padding] +class AvgPool2D(Function): + def __init__(self, kernel_size=(2, 2), stride = 2): + super().__init__() + self.kernel_size = (kernel_size, kernel_size) if isinstance(kernel_size, int) else kernel_size + self.stride = stride + self.input_shape = 0 + self.grad = 0 + + def forward(self, X): + self.input_shape = X.shape + N, C, H, W = X.shape + KH, KW = self.kernel_size + + self.grad = np.zeros_like(X) + out_shape = (N, C, 1 + int((H - KH)/self.stride), 1 + int((W - KW)/self.stride)) + Y = np.zeros(out_shape) + #P = np.zeros_like(X) + # for n in range(N): + for h, w in product(range(0, out_shape[2]), range(0, out_shape[3])): + h_offset, w_offset = h*self.stride, w*self.stride + rec_field = X[:, :, h_offset:h_offset+KH, w_offset:w_offset+KW] + Y[:, :, h, w] = np.mean(rec_field, axis=(2, 3)) + self.grad[:, :, h_offset:h_offset+KH, w_offset:w_offset+KW] = (np.ones_like(rec_field) * np.mean(rec_field, axis=(2, 3))[0][0]) + # for kh, kw in product(range(KH), range(KW)): + # #this contains the mask that will be multiplied to the max value error + # grad[:, :, h_offset+kh, w_offset+kw] = P[:, :, h_offset+kh, w_offset+kw] + # storing the gradient + assert(not isnan(np.max(Y))) + return Y + + def backward(self, dY): + KH, KW = self.kernel_size + N, C, H, W = dY.shape + + for h, w in product(range(0, H), range(0, W)): + h_offset, w_offset = h*self.stride, w*self.stride + rec_field = self.grad[:, :, h_offset:h_offset+KH, w_offset:w_offset+KW] + self.grad[:, :, h_offset:h_offset+KH, w_offset:w_offset+KW] = (np.ones_like(rec_field) * dY[:, :, h, w].reshape(N, C, 1, 1)) + + return (self.grad/(self.kernel_size[0]*self.kernel_size[0])) + + class MaxPool2D(Function): - def __init__(self, kernel_size=(2, 2)): + def __init__(self, kernel_size=(2, 2), stride = 2): super().__init__() self.kernel_size = (kernel_size, kernel_size) if isinstance(kernel_size, int) else kernel_size + self.stride = stride def forward(self, X): + # print("Max_pool forward started\n") N, C, H, W = X.shape KH, KW = self.kernel_size grad = np.zeros_like(X) - Y = np.zeros((N, C, H//KH, W//KW)) - + out_shape = (N, C, 1 + int((H - KH)/self.stride), 1 + int((W - KW)/self.stride)) + Y = np.zeros(out_shape) # for n in range(N): - for h, w in product(range(0, H//KH), range(0, W//KW)): - h_offset, w_offset = h*KH, w*KW + for h, w in product(range(0, out_shape[2]), range(0, out_shape[3])): + h_offset, w_offset = h*self.stride, w*self.stride rec_field = X[:, :, h_offset:h_offset+KH, w_offset:w_offset+KW] Y[:, :, h, w] = np.max(rec_field, axis=(2, 3)) - for kh, kw in product(range(KH), range(KW)): - grad[:, :, h_offset+kh, w_offset+kw] = (X[:, :, h_offset+kh, w_offset+kw] >= Y[:, :, h, w]) + grad[:, :, h_offset:h_offset+KH, w_offset:w_offset+KW] = (X[:, :, h_offset:h_offset+KH, w_offset:w_offset+KW] >= np.max(rec_field, axis=(2, 3))[0][0]) + + # for kh, kw in product(range(KH), range(KW)): + # #this contains the mask that will be multiplied to the max value error + # grad[:, :, h_offset+kh, w_offset+kw] = (X[:, :, h_offset+kh, w_offset+kw] >= Y[:, :, h, w]) # storing the gradient self.local_grads['X'] = grad - + # print("Y", Y.shape) + assert(not isnan(np.max(Y))) + # print("Max_pool forward ended\n") return Y def backward(self, dY): - # print("Maxbool backward") - # print(dY.shape) - dY = np.repeat(np.repeat(dY, repeats=self.kernel_size[0], axis=2), repeats=self.kernel_size[1], axis=3) - #print("line 141",dY) - return self.local_grads['X']*dY + # print("\nMax_pool backward started") + KH, KW = self.kernel_size + N, C, H, W = dY.shape + for h, w in product(range(0, H), range(0, W)): + h_offset, w_offset = h*self.stride, w*self.stride + rec_field = self.local_grads['X'][:, :, h_offset:h_offset+KH, w_offset:w_offset+KW] + self.local_grads['X'][:, :, h_offset:h_offset+KH, w_offset:w_offset+KW] = (rec_field * dY[:, :, h, w].reshape(N, C, 1, 1)) + + assert(not isnan(np.max(self.local_grads['X']))) + # print("Max_pool backward ended\n") + return self.local_grads['X'] def calculate_local_grads(self, X): - # small hack: because for MaxPool calculating the gradient is simpler during - # the forward pass, it is calculated there and this function just returns the - # grad dictionary + return self.local_grads class Flatten(Function): def forward(self, X): + # print("\nFlatten forward started") self.cache['shape'] = X.shape n_batch = X.shape[0] - return X.reshape(-1, n_batch) + X = X.reshape(-1, n_batch) + # print("X", X.shape) + assert(not isnan(np.max(X))) + # print("Flatten forward ended\n") + return X def backward(self, dY): - return dY.reshape(self.cache['shape']) + # print("\nFlatten backward started") + dY = dY.reshape(self.cache['shape']) + assert(not isnan(np.max(dY))) + return dY diff --git a/dataset.py b/dataset.py index fd43115..0a4653e 100644 --- a/dataset.py +++ b/dataset.py @@ -7,7 +7,6 @@ # for loading CIFER-10 dataset def unpickle(file): - with open(file, 'rb') as fo: dict = cPickle.load(fo,encoding='bytes') return dict diff --git a/lenet.py b/lenet.py index 387b716..f8d8594 100644 --- a/lenet.py +++ b/lenet.py @@ -9,7 +9,7 @@ import time # MNIST Dataset -batch_size = 1 +batch_size = 2 dataset = MNIST_dataset("train.csv") dataloader = Data_Loader(dataset, batch_size) @@ -17,39 +17,42 @@ lenet = Model() lenet.add(conv(1, 6, 5, padding=2)) -lenet.add(Sigmoid()) -lenet.add(MaxPool2D(kernel_size=2)) +lenet.add(ReLU()) +lenet.add(AvgPool2D(kernel_size=2)) lenet.add(conv(6, 16, 5)) -lenet.add(Sigmoid()) -lenet.add(MaxPool2D(kernel_size=2)) +lenet.add(ReLU()) +lenet.add(AvgPool2D(kernel_size=2)) lenet.add(Flatten()) lenet.add(Dense(16*5*5, 120)) -lenet.add(Sigmoid()) +lenet.add(ReLU()) lenet.add(Dense(120, 84)) -lenet.add(Sigmoid()) +lenet.add(ReLU()) lenet.add(Dense(84, 10)) lenet.set_loss(CrossEntropyLoss()) -optimizer = Adam(lenet.parameters(), learning_rate=0.01) -lr_schedular = StepLR(optimizer, step_size=1, gamma=0.1) +optimizer = GradientDecent(lenet.parameters(), learning_rate=0.1) epochs = 10 for epoch in range(epochs): i = 0 for image, label in dataloader: - # if i == 1700: + # if i == 1: # break - image = image/255 - image = image.reshape(batch_size, 1, 28, 28) + + for batch_idx in range(image.shape[1]): + img = image[:, batch_idx].reshape(1, 28, 28) + img = img/255 + images.append(img) + images = np.asarray(images) + i = i + 1 print("Iteration no.", i) - predicted = lenet(image) + predicted = lenet(images) loss = lenet.loss(predicted, label) lenet.backward() optimizer.step() print("loss= ", loss) #time.sleep(0.1) print("===========") - lr_schedular.step() \ No newline at end of file diff --git a/loss.py b/loss.py index cc426f2..ef8a31c 100644 --- a/loss.py +++ b/loss.py @@ -1,7 +1,7 @@ import numpy as np import sys import math - +from activation_functions import * from abstract_classes import Function class Loss(Function): @@ -34,14 +34,8 @@ def forward(self, Y_hat, Y): yhat = (ndim, nbatch) y = (1, nbatch) """ - # calculating crossentropy - # exp_x = np.maximum(np.exp(Y_hat), np.exp(Y_hat) + 1e-1) - max_1 = np.max(Y_hat, axis=0, keepdims=True) - max_1 = np.subtract(Y_hat, max_1) - exp_x = np.exp(max_1) - probs = np.divide(exp_x, np.sum(exp_x, axis=0, keepdims=True)) - - print(probs) + probs = softMax(Y_hat) + # print(probs) # print() y = Y.T diff --git a/testing_pooling.py b/testing_pooling.py new file mode 100644 index 0000000..9fb538a --- /dev/null +++ b/testing_pooling.py @@ -0,0 +1,145 @@ +import numpy as np +from itertools import product + +def maxp_Forward(X, stride, kernel_size): + N,C,H,W = X.shape + KH, KW = kernel_size + out_shape = (N, C, 1 + int((H - KH)/stride), 1 + int((W - KW)/stride)) + + Y = np.zeros(out_shape) + + for i in range(N): # loop over the training examples + for h in range(out_shape[2]): # loop on the vertical axis of the output volume + for w in range(out_shape[3]): # loop on the horizontal axis of the output volume + for c in range (C): # loop over the channels of the output volume + + # Find the corners of the current "slice" (≈4 lines) + vert_start = h*stride + vert_end = h*stride + KH + horiz_start = w*stride + horiz_end = w*stride + KW + + # Use the corners to define the current slice on the ith training example of A_prev, channel c. (≈1 line) + X_slice = X[i, vert_start:vert_end, horiz_start:horiz_end,c] + + # Compute the pooling operation on the slice. Use an if statment to differentiate the modes. Use np.max/np.mean. + Y[i, c, h, w] = np.max(X_slice) + + return Y + +A_prev = np.random.randn(2, 3, 4, 4) +#print(A_prev,"\n") +Y=maxp_Forward(A_prev, 2, (3,3)) +#print(Y) +def create_mask_from_window(x): + mask = x == np.max(x) + + return mask + +def distribute_value(dz, shape): + # Retrieve dimensions from shape (≈1 line) + (n_H, n_W) = shape + + # Compute the value to distribute on the matrix (≈1 line) + average = dz / (n_H * n_W) + + # Create a matrix where every entry is the "average" value (≈1 line) + a = np.ones(shape) * average + ### END CODE HERE ### + + return a + +arr= np.array([[[[5,4],[3,2],[1,2]]]]) +mask= create_mask_from_window(arr) +#print(mask) +#print(arr.shape) +#print(np.repeat(arr, repeats=2, axis=2)) +dY = np.repeat(np.repeat(arr, repeats=2, axis=2),repeats=2, axis=3) +#print(dY) + +p = (5>=10) +#print(p) + + +def min_forward(X, kernel_size, out_channels, stride): + N, C, H, W = X.shape + KH, KW = kernel_size + grad = np.zeros_like(X) + out_shape = (N, out_channels, 1 + int((H - KH)/stride), 1 + int((W - KW)/stride)) + Y = np.zeros(out_shape) + P = np.zeros_like(X) + # for n in range(N): + for h, w in product(range(0, out_shape[2]), range(0, out_shape[3])): + h_offset, w_offset = h*stride, w*stride + rec_field = X[:, :, h_offset:h_offset+KH, w_offset:w_offset+KW] + # print(P[:, :, h_offset:h_offset+KH, w_offset:w_offset+KW].shape) + Y[:, :, h, w] = np.mean(rec_field, axis=(2, 3)) + P[:, :, h_offset:h_offset+KH, w_offset:w_offset+KW] = (np.ones_like(rec_field) * np.mean(rec_field, axis=(2, 3))[0][0]) + # = + for kh, kw in product(range(KH), range(KW)): + #this contains the mask that will be multiplied to the max value error + grad[:, :, h_offset+kh, w_offset+kw] = P[:, :, h_offset+kh, w_offset+kw] + + # storing the gradient + print(P) + print(grad) + print("Y: ", Y) + return Y +# arr= np.array([[[[31,15,28,124],[0,100,70,38],[12,12,7,2],[12,12,45,6]]]]) +# forward(arr, (2,2),1, 2) + +dY = np.array([[[[7, 8],[-1, 2]]]]) + +def min_backward(dY): + KH, KW = (2, 2) + N, C, H, W = dY.shape + grad = np.zeros_like([[[[31,15,28,124],[0,100,70,38],[12,12,7,2],[12,12,45,6]]]]) + + for h, w in product(range(0, H), range(0, W)): + h_offset, w_offset = h*2, w*2 + rec_field = grad[:, :, h_offset:h_offset+KH, w_offset:w_offset+KW] + grad[:, :, h_offset:h_offset+KH, w_offset:w_offset+KW] = (np.ones_like(rec_field) * dY[:, :, h, w].reshape(N, C, 1, 1)) + + # for n in range(N): + # for f in range(C): + # for h in range(H): + # for w in range(W): + # h_offset, w_offset = h*2, w*2 + # rec_field = grad[n, f, h_offset:h_offset+KH, w_offset:w_offset+KW] + # grad[n, f, h_offset:h_offset+KH, w_offset:w_offset+KW] = (np.ones_like(rec_field) * dY[n, f, h, w].reshape(N, C, 1, 1)) + + return (grad/(2*2)) + +print(min_backward(dY)) + +input_arr = np.asarray([[[[1, 1, 2, 4], [5, 6, 7, 8], [3, 2, 1, 0], [1, 2, 3, 4]]]]) + +def max_forward(X): + N, C, H, W = X.shape + KH, KW = (2, 2) + grad = np.zeros_like(X) + out_shape = (N, C, 1 + int((H - KH) / 2), 1 + int((W - KW)/2)) + Y = np.zeros(out_shape) + # for n in range(N): + for h, w in product(range(0, out_shape[2]), range(0, out_shape[3])): + h_offset, w_offset = h*2, w*2 + rec_field = X[:, :, h_offset:h_offset+KH, w_offset:w_offset+KW] + Y[:, :, h, w] = np.max(rec_field, axis=(2, 3)) + grad[:, :, h_offset:h_offset+KH, w_offset:w_offset+KW] = (X[:, :, h_offset:h_offset+KH, w_offset:w_offset+KW] >= np.max(rec_field, axis=(2, 3))[0][0]) + + print(Y) + # storing the gradient + return grad + + +def max_backward(X, dY): + KH, KW = (2, 2) + N, C, H, W = (1, 1, 2, 2) + for h, w in product(range(0, H), range(0, W)): + h_offset, w_offset = h * 2, w * 2 + rec_field = X[:, :, h_offset:h_offset+KH, w_offset:w_offset+KW] + X[:, :, h_offset:h_offset+KH, w_offset:w_offset+KW] = (rec_field * dY[:, :, h, w].reshape(N, C, 1, 1)) + print(X) + + +max_backward(max_forward(input_arr), np.asarray([[[[6, 8], [3, 4]]]])) diff --git a/train1.csv b/train1.csv deleted file mode 100644 index f7ac5a5..0000000 --- a/train1.csv +++ /dev/null @@ -1,9 +0,0 @@ -label,pixel0,pixel1,pixel2,pixel3,pixel4 -1,0,0,0,1,0 -4,0,1,0,0,1 -3,0,0,0,0,0 -2,0,0,0,0,1 -2,2,2,2,22,2 -2,4,5,5,5,5 -7,7,7,7,7,7 -7,7,7,7,7,7