kerasSSSeagrass.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import keras\n",
    "import os\n",
    "import cv2\n",
    "import numpy as np\n",
    "from PIL import Image\n",
    "import json\n",
    "import keras.backend as K\n",
    "import tensorflow as tf\n",
    "import sklearn.preprocessing\n",
    "from keras.models import Sequential\n",
    "from keras import metrics\n",
    "from keras.layers import Dense, Softmax, Conv2D, Input, Flatten, Lambda, MaxPooling2D, BatchNormalization, Conv2DTranspose, Dropout\n",
    "from IPython.display import display \n",
    "import metricsSemSeg\n",
    "from metricsSemSeg import pixel_accuracy, mean_accuracy, mean_IU, frequency_weighted_IU\n",
    "\n",
    "CLASSTORGB = [[0,0,0],[255,255,255]]\n",
    "NNName = \"NASNet\"\n",
    "DATASET = \"Seagrass\"\n",
    "CLASSES = 2\n",
    "BSIZE = 2\n",
    "X = 512\n",
    "Y = 256\n",
    "CLASSWEIGHTS = [0.1, 1.0]\n",
    "TRAINSIZE = 1846/BSIZE#3424/BSIZE#4223/BSIZE\n",
    "VALSIZE = 264/BSIZE#498/BSIZE#610/BSIZE\n",
    "TESTSIZE = 525/BSIZE#975/BSIZE#1204/BSIZE\n",
    "TRAINPATH = \"images/\"\n",
    "LABELPATH = \"ground-truth/\"\n",
    "DATASETPATH = \"../data/\"+DATASET+\"/\"\n",
    "EPOCHS = 100\n",
    "LR = 0.001\n",
    "filepath = \"../models/keras/\"+NNName+str(X)+str(Y)+DATASET+\"LR\"+str(LR)+\"batch\"+str(BSIZE)+\".h5\"\n",
    "\n",
    "import tensorflow as tf\n",
    "from keras.backend.tensorflow_backend import set_session\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "#config.gpu_options.per_process_gpu_memory_fraction = 0.3\n",
    "set_session(tf.Session(config=config))\n",
    "\n",
    "\n",
    "# semantic segmentation metrics\n",
    "class metricsSS(object):\n",
    "\n",
    "    def __init__(self, num_classes, _bsize):\n",
    "        super().__init__()\n",
    "        self.classes = num_classes\n",
    "        self.batchSize = _bsize\n",
    "        self.pA = metricsSemSeg.pixel_accuracy\n",
    "        self.mA = metricsSemSeg.mean_accuracy\n",
    "        self.mIoU = metricsSemSeg.mean_IU\n",
    "        self.fwmIoU = metricsSemSeg.frequency_weighted_IU\n",
    "\n",
    "    def preProcessKerasInput(self, _pred, _true):\n",
    "            pred = K.argmax(_pred, axis=2)\n",
    "            true = K.argmax(_true, axis=2)\n",
    "            #pred = K.cast(pred, tf.float32)\n",
    "            #true = K.cast(true, tf.float32)\n",
    "            return pred, true\n",
    "\n",
    "    def meanIoU(self, y_true, y_pred):\n",
    "        metric = 0.0\n",
    "        for b in range(self.batchSize):\n",
    "            pred, true = self.preProcessKerasInput(y_pred[b], y_true[b])\n",
    "            metric += tf.py_func(self.mIoU, [pred, true], tf.float32)\n",
    "\n",
    "        return metric/self.batchSize\n",
    "\n",
    "    def frequencyWeightedUI(self, y_true, y_pred):\n",
    "        metric = 0.0\n",
    "        for b in range(self.batchSize):\n",
    "            pred, true = self.preProcessKerasInput(y_pred[b], y_true[b])\n",
    "            metric += tf.py_func(self.fwmIoU, [pred, true], tf.float32)\n",
    "\n",
    "        return metric/self.batchSize\n",
    "\n",
    "\n",
    "    def pixelAccuracy(self, y_true, y_pred):\n",
    "        metric = 0.0\n",
    "        for b in range(self.batchSize):\n",
    "            pred, true = self.preProcessKerasInput(y_pred[b], y_true[b])\n",
    "            metric += tf.py_func(self.pA, [pred, true], tf.float32)\n",
    "\n",
    "        return metric/self.batchSize\n",
    "\n",
    "\n",
    "    def meanAccuracy(self, y_true, y_pred):\n",
    "        metric = 0.0\n",
    "        for b in range(self.batchSize):\n",
    "            pred, true = self.preProcessKerasInput(y_pred[b], y_true[b])\n",
    "            metric += tf.py_func(self.mA, [pred, true], tf.float32)\n",
    "\n",
    "        return metric/self.batchSize\n",
    "\n",
    "\n",
    "def deconv(f, model):\n",
    "    model.add(Conv2DTranspose(f, 3, strides=2, padding='same', activation=\"relu\"))\n",
    "\n",
    "\n",
    "def conv(first=False, units=128, f=3, dilation=1, last=False):\n",
    "    if first :  \n",
    "        return Conv2D(\n",
    "            units,\n",
    "            f,\n",
    "            input_shape=(Y, X, 3),\n",
    "            strides=(1, 1),\n",
    "            padding='same',\n",
    "            activation=\"relu\" if not last else \"softmax\",\n",
    "            use_bias=True,\n",
    "            dilation_rate=dilation,\n",
    "            data_format=\"channels_last\",\n",
    "            kernel_initializer=\"glorot_normal\",\n",
    "            bias_initializer=keras.initializers.Constant(value=0.1)\n",
    "        )\n",
    "    else:\n",
    "        return Conv2D(\n",
    "            units if not last else CLASSES,\n",
    "            3 if not last else 1,\n",
    "            strides=(1, 1),\n",
    "            padding='same',\n",
    "            activation=\"relu\",\n",
    "            use_bias=True,\n",
    "            data_format=\"channels_last\",\n",
    "            kernel_initializer=\"glorot_normal\",\n",
    "            bias_initializer=keras.initializers.Constant(value=0.1)\n",
    "        )\n",
    "\n",
    "def deConv(depth):\n",
    "    return Conv2DTranspose(\n",
    "        depth,\n",
    "        3,\n",
    "        strides=(2, 2),\n",
    "        padding='same',\n",
    "        data_format=\"channels_last\",\n",
    "        kernel_initializer=\"glorot_normal\",\n",
    "        bias_initializer=keras.initializers.Constant(value=0.1)\n",
    "    )\n",
    "\n",
    "    \n",
    "def netFCN():\n",
    "    model = Sequential()\n",
    "    # layers\n",
    "    # encoding\n",
    "    model.add(conv(first=True))\n",
    "    model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Dropout(rate=0.3))\n",
    "\n",
    "\n",
    "    model.add(conv(units=128))\n",
    "    model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Dropout(rate=0.3))\n",
    "\n",
    "    model.add(conv(units=256))\n",
    "    #model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Dropout(rate=0.3))\n",
    "    \n",
    "    model.add(conv(units=512))\n",
    "    L = BatchNormalization()\n",
    "    model.add(L)\n",
    "\n",
    "    # decoding\n",
    "    model.add(deConv(512))\n",
    "    model.add(deConv(256))\n",
    "    #model.add(deConv(128))\n",
    "    #model.add(keras.layers.UpSampling2D(size=(2, 2), data_format=None, interpolation='nearest'))\n",
    "    #model.add(keras.layers.UpSampling2D(size=(2, 2), data_format=None, interpolation='nearest'))\n",
    "\n",
    "    #L = Lambda(lambda input: pixelDeconv(tf.convert_to_tensor(input), 512, \"dec1\"))\n",
    "    #model.add(L)\n",
    "    #L = Lambda(lambda input: pixelDeconv(tf.convert_to_tensor(input), 256, \"dec2\"))\n",
    "    #model.add(L)\n",
    "    #L = Lambda(pixelDeconv(L.output, 128, \"dec3\"))\n",
    "    #model.add(L)\n",
    "    \n",
    "    #model.add(Dropout(0.5))\n",
    "    model.add(conv(last=True))\n",
    "    model.add(Softmax())\n",
    "\n",
    "    return model\n",
    "\n",
    "def dilNet():\n",
    "    model = Sequential()\n",
    "\n",
    "    model.add(conv(first=True))\n",
    "    model.add(conv(units=128))\n",
    "    model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "    model.add(BatchNormalization())\n",
    "\n",
    "    model.add(conv(units=128))\n",
    "    model.add(conv(units=256))\n",
    "    model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "    model.add(BatchNormalization())\n",
    "        \n",
    "    model.add(conv(units=256))\n",
    "    model.add(conv(units=256))\n",
    "    model.add(conv(units=256, f=1))\n",
    "    model.add(BatchNormalization())\n",
    "\n",
    "    model.add(conv(units=512, dilation=2))\n",
    "    model.add(conv(units=512, dilation=2))\n",
    "    model.add(conv(units=512, f=1, dilation=2))\n",
    "    model.add(BatchNormalization())\n",
    "\n",
    "    model.add(conv(units=512, dilation=4))\n",
    "    model.add(conv(units=512, dilation=4))\n",
    "    model.add(conv(units=512, f=1, dilation=4))\n",
    "    model.add(BatchNormalization())\n",
    "\n",
    "    model.add(deConv(512))\n",
    "    model.add(deConv(256))\n",
    "\n",
    "    #model.add(keras.layers.UpSampling2D(size=(2, 2), data_format=None, interpolation='nearest'))\n",
    "    #model.add(keras.layers.UpSampling2D(size=(2, 2), data_format=None, interpolation='nearest'))\n",
    "\n",
    "    model.add(conv(last=True))\n",
    "    model.add(Softmax())    \n",
    "    return model\n",
    "\n",
    "def mobileNet_V2_SS(inputShape):\n",
    "    model = Sequential()\n",
    "    model.add(keras.applications.mobilenet_v2.MobileNetV2(\n",
    "        input_shape=inputShape,\n",
    "        alpha=1.0,\n",
    "        depth_multiplier=1,\n",
    "        include_top=False,\n",
    "        weights=None\n",
    "        \n",
    "    ))\n",
    "    \n",
    "    deconv(320, model)\n",
    "    deconv(256, model)\n",
    "    deconv(256, model)\n",
    "    deconv(256, model)\n",
    "    deconv(128, model)\n",
    "    model.add(conv(last=True))\n",
    "    model.add(Softmax())\n",
    "    \n",
    "    return model\n",
    "\n",
    "\n",
    "\n",
    "def inceptionResnet_SS(inputShape):\n",
    "    model = Sequential()\n",
    "    model.add(keras.applications.inception_resnet_v2.InceptionResNetV2(\n",
    "        input_shape=inputShape,\n",
    "        include_top=False,\n",
    "        weights=None\n",
    "    ))\n",
    "    \n",
    "    deconv(320, model)\n",
    "    deconv(256, model)\n",
    "    deconv(256, model)\n",
    "    deconv(256, model)\n",
    "    deconv(128, model)\n",
    "    model.add(conv(last=True))\n",
    "    model.add(Softmax())\n",
    "    \n",
    "    return model\n",
    "              \n",
    "def xception_SS(inputShape):\n",
    "    model = Sequential()\n",
    "    model.add(keras.applications.xception.Xception(\n",
    "        input_shape=inputShape,\n",
    "        include_top=False,\n",
    "        weights=None\n",
    "    ))\n",
    "    \n",
    "    deconv(320, model)\n",
    "    deconv(256, model)\n",
    "    deconv(256, model)\n",
    "    deconv(256, model)\n",
    "    deconv(128, model)\n",
    "    model.add(conv(last=True))\n",
    "    model.add(Softmax())\n",
    "    \n",
    "    return model\n",
    "              \n",
    "def nasNet_SS(inputShape):\n",
    "    model = Sequential()\n",
    "    model.add(keras.applications.nasnet.NASNetMobile(\n",
    "        input_shape=inputShape,\n",
    "        include_top=False,\n",
    "        weights=None\n",
    "    ))\n",
    "    \n",
    "    deconv(320, model)\n",
    "    deconv(256, model)\n",
    "    deconv(256, model)\n",
    "    deconv(256, model)\n",
    "    deconv(128, model)\n",
    "    model.add(conv(last=True))\n",
    "    model.add(Softmax())\n",
    "    \n",
    "    return model\n",
    "\n",
    "class DataSequence(keras.utils.Sequence):\n",
    "\n",
    "    def __init__(self, batchSize, trainType):\n",
    "        self.dataSetPath = DATASETPATH\n",
    "        \n",
    "        if trainType == \"train\":\n",
    "            jsonData = json.load(open(DATASETPATH+\"train.json\"))\n",
    "            self.x = list(map(lambda i:os.path.basename(i[\"image\"]) if i[\"depth\"] <= float(2) else None, jsonData))\n",
    "            self.y = list(map(lambda i:os.path.basename(i[\"ground-truth\"]) if i[\"depth\"] <= float(2) else None, jsonData))\n",
    "            self.x = list(filter(lambda i:i != None, self.x))\n",
    "            self.y = list(filter(lambda i:i != None, self.y))\n",
    "            \n",
    "            \n",
    "            \n",
    "        elif trainType == \"validation\":\n",
    "            jsonData = json.load(open(DATASETPATH+\"validate.json\"))\n",
    "            self.x = list(map(lambda i:os.path.basename(i[\"image\"]) if i[\"depth\"] <= float(2) else None, jsonData))\n",
    "            self.y = list(map(lambda i:os.path.basename(i[\"ground-truth\"]) if i[\"depth\"] <= float(2) else None, jsonData))\n",
    "            self.x = list(filter(lambda i:i != None, self.x))\n",
    "            self.y = list(filter(lambda i:i != None, self.y))\n",
    "            \n",
    "        elif trainType == \"test\":\n",
    "            jsonData = json.load(open(DATASETPATH+\"test.json\"))\n",
    "            self.x = list(map(lambda i:os.path.basename(i[\"image\"]) if i[\"depth\"] <= float(2) else None, jsonData))\n",
    "            self.y = list(map(lambda i:os.path.basename(i[\"ground-truth\"]) if i[\"depth\"] <= float(2) else None, jsonData))\n",
    "            self.x = list(filter(lambda i:i != None, self.x))\n",
    "            self.y = list(filter(lambda i:i != None, self.y))\n",
    "            \n",
    "        else:\n",
    "            raise \"unknown dataset type, valid are train, validation and test\"\n",
    "    \n",
    "        \n",
    "        self.batch_size = batchSize\n",
    "        print(\"Loaded data with size of \", len(self))\n",
    "    def __len__(self):\n",
    "        return int(len(self.x) / self.batch_size)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        \n",
    "        imgBatch = None\n",
    "        labelImgBatch = None\n",
    "        \n",
    "        for b in range(self.batch_size):\n",
    "            #  loading train and label image\n",
    "            \n",
    "            elIdx = idx*self.batch_size+b\n",
    "            img = cv2.imread(self.dataSetPath+TRAINPATH+self.x[elIdx])\n",
    "            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
    "            img = cv2.resize(img, (X, Y), interpolation=cv2.INTER_NEAREST)\n",
    "\n",
    "            labelImg = cv2.imread(self.dataSetPath+LABELPATH+self.y[elIdx])\n",
    "            labelImg = cv2.cvtColor(labelImg, cv2.COLOR_BGR2RGB)\n",
    "            labelImg = cv2.resize(labelImg, (X, Y), interpolation=cv2.INTER_NEAREST)\n",
    "\n",
    "\n",
    "            if CLASSES == 2:\n",
    "                labelImg[(labelImg  >= 128).all(-1)] = [255,255,255]\n",
    "                labelImg[(labelImg  <= 127).all(-1)] = [0,0,0]\n",
    "\n",
    "\n",
    "            #print(self.dataSetPath+TRAINPATH+self.x[idx], self.dataSetPath+LABELPATH+self.y[idx])\n",
    "            #display(Image.fromarray(img, \"RGB\"))\n",
    "            #display(Image.fromarray(labelImg, \"RGB\"))\n",
    "\n",
    "            # process train and label image\n",
    "            img = ((img - img.mean()) / img.std()).astype(np.float32)\n",
    "            img = np.array(img)\n",
    "            img = np.reshape(img, (1,Y,X,3))\n",
    "\n",
    "\n",
    "\n",
    "            for rgbIdx, rgbV in enumerate(CLASSTORGB):\n",
    "                labelImg[(labelImg == rgbV).all(-1)] = rgbIdx\n",
    "\n",
    "\n",
    "            labelImg = labelImg[:,:,0].astype(np.int32)\n",
    "            labelImg = np.reshape(labelImg, (1,Y,X,1))\n",
    "\n",
    "\n",
    "            # dont know why but there are(some datasets) rgb values which are not assigned to a class\n",
    "            # because of this these values are not replaced with their assigned class and\n",
    "            # have to be removed as in assigned to class zero aka black\n",
    "            #print(\"UNIQUE RGB VALUES\", np.unique(np.array(img).reshape((int(6291456/3), 3)), axis=0))\n",
    "\n",
    "            labelImg[(labelImg >= CLASSES)] = 0\n",
    "\n",
    "            #onehot = keras.utils.to_categorical(labelImg, num_classes=2, dtype='float32')\n",
    "            #print(img.shape, labelImg.shape)\n",
    "            #calculcate the weights for the current image\n",
    "            #sampleWeights = onehot * [0.1, 1.0]\n",
    "            #sampleWeights = np.sum(sampleWeights, axis=3)\n",
    "            #sampleWeights = sampleWeights.reshape((self.batch_size,224,224,1))\n",
    "        \n",
    "            if imgBatch is None:\n",
    "                imgBatch = img\n",
    "                labelImgBatch = labelImg\n",
    "            else:\n",
    "                imgBatch = np.concatenate((imgBatch, img), axis=0)\n",
    "                labelImgBatch = np.concatenate((labelImgBatch, labelImg), axis=0)\n",
    "            \n",
    "        return imgBatch, labelImgBatch#, sampleWeights\n",
    "    \n",
    "\n",
    "    \n",
    "def pred(fileName, debug=True):\n",
    "    predImg = cv2.imread(\"../results/\"+fileName)\n",
    "    predImg = cv2.cvtColor(predImg, cv2.COLOR_BGR2RGB)\n",
    "    predImg = cv2.resize(predImg, (X, Y), interpolation=cv2.INTER_NEAREST)\n",
    "    predImg = np.expand_dims(((predImg  - predImg.mean()) / predImg.std()).astype(np.float32), axis=0)\n",
    "    predClasses = model.predict(predImg)\n",
    "    predClasses = np.argmax(predClasses, axis=3).flatten()\n",
    "    #print(predClasses.shape, np.bincount(predClasses), predClasses)\n",
    "    predImg = np.zeros((X*Y,3))\n",
    "\n",
    "    for idx, p in enumerate(predClasses):\n",
    "        predImg[idx] = CLASSTORGB[p]\n",
    "         \n",
    "   \n",
    "    predImg = predImg.reshape((Y, X, 3)).astype(\"uint8\")\n",
    "    \n",
    "    if debug:\n",
    "        display(Image.fromarray(predImg, \"RGB\"))\n",
    "\n",
    "def evaluate(data, model):\n",
    "\n",
    "    from metricsSemSeg import pixel_accuracy, mean_accuracy, mean_IU, frequency_weighted_IU\n",
    "\n",
    "    totalCorrect = 0\n",
    "    totalCount = TESTSIZE*X*Y*3\n",
    "\n",
    "    totalPAcc = 0.0\n",
    "    totalMAcc = 0.0\n",
    "    totalMIU = 0.0\n",
    "    totalFWIU = 0.0\n",
    "\n",
    "    i = 0\n",
    "\n",
    "    \n",
    "    for idx in range(len(data)):#data.getNextBatchValidation(BSIZE, max_images):# data.config[\"validationSize\"]):\n",
    "            x, y = data[idx]\n",
    "\n",
    "            predClasses = model.predict_on_batch(x)\n",
    "            for b in range(BSIZE):\n",
    "                pred = np.squeeze(predClasses[b])\n",
    "                pred = np.argmax(pred, axis=2)\n",
    "                labelData = np.squeeze(y[b])\n",
    "\n",
    "                if i % 100/BSIZE == 0:\n",
    "                    print(\"Image \", i, \" evaluated...\")\n",
    "\n",
    "                totalPAcc = pixel_accuracy(pred, labelData) if totalPAcc == 0.0 else  (totalPAcc + pixel_accuracy(pred, labelData))/2\n",
    "                totalMAcc = mean_accuracy(pred, labelData) if totalMAcc == 0.0 else  (totalMAcc + mean_accuracy(pred, labelData))/2\n",
    "                totalMIU = mean_IU(pred, labelData) if totalMIU == 0.0 else  (totalMIU + mean_IU(pred, labelData))/2\n",
    "                totalFWIU = frequency_weighted_IU(pred, labelData) if totalFWIU == 0.0 else  (totalFWIU + frequency_weighted_IU(pred, labelData))/2\n",
    "\n",
    "                i = i+1\n",
    "\n",
    "\n",
    "    print(\"Pixel accuracy: \", totalPAcc ,\" || Mean accuracy: \", totalMAcc ,\" || Mean intersection union:\", totalMIU ,\" || frequency weighted IU: \", totalFWIU)\n",
    "\n",
    "class regularPred(keras.callbacks.Callback):\n",
    "    \n",
    "    def __init__(self):\n",
    "        self.data = data = DataSequence(BSIZE, \"test\")\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        pred(\"predictSeagrass.jpg\")\n",
    "        if epoch % 5 == 0:\n",
    "            evaluate(self.data, self.model)\n",
    "    #def on_batch_end(self, batch, logs={}):\n",
    "    #    if batch % 500 == 0:\n",
    "    #        pred(\"predict.jpg\")\n",
    "        \n",
    "    \n",
    "# in case class weights are needed\n",
    "def sparse_crossentropy_weighted(ground_truth, predictions):\n",
    "    \n",
    "    ground_truth = tf.cast(ground_truth, tf.int32)\n",
    "        \n",
    "    #onehot_labels = tf.one_hot(tf.squeeze(ground_truth,3), CLASSES)\n",
    "    #weights = onehot_labels * CLASSWEIGHTS\n",
    "    #weights = tf.reduce_sum(weights, 3)\n",
    "    \n",
    "    return tf.reduce_mean(\n",
    "        tf.losses.sparse_softmax_cross_entropy(\n",
    "                        labels=ground_truth,\n",
    "                        logits=predictions))\n",
    "     #                   weights=weights))\n",
    "\n",
    "def getModel(modelName):\n",
    "    import os\n",
    "    from keras.models import load_model\n",
    "    from nets.deepLabv3Keras import BilinearUpsampling, relu6\n",
    "    metricsPA = metricsSS(CLASSES, BSIZE).pixelAccuracy\n",
    "    metricsMA = metricsSS(CLASSES, BSIZE).meanAccuracy\n",
    "    metricsMIOU = metricsSS(CLASSES, BSIZE).meanIoU\n",
    "    metricsFWMIOU = metricsSS(CLASSES, BSIZE).frequencyWeightedUI\n",
    "    \n",
    "    if os.path.isfile(filepath):\n",
    "        model = load_model(filepath, custom_objects={\n",
    "                                        \"sparse_crossentropy_weighted\":sparse_crossentropy_weighted,\n",
    "                                        'pixelAccuracy': metricsPA,\n",
    "                                        \"meanAccuracy\": metricsMA,\n",
    "                                        \"meanIoU\": metricsMIOU,\n",
    "                                        \"frequencyWeightedUI\": metricsFWMIOU,\n",
    "                                        \"BilinearUpsampling\": BilinearUpsampling,\n",
    "                                        \"relu6\":relu6\n",
    "                                    }\n",
    "                          )\n",
    "\n",
    "        print(\"Model loaded from h5 file\")\n",
    "    else:\n",
    "        if modelName == \"deeplabv3\":\n",
    "            from nets.deepLabv3Keras import Deeplabv3\n",
    "            model = Deeplabv3(weights=None, backbone=\"mobilenetv2\", input_shape=(Y,X,3), classes=CLASSES, OS=8)  \n",
    "        elif modelName == \"mobilenetv2\":\n",
    "            model = mobileNet_V2_SS((Y,X,3))\n",
    "        elif modelName ==\"FCN\":\n",
    "            model = netFCN()\n",
    "        elif modelName ==\"dilNet\":\n",
    "            model = dilNet()\n",
    "        elif modelName ==\"NASNet\":\n",
    "            model = nasNet_SS((Y,X,3))\n",
    "        elif modelName ==\"inception\":\n",
    "            model = inceptionResnet_SS((Y,X,3))\n",
    "        elif modelName ==\"xception\":\n",
    "            model = xception_SS((Y,X,3)) \n",
    "        else:\n",
    "            raise \"no right modelname given\"\n",
    "\n",
    "        print(\"Fresh model loaded from architecture\")\n",
    "        \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "\n",
    "\n",
    "model = getModel(NNName)\n",
    "        \n",
    "#model.summary()\n",
    "model.compile(\n",
    "                loss=sparse_crossentropy_weighted,\n",
    "                optimizer=keras.optimizers.Nadam(lr=LR),\n",
    "                metrics=[]\n",
    "            )\n",
    "\n",
    "print(\"Model compiled...\")\n",
    "\n",
    "modelCheckpointer = keras.callbacks.ModelCheckpoint(\n",
    "                                    filepath,\n",
    "                                    monitor='val_loss',\n",
    "                                    verbose=1,\n",
    "                                    save_best_only=True,\n",
    "                                    save_weights_only=False,\n",
    "                                    mode='min',\n",
    "                                    period=1)\n",
    "\n",
    "earlyStopper = keras.callbacks.EarlyStopping(\n",
    "                                          monitor='val_loss',\n",
    "                                          min_delta=0.0001,\n",
    "                                          patience=20,\n",
    "                                          verbose=1,\n",
    "                                          mode='min'\n",
    "                                    )\n",
    "lrReducer = keras.callbacks.ReduceLROnPlateau(\n",
    "                                    monitor='val_loss',\n",
    "                                    factor=0.1,\n",
    "                                    patience=5,\n",
    "                                    verbose=1,\n",
    "                                    mode=\"min\",\n",
    "                                    min_delta=0.0001,\n",
    "                                    cooldown=0,\n",
    "                                    min_lr=0)\n",
    "\n",
    "history = model.fit_generator(\n",
    "        DataSequence(BSIZE, \"train\"),\n",
    "        steps_per_epoch=TRAINSIZE,\n",
    "        epochs=EPOCHS,\n",
    "        verbose=1,\n",
    "        use_multiprocessing=True,\n",
    "        shuffle=True,\n",
    "        validation_data=DataSequence(BSIZE,\"validation\"),\n",
    "        validation_steps=VALSIZE,\n",
    "        callbacks=[\n",
    "            regularPred(),\n",
    "            modelCheckpointer,\n",
    "            earlyStopper,\n",
    "            lrReducer\n",
    "            ]\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "testloss = model.evaluate_generator(\n",
    "    DataSequence(BSIZE,\"test\"),\n",
    "    steps=TESTSIZE,\n",
    "    use_multiprocessing=True,\n",
    "    verbose=1\n",
    ")\n",
    "\n",
    "print(testloss)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#https://machinelearningmastery.com/display-deep-learning-model-training-history-in-keras/\n",
    "import matplotlib.pyplot as plt\n",
    "# summarize history for accuracy\n",
    "#plt.plot(history.history['pixelAccuracy'])\n",
    "#plt.plot(history.history['val_pixelAccuracy'])\n",
    "#plt.title('pixel accuracy')\n",
    "#plt.ylabel('accuracy')\n",
    "#plt.xlabel('epoch')\n",
    "#plt.legend(['train',\"val\"], loc='upper left')\n",
    "#plt.show()\n",
    "# summarize history for loss\n",
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('model loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', \"val\"], loc='upper left')\n",
    "plt.show()\n",
    "# summarize history for mean iou\n",
    "#plt.plot(history.history['meanIoU'])\n",
    "#plt.plot(history.history['val_meanIoU'])\n",
    "#plt.title('model mean IoU')\n",
    "#plt.ylabel('mean_iou')\n",
    "#plt.xlabel('epoch')\n",
    "#plt.legend(['train', \"val\"], loc='upper left')\n",
    "#plt.show()\n",
    "#plt.plot(history.history['meanAccuracy'])\n",
    "#plt.plot(history.history['val_meanAccuracy'])\n",
    "#plt.title('model meanAccuracy')\n",
    "#plt.ylabel('meanAccuracy')\n",
    "#plt.xlabel('epoch')\n",
    "#plt.legend(['train', \"val\"], loc='upper left')\n",
    "#plt.show()\n",
    "#plt.plot(history.history['frequencyWeightedUI'])\n",
    "#plt.plot(history.history['val_frequencyWeightedUI'])\n",
    "#plt.title('model frequencyWeightedUI')\n",
    "#plt.ylabel('frequencyWeightedUI')\n",
    "#plt.xlabel('epoch')\n",
    "#plt.legend(['train', \"val\"], loc='upper left')\n",
    "#plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from data import Data\n",
    "from metricsSemSeg import pixel_accuracy, mean_accuracy, mean_IU, frequency_weighted_IU\n",
    "\n",
    "config = json.load(open(\"nets/netFCNConfig.json\"))\n",
    "# load data object initially which provides training and test data loader\n",
    "data = Data(\"../data/\"+DATASET+\"/configData\"+DATASET+\".json\")\n",
    "totalCorrect = 0\n",
    "totalCount = TESTSIZE*X*Y*3\n",
    "\n",
    "totalPAcc = 0.0\n",
    "totalMAcc = 0.0\n",
    "totalMIU = 0.0\n",
    "totalFWIU = 0.0\n",
    "\n",
    "i = 0\n",
    "model = getModel(NNName)\n",
    "\n",
    "data = DataSequence(BSIZE, \"test\")\n",
    "for idx in range(len(data)):\n",
    "        x, y = data[idx]\n",
    "    \n",
    "\n",
    "        predClasses = model.predict_on_batch(x)\n",
    "        for b in range(BSIZE):\n",
    "            pred = np.squeeze(predClasses[b])\n",
    "            pred = np.argmax(pred, axis=2)\n",
    "            labelData = np.squeeze(y[b])\n",
    "\n",
    "            if i % 100/BSIZE == 0:\n",
    "                print(\"Image \", i, \" evaluated...\")\n",
    "\n",
    "\n",
    "            totalPAcc = pixel_accuracy(pred, labelData) if totalPAcc == 0.0 else  (totalPAcc + pixel_accuracy(pred, labelData))/2\n",
    "            totalMAcc = mean_accuracy(pred, labelData) if totalMAcc == 0.0 else  (totalMAcc + mean_accuracy(pred, labelData))/2\n",
    "            totalMIU = mean_IU(pred, labelData) if totalMIU == 0.0 else  (totalMIU + mean_IU(pred, labelData))/2\n",
    "            totalFWIU = frequency_weighted_IU(pred, labelData) if totalFWIU == 0.0 else  (totalFWIU + frequency_weighted_IU(pred, labelData))/2\n",
    "\n",
    "            i = i+1\n",
    "\n",
    "\n",
    "print(\"Pixel accuracy: \", totalPAcc ,\" || Mean accuracy: \", totalMAcc ,\" || Mean intersection union:\", totalMIU ,\" || frequency weighted IU: \", totalFWIU)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from data import Data\n",
    "import matplotlib as mpl\n",
    "#mpl.use('TkAgg')\n",
    "import matplotlib.pyplot as plt\n",
    "# load data object initially which provides training and test data loader\n",
    "#data = Data(\"../data/\"+DATASET+\"/configData\"+DATASET+\".json\")\n",
    "\n",
    "\n",
    "max_images = 60\n",
    "model = getModel(NNName)\n",
    "#sanity check\n",
    "\n",
    "x_valid = []\n",
    "y_valid = []\n",
    "x_preds = []\n",
    "y_preds = []\n",
    "\n",
    "print(\"loading data...\")\n",
    "\n",
    "data = DataSequence(BSIZE, \"validation\")\n",
    "print(len(data))\n",
    "for idx in range(int(max_images/BSIZE)):#data.getNextBatchValidation(BSIZE, max_images):# data.config[\"validationSize\"]):\n",
    "        x, y = data[idx]\n",
    "        predClasses = model.predict_on_batch(x)\n",
    "        print(predClasses.shape)\n",
    "        for e in range(BSIZE):\n",
    "            pred = np.squeeze(predClasses[e])\n",
    "            pred = np.argmax(pred, axis=2)\n",
    "            x_preds.append(pred.squeeze())\n",
    "            x_valid.append(x[e].squeeze())\n",
    "            y_valid.append(y[e].squeeze())\n",
    "\n",
    "\n",
    "x_valid = np.array(x_valid)\n",
    "y_valid = np.array(y_valid)\n",
    "x_preds = np.array(x_preds)\n",
    "y_preds = np.array(y_preds)\n",
    "\n",
    "print(x_valid.shape, y_valid.shape, x_preds.shape, y_preds.shape)\n",
    "\n",
    "grid_width = 15\n",
    "\n",
    "grid_height = int(max_images / grid_width)\n",
    "fig, axs = plt.subplots(grid_height, grid_width, figsize=(grid_width, grid_height))\n",
    "for idx, i in enumerate(x_valid):\n",
    "    img = (x_valid[idx] * 255).astype(np.uint8)\n",
    "    mask = (y_valid[idx]  * 255).astype(np.uint8)\n",
    "    ax = axs[int(idx / grid_width), idx % grid_width]\n",
    "    #ax.imshow(img, cmap=\"Greys\")\n",
    "    ax.imshow(mask, alpha=0.6, cmap=\"Greens\")\n",
    "    #ax.imshow(pred, alpha=0.6, cmap=\"OrRd\")\n",
    "    ax.set_yticklabels([])\n",
    "    ax.set_xticklabels([])\n",
    "\n",
    "\n",
    "plt.suptitle(\"Green: salt\")\n",
    "plt.show()\n",
    "\n",
    "#display predictions\n",
    "grid_height = int(max_images / grid_width)\n",
    "fig, axs = plt.subplots(grid_height, grid_width, figsize=(grid_width, grid_height))\n",
    "for idx, i in enumerate(x_valid):\n",
    "    img = (x_valid[idx] * 255).astype(np.uint8)\n",
    "    pred = (x_preds[idx] * 255).astype(np.uint8)\n",
    "    ax = axs[int(idx / grid_width), idx % grid_width]\n",
    "    #ax.imshow(img, cmap=\"Greys\")\n",
    "    #ax.imshow(mask, alpha=0.6, cmap=\"Greens\")\n",
    "    ax.imshow(pred, alpha=0.6, cmap=\"OrRd\")\n",
    "    ax.set_yticklabels([])\n",
    "    ax.set_xticklabels([])\n",
    "\n",
    "\n",
    "plt.suptitle(\"Red: prediction\")\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}