VGG16 Training & Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/stefano/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_1 is not trainable\n",
      "block1_conv1 is not trainable\n",
      "block1_conv2 is not trainable\n",
      "block1_pool is not trainable\n",
      "block2_conv1 is not trainable\n",
      "block2_conv2 is not trainable\n",
      "block2_pool is not trainable\n",
      "block3_conv1 is not trainable\n",
      "block3_conv2 is not trainable\n",
      "block3_conv3 is not trainable\n",
      "block3_pool is not trainable\n",
      "block4_conv1 is not trainable\n",
      "block4_conv2 is not trainable\n",
      "block4_conv3 is not trainable\n",
      "block4_pool is not trainable\n",
      "block5_conv1 is not trainable\n",
      "block5_conv2 is not trainable\n",
      "block5_conv3 is trainable\n",
      "block5_pool is trainable\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, None, None, 3)     0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, None, None, 64)    1792      \n",
      "_________________________________________________________________\n",
      "block1_conv2 (Conv2D)        (None, None, None, 64)    36928     \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, None, None, 64)    0         \n",
      "_________________________________________________________________\n",
      "block2_conv1 (Conv2D)        (None, None, None, 128)   73856     \n",
      "_________________________________________________________________\n",
      "block2_conv2 (Conv2D)        (None, None, None, 128)   147584    \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, None, None, 128)   0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, None, None, 256)   295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, None, None, 256)   590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, None, None, 256)   590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, None, None, 256)   0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, None, None, 512)   1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, None, None, 512)   2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, None, None, 512)   2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, None, None, 512)   0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, None, None, 512)   2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, None, None, 512)   2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, None, None, 512)   2359808   \n",
      "_________________________________________________________________\n",
      "global_average_pooling2d_1 ( (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 512)               2048      \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 512)               262656    \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1)                 513       \n",
      "=================================================================\n",
      "Total params: 14,979,905\n",
      "Trainable params: 264,193\n",
      "Non-trainable params: 14,715,712\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras.applications import VGG16\n",
    "from keras.layers import GlobalAveragePooling2D,MaxPooling2D, Dense, Dropout, BatchNormalization, Flatten\n",
    "from keras.models import Model\n",
    "\n",
    "def vgg16_model(trainable=True):\n",
    "    base_model = VGG16(False, \"imagenet\")\n",
    "    train_from_layer = -2\n",
    "    for layer in base_model.layers[:train_from_layer]:\n",
    "        layer.trainable = False\n",
    "        print(\"{} is not trainable\".format(layer.name))\n",
    "    for layer in base_model.layers[train_from_layer:]:\n",
    "        #layer.trainable = True\n",
    "        layer.trainable = False\n",
    "        print(\"{} is trainable\".format(layer.name))\n",
    "    last_conv_layer = base_model.get_layer(\"block5_conv3\")\n",
    "    x = GlobalAveragePooling2D()(last_conv_layer.output)\n",
    "    #x = Flatten()(last_conv_layer.output)\n",
    "    x = BatchNormalization(axis=-1)(x)\n",
    "    x = Dropout(0.5)(x)\n",
    "    x = Dense(512, activation=\"relu\")(x)        \n",
    "    predictions = Dense(1, activation=\"sigmoid\")(x)\n",
    "    return Model(base_model.input, predictions)\n",
    "\n",
    "\n",
    "\n",
    "model = vgg16_model(False)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tool\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tool.add(2,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(14835, 2555, 1112)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import cv2\n",
    "import numpy as np\n",
    "import models\n",
    "from sklearn.metrics import classification_report\n",
    "from keras.utils import to_categorical\n",
    "from keras.preprocessing.image import ImageDataGenerator, img_to_array\n",
    "import os\n",
    "\n",
    "X_train = []\n",
    "X_test = []\n",
    "Y_train = []\n",
    "Y_test = []\n",
    "X_val = []\n",
    "Y_val = []\n",
    "X_tr_ft = []\n",
    "X_te_ft = []\n",
    "size = (140,140)\n",
    "path = '../../DataDeepSolaris/pos_train/'\n",
    "for file in os.listdir(path):\n",
    "    if file.endswith('png') or file.endswith('jpg'):\n",
    "        img = cv2.imread(path+file)\n",
    "        imgg = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n",
    "        imgg = cv2.resize(imgg, (75,75))\n",
    "        dft_img = np.fft.fft2(imgg)\n",
    "        img = cv2.resize(img, size)\n",
    "        img = img/255\n",
    "        img = img_to_array(img)\n",
    "        X_tr_ft.append(dft_img)\n",
    "        X_train.append(img)\n",
    "        Y_train.append(1)\n",
    "        \n",
    "path = '../../DataDeepSolaris/pos_test/'\n",
    "for file in os.listdir(path):\n",
    "    if file.endswith('png') or file.endswith('jpg'):\n",
    "        img = cv2.imread(path+file)\n",
    "        imgg = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n",
    "        imgg = cv2.resize(imgg, (75,75))\n",
    "        dft_img = np.fft.fft2(imgg)\n",
    "        img = cv2.resize(img, size)\n",
    "        img = img/255\n",
    "        img = img_to_array(img)\n",
    "        if np.random.uniform()>0.3:\n",
    "            X_test.append(img)\n",
    "            Y_test.append(1)\n",
    "            X_te_ft.append(dft_img)\n",
    "        else:\n",
    "            X_val.append(img)\n",
    "            Y_val.append(1)\n",
    "        \n",
    "path = '../../DataDeepSolaris/neg_train/'\n",
    "for file in os.listdir(path):\n",
    "    if file.endswith('png') or file.endswith('jpg'):\n",
    "        img = cv2.imread(path+file)\n",
    "        imgg = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n",
    "        imgg = cv2.resize(imgg, (75,75))\n",
    "        dft_img = np.fft.fft2(imgg)\n",
    "        img = cv2.resize(img, size)\n",
    "        img = img/255\n",
    "        img = img_to_array(img)\n",
    "        X_tr_ft.append(dft_img)\n",
    "        X_train.append(img)\n",
    "        Y_train.append(0)\n",
    "        \n",
    "path = '../../DataDeepSolaris/neg_test/'\n",
    "for file in os.listdir(path):\n",
    "    if file.endswith('png') or file.endswith('jpg'):\n",
    "        img = cv2.imread(path+file)\n",
    "        imgg = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n",
    "        imgg = cv2.resize(imgg, (75,75))\n",
    "        dft_img = np.fft.fft2(imgg)\n",
    "        img = cv2.resize(img, size)\n",
    "        img = img/255\n",
    "        img = img_to_array(img)\n",
    "        if np.random.uniform()>0.3:\n",
    "            X_te_ft.append(dft_img)\n",
    "            X_test.append(img)\n",
    "            Y_test.append(0)\n",
    "        else:\n",
    "            X_val.append(img)\n",
    "            Y_val.append(0)\n",
    "        \n",
    "X_train = np.array(X_train)\n",
    "X_test = np.array(X_test)\n",
    "X_tr_ft = np.array(X_tr_ft).reshape(-1,75,75,1)\n",
    "X_te_ft = np.array(X_te_ft).reshape(-1,75,75,1)\n",
    "Y_train_c = to_categorical(Y_train)\n",
    "Y_test_c = to_categorical(Y_test)\n",
    "X_val = np.array(X_val)\n",
    "Y_val_c = to_categorical(Y_val)\n",
    "\n",
    "len(X_train), len(X_test), len(X_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded ../../DataDeepSolaris/pos_train/\n",
      "Loaded ../../DataDeepSolaris/pos_test/\n",
      "Loaded ../../DataDeepSolaris/neg_train/\n",
      "Loaded ../../DataDeepSolaris/neg_test/\n"
     ]
    }
   ],
   "source": [
    "X_train1 = []\n",
    "X_train2 = []\n",
    "X_test1 = []\n",
    "X_test2 = []\n",
    "Y_train = []\n",
    "Y_test = []\n",
    "size = (140,140)\n",
    "path = '../../DataDeepSolaris/neg_test/'\n",
    "\n",
    "path = '../../DataDeepSolaris/pos_train/'\n",
    "\n",
    "for file in os.listdir(path):\n",
    "    if file.endswith('png') or file.endswith('jpg'):\n",
    "        img = cv2.imread(path+file)\n",
    "        imgg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
    "        imgg = cv2.resize(imgg, (75,75))\n",
    "        #dft_img = np.fft.fft2(imgg)\n",
    "        #img = cv2.resize(img, size)\n",
    "        img = imgg/255\n",
    "        img = img_to_array(img)\n",
    "        X_train1.append(img)\n",
    "        #X_train2.append(dft_img)\n",
    "        Y_train.append(1)\n",
    "print ('Loaded', path)      \n",
    "path = '../../DataDeepSolaris/pos_test/'\n",
    "\n",
    "for file in os.listdir(path):\n",
    "    if file.endswith('png') or file.endswith('jpg'):\n",
    "        img = cv2.imread(path+file)\n",
    "        imgg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
    "        imgg = cv2.resize(imgg, (75,75))\n",
    "        #dft_img = np.fft.fft2(imgg)\n",
    "        #img = cv2.resize(img, size)\n",
    "        img = imgg/255\n",
    "        img = img_to_array(img)\n",
    "        X_test1.append(img)\n",
    "        #X_test2.append(dft_img)\n",
    "        Y_test.append(1)\n",
    "        \n",
    "print ('Loaded', path)      \n",
    "path = '../../DataDeepSolaris/neg_train/'\n",
    "\n",
    "for file in os.listdir(path):\n",
    "    if file.endswith('png') or file.endswith('jpg'):\n",
    "        img = cv2.imread(path+file)\n",
    "        imgg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
    "        imgg = cv2.resize(imgg, (75,75))\n",
    "        #dft_img = np.fft.fft2(imgg)\n",
    "        #img = cv2.resize(img, size)\n",
    "        img = imgg/255\n",
    "        img = img_to_array(img)\n",
    "        X_train1.append(img)\n",
    "        #X_train2.append(dft_img)\n",
    "        Y_train.append(0)\n",
    "\n",
    "print ('Loaded', path)\n",
    "\n",
    "path = '../../DataDeepSolaris/neg_test/'\n",
    "\n",
    "for file in os.listdir(path):\n",
    "    if file.endswith('png') or file.endswith('jpg'):\n",
    "        img = cv2.imread(path+file)\n",
    "        imgg = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
    "        imgg = cv2.resize(imgg, (75,75))\n",
    "        #dft_img = np.fft.fft2(imgg)\n",
    "        #img = cv2.resize(img, size)\n",
    "        img = imgg/255\n",
    "        img = img_to_array(img)\n",
    "        X_test1.append(img)\n",
    "        #X_test2.append(dft_img)\n",
    "        Y_test.append(0)\n",
    "print ('Loaded', path)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(14835, 75, 75, 3)\n"
     ]
    }
   ],
   "source": [
    "print(np.shape(np.array(X_train1)))\n",
    "\n",
    "X_train1 = np.array(X_train1).reshape(-1,75,75,3)\n",
    "X_test1 = np.array(X_test1).reshape(-1,75,75,3)\n",
    "#X_train2 = np.array(X_train2).reshape(-1,75,75,3)\n",
    "#X_test2 = np.array(X_test2).reshape(-1,75,75,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from Datasets import Datasets\n",
    "\n",
    "dataset_name = \"Bradbury\"\n",
    "dataset = Datasets.datasets()[dataset_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.applications.xception import preprocess_input\n",
    "import numpy as np\n",
    "\n",
    "# Add preprocessing\n",
    "train_images = X_train1 #dataset[0].images\n",
    "train_labels  = Y_train #dataset[0].labels\n",
    "\n",
    "test_images = X_test1 #dataset[1].images\n",
    "test_labels = Y_test #dataset[1].labels\n",
    "\n",
    "#validation_images = dataset[2].images\n",
    "#validation_labels = dataset[2].labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "from ProjectPaths import ProjectPaths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "\n",
    "data_generator = ImageDataGenerator(rotation_range=20,\n",
    "    width_shift_range=0.2,\n",
    "    height_shift_range=0.2,\n",
    "    zoom_range = 0.3,\n",
    "    #brightness_range = [0.5, 1.5],\n",
    "    horizontal_flip=True)\n",
    "\n",
    "test_datagen = ImageDataGenerator()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 64\n",
    "\n",
    "train_generator = data_generator.flow(train_images, train_labels, batch_size=batch_size)\n",
    "test_generator = test_datagen.flow(test_images, test_labels, batch_size=batch_size, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.callbacks import LearningRateScheduler\n",
    "\n",
    "def schedule_lr(epoch):\n",
    "    return 0.01\n",
    "\n",
    "lrCallback = LearningRateScheduler(schedule_lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.optimizers import SGD\n",
    "\n",
    "epochs = 100\n",
    "learning_rate = 0.1\n",
    "decay_rate = learning_rate / epochs\n",
    "momentum = 0.8\n",
    "sgd = SGD(lr=learning_rate, momentum=momentum, decay=decay_rate, nesterov=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from PerformanceMetrics import PerformanceMetrics\n",
    "\n",
    "model.compile(optimizer=\"adadelta\", loss='binary_crossentropy',\n",
    "                  metrics=['accuracy', PerformanceMetrics.precision,\n",
    "                           PerformanceMetrics.recall, PerformanceMetrics.fmeasure])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "231/231 [==============================] - 150s 649ms/step - loss: 0.3568 - acc: 0.8673 - precision: 0.5297 - recall: 0.2355 - fmeasure: 0.3068 - val_loss: 0.3118 - val_acc: 0.8765 - val_precision: 0.1413 - val_recall: 0.0465 - val_fmeasure: 0.0698\n",
      "\n",
      "Epoch 00001: val_acc improved from -inf to 0.87647, saving model to vgg16_3t_wmp_wr_aachen__01_0.88.hdf5\n",
      "Epoch 2/100\n",
      "231/231 [==============================] - 144s 625ms/step - loss: 0.3341 - acc: 0.8759 - precision: 0.6144 - recall: 0.2413 - fmeasure: 0.3273 - val_loss: 0.3034 - val_acc: 0.8791 - val_precision: 0.1413 - val_recall: 0.0382 - val_fmeasure: 0.0600\n",
      "\n",
      "Epoch 00002: val_acc improved from 0.87647 to 0.87908, saving model to vgg16_3t_wmp_wr_aachen__02_0.88.hdf5\n",
      "Epoch 3/100\n",
      "231/231 [==============================] - 147s 637ms/step - loss: 0.3209 - acc: 0.8828 - precision: 0.6658 - recall: 0.2738 - fmeasure: 0.3679 - val_loss: 0.3054 - val_acc: 0.8804 - val_precision: 0.1413 - val_recall: 0.0379 - val_fmeasure: 0.0596\n",
      "\n",
      "Epoch 00003: val_acc improved from 0.87908 to 0.88042, saving model to vgg16_3t_wmp_wr_aachen__03_0.88.hdf5\n",
      "Epoch 4/100\n",
      "231/231 [==============================] - 148s 640ms/step - loss: 0.3152 - acc: 0.8821 - precision: 0.6685 - recall: 0.2675 - fmeasure: 0.3621 - val_loss: 0.3215 - val_acc: 0.8716 - val_precision: 0.1413 - val_recall: 0.0473 - val_fmeasure: 0.0707\n",
      "\n",
      "Epoch 00004: val_acc did not improve from 0.88042\n",
      "Epoch 5/100\n",
      "231/231 [==============================] - 149s 646ms/step - loss: 0.3193 - acc: 0.8813 - precision: 0.6768 - recall: 0.2452 - fmeasure: 0.3416 - val_loss: 0.3086 - val_acc: 0.8796 - val_precision: 0.1413 - val_recall: 0.0379 - val_fmeasure: 0.0596\n",
      "\n",
      "Epoch 00005: val_acc did not improve from 0.88042\n",
      "Epoch 6/100\n",
      "231/231 [==============================] - 146s 631ms/step - loss: 0.3146 - acc: 0.8818 - precision: 0.6665 - recall: 0.2438 - fmeasure: 0.3392 - val_loss: 0.3105 - val_acc: 0.8854 - val_precision: 0.1413 - val_recall: 0.0426 - val_fmeasure: 0.0653\n",
      "\n",
      "Epoch 00006: val_acc improved from 0.88042 to 0.88539, saving model to vgg16_3t_wmp_wr_aachen__06_0.89.hdf5\n",
      "Epoch 7/100\n",
      "231/231 [==============================] - 144s 626ms/step - loss: 0.3096 - acc: 0.8866 - precision: 0.6822 - recall: 0.2621 - fmeasure: 0.3610 - val_loss: 0.3094 - val_acc: 0.8810 - val_precision: 0.1413 - val_recall: 0.0382 - val_fmeasure: 0.0600\n",
      "\n",
      "Epoch 00007: val_acc did not improve from 0.88539\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f050e626c88>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from keras.callbacks import ModelCheckpoint, TensorBoard, EarlyStopping\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "model_name = \"vgg16_3t_wmp_wr_{}\".format('aachen')\n",
    "\n",
    "checkpoint_dir = './model' #ProjectPaths.instance().checkpoint_dir_for(model_name, batch_size, epochs)\n",
    "if not os.path.exists(checkpoint_dir):\n",
    "    os.mkdir(checkpoint_dir)\n",
    "\n",
    "\n",
    "early_stopping_callback = EarlyStopping(patience=5)\n",
    "model_checkpoint_callback = ModelCheckpoint(model_name+\"__{epoch:02d}_{val_acc:.2f}.hdf5\", monitor='val_acc', verbose=True,\n",
    "                                                save_weights_only=True,\n",
    "                                                save_best_only=True)\n",
    "\n",
    "log_dir = os.path.join('./', model_name)\n",
    "tensorboard_callback = TensorBoard(log_dir=log_dir, histogram_freq=0,  write_graph=False, write_images=False)\n",
    "\n",
    "\n",
    "model.fit_generator(train_generator,\n",
    "                    steps_per_epoch=len(train_labels) // batch_size, \n",
    "                    epochs=epochs,\n",
    "                    callbacks=[early_stopping_callback, model_checkpoint_callback, tensorboard_callback],\n",
    "                    validation_data=test_generator,\n",
    "                    validation_steps=len(test_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.load_weights('vgg16_3t_wmp_wr_aachen__06_0.89.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 6208/14835 [===========>..................] - ETA: 2:16"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-b3763579b82d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpandas\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mbatch_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m64\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mtrain_eval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_images\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_labels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mtest_eval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_images\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_labels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mvalidation_eval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalidation_images\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_labels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mevaluate\u001b[0;34m(self, x, y, batch_size, verbose, sample_weight, steps)\u001b[0m\n\u001b[1;32m   1777\u001b[0m                                \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1778\u001b[0m                                \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1779\u001b[0;31m                                steps=steps)\n\u001b[0m\u001b[1;32m   1780\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1781\u001b[0m     def predict(self, x,\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_test_loop\u001b[0;34m(self, f, ins, batch_size, verbose, steps)\u001b[0m\n\u001b[1;32m   1424\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1425\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1426\u001b[0;31m                 \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1427\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1428\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0mbatch_index\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2480\u001b[0m         \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2481\u001b[0m         updated = session.run(fetches=fetches, feed_dict=feed_dict,\n\u001b[0;32m-> 2482\u001b[0;31m                               **self.session_kwargs)\n\u001b[0m\u001b[1;32m   2483\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2484\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    885\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    886\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 887\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    888\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    889\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1108\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1109\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1110\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1111\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1112\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1284\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1285\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1286\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1287\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1288\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1290\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1291\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1292\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1293\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1294\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1275\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1276\u001b[0m       return self._call_tf_sessionrun(\n\u001b[0;32m-> 1277\u001b[0;31m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[1;32m   1278\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1279\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[1;32m   1365\u001b[0m     return tf_session.TF_SessionRun_wrapper(\n\u001b[1;32m   1366\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1367\u001b[0;31m         run_metadata)\n\u001b[0m\u001b[1;32m   1368\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1369\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_tf_sessionprun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "batch_size = 64\n",
    "train_eval = model.evaluate(train_images, train_labels, batch_size)\n",
    "test_eval = model.evaluate(test_images, test_labels, batch_size)\n",
    "validation_eval = model.evaluate(validation_images, validation_labels, batch_size)\n",
    "\n",
    "np_model_evaluations = np.array([train_eval, test_eval, validation_eval])\n",
    "\n",
    "evaluations = pd.DataFrame(np_model_evaluations, columns=model.metrics_names)\n",
    "print(evaluations.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.metrics import roc_curve\n",
    "\n",
    "def find_optimal_cutoff(target, predicted):\n",
    "   \"\"\" Find the optimal probability cutoff point for a classification model related to event rate\n",
    "   Parameters\n",
    "   ----------\n",
    "   target : Matrix with dependent or target data, where rows are observations\n",
    "\n",
    "   predicted : Matrix with predicted data, where rows are observations\n",
    "\n",
    "   Returns\n",
    "   -------\n",
    "   list type, with optimal cutoff value\n",
    "\n",
    "   \"\"\"\n",
    "   fpr, tpr, threshold = roc_curve(target, predicted)\n",
    "   i = np.arange(len(tpr))\n",
    "   roc = pd.DataFrame({'tf' : pd.Series(tpr-(1-fpr), index=i), 'threshold' : pd.Series(threshold, index=i)})\n",
    "   roc_t = roc.ix[(roc.tf-0).abs().argsort()[:1]]\n",
    "\n",
    "   return list(roc_t['threshold'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "batch_size= 64\n",
    "test_predictions = model.predict(test_images, batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/stefano/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:21: DeprecationWarning: \n",
      ".ix is deprecated. Please use\n",
      ".loc for label based indexing or\n",
      ".iloc for positional indexing\n",
      "\n",
      "See the documentation here:\n",
      "http://pandas.pydata.org/pandas-docs/stable/indexing.html#ix-indexer-is-deprecated\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.35933247208595276]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cut_off = find_optimal_cutoff(Y_test, test_predictions)\n",
    "cut_off"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1854, 1302],\n",
       "       [ 212,  299]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predicted_labels = test_predictions > cut_off\n",
    "confusion_matrix(test_labels, predicted_labels)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.90      0.59      0.71      3156\n",
      "          1       0.19      0.59      0.28       511\n",
      "\n",
      "avg / total       0.80      0.59      0.65      3667\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "print(classification_report(test_labels, predicted_labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Aachen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ac_dataset = Datasets.datasets()[\"AcMüDüHo\"]\n",
    "\n",
    "train_images = ac_dataset[0].images\n",
    "train_labels = ac_dataset[0].labels\n",
    "\n",
    "test_images = ac_dataset[1].images\n",
    "test_labels = ac_dataset[1].labels\n",
    "\n",
    "validation_images = ac_dataset[2].images\n",
    "validation_labels = ac_dataset[2].labels\n",
    "\n",
    "eval_images = np.concatenate((train_images, test_images, validation_images), axis=0)\n",
    "eval_labels = np.concatenate((train_labels, test_labels, validation_labels), axis=0)\n",
    "\n",
    "\n",
    "print(train_images.shape, test_images.shape, validation_images.shape)\n",
    "print(eval_images.shape, eval_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "eval_predictions = model.predict(eval_images, batch_size) > cut_off\n",
    "confusion_matrix(eval_labels, eval_predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "print(classification_report(eval_labels, eval_predictions))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fresno"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fresno_dataset = Datasets.datasets()[\"Fresno\"]\n",
    "\n",
    "train_images = fresno_dataset[0].images\n",
    "train_labels = fresno_dataset[0].labels\n",
    "\n",
    "test_images = fresno_dataset[1].images\n",
    "test_labels = fresno_dataset[1].labels\n",
    "\n",
    "validation_images = fresno_dataset[2].images\n",
    "validation_labels = fresno_dataset[2].labels\n",
    "\n",
    "fresno_eval_images = np.concatenate((train_images, test_images, validation_images), axis=0)\n",
    "fresno_eval_labels = np.concatenate((train_labels, test_labels, validation_labels), axis=0)\n",
    "\n",
    "\n",
    "print(train_images.shape, test_images.shape, validation_images.shape)\n",
    "print(fresno_eval_images.shape, fresno_eval_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fresno_eval_predictions = model.predict(fresno_eval_images, batch_size) > cut_off\n",
    "confusion_matrix(fresno_eval_labels, fresno_eval_predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "print(classification_report(fresno_eval_labels, fresno_eval_predictions))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Heerlen Preprocessing\n",
    "\n",
    "Now we use the trained model to preprocess the images from Heerlen and see how well it performs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "heerlen_dir = os.path.join(ProjectPaths.instance().image_dir, \"Heerlen75x75\")\n",
    "image_files = [os.path.join(heerlen_dir, filename) for filename in os.listdir(heerlen_dir) if filename.endswith(\"rgb_2016.tiff\") or filename.endswith(\"rgb_2017.tiff\")]\n",
    "image_files = sorted(image_files)\n",
    "image_files[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "len(image_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_dir(path):\n",
    "    if os.path.exists(path):\n",
    "        return\n",
    "    os.makedirs(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "preprocessed_dir = os.path.join(ProjectPaths.instance().image_dir, \"Heerlen75x75_preprocessed\")\n",
    "positives_dir = os.path.join(preprocessed_dir, \"Positives\")\n",
    "negatives_dir = os.path.join(preprocessed_dir, \"Negatives\")\n",
    "\n",
    "create_dir(positives_dir)\n",
    "create_dir(negatives_dir)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.preprocessing.image import load_img, img_to_array\n",
    "from keras.applications.vgg16 import preprocess_input\n",
    "\n",
    "def prepare_image(filename):\n",
    "    img = load_img(filename) \n",
    "    img_array = img_to_array(img)\n",
    "    return img_array[:,:, ::-1]\n",
    "\n",
    "def image_generator(image_files, batch_size):\n",
    "    batch = []\n",
    "    for i, filename in enumerate(image_files):\n",
    "        if i > 0 and (i % batch_size == 0):\n",
    "            old_batch = batch\n",
    "            batch = []\n",
    "            yield np.array(old_batch)\n",
    "        batch.append(prepare_image(filename))\n",
    "    #if len(batch) > 0:\n",
    "    #    repeat_last = len(batch) - batch_size\n",
    "    #    repeated_images = [batch[-1] for i in range(repeat_last)]\n",
    "    #   yield np.array(batch + repeated_images)\n",
    "\n",
    "heerlen_image_generator = image_generator(image_files, batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predict labels for the images in the source directory and write them to positive or negative directory based on the prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictions = model.predict_generator(heerlen_image_generator, steps=len(image_files) // batch_size)\n",
    "predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictions = predictions.reshape(predictions.shape[0])\n",
    "predictions.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from shutil import copyfile\n",
    "\n",
    "for i, file_path in enumerate(image_files):        \n",
    "    filename = os.path.basename(file_path)\n",
    "    if i >= predictions.shape[0]:\n",
    "        break\n",
    "    \n",
    "    prediction = predictions[i]    \n",
    "    if prediction > cut_off:\n",
    "        output_path = os.path.join(positives_dir, filename)\n",
    "    else:        \n",
    "        output_path = os.path.join(negatives_dir, filename)\n",
    "    copyfile(file_path, output_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "len(os.listdir(positives_dir)), len(os.listdir(negatives_dir))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "positive_images = [load_img(os.path.join(positives_dir, image)) for i, image in enumerate(os.listdir(positives_dir)) if i < 25]\n",
    "\n",
    "_, ax = plt.subplots(5,5, figsize=(10,10))\n",
    "\n",
    "j = 0\n",
    "for r in range(5):\n",
    "    for c in range(5):\n",
    "        ax[r,c].imshow(positive_images[j])     \n",
    "        j += 1 \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "negative_images = [load_img(os.path.join(negatives_dir, image)) for i, image in enumerate(os.listdir(negatives_dir)) if i < 25]\n",
    "\n",
    "_, ax = plt.subplots(5,5, figsize=(10,10))\n",
    "\n",
    "j = 0\n",
    "for r in range(5):\n",
    "    for c in range(5):\n",
    "        ax[r,c].imshow(negative_images[j])     \n",
    "        j += 1 \n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}