infer-transfer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import torch\n",
    "import torchvision\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "import time\n",
    "\n",
    "from KittiDataset import KittiDataset\n",
    "from Model import EnDeWithPooling, EnDeConvLSTM_ws, SkipLSTMEnDe\n",
    "from torchvision import transforms\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def saveTransformedImages(imageTensor):\n",
    "    to_pil = torchvision.transforms.ToPILImage()\n",
    "    im = to_pil(imageTensor)\n",
    "    mn, mx = np.min(im), np.max(im)\n",
    "    im = (im - mn) / (mx - mn)\n",
    "    print(im)\n",
    "    plt.imshow(im, cmap='gray')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plotTrajectory(xValsGT, yValsGT, xValsPred, yValsPred, seqLen, im_path, numFrames=None):\n",
    "    fig = plt.figure(figsize=(8, 8))\n",
    "    plt.scatter(yValsGT, xValsGT, c='r', marker='o', label='Ground Truth')\n",
    "    plt.scatter(yValsPred, xValsPred, c='g', marker='x', label='Prediction')\n",
    "    axes = plt.gca()\n",
    "    axes.set_xlim([1, 512])\n",
    "    axes.set_ylim([1, 512])\n",
    "    plt.xlabel('X-Axis')\n",
    "    plt.ylabel('Y-Axis')\n",
    "    plt.legend(loc='upper right')\n",
    "    if numFrames == None:\n",
    "        plt.title('Trajectory')\n",
    "    else:\n",
    "        plot_title = 'Trajectory (' + str(numFrames // 10 - 1) + \"s)\"\n",
    "        plt.title(plot_title)\n",
    "    plt.savefig(im_path)\n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def heatmapAccuracy(outputMap, labelMap, thr=1.5):\n",
    "    pred = np.unravel_index(outputMap.argmax(), outputMap.shape)\n",
    "    gt = np.unravel_index(labelMap.argmax(), labelMap.shape)\n",
    "\n",
    "    dist = math.sqrt((pred[0] - gt[0]) ** 2 + (pred[1] - gt[1]) ** 2)\n",
    "    if dist <= thr:\n",
    "        return 1, dist, (pred[0], pred[1]), (gt[0], gt[1])\n",
    "    return 0, dist, (pred[0], pred[1]), (gt[0], gt[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def largest_indices(ary, n):\n",
    "    \"\"\"Returns the n largest indices from a numpy array.\"\"\"\n",
    "    flat = ary.flatten()\n",
    "    indices = np.argpartition(flat, -n)[-n:]\n",
    "    indices = indices[np.argsort(-flat[indices])]\n",
    "    return np.unravel_index(indices, ary.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def multiAccuracy(outputMap, labelMap, topK=5):\n",
    "    pred = largest_indices(outputMap, topK)\n",
    "    gt = np.unravel_index(labelMap.argmax(), labelMap.shape)\n",
    "    dist_arr = []\n",
    "    for i in range(len(pred[0])):\n",
    "        dist = math.sqrt((pred[0][i] - gt[0]) ** 2 + (pred[1][i] - gt[1]) ** 2)\n",
    "        dist_arr.append(dist)\n",
    "    \n",
    "    min_val = np.min(dist_arr)\n",
    "    min_idx = np.argmin(dist_arr)\n",
    "    within_radius = 0\n",
    "    if min_val <= 4:\n",
    "        within_radius = 1\n",
    "    return 0, min_val, (pred[0][min_idx], pred[1][min_idx]), (gt[0], gt[1]), within_radius"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.set_default_tensor_type(torch.cuda.FloatTensor)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set correct repo path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "repo_path = \"/home/fbd/rrc/submission/INFER-code\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Best\n",
    "checkpoint_path = os.path.join(repo_path, \"models\", \"cityscapes-transfer\", \"checkpoint_future.tar\")\n",
    "\n",
    "checkpoint = torch.load(checkpoint_path)\n",
    "model = SkipLSTMEnDe(activation=\"relu\", initType=\"default\", numChannels=5, imageHeight=256, imageWidth=256, batchnorm=False, softmax=False)\n",
    "model.load_state_dict(checkpoint[\"model_state_dict\"])\n",
    "model = model.cuda()\n",
    "model.convlstm = model.convlstm.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = \"/home/fbd/rrc/submission/INFER-datasets/cityscapes\"\n",
    "val_dir = os.path.join(data_dir, \"test.csv\")\n",
    "\n",
    "val_dataset = KittiDataset(data_dir, height=256, width=256, train=False, infoPath=val_dir, augmentation=False, groundTruth=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Future Prediction (Final, Validation)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 0.8s preconditioning, 1s future"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "upsample_512 = torch.nn.Upsample(scale_factor=2, mode='bilinear')\n",
    "labelTransform = transforms.Compose([\n",
    "    transforms.ToTensor()\n",
    "])\n",
    "targetGTDir = os.path.join(data_dir, 'targetGT')\n",
    "valLoss1, valLoss2, valLoss3, valLoss4, valLoss = [], [], [], [], []\n",
    "futureFrames = 14\n",
    "topK = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fbd/anaconda3/envs/torchenv/lib/python3.5/site-packages/torch/nn/functional.py:1749: UserWarning: Default upsampling behavior when mode=bilinear is changed to align_corners=False since 0.4.0. Please specify align_corners=True if the old behavior is desired. See the documentation of nn.Upsample for details.\n",
      "  \"See the documentation of nn.Upsample for details.\".format(mode))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SeqNum: 1, KittiSeqNum: 0, VehicleNum: 3, numFrames: 30, loss: 4.5682446146884175, len(seqLoss): 8\n",
      "SeqNum: 2, KittiSeqNum: 0, VehicleNum: 4, numFrames: 30, loss: 4.7279944297634255, len(seqLoss): 8\n",
      "SeqNum: 3, KittiSeqNum: 18, VehicleNum: 6, numFrames: 30, loss: 4.86975096479574, len(seqLoss): 8\n",
      "SeqNum: 4, KittiSeqNum: 18, VehicleNum: 7, numFrames: 30, loss: 5.034616995972669, len(seqLoss): 8\n",
      "SeqNum: 5, KittiSeqNum: 26, VehicleNum: 21, numFrames: 30, loss: 1.375, len(seqLoss): 8\n",
      "SeqNum: 6, KittiSeqNum: 32, VehicleNum: 14, numFrames: 30, loss: 2.4705114938006396, len(seqLoss): 8\n",
      "SeqNum: 7, KittiSeqNum: 42, VehicleNum: 15, numFrames: 30, loss: 4.9992640141559415, len(seqLoss): 8\n",
      "SeqNum: 8, KittiSeqNum: 50, VehicleNum: 2, numFrames: 30, loss: 4.3017766952966365, len(seqLoss): 8\n",
      "SeqNum: 9, KittiSeqNum: 1, VehicleNum: 25, numFrames: 30, loss: 9.064940543450463, len(seqLoss): 8\n",
      "SeqNum: 10, KittiSeqNum: 10, VehicleNum: 20, numFrames: 30, loss: 0.875, len(seqLoss): 8\n",
      "SeqNum: 11, KittiSeqNum: 11, VehicleNum: 12, numFrames: 30, loss: 1.6970614028176843, len(seqLoss): 8\n",
      "SeqNum: 12, KittiSeqNum: 44, VehicleNum: 21, numFrames: 30, loss: 1.7580618877807475, len(seqLoss): 8\n",
      "SeqNum: 13, KittiSeqNum: 52, VehicleNum: 8, numFrames: 30, loss: 2.353553390593274, len(seqLoss): 8\n",
      "SeqNum: 14, KittiSeqNum: 6, VehicleNum: 6, numFrames: 30, loss: 0.5517766952966369, len(seqLoss): 8\n",
      "SeqNum: 15, KittiSeqNum: 24, VehicleNum: 1, numFrames: 30, loss: 5.103530503098334, len(seqLoss): 8\n",
      "SeqNum: 16, KittiSeqNum: 30, VehicleNum: 6, numFrames: 30, loss: 2.359633199996396, len(seqLoss): 8\n",
      "SeqNum: 17, KittiSeqNum: 30, VehicleNum: 7, numFrames: 30, loss: 2.6070673442973007, len(seqLoss): 8\n",
      "SeqNum: 18, KittiSeqNum: 37, VehicleNum: 5, numFrames: 30, loss: 1.25, len(seqLoss): 8\n",
      "SeqNum: 19, KittiSeqNum: 48, VehicleNum: 3, numFrames: 30, loss: 1.8952847075210475, len(seqLoss): 8\n",
      "SeqNum: 20, KittiSeqNum: 17, VehicleNum: 7, numFrames: 30, loss: 4.919896700389682, len(seqLoss): 8\n",
      "SeqNum: 21, KittiSeqNum: 17, VehicleNum: 8, numFrames: 30, loss: 8.612162205531185, len(seqLoss): 8\n",
      "SeqNum: 22, KittiSeqNum: 17, VehicleNum: 9, numFrames: 30, loss: 0.8017766952966369, len(seqLoss): 8\n",
      "SeqNum: 23, KittiSeqNum: 25, VehicleNum: 45, numFrames: 30, loss: 3.5416841671027304, len(seqLoss): 8\n",
      "SeqNum: 24, KittiSeqNum: 38, VehicleNum: 13, numFrames: 30, loss: 2.4960777239642, len(seqLoss): 8\n",
      "SeqNum: 25, KittiSeqNum: 38, VehicleNum: 14, numFrames: 30, loss: 10.567159786693122, len(seqLoss): 8\n",
      "SeqNum: 26, KittiSeqNum: 49, VehicleNum: 17, numFrames: 30, loss: 2.7468546075262057, len(seqLoss): 8\n"
     ]
    }
   ],
   "source": [
    "debug, prevOut, state = True, None, None\n",
    "prevChannels = None\n",
    "xValsGT, yValsGT, xValsPred, yValsPred = [], [], [], []\n",
    "seqLoss, seqVals = [], []\n",
    "seqNum, seqLen = 0, 0\n",
    "\n",
    "start_time = time.time()\n",
    "model.eval()\n",
    "\n",
    "for i in range(len(val_dataset)):\n",
    "    grid, kittiSeqNum, vehicleId, frame1, frame2, endOfSequence, offset, numFrames, augmentation = val_dataset[i]\n",
    "    \n",
    "    if endOfSequence is False:\n",
    "        if int(offset) % 2 == 0:\n",
    "            continue\n",
    "\n",
    "    # The Last Channel is the target frame and first n - 1 are source frames\n",
    "    inp = grid[:-1, :].unsqueeze(0).to(device)\n",
    "    currLabel = grid[-1:, :].unsqueeze(0).to(device)\n",
    "    \n",
    "    if offset < futureFrames:\n",
    "        prevChannels = inp\n",
    "\n",
    "    if offset >= futureFrames:\n",
    "        new_inp = inp.clone().squeeze(0)\n",
    "        mn, mx = torch.min(prevOut), torch.max(prevOut)\n",
    "        prevOut = (prevOut - mn) / (mx - mn)\n",
    "        new_inp[0] = prevOut\n",
    "        new_inp[4] = prevChannels[0, 4, :, :]\n",
    "        inp = new_inp.unsqueeze(0).cuda()\n",
    "\n",
    "    # Forward the input and obtain the result\n",
    "    out = model.forward(inp, state)\n",
    "    state = (model.h, model.c, model.h1, model.c1, model.h2, model.c2)\n",
    "    currOutputMap = out.clone()\n",
    "    newOutputMap = upsample_512(currOutputMap)\n",
    "    nextTargetImg = Image.open(os.path.join(targetGTDir, str(kittiSeqNum).zfill(4), \n",
    "                                            str(frame2).zfill(6), str(vehicleId).zfill(6) + '.png'))\n",
    "    \n",
    "    nextTargetTensor = labelTransform(nextTargetImg).unsqueeze(0)\n",
    "    \n",
    "    prevOut = currOutputMap.detach().cpu().squeeze(0).squeeze(0)\n",
    "    currOutputMap = currOutputMap.detach().cpu().numpy().squeeze(0).squeeze(0)\n",
    "    currLabel = currLabel.detach().cpu().numpy().squeeze(0).squeeze(0)\n",
    "    _, dist, predCoordinates, gtCoordinates = heatmapAccuracy(currOutputMap, currLabel)\n",
    "    \n",
    "    # Upsampled outputs and inputs\n",
    "    currOutputMap1 = newOutputMap.detach().cpu().numpy().squeeze(0).squeeze(0)\n",
    "    currLabel1 = nextTargetTensor.detach().cpu().numpy().squeeze(0).squeeze(0)\n",
    "    \n",
    "    _, dist1, predCoordinates1, gtCoordinates1 = heatmapAccuracy(currOutputMap1, currLabel1)\n",
    "    _, dist2, predCoordinates2, gtCoordinates2, within_radius = multiAccuracy(currOutputMap1, currLabel1, topK=topK)\n",
    "    \n",
    "    if offset >= futureFrames:\n",
    "        seqLoss.append(dist2)\n",
    "\n",
    "    seqLen += 1\n",
    "    xValsGT.append(gtCoordinates1[0])\n",
    "    yValsGT.append(gtCoordinates1[1])\n",
    "    xValsPred.append(predCoordinates1[0])\n",
    "    yValsPred.append(predCoordinates1[1])\n",
    "    \n",
    "    if endOfSequence:\n",
    "        seqVals.append(seqLen)\n",
    "        xValsGT, yValsGT, xValsPred, yValsPred = [], [], [], []\n",
    "        seqNum += 1\n",
    "        state = None\n",
    "        valLoss.append(np.mean(seqLoss))\n",
    "        print(\"SeqNum: {}, KittiSeqNum: {}, VehicleNum: {}, numFrames: {}, loss: {}, len(seqLoss): {}\".format(seqNum, kittiSeqNum, vehicleId, numFrames, np.mean(seqLoss), len(seqLoss)))\n",
    "        seqLoss = []\n",
    "\n",
    "end_time = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Avg Loss: 3.674949260378043\n",
      "Avg Loss in m: 0.9187373150945107\n",
      "Num Seq: 26\n"
     ]
    }
   ],
   "source": [
    "print(\"Avg Loss: {}\".format(np.mean(valLoss)))\n",
    "print(\"Avg Loss in m: {}\".format(np.mean(valLoss) * 0.25))\n",
    "print(\"Num Seq: {}\".format(len(valLoss)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Kitti Future Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint_path = os.path.join(repo_path, \"models\", \"cityscapes-transfer\", \"checkpoint_future.tar\")\n",
    "\n",
    "checkpoint = torch.load(checkpoint_path)\n",
    "model = SkipLSTMEnDe(activation=\"relu\", initType=\"default\", numChannels=5, imageHeight=256, imageWidth=256, batchnorm=False, softmax=False)\n",
    "model.load_state_dict(checkpoint[\"model_state_dict\"])\n",
    "model = model.cuda()\n",
    "model.convlstm = model.convlstm.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = \"/home/fbd/rrc/submission/INFER-datasets/kitti\"\n",
    "val_dir = os.path.join(data_dir, \"val.csv\")\n",
    "val_dataset = KittiDataset(data_dir, height=256, width=256, train=False, infoPath=val_dir, augmentation=False, groundTruth=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "upsample_512 = torch.nn.Upsample(scale_factor=2, mode='bilinear')\n",
    "labelTransform = transforms.Compose([\n",
    "    transforms.ToTensor()\n",
    "])\n",
    "targetGTDir = os.path.join(data_dir, 'targetGT')\n",
    "valLoss1, valLoss2, valLoss3, valLoss4, valLoss, valLoss8, valLoss9 = [], [], [], [], [], [], []\n",
    "futureFrames = 8\n",
    "topK = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fbd/anaconda3/envs/torchenv/lib/python3.5/site-packages/torch/nn/functional.py:1749: UserWarning: Default upsampling behavior when mode=bilinear is changed to align_corners=False since 0.4.0. Please specify align_corners=True if the old behavior is desired. See the documentation of nn.Upsample for details.\n",
      "  \"See the documentation of nn.Upsample for details.\".format(mode))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SeqNum: 1, KittiSeqNum: 0, VehicleNum: 0, numFrames: 21, loss: 17.120213422955146, len(seqLoss): 12\n",
      "SeqNum: 2, KittiSeqNum: 4, VehicleNum: 2, numFrames: 21, loss: 0.985702260395516, len(seqLoss): 12\n",
      "SeqNum: 3, KittiSeqNum: 4, VehicleNum: 7, numFrames: 21, loss: 8.89134361076261, len(seqLoss): 12\n",
      "SeqNum: 4, KittiSeqNum: 4, VehicleNum: 8, numFrames: 21, loss: 7.216872177929858, len(seqLoss): 12\n",
      "SeqNum: 5, KittiSeqNum: 8, VehicleNum: 8, numFrames: 21, loss: 0.4166666666666667, len(seqLoss): 12\n",
      "SeqNum: 6, KittiSeqNum: 8, VehicleNum: 17, numFrames: 21, loss: 4.00455705869119, len(seqLoss): 12\n",
      "SeqNum: 7, KittiSeqNum: 16, VehicleNum: 1, numFrames: 21, loss: 0.0, len(seqLoss): 12\n",
      "SeqNum: 8, KittiSeqNum: 16, VehicleNum: 3, numFrames: 21, loss: 0.0, len(seqLoss): 12\n",
      "SeqNum: 9, KittiSeqNum: 20, VehicleNum: 0, numFrames: 21, loss: 0.0, len(seqLoss): 12\n",
      "SeqNum: 10, KittiSeqNum: 20, VehicleNum: 0, numFrames: 21, loss: 3.75, len(seqLoss): 12\n",
      "SeqNum: 11, KittiSeqNum: 20, VehicleNum: 4, numFrames: 21, loss: 1.6769254688014719, len(seqLoss): 12\n",
      "SeqNum: 12, KittiSeqNum: 20, VehicleNum: 5, numFrames: 21, loss: 0.08333333333333333, len(seqLoss): 12\n",
      "SeqNum: 13, KittiSeqNum: 20, VehicleNum: 5, numFrames: 21, loss: 2.2269084131670773, len(seqLoss): 12\n",
      "SeqNum: 14, KittiSeqNum: 20, VehicleNum: 5, numFrames: 21, loss: 5.852511890376643, len(seqLoss): 12\n",
      "SeqNum: 15, KittiSeqNum: 20, VehicleNum: 12, numFrames: 21, loss: 0.8880711874576984, len(seqLoss): 12\n",
      "SeqNum: 16, KittiSeqNum: 20, VehicleNum: 12, numFrames: 21, loss: 2.171449515815516, len(seqLoss): 12\n",
      "SeqNum: 17, KittiSeqNum: 20, VehicleNum: 16, numFrames: 21, loss: 2.333346660658742, len(seqLoss): 12\n",
      "SeqNum: 18, KittiSeqNum: 20, VehicleNum: 16, numFrames: 21, loss: 0.985702260395516, len(seqLoss): 12\n",
      "SeqNum: 19, KittiSeqNum: 20, VehicleNum: 16, numFrames: 21, loss: 0.9560113295832983, len(seqLoss): 12\n",
      "SeqNum: 20, KittiSeqNum: 20, VehicleNum: 65, numFrames: 21, loss: 1.530969240967604, len(seqLoss): 12\n",
      "SeqNum: 21, KittiSeqNum: 20, VehicleNum: 69, numFrames: 21, loss: 1.9309777335965597, len(seqLoss): 12\n",
      "SeqNum: 22, KittiSeqNum: 20, VehicleNum: 91, numFrames: 21, loss: 1.3603796100280634, len(seqLoss): 12\n",
      "SeqNum: 23, KittiSeqNum: 20, VehicleNum: 122, numFrames: 21, loss: 4.868953014355358, len(seqLoss): 12\n",
      "SeqNum: 24, KittiSeqNum: 20, VehicleNum: 126, numFrames: 21, loss: 6.582696064457413, len(seqLoss): 12\n",
      "SeqNum: 25, KittiSeqNum: 84, VehicleNum: 15, numFrames: 21, loss: 1.0345177968644246, len(seqLoss): 12\n",
      "SeqNum: 26, KittiSeqNum: 84, VehicleNum: 16, numFrames: 21, loss: 0.8333333333333334, len(seqLoss): 12\n",
      "SeqNum: 27, KittiSeqNum: 84, VehicleNum: 42, numFrames: 21, loss: 1.3387079251871647, len(seqLoss): 12\n",
      "SeqNum: 28, KittiSeqNum: 84, VehicleNum: 42, numFrames: 21, loss: 0.5345177968644246, len(seqLoss): 12\n",
      "SeqNum: 29, KittiSeqNum: 5, VehicleNum: 2, numFrames: 21, loss: 5.530490283085045, len(seqLoss): 12\n",
      "SeqNum: 30, KittiSeqNum: 5, VehicleNum: 5, numFrames: 21, loss: 6.309312827549192, len(seqLoss): 12\n",
      "SeqNum: 31, KittiSeqNum: 5, VehicleNum: 7, numFrames: 21, loss: 3.220637307187151, len(seqLoss): 12\n",
      "SeqNum: 32, KittiSeqNum: 5, VehicleNum: 9, numFrames: 21, loss: 2.1478392629691765, len(seqLoss): 12\n",
      "SeqNum: 33, KittiSeqNum: 5, VehicleNum: 31, numFrames: 21, loss: 0.75, len(seqLoss): 12\n",
      "SeqNum: 34, KittiSeqNum: 9, VehicleNum: 16, numFrames: 21, loss: 1.991742813872919, len(seqLoss): 12\n",
      "SeqNum: 35, KittiSeqNum: 9, VehicleNum: 20, numFrames: 21, loss: 1.1035533905932737, len(seqLoss): 12\n",
      "SeqNum: 36, KittiSeqNum: 9, VehicleNum: 26, numFrames: 21, loss: 3.9871744901119257, len(seqLoss): 12\n",
      "SeqNum: 37, KittiSeqNum: 9, VehicleNum: 51, numFrames: 21, loss: 1.108380256645481, len(seqLoss): 12\n",
      "SeqNum: 38, KittiSeqNum: 9, VehicleNum: 78, numFrames: 21, loss: 4.678279079872502, len(seqLoss): 12\n",
      "SeqNum: 39, KittiSeqNum: 9, VehicleNum: 82, numFrames: 21, loss: 6.314586385742635, len(seqLoss): 12\n",
      "SeqNum: 40, KittiSeqNum: 48, VehicleNum: 5, numFrames: 21, loss: 2.5, len(seqLoss): 12\n",
      "SeqNum: 41, KittiSeqNum: 91, VehicleNum: 23, numFrames: 21, loss: 0.0, len(seqLoss): 12\n",
      "SeqNum: 42, KittiSeqNum: 10, VehicleNum: 0, numFrames: 21, loss: 1.3333333333333333, len(seqLoss): 12\n",
      "SeqNum: 43, KittiSeqNum: 10, VehicleNum: 3, numFrames: 21, loss: 1.7798737893643546, len(seqLoss): 12\n",
      "SeqNum: 44, KittiSeqNum: 18, VehicleNum: 1, numFrames: 21, loss: 2.0541901283227406, len(seqLoss): 12\n",
      "SeqNum: 45, KittiSeqNum: 18, VehicleNum: 2, numFrames: 21, loss: 0.6666666666666666, len(seqLoss): 12\n",
      "SeqNum: 46, KittiSeqNum: 18, VehicleNum: 2, numFrames: 21, loss: 0.0, len(seqLoss): 12\n",
      "SeqNum: 47, KittiSeqNum: 18, VehicleNum: 3, numFrames: 21, loss: 0.7357022603955158, len(seqLoss): 12\n",
      "SeqNum: 48, KittiSeqNum: 18, VehicleNum: 6, numFrames: 21, loss: 2.130051941486379, len(seqLoss): 12\n",
      "SeqNum: 49, KittiSeqNum: 51, VehicleNum: 25, numFrames: 21, loss: 2.814255750843888, len(seqLoss): 12\n",
      "SeqNum: 50, KittiSeqNum: 51, VehicleNum: 41, numFrames: 21, loss: 2.2831954698056807, len(seqLoss): 12\n",
      "SeqNum: 51, KittiSeqNum: 7, VehicleNum: 11, numFrames: 21, loss: 0.08333333333333333, len(seqLoss): 12\n",
      "SeqNum: 52, KittiSeqNum: 7, VehicleNum: 14, numFrames: 21, loss: 1.5940825170409967, len(seqLoss): 12\n",
      "SeqNum: 53, KittiSeqNum: 7, VehicleNum: 22, numFrames: 21, loss: 4.444014590735412, len(seqLoss): 12\n",
      "SeqNum: 54, KittiSeqNum: 7, VehicleNum: 22, numFrames: 21, loss: 0.6666666666666666, len(seqLoss): 12\n",
      "SeqNum: 55, KittiSeqNum: 7, VehicleNum: 23, numFrames: 21, loss: 3.4450145542430435, len(seqLoss): 12\n",
      "SeqNum: 56, KittiSeqNum: 7, VehicleNum: 32, numFrames: 21, loss: 1.1232257220515895, len(seqLoss): 12\n",
      "SeqNum: 57, KittiSeqNum: 7, VehicleNum: 44, numFrames: 21, loss: 0.3333333333333333, len(seqLoss): 12\n",
      "SeqNum: 58, KittiSeqNum: 7, VehicleNum: 44, numFrames: 21, loss: 0.3875234616560737, len(seqLoss): 12\n",
      "SeqNum: 59, KittiSeqNum: 7, VehicleNum: 45, numFrames: 21, loss: 0.6666666666666666, len(seqLoss): 12\n",
      "SeqNum: 60, KittiSeqNum: 7, VehicleNum: 56, numFrames: 21, loss: 20.80825692703798, len(seqLoss): 12\n",
      "SeqNum: 61, KittiSeqNum: 11, VehicleNum: 0, numFrames: 21, loss: 1.0345177968644246, len(seqLoss): 12\n",
      "SeqNum: 62, KittiSeqNum: 11, VehicleNum: 0, numFrames: 21, loss: 1.4561256729903622, len(seqLoss): 12\n",
      "SeqNum: 63, KittiSeqNum: 11, VehicleNum: 10, numFrames: 21, loss: 1.3596050087709441, len(seqLoss): 12\n",
      "SeqNum: 64, KittiSeqNum: 11, VehicleNum: 12, numFrames: 21, loss: 4.219378250565231, len(seqLoss): 12\n",
      "SeqNum: 65, KittiSeqNum: 15, VehicleNum: 18, numFrames: 21, loss: 0.0, len(seqLoss): 12\n",
      "SeqNum: 66, KittiSeqNum: 15, VehicleNum: 18, numFrames: 21, loss: 3.26307588752687, len(seqLoss): 12\n",
      "SeqNum: 67, KittiSeqNum: 15, VehicleNum: 19, numFrames: 21, loss: 2.8186542485321255, len(seqLoss): 12\n",
      "SeqNum: 68, KittiSeqNum: 15, VehicleNum: 20, numFrames: 21, loss: 3.254509890622254, len(seqLoss): 12\n",
      "SeqNum: 69, KittiSeqNum: 15, VehicleNum: 21, numFrames: 21, loss: 5.41059289279715, len(seqLoss): 12\n",
      "SeqNum: 70, KittiSeqNum: 15, VehicleNum: 21, numFrames: 21, loss: 1.8549316263779172, len(seqLoss): 12\n",
      "SeqNum: 71, KittiSeqNum: 15, VehicleNum: 23, numFrames: 21, loss: 3.172720547469479, len(seqLoss): 12\n",
      "SeqNum: 72, KittiSeqNum: 57, VehicleNum: 1, numFrames: 21, loss: 0.0, len(seqLoss): 12\n",
      "SeqNum: 73, KittiSeqNum: 57, VehicleNum: 3, numFrames: 21, loss: 4.6854081795024305, len(seqLoss): 12\n",
      "SeqNum: 74, KittiSeqNum: 57, VehicleNum: 8, numFrames: 21, loss: 0.0, len(seqLoss): 12\n",
      "SeqNum: 75, KittiSeqNum: 57, VehicleNum: 10, numFrames: 21, loss: 0.0, len(seqLoss): 12\n",
      "SeqNum: 76, KittiSeqNum: 57, VehicleNum: 10, numFrames: 21, loss: 0.0, len(seqLoss): 12\n"
     ]
    }
   ],
   "source": [
    "debug, prevOut, state = True, None, None\n",
    "xValsGT, yValsGT, xValsPred, yValsPred = [], [], [], []\n",
    "prevChannels = None\n",
    "seqLoss, seqVals = [], []\n",
    "seqNum, seqLen = 0, 0\n",
    "\n",
    "start_time = time.time()\n",
    "model.eval()\n",
    "\n",
    "for i in range(len(val_dataset)):\n",
    "    grid, kittiSeqNum, vehicleId, frame1, frame2, endOfSequence, offset, numFrames, augmentation = val_dataset[i]\n",
    "    \n",
    "    # The Last Channel is the target frame and first n - 1 are source frames\n",
    "    inp = grid[:-1, :].unsqueeze(0).to(device)\n",
    "    currLabel = grid[-1:, :].unsqueeze(0).to(device)\n",
    "    \n",
    "    if offset < futureFrames:\n",
    "        prevChannels = inp    \n",
    "\n",
    "    if offset >= futureFrames:\n",
    "        new_inp = inp.clone().squeeze(0)\n",
    "        mn, mx = torch.min(prevOut), torch.max(prevOut)\n",
    "        prevOut = (prevOut - mn) / (mx - mn)\n",
    "        new_inp[0] = prevOut\n",
    "        new_inp[4] = prevChannels[0, 4, :, :]        \n",
    "        inp = new_inp.unsqueeze(0).cuda()\n",
    "\n",
    "    # Forward the input and obtain the result\n",
    "    out = model.forward(inp, state)\n",
    "    state = (model.h, model.c, model.h1, model.c1, model.h2, model.c2)\n",
    "    currOutputMap = out.clone()\n",
    "    newOutputMap = upsample_512(currOutputMap)\n",
    "    nextTargetImg = Image.open(os.path.join(targetGTDir, str(kittiSeqNum).zfill(4), \n",
    "                                            str(frame2).zfill(6), str(vehicleId).zfill(6) + '.png'))\n",
    "    \n",
    "    nextTargetTensor = labelTransform(nextTargetImg).unsqueeze(0)\n",
    "    \n",
    "    prevOut = currOutputMap.detach().cpu().squeeze(0).squeeze(0)\n",
    "    currOutputMap = currOutputMap.detach().cpu().numpy().squeeze(0).squeeze(0)\n",
    "    currLabel = currLabel.detach().cpu().numpy().squeeze(0).squeeze(0)\n",
    "    \n",
    "    # Upsampled outputs and inputs\n",
    "    currOutputMap1 = newOutputMap.detach().cpu().numpy().squeeze(0).squeeze(0)\n",
    "    currLabel1 = nextTargetTensor.detach().cpu().numpy().squeeze(0).squeeze(0)\n",
    "    \n",
    "    _, dist2, predCoordinates2, gtCoordinates2, within_radius = multiAccuracy(currOutputMap1, currLabel1, topK=topK)    \n",
    "    \n",
    "    if offset >= futureFrames:\n",
    "        seqLoss.append(dist2)\n",
    "\n",
    "    seqLen += 1\n",
    "    xValsGT.append(gtCoordinates1[0])\n",
    "    yValsGT.append(gtCoordinates1[1])\n",
    "    xValsPred.append(predCoordinates1[0])\n",
    "    yValsPred.append(predCoordinates1[1])\n",
    "    \n",
    "    if endOfSequence:\n",
    "        seqVals.append(seqLen)\n",
    "        xValsGT, yValsGT, xValsPred, yValsPred = [], [], [], []\n",
    "        seqNum +=1\n",
    "        state = None\n",
    "        valLoss.append(np.mean(seqLoss))\n",
    "        valLoss8.append(np.mean(seqLoss[:8]))\n",
    "        valLoss9.append(np.mean(seqLoss[:9]))        \n",
    "        valLoss1.append(np.mean(seqLoss[:10]))\n",
    "        print(\"SeqNum: {}, KittiSeqNum: {}, VehicleNum: {}, numFrames: {}, loss: {}, len(seqLoss): {}\".format(seqNum, kittiSeqNum, vehicleId, numFrames, np.mean(seqLoss), len(seqLoss)))\n",
    "        seqLoss = []\n",
    "\n",
    "end_time = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8s: 0.4919861827300455\n",
      "0.9s: 0.537135809450425\n",
      "1s: 0.5770326002443692\n"
     ]
    }
   ],
   "source": [
    "print(\"0.8s: {}\".format(np.mean(valLoss8) * 0.25))\n",
    "print(\"0.9s: {}\".format(np.mean(valLoss9) * 0.25))\n",
    "print(\"1s: {}\".format(np.mean(valLoss1) * 0.25))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}