dummy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import dlib\n",
    "import cv2\n",
    "from skimage.io import imshow\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import torch\n",
    "from torch import nn\n",
    "from torchvision.datasets import ImageFolder\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import transforms as tfms\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import os\n",
    "import numpy as np\n",
    "import random\n",
    "import argparse\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "\n",
    "SEED = 10\n",
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "\n",
    "# Dataset Parameters\n",
    "img_size = 224\n",
    "size = 15\n",
    "question_size = 10 # 6 for one-hot vector of color, 1 for question type, 3 for question subtype\n",
    "q_type_idx = 6\n",
    "sub_q_type_idx = 7\n",
    "nb_questions = 10\n",
    "# Possibles Answers : [yes, no, rectangle, circle, 1, 2, 3, 4, 5, 6]\n",
    "\n",
    "colors = [\n",
    "    (0,0,255), # red\n",
    "    (0,255,0), # green\n",
    "    (255,0,0), # blue\n",
    "    (0,156,255), # orange\n",
    "    (128,128,128), # gray\n",
    "    (0,255,255) # yellow\n",
    "]\n",
    "\n",
    "def center_generate(objects):\n",
    "    '''Generate centers of objects'''\n",
    "    while True:\n",
    "        pas = True\n",
    "        center = np.random.randint(0+size, img_size - size, 2)        \n",
    "        if len(objects) > 0:\n",
    "            for name, c, shape, _ in objects:\n",
    "                if ((center - c) ** 2).sum() < ((size * 2) ** 2):\n",
    "                    pas = False\n",
    "        if pas:\n",
    "            return center\n",
    "\n",
    "\n",
    "def build_sample():\n",
    "    '''Returns an image (with its bbox and attributes) and the corresponding questions, programs and answers'''\n",
    "    \n",
    "    # Create objects\n",
    "    objects = [] # [(color, center, shape, xmin, ymin, xmax, ymax), (...), ...]\n",
    "    img = np.ones((img_size,img_size,3)) * 255\n",
    "    for color_id, color in enumerate(colors):  \n",
    "        center = center_generate(objects)\n",
    "        if random.random()<0.5:\n",
    "            start = (center[0]-size, center[1]-size)\n",
    "            end = (center[0]+size, center[1]+size)\n",
    "            cv2.rectangle(img, start, end, color, -1)\n",
    "            objects.append((color_id, center, 'r', (start[0], start[1], end[0], end[1]))) \n",
    "        else:\n",
    "            start = (center[0]-size, center[1]-size)\n",
    "            end = (center[0]+size, center[1]+size)\n",
    "            center_ = (center[0], center[1])\n",
    "            cv2.circle(img, center_, size, color, -1)\n",
    "            objects.append((color_id, center, 'c', (start[0], start[1], end[0], end[1])))\n",
    "\n",
    "\n",
    "    rel_questions = []\n",
    "    norel_questions = []\n",
    "    rel_answers = []\n",
    "    norel_answers = []\n",
    "    \n",
    "    # Non-Relational Questions\n",
    "    for _ in range(nb_questions):\n",
    "        \n",
    "        question = np.zeros((question_size))\n",
    "        color = random.randint(0,5)\n",
    "        question[color] = 1\n",
    "        question[q_type_idx] = 0\n",
    "        subtype = random.randint(0,2)\n",
    "        question[subtype+sub_q_type_idx] = 1\n",
    "        norel_questions.append(question)\n",
    "        \n",
    "        if subtype == 0:\n",
    "            # query shape -> rectangle/circle\n",
    "            if objects[color][2] == 'r':\n",
    "                answer = 2\n",
    "            else:\n",
    "                answer = 3\n",
    "\n",
    "        elif subtype == 1:\n",
    "            # query is left side (horizontal position) -> yes/no\n",
    "            if objects[color][1][0] < img_size / 2:\n",
    "                answer = 0\n",
    "            else:\n",
    "                answer = 1\n",
    "\n",
    "        elif subtype == 2:\n",
    "            # query is up side (vertical position) -> yes/no\n",
    "            if objects[color][1][1] < img_size / 2:\n",
    "                answer = 0\n",
    "            else:\n",
    "                answer = 1\n",
    "        norel_answers.append(answer)\n",
    "    \n",
    "    # Relational Questions\n",
    "    for _ in range(nb_questions):\n",
    "        \n",
    "        question = np.zeros((question_size))\n",
    "        color = random.randint(0,5)\n",
    "        question[color] = 1\n",
    "        question[q_type_idx] = 1\n",
    "        subtype = random.randint(0,2)\n",
    "        question[subtype+sub_q_type_idx] = 1\n",
    "        rel_questions.append(question)\n",
    "\n",
    "        if subtype == 0:\n",
    "            # closest to -> rectangle/circle\n",
    "            my_obj = objects[color][1]\n",
    "            distances = np.array([np.linalg.norm(np.array(my_obj) - np.array(obj[1])) for obj in objects])\n",
    "            sorted_dists = distances.argsort()\n",
    "            idx = sorted_dists[0] if distances[sorted_dists[0]] != 0 else sorted_dists[1]\n",
    "#             dist_list = [((my_obj - obj[1]) ** 2).sum() for obj in objects]\n",
    "#             dist_list[dist_list.index(0)] = 999\n",
    "#             closest = dist_list.index(min(dist_list))\n",
    "            if objects[idx][2] == 'r':\n",
    "                answer = 2\n",
    "            else:\n",
    "                answer = 3\n",
    "                \n",
    "        elif subtype == 1:\n",
    "            # furthest from -> rectangle/circle\n",
    "            my_obj = objects[color][1]\n",
    "            distances = np.array([np.linalg.norm(np.array(my_obj) - np.array(obj[1])) for obj in objects])\n",
    "            sorted_dists = distances.argsort()\n",
    "            idx = sorted_dists[-1] if distances[sorted_dists[-1]] != 0 else sorted_dists[-2]\n",
    "#             dist_list = [((my_obj - obj[1]) ** 2).sum() for obj in objects]\n",
    "#             furthest = dist_list.index(max(dist_list))\n",
    "            if objects[idx][2] == 'r':\n",
    "                answer = 2\n",
    "            else:\n",
    "                answer = 3\n",
    "\n",
    "        elif subtype == 2:\n",
    "            # count -> 1~6\n",
    "            my_obj = objects[color][2]\n",
    "            count = -1\n",
    "            for obj in objects:\n",
    "                if obj[2] == my_obj:\n",
    "                    count += 1 \n",
    "            answer = count + 4\n",
    "\n",
    "        rel_answers.append(answer)\n",
    "\n",
    "    relations = (rel_questions, rel_answers)\n",
    "    norelations = (norel_questions, norel_answers)\n",
    "    img = img / 255.\n",
    "    sample = (img, objects, relations, norelations)\n",
    "    \n",
    "    return sample\n",
    "\n",
    "def convert_sample(sample):\n",
    "    '''Converts question/answer vector to natural language questions and programs'''\n",
    "    \n",
    "    img, objects, (rel_questions, rel_answers), (norel_questions, norel_answers) = sample\n",
    "    colors = ['red', 'green', 'blue', 'orange', 'gray', 'yellow']\n",
    "    answer_sheet = ['yes', 'no', 'rectangle', 'circle', '1', '2', '3', '4', '5', '6']\n",
    "    questions = rel_questions + norel_questions\n",
    "    answers = rel_answers + norel_answers\n",
    "    \n",
    "    queries = []\n",
    "    programs = []\n",
    "    text_answers = []\n",
    "\n",
    "    for i, (question, answer) in enumerate(zip(questions, answers)):\n",
    "        query = f'Q{i}. '\n",
    "        color = colors[question.tolist()[0:6].index(1)]\n",
    "        \n",
    "        # Non-relational questions\n",
    "        if question[q_type_idx] == 0:\n",
    "            if question[sub_q_type_idx] == 1:\n",
    "                queries.append(f'What is the shape of the {color} object?')\n",
    "                programs.append(f'filter {color} <nxt> query shape')\n",
    "                \n",
    "            elif question[sub_q_type_idx+1] == 1:\n",
    "                queries.append(f'Is there a {color} object on the left?')\n",
    "                programs.append(f'filter {color} <nxt> query position <nxt> isLeft')\n",
    "                \n",
    "            elif question[sub_q_type_idx+2] == 1:\n",
    "                queries.append(f'Is there a {color} object on the top?')\n",
    "                programs.append(f'filter {color} <nxt> query position <nxt> isTop')\n",
    "            \n",
    "        # Relational questions\n",
    "        elif question[q_type_idx] == 1:\n",
    "            if question[sub_q_type_idx] == 1:\n",
    "                queries.append(f'What is the closest shape to the {color} object?')\n",
    "                programs.append(f'filter {color} <nxt> relate closest <nxt> query shape')\n",
    "                \n",
    "            elif question[sub_q_type_idx+1] == 1:\n",
    "                queries.append(f'What is the furthest shape from the {color} object?')\n",
    "                programs.append(f'filter {color} <nxt> relate furthest <nxt> query shape')\n",
    "                \n",
    "            elif question[sub_q_type_idx+2] == 1:\n",
    "                queries.append(f'How many objects of the same shape as the {color} object are there?')\n",
    "                programs.append(f'filter {color} <nxt> query shape <nxt> filter <nxt> count')\n",
    "        \n",
    "        ans = answer_sheet[answer]\n",
    "        text_answers.append(ans)\n",
    "        \n",
    "    return img, objects, queries, programs, text_answers\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate a sample i.e.\n",
    "# a tuple of 1 image, 6 objects' properties, 20 QA vectors (10 relational and 10 non-relational)\n",
    "sample = build_sample()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Break down the sample\n",
    "img, objects, queries, programs, answers = convert_sample(sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "img = (img * 255).astype('uint8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7047f63bad70>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd0AAAHVCAYAAAC0biEDAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAqeUlEQVR4nO3df3BU9b3/8ddGyPIr2ZhAstkSIFDLj/LjKmrM2FIpKRApyiVtBWmFygW1AStRy80dFeHeueHCvepoKfQPAe8oapnhx0BvuRN+hNRLQAQzFKoZwkR+lCRYmOySUJaEfL5/+M3WJYEksPvZ3fB8zJyZ7DmfPXnvye557fucsxuHMcYIAACEXVykCwAA4HZB6AIAYAmhCwCAJYQuAACWELoAAFhC6AIAYAmhCwCAJYQuAACWELoAAFhC6AIAYEnEQnfVqlUaNGiQevTooaysLH388ceRKgUAACsiEroffvihCgoKtGTJEh0+fFhjxozRpEmTdO7cuUiUAwCAFY5I/MODrKws3Xffffr1r38tSWpublZGRoYWLlyof/7nf273/s3NzTp79qwSEhLkcDjCXS4AADdkjNHFixfl8XgUF3f9frabxZokSVeuXNGhQ4dUWFgYmBcXF6ecnByVlZW1eR+/3y+/3x+4/Ze//EUjRowIe60AAHTG6dOn1b9//+sutx66f/3rX3X16lWlpaUFzU9LS9Pnn3/e5n2Kioq0dOnSVvNPnz6txMTEsNQJBHOFYZ3eMKwTQCT4fD5lZGQoISHhhuOsh+7NKCwsVEFBQeB2y4NLTEwkdBHDeO4CXU17pzyth27fvn11xx13qLa2Nmh+bW2t3G53m/dxOp1yOp02ygO+JtzXC1y7fuuXVwCwzPrVy/Hx8Ro7dqx27doVmNfc3Kxdu3YpOzvbdjkAAFgTkcPLBQUFmj17tu69917df//9euONN9TQ0KCf//znkSgH+P8ifSU8nS/Q1UUkdB977DF9+eWXeuWVV1RTU6N/+Id/0I4dO1pdXAUAQFcSkc/p3iqfzyeXyyWv18uFVAiBSHe47Ym5lyhw2+loLvHdy7iNORT9gSvFRo0AOoLQBQDAkpj4nC4QWrHYObbUzKFmIJbR6QIAYAmhCwCAJYQuAACWcE4Xt5FYPJd7ra5zbrfln5gsWbIkwpUA9tDpAgBgCV+OgdtIV+h0W0T3y7atf8V5K+iGEe34cgwAAKIMnS5uA12pw71WdLx8Q93ZtofOF9GGThcAgChDp4vbAJ1uqNnubNtD54tIo9MFACDK0OniNkCnGyrR1uFei44XkdLRXCJ0cRsgdG9VtIfttQhf2MbhZQAAogydLm4DdLq3Ita63K+j44UtdLoAAEQZQhddXFfucqWvHl9Xf4xA10HoAgBgCed0cRvoyp1g+F6+sXwu91qc20W4cU4XAIAoQ+gCAGAJoQsAgCWc08VtgHO6ndGVzuV+Hed1EU6c0wUAIMoQugAAWELoAgBgCaELAIAlhC4AAJZ0i3QBQPi1XOHbla5ijrkPHQAQnS4AANYQugAAWMKXY+A2FMuHmfkHB7eKL8lAOPDlGAAARJmQh25RUZHuu+8+JSQkKDU1VdOmTVNFRUXQmIceekgOhyNoevrpp0NdCoBOWLJkSZfuArv640NsCHno7t27V/n5+dq/f7+Ki4vV2NioiRMnqqGhIWjcvHnzVF1dHZhWrFgR6lIAAIgqIf/I0I4dO4Jur1+/XqmpqTp06JDGjRsXmN+rVy+53e5Q/3qgA2LxI0Qxd+kFgDaE/Zyu1+uVJCUnJwfNf++999S3b1+NHDlShYWFunTp0nXX4ff75fP5giYAAGJNWL8co7m5Wc8995wefPBBjRw5MjD/8ccf18CBA+XxeHTkyBEtXrxYFRUV2rRpU5vrKSoqum2urIRNsdLx0uUCXUVYQzc/P19Hjx7VRx99FDR//vz5gZ9HjRql9PR0TZgwQSdOnNCQIUNaraewsFAFBQWB2z6fTxkZGeErHACAMAhb6C5YsEDbt29XaWmp+vfvf8OxWVlZkqTKyso2Q9fpdMrpdIalTiB6O146XKCrCXnoGmO0cOFCbd68WSUlJcrMzGz3PuXl5ZKk9PT0UJcDAEDUCHno5ufna8OGDdq6dasSEhJUU1MjSXK5XOrZs6dOnDihDRs26OGHH1ZKSoqOHDmiRYsWady4cRo9enSoywE64drO0nbnS2cLdHUhD93Vq1dL+uoLML5u3bp1mjNnjuLj47Vz50698cYbamhoUEZGhvLy8vTSSy+FuhQAAKIK370MdEg4ut7oful1pU8M8E1UCDe+exkAgCjDP7EHOqSjXWlLRxzdXSyAyODwMoAbiuXDzBxWhi0cXgYAIMoQugAAWELoAgBgCed0AbQr1s7rci4XtnFOFwCAKEOnC6DDor3jpcNFpNDpAgAQZeh0AXRatHW8dLiINDpdAACiDJ0ugFtmu/Ols0W0odMFACDK0OkCCLlQd750toh2dLoAAEQZOl0AEdHSDdPFoivoaC4RugAA3CIOLwMAEGUIXQAALCF0AQCwhNAFAMASQhcAAEsIXQAALCF0AQCwhNAFAMASQhcAAEsIXQAALCF0AQCwhNAFAMASQhcAAEsIXQAALCF0AQCwhNAFAMASQhcAAEsIXQAALAl56L766qtyOBxB07BhwwLLL1++rPz8fKWkpKhPnz7Ky8tTbW1tqMsAACDqhKXT/fa3v63q6urA9NFHHwWWLVq0SNu2bdPGjRu1d+9enT17VtOnTw9HGQAARJVuYVlpt25yu92t5nu9Xr399tvasGGDvv/970uS1q1bp+HDh2v//v164IEHwlEOAABRISyd7vHjx+XxeDR48GDNmjVLp06dkiQdOnRIjY2NysnJCYwdNmyYBgwYoLKysuuuz+/3y+fzBU0AAMSakIduVlaW1q9frx07dmj16tWqqqrSd7/7XV28eFE1NTWKj49XUlJS0H3S0tJUU1Nz3XUWFRXJ5XIFpoyMjFCXDQBA2IX88HJubm7g59GjRysrK0sDBw7U7373O/Xs2fOm1llYWKiCgoLAbZ/PR/ACAGJO2D8ylJSUpG9961uqrKyU2+3WlStXVFdXFzSmtra2zXPALZxOpxITE4MmAABiTdhDt76+XidOnFB6errGjh2r7t27a9euXYHlFRUVOnXqlLKzs8NdCgAAERXyw8svvPCCpk6dqoEDB+rs2bNasmSJ7rjjDs2cOVMul0tz585VQUGBkpOTlZiYqIULFyo7O5srlwEAXV7IQ/fMmTOaOXOmzp8/r379+uk73/mO9u/fr379+kmSXn/9dcXFxSkvL09+v1+TJk3Sb37zm1CXAQBA1HEYY0yki+gsn88nl8slr9fL+V0AQMR1NJf47mUAACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsKRbpAuIFQ45Il2CNUYm0iUAQJdEpwsAgCWELgAAlhC6AABYQugCAGAJoQsAgCWELgAAlhC6AABYQugCAGAJoQsAgCWELgAAlhC6AABYQugCAGAJoQsAgCWELgAAlhC6AABYQugCAGAJoQsAgCUhD91BgwbJ4XC0mvLz8yVJDz30UKtlTz/9dKjLAAAg6nQL9QoPHjyoq1evBm4fPXpUP/jBD/TjH/84MG/evHlatmxZ4HavXr1CXQYAAFEn5KHbr1+/oNvLly/XkCFD9L3vfS8wr1evXnK73aH+1QAARLWwntO9cuWK3n33XT355JNyOByB+e+995769u2rkSNHqrCwUJcuXbrhevx+v3w+X9AEAECsCXmn+3VbtmxRXV2d5syZE5j3+OOPa+DAgfJ4PDpy5IgWL16siooKbdq06brrKSoq0tKlS8NZKgAAYecwxphwrXzSpEmKj4/Xtm3brjtm9+7dmjBhgiorKzVkyJA2x/j9fvn9/sBtn8+njIwMeb1eJSYmhrzutjjkaH9QF2EUtqcEAHRJPp9PLper3VwKW6d78uRJ7dy584YdrCRlZWVJ0g1D1+l0yul0hrxGAABsCts53XXr1ik1NVVTpky54bjy8nJJUnp6erhKAQAgKoSl021ubta6des0e/Zsdev2919x4sQJbdiwQQ8//LBSUlJ05MgRLVq0SOPGjdPo0aPDUQoAAFEjLKG7c+dOnTp1Sk8++WTQ/Pj4eO3cuVNvvPGGGhoalJGRoby8PL300kvhKAMAgKgS1gupwqWjJ6xDiQupAADX09Fc4ruXAQCwhNAFAMASQhcAAEsIXQAALCF0AQCwhNAFAMASQhcAAEsIXQAALCF0AQCwhNAFAMASQhcAAEsIXQAALCF0AQCwhNAFAMASQhcAAEsIXQAALCF0AQCwhNAFAMASQhcAAEsIXQAALCF0AQCwhNAFAMASQhcAAEsIXQAALCF0AQCwhNAFAMASQhcAAEsIXQAALCF0AQCwpFukC4gVRibSJQAAYhydLgAAlhC6AABYQugCAGAJoQsAgCWELgAAlhC6AABY0unQLS0t1dSpU+XxeORwOLRly5ag5cYYvfLKK0pPT1fPnj2Vk5Oj48ePB425cOGCZs2apcTERCUlJWnu3Lmqr6+/pQcCAEC063ToNjQ0aMyYMVq1alWby1esWKE333xTa9as0YEDB9S7d29NmjRJly9fDoyZNWuWjh07puLiYm3fvl2lpaWaP3/+zT8KAABigMMYc9Pf+uBwOLR582ZNmzZN0lddrsfj0fPPP68XXnhBkuT1epWWlqb169drxowZ+uyzzzRixAgdPHhQ9957ryRpx44devjhh3XmzBl5PJ52f6/P55PL5ZLX61ViYuLNlg8AQEh0NJdCek63qqpKNTU1ysnJCcxzuVzKyspSWVmZJKmsrExJSUmBwJWknJwcxcXF6cCBA22u1+/3y+fzBU0AAMSakIZuTU2NJCktLS1oflpaWmBZTU2NUlNTg5Z369ZNycnJgTHXKioqksvlCkwZGRmhLBsAACti4urlwsJCeb3ewHT69OlIlwQAQKeFNHTdbrckqba2Nmh+bW1tYJnb7da5c+eCljc1NenChQuBMddyOp1KTEwMmgAAiDUhDd3MzEy53W7t2rUrMM/n8+nAgQPKzs6WJGVnZ6uurk6HDh0KjNm9e7eam5uVlZUVynIAAIgqnf7XfvX19aqsrAzcrqqqUnl5uZKTkzVgwAA999xz+rd/+zfdddddyszM1MsvvyyPxxO4wnn48OGaPHmy5s2bpzVr1qixsVELFizQjBkzOnTlMgAAsarTofvJJ59o/PjxgdsFBQWSpNmzZ2v9+vX61a9+pYaGBs2fP191dXX6zne+ox07dqhHjx6B+7z33ntasGCBJkyYoLi4OOXl5enNN98MwcMBACB63dLndCOFz+kCAKJJRD6nCwAAro/QBQDAEkIXAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AEeFwfDUBt5NOh25paammTp0qj8cjh8OhLVu2BJY1NjZq8eLFGjVqlHr37i2Px6MnnnhCZ8+eDVrHoEGD5HA4gqbly5ff8oMBACCadTp0GxoaNGbMGK1atarVskuXLunw4cN6+eWXdfjwYW3atEkVFRV65JFHWo1dtmyZqqurA9PChQtv7hEAiDotXeyNppsZC8S6bp29Q25urnJzc9tc5nK5VFxcHDTv17/+te6//36dOnVKAwYMCMxPSEiQ2+3u7K+PbS17D2MiWwcQYuEOxmvXz0sIsSrs53S9Xq8cDoeSkpKC5i9fvlwpKSm6++67tXLlSjU1NV13HX6/Xz6fL2gCACDWdLrT7YzLly9r8eLFmjlzphITEwPzn332Wd1zzz1KTk7Wvn37VFhYqOrqar322mttrqeoqEhLly4NZ6m3rjNv9Ts6lrfziFKRPuRL54tY5TDm5p+uDodDmzdv1rRp01ota2xsVF5ens6cOaOSkpKg0L3W2rVr9dRTT6m+vl5Op7PVcr/fL7/fH7jt8/mUkZEhr9d7w/VaFY69EHsSRKlIh+61eKkg0nw+n1wuV7u5FJZOt7GxUT/5yU908uRJ7d69u91gzMrKUlNTk7744gsNHTq01XKn09lmGEeUjb3O138HexVEgWgL2xZcLoFYEfLQbQnc48ePa8+ePUpJSWn3PuXl5YqLi1NqamqoywEAIGp0OnTr6+tVWVkZuF1VVaXy8nIlJycrPT1dP/rRj3T48GFt375dV69eVU1NjSQpOTlZ8fHxKisr04EDBzR+/HglJCSorKxMixYt0k9/+lPdeeedoXtkoRbpt/icxEIERfrp31F0vIh2nT6nW1JSovHjx7eaP3v2bL366qvKzMxs83579uzRQw89pMOHD+sXv/iFPv/8c/n9fmVmZupnP/uZCgoKOnwIuaPHzkMq2vY67FVgUbQ9/dvDywO2dTSXbulCqkixGrrRvreJvT8fYky0vwRuhJcHbOloLvHdywAAWBLWz+nGtFh5e89JLACIGXS6AABYQqfblljpcr+OjhchFosvg2vxskC0odMFAMASQhcAAEsIXQAALOGc7tdxEgvoEi+DazkcvCQQHeh0AQCwhNAFAMASQlf66thTVzum1tUeDwB0AYQuAACWELoAAFhC6AIAYAmhCwCAJYQuAACW8OUYACR1/Qve+d4YRAM6XQAALKHTBSDp7x1gV+146XARDeh0AQCw5PbudLvqW/oWnMQCgKhCpwsAgCW3d6fLSSwAgEV0ugAAWELoAgBgCaELAIAlhC4AAJYQugCCGNP1rsHrao8HsYvQBQDAEkIXAABLCF0AACy5vb8co0VX/JIMTmLhFnWFlwUvA0QbOl0AACwhdAEAsITQBQDAEs7pfh0nsYBWjIm9lwQvA0SrTne6paWlmjp1qjwejxwOh7Zs2RK0fM6cOXI4HEHT5MmTg8ZcuHBBs2bNUmJiopKSkjR37lzV19ff0gMBACDadTp0GxoaNGbMGK1ateq6YyZPnqzq6urA9P777wctnzVrlo4dO6bi4mJt375dpaWlmj9/fuerB2BFrHxLVazUidtXpw8v5+bmKjc394ZjnE6n3G53m8s+++wz7dixQwcPHtS9994rSXrrrbf08MMP6z//8z/l8Xg6WxIAADEhLBdSlZSUKDU1VUOHDtUzzzyj8+fPB5aVlZUpKSkpELiSlJOTo7i4OB04cKDN9fn9fvl8vqAprGLxrTJv8WFBtD7NorUu4FohD93Jkyfrv//7v7Vr1y79x3/8h/bu3avc3FxdvXpVklRTU6PU1NSg+3Tr1k3Jycmqqalpc51FRUVyuVyBKSMjI9RltxYrr+JYqRNdSsvTLlJPv0j/fuBmhfzq5RkzZgR+HjVqlEaPHq0hQ4aopKREEyZMuKl1FhYWqqCgIHDb5/PZCV4AAEIo7J/THTx4sPr27avKykpJktvt1rlz54LGNDU16cKFC9c9D+x0OpWYmBg0WROtb6WjtS7clsLdedLZoqsIe+ieOXNG58+fV3p6uiQpOztbdXV1OnToUGDM7t271dzcrKysrHCXAwBAxHT68HJ9fX2ga5WkqqoqlZeXKzk5WcnJyVq6dKny8vLkdrt14sQJ/epXv9I3v/lNTZo0SZI0fPhwTZ48WfPmzdOaNWvU2NioBQsWaMaMGdF95fK1b69tf1sAb+8RQzrydG15CfHUxu3EYUznnvIlJSUaP358q/mzZ8/W6tWrNW3aNH366aeqq6uTx+PRxIkT9a//+q9KS0sLjL1w4YIWLFigbdu2KS4uTnl5eXrzzTfVp0+fDtXg8/nkcrnk9XrtHmr+OkIXuCWELrqSjuZSp0M3GkRF6F4rHCEce38aALgtdTSX+IcHAABYwj88CBVOYgEA2kGnCwCAJXS6NtHhAsBtjU4XAABL6HQBIFTetvxRwkiZy1G7m0WnCwCAJYQuAACWELoAAFhC6AIAYAmhCwCAJYQuAACWELoAAFhC6AIAYAmhCwCAJYQuAACWELoAAFhC6AIAYAmhCwCAJYQuAACWELoAAFhC6AIAYAmhCwCAJYQuAACWELoAAFhC6AIAYAmhCwCAJYQuAACWELoAAFhC6AIAYAmhCwCAJYQuAACWELoAAFhC6AIAYAmhCwCAJYQuAACWdDp0S0tLNXXqVHk8HjkcDm3ZsiVoucPhaHNauXJlYMygQYNaLV++fPktPxgAAKJZp0O3oaFBY8aM0apVq9pcXl1dHTStXbtWDodDeXl5QeOWLVsWNG7hwoU39wgAAIgR3Tp7h9zcXOXm5l53udvtDrq9detWjR8/XoMHDw6an5CQ0GosAABdWVjP6dbW1ur3v/+95s6d22rZ8uXLlZKSorvvvlsrV65UU1PTddfj9/vl8/mCJgAAYk2nO93OeOedd5SQkKDp06cHzX/22Wd1zz33KDk5Wfv27VNhYaGqq6v12muvtbmeoqIiLV26NJylAgAQdg5jjLnpOzsc2rx5s6ZNm9bm8mHDhukHP/iB3nrrrRuuZ+3atXrqqadUX18vp9PZarnf75ff7w/c9vl8ysjIkNfrVWJi4s2WDwCh9bYj0hXYMfemY6PL8vl8crlc7eZS2DrdP/7xj6qoqNCHH37Y7tisrCw1NTXpiy++0NChQ1stdzqdbYYxAACxJGzndN9++22NHTtWY8aMaXdseXm54uLilJqaGq5yAACIuE53uvX19aqsrAzcrqqqUnl5uZKTkzVgwABJX7XZGzdu1H/913+1un9ZWZkOHDig8ePHKyEhQWVlZVq0aJF++tOf6s4777yFhwIAQHTrdOh+8sknGj9+fOB2QUGBJGn27Nlav369JOmDDz6QMUYzZ85sdX+n06kPPvhAr776qvx+vzIzM7Vo0aLAegAA6Kpu6UKqSOnoCWsAsIoLqW5bHc0lvnsZAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsITQBQDAkm6RLgAAugz+uTvaQacLAIAlhC4AAJYQugAAWELoAgBgCaELAIAlhC4AAJYQugAAWELoAgBgCaELAIAlhC4AAJYQugAAWELoAgBgCaELAIAlhC4AAJYQugAAWELoAgBgCaELAIAlhC4AAJYQugAAWNKp0C0qKtJ9992nhIQEpaamatq0aaqoqAgac/nyZeXn5yslJUV9+vRRXl6eamtrg8acOnVKU6ZMUa9evZSamqoXX3xRTU1Nt/5oAACIYp0K3b179yo/P1/79+9XcXGxGhsbNXHiRDU0NATGLFq0SNu2bdPGjRu1d+9enT17VtOnTw8sv3r1qqZMmaIrV65o3759euedd7R+/Xq98soroXtUAABEIYcxxtzsnb/88kulpqZq7969GjdunLxer/r166cNGzboRz/6kSTp888/1/Dhw1VWVqYHHnhAf/jDH/TDH/5QZ8+eVVpamiRpzZo1Wrx4sb788kvFx8e3+3t9Pp9cLpe8Xq8SExNvtnwAAEKio7l0S+d0vV6vJCk5OVmSdOjQITU2NionJycwZtiwYRowYIDKysokSWVlZRo1alQgcCVp0qRJ8vl8OnbsWJu/x+/3y+fzBU0AAMSamw7d5uZmPffcc3rwwQc1cuRISVJNTY3i4+OVlJQUNDYtLU01NTWBMV8P3JblLcvaUlRUJJfLFZgyMjJutmwAACLmpkM3Pz9fR48e1QcffBDKetpUWFgor9cbmE6fPh323wkAQKh1u5k7LViwQNu3b1dpaan69+8fmO92u3XlyhXV1dUFdbu1tbVyu92BMR9//HHQ+lqubm4Zcy2n0ymn03kzpQIAEDU61ekaY7RgwQJt3rxZu3fvVmZmZtDysWPHqnv37tq1a1dgXkVFhU6dOqXs7GxJUnZ2tv70pz/p3LlzgTHFxcVKTEzUiBEjbuWxAAAQ1TrV6ebn52vDhg3aunWrEhISAudgXS6XevbsKZfLpblz56qgoEDJyclKTEzUwoULlZ2drQceeECSNHHiRI0YMUI/+9nPtGLFCtXU1Oill15Sfn4+3SwAoEvr1EeGHA5Hm/PXrVunOXPmSPrqyzGef/55vf/++/L7/Zo0aZJ+85vfBB06PnnypJ555hmVlJSod+/emj17tpYvX65u3Tr2HoCPDAEAoklHc+mWPqcbKYQuACCaWPmcLgAA6DhCFwAASwhdAAAsIXQBALCE0AUAwBJCFwAASwhdAAAsIXQBALCE0AUAwBJCFwAASwhdAAAsIXQBALCE0AUAwBJCFwAASwhdAAAsIXQBALCE0AUAwBJCFwAASwhdAAAsIXQBALCE0AUAwBJCFwAASwhdAAAsIXQBALCE0AUAwBJCFwAASwhdAAAsIXQBALCE0AUAwBJCFwAASwhdAAAsIXQBALCE0AUAwBJCFwAASwhdAAAsIXQBALCkW6QLuBnGGEmSz+eLcCUAAPw9j1ry6XpiMnQvXrwoScrIyIhwJQAA/N3Fixflcrmuu9xh2ovlKNTc3KyKigqNGDFCp0+fVmJiYqRL6hSfz6eMjAxqt4zaI4PaI4Pa7TLG6OLFi/J4PIqLu/6Z25jsdOPi4vSNb3xDkpSYmBgzf5RrUXtkUHtkUHtkULs9N+pwW3AhFQAAlhC6AABYErOh63Q6tWTJEjmdzkiX0mnUHhnUHhnUHhnUHp1i8kIqAABiUcx2ugAAxBpCFwAASwhdAAAsIXQBALCE0AUAwJKYDd1Vq1Zp0KBB6tGjh7KysvTxxx9HuqQgRUVFuu+++5SQkKDU1FRNmzZNFRUVQWMeeughORyOoOnpp5+OUMV/9+qrr7aqa9iwYYHlly9fVn5+vlJSUtSnTx/l5eWptrY2ghUHGzRoUKv6HQ6H8vPzJUXXdi8tLdXUqVPl8XjkcDi0ZcuWoOXGGL3yyitKT09Xz549lZOTo+PHjweNuXDhgmbNmqXExEQlJSVp7ty5qq+vj2jtjY2NWrx4sUaNGqXevXvL4/HoiSee0NmzZ4PW0dbfavny5RGtXZLmzJnTqq7JkycHjYnG7S6pzee+w+HQypUrA2Misd07sk/syL7l1KlTmjJlinr16qXU1FS9+OKLampqCmvtoRSTofvhhx+qoKBAS5Ys0eHDhzVmzBhNmjRJ586di3RpAXv37lV+fr7279+v4uJiNTY2auLEiWpoaAgaN2/ePFVXVwemFStWRKjiYN/+9reD6vroo48CyxYtWqRt27Zp48aN2rt3r86ePavp06dHsNpgBw8eDKq9uLhYkvTjH/84MCZatntDQ4PGjBmjVatWtbl8xYoVevPNN7VmzRodOHBAvXv31qRJk3T58uXAmFmzZunYsWMqLi7W9u3bVVpaqvnz50e09kuXLunw4cN6+eWXdfjwYW3atEkVFRV65JFHWo1dtmxZ0N9i4cKFEa29xeTJk4Pqev/994OWR+N2lxRUc3V1tdauXSuHw6G8vLygcba3e0f2ie3tW65evaopU6boypUr2rdvn9555x2tX79er7zySlhrDykTg+6//36Tn58fuH316lXj8XhMUVFRBKu6sXPnzhlJZu/evYF53/ve98wvf/nLyBV1HUuWLDFjxoxpc1ldXZ3p3r272bhxY2DeZ599ZiSZsrIySxV2zi9/+UszZMgQ09zcbIyJ3u0uyWzevDlwu7m52bjdbrNy5crAvLq6OuN0Os37779vjDHmz3/+s5FkDh48GBjzhz/8wTgcDvOXv/wlYrW35eOPPzaSzMmTJwPzBg4caF5//fXwFteOtmqfPXu2efTRR697n1ja7o8++qj5/ve/HzQvGrb7tfvEjuxb/ud//sfExcWZmpqawJjVq1ebxMRE4/f77T6AmxRzne6VK1d06NAh5eTkBObFxcUpJydHZWVlEazsxrxeryQpOTk5aP57772nvn37auTIkSosLNSlS5ciUV4rx48fl8fj0eDBgzVr1iydOnVKknTo0CE1NjYGbf9hw4ZpwIABUbn9r1y5onfffVdPPvmkHA5HYH60bvevq6qqUk1NTdC2drlcysrKCmzrsrIyJSUl6d577w2MycnJUVxcnA4cOGC95hvxer1yOBxKSkoKmr98+XKlpKTo7rvv1sqVK6PmUGFJSYlSU1M1dOhQPfPMMzp//nxgWaxs99raWv3+97/X3LlzWy2L9Ha/dp/YkX1LWVmZRo0apbS0tMCYSZMmyefz6dixYxarv3kx91+G/vrXv+rq1atBG12S0tLS9Pnnn0eoqhtrbm7Wc889pwcffFAjR44MzH/88cc1cOBAeTweHTlyRIsXL1ZFRYU2bdoUwWqlrKwsrV+/XkOHDlV1dbWWLl2q7373uzp69KhqamoUHx/faseZlpammpqayBR8A1u2bFFdXZ3mzJkTmBet2/1aLduzred6y7KamhqlpqYGLe/WrZuSk5Oj6u9x+fJlLV68WDNnzgz6rzHPPvus7rnnHiUnJ2vfvn0qLCxUdXW1XnvttQhW+9Wh5enTpyszM1MnTpzQv/zLvyg3N1dlZWW64447Yma7v/POO0pISGh1+ifS272tfWJH9i01NTVtvh5alsWCmAvdWJSfn6+jR48GnReVFHT+Z9SoUUpPT9eECRN04sQJDRkyxHaZAbm5uYGfR48eraysLA0cOFC/+93v1LNnz4jVdTPefvtt5ebmyuPxBOZF63bvqhobG/WTn/xExhitXr06aFlBQUHg59GjRys+Pl5PPfWUioqKIvq9uzNmzAj8PGrUKI0ePVpDhgxRSUmJJkyYELG6Omvt2rWaNWuWevToETQ/0tv9evvE20HMHV7u27ev7rjjjlZXtNXW1srtdkeoqutbsGCBtm/frj179qh///43HJuVlSVJqqystFFahyUlJelb3/qWKisr5Xa7deXKFdXV1QWNicbtf/LkSe3cuVP/9E//dMNx0brdW7bnjZ7rbre71QWETU1NunDhQlT8PVoC9+TJkyouLm73f6NmZWWpqalJX3zxhZ0CO2jw4MHq27dv4DkS7dtdkv74xz+qoqKi3ee/ZHe7X2+f2JF9i9vtbvP10LIsFsRc6MbHx2vs2LHatWtXYF5zc7N27dql7OzsCFYWzBijBQsWaPPmzdq9e7cyMzPbvU95ebkkKT09PczVdU59fb1OnDih9PR0jR07Vt27dw/a/hUVFTp16lRUbX9JWrdunVJTUzVlypQbjovW7Z6ZmSm32x20rX0+nw4cOBDY1tnZ2aqrq9OhQ4cCY3bv3q3m5ubAm4lIaQnc48ePa+fOnUpJSWn3PuXl5YqLi2t16DbSzpw5o/PnzweeI9G83Vu8/fbbGjt2rMaMGdPuWBvbvb19Ykf2LdnZ2frTn/4U9Ian5c3ciBEjwlZ7SEX4Qq6b8sEHHxin02nWr19v/vznP5v58+ebpKSkoCvaIu2ZZ54xLpfLlJSUmOrq6sB06dIlY4wxlZWVZtmyZeaTTz4xVVVVZuvWrWbw4MFm3LhxEa7cmOeff96UlJSYqqoq83//938mJyfH9O3b15w7d84YY8zTTz9tBgwYYHbv3m0++eQTk52dbbKzsyNcdbCrV6+aAQMGmMWLFwfNj7btfvHiRfPpp5+aTz/91Egyr732mvn0008DV/guX77cJCUlma1bt5ojR46YRx991GRmZpq//e1vgXVMnjzZ3H333ebAgQPmo48+MnfddZeZOXNmRGu/cuWKeeSRR0z//v1NeXl50Gug5SrTffv2mddff92Ul5ebEydOmHfffdf069fPPPHEExGt/eLFi+aFF14wZWVlpqqqyuzcudPcc8895q677jKXL18OrCMat3sLr9drevXqZVavXt3q/pHa7u3tE41pf9/S1NRkRo4caSZOnGjKy8vNjh07TL9+/UxhYWFYaw+lmAxdY4x56623zIABA0x8fLy5//77zf79+yNdUhBJbU7r1q0zxhhz6tQpM27cOJOcnGycTqf55je/aV588UXj9XojW7gx5rHHHjPp6ekmPj7efOMb3zCPPfaYqaysDCz/29/+Zn7xi1+YO++80/Tq1cv84z/+o6muro5gxa397//+r5FkKioqguZH23bfs2dPm8+T2bNnG2O++tjQyy+/bNLS0ozT6TQTJkxo9ZjOnz9vZs6cafr06WMSExPNz3/+c3Px4sWI1l5VVXXd18CePXuMMcYcOnTIZGVlGZfLZXr06GGGDx9u/v3f/z0o2CJR+6VLl8zEiRNNv379TPfu3c3AgQPNvHnzWr2pj8bt3uK3v/2t6dmzp6mrq2t1/0ht9/b2icZ0bN/yxRdfmNzcXNOzZ0/Tt29f8/zzz5vGxsaw1h5K/D9dAAAsiblzugAAxCpCFwAASwhdAAAsIXQBALCE0AUAwBJCFwAASwhdAAAsIXQBALCE0AUAwBJCFwAASwhdAAAs+X8t9ZhRyBExZgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, array([ 24, 140]), 'c', (9, 125, 39, 155)),\n",
       " (1, array([30, 79]), 'r', (15, 64, 45, 94)),\n",
       " (2, array([128, 138]), 'c', (113, 123, 143, 153)),\n",
       " (3, array([171, 172]), 'r', (156, 157, 186, 187)),\n",
       " (4, array([128,  23]), 'c', (113, 8, 143, 38)),\n",
       " (5, array([88, 15]), 'c', (73, 0, 103, 30))]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_dataset(num_samples, data_dir, prefix='train'):\n",
    "    '''Builds a Full Dataset with Images, Detector Data, Attribute Data, Queries, Programs and Answers'''\n",
    "    \n",
    "    # Generate Samples\n",
    "    samples = [build_sample() for _ in range(num_samples)]\n",
    "    \n",
    "    # Init dataframes\n",
    "    img_det_df = pd.DataFrame(columns=['filename', 'width', 'height', 'class', 'xmin', 'ymin', 'xmax', 'ymax'])\n",
    "    que2prog_df = pd.DataFrame(columns=['filename', 'answer', 'query_text', 'program_text'])\n",
    "    \n",
    "    img_dir = os.path.join(data_dir, 'images')\n",
    "    shape_map = {'r': 'rectangle', 'c': 'circle'}\n",
    "    \n",
    "    try:\n",
    "        os.makedirs(data_dir)\n",
    "    except:\n",
    "        pass\n",
    "    \n",
    "    try:\n",
    "        os.makedirs(img_dir)\n",
    "    except:\n",
    "        pass\n",
    "    \n",
    "    for i, sample in enumerate(tqdm(samples)):\n",
    "        # Get Data\n",
    "        img, objects, queries, programs, answers = convert_sample(sample)\n",
    "        \n",
    "        # Save Image\n",
    "        filename = f'{i}.jpg'\n",
    "        img_path = os.path.join(img_dir, filename)\n",
    "        cv2.imwrite(img_path, img * 255)\n",
    "        \n",
    "        # Append image data to dataframes\n",
    "        for obj in objects:\n",
    "            # Get object params\n",
    "            color_id, shape, bbox = obj[0], shape_map[obj[2]], obj[3]\n",
    "            \n",
    "            img_det_df = img_det_df.append({'filename': filename, \n",
    "                                            'width': img_size, \n",
    "                                            'height': img_size, \n",
    "                                            'class': 'obj', \n",
    "                                            'xmin': bbox[0], 'ymin': bbox[1],\n",
    "                                            'xmax': bbox[2], 'ymax': bbox[3]}, ignore_index=True)\n",
    "        \n",
    "        # Append text data to dataframe\n",
    "        for answer, query, program in zip(answers, queries, programs):\n",
    "            que2prog_df = que2prog_df.append({'filename': filename,\n",
    "                                              'answer': answer,\n",
    "                                              'query_text': query,\n",
    "                                              'program_text': program}, ignore_index=True)\n",
    "    \n",
    "    # Save to csv files\n",
    "    img_det_df.to_csv(os.path.join(data_dir, f'{prefix}_img_det.csv'), index=False)\n",
    "    que2prog_df.to_csv(os.path.join(data_dir, f'{prefix}_q2p.csv'), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/100 [00:00<?, ?it/s]\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'DataFrame' object has no attribute 'append'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_251043/1739084571.py\u001b[0m in \u001b[0;36m?\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Generate the dataset\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mn_train_imgs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m100\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mn_test_imgs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mbuild_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_train_imgs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'data/train'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprefix\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'train'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0mbuild_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_test_imgs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'data/test'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprefix\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'test'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/tmp/ipykernel_251043/1008171856.py\u001b[0m in \u001b[0;36m?\u001b[0;34m(num_samples, data_dir, prefix)\u001b[0m\n\u001b[1;32m     34\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mobjects\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m             \u001b[0;31m# Get object params\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m             \u001b[0mcolor_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbbox\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshape_map\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 38\u001b[0;31m             img_det_df = img_det_df.append({'filename': filename, \n\u001b[0m\u001b[1;32m     39\u001b[0m                                             \u001b[0;34m'width'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mimg_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m                                             \u001b[0;34m'height'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mimg_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m                                             \u001b[0;34m'class'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m'obj'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.10/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m?\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m   6295\u001b[0m             \u001b[0;32mand\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_accessors\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   6296\u001b[0m             \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_info_axis\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_can_hold_identifiers_and_holds_name\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   6297\u001b[0m         ):\n\u001b[1;32m   6298\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 6299\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getattribute__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m: 'DataFrame' object has no attribute 'append'"
     ]
    }
   ],
   "source": [
    "# Generate the dataset\n",
    "n_train_imgs = 100\n",
    "n_test_imgs = 10\n",
    "\n",
    "build_dataset(n_train_imgs, data_dir='data/train', prefix='train')\n",
    "build_dataset(n_test_imgs, data_dir='data/test', prefix='test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_image_data(csv_file, img_dir):\n",
    "    '''Load the image detection data for training the object detector'''\n",
    "    # Annotations\n",
    "    annot = pd.read_csv(csv_file)\n",
    "    \n",
    "    data = {}\n",
    "    for i in range(len(os.listdir(img_dir))):\n",
    "        # Get all rows belonging to the same scene\n",
    "        scene = annot[annot['filename'] == f'{i}.jpg']\n",
    "        # Read the image\n",
    "        img = cv2.imread(os.path.join(img_dir, f'{i}.jpg'))\n",
    "        \n",
    "        # Add bounding box information for dlib\n",
    "        bboxes = []\n",
    "        for row in scene.values:\n",
    "            x1, y1, x2, y2 = row[-4:]\n",
    "            dlib_box = dlib.rectangle(left=x1 , top=y1, right=x2, bottom=y2)\n",
    "            bboxes.append(dlib_box)\n",
    "        \n",
    "        data[i] = (img, bboxes)\n",
    "    \n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'load_image_data' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[10], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# Load the Image Detection Data\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m det_data \u001b[38;5;241m=\u001b[39m \u001b[43mload_image_data\u001b[49m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdata/train/train_img_det.csv\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdata/train/images\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
      "\u001b[0;31mNameError\u001b[0m: name 'load_image_data' is not defined"
     ]
    }
   ],
   "source": [
    "# Load the Image Detection Data\n",
    "det_data = load_image_data('data/train/train_img_det.csv', 'data/train/images')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'det_data' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[11], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mdet_data\u001b[49m[\u001b[38;5;241m4\u001b[39m][\u001b[38;5;241m1\u001b[39m]\n",
      "\u001b[0;31mNameError\u001b[0m: name 'det_data' is not defined"
     ]
    }
   ],
   "source": [
    "det_data[4][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import dlib\n",
    "import cv2\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import torch\n",
    "from torch import nn\n",
    "from torchvision.datasets import ImageFolder\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import transforms as tfms\n",
    "from PIL import Image\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "\n",
    "def train_detector(train_data, filename='detector.svm'):\n",
    "    '''Trains an object detector (HOG + SVM) and saves the model'''\n",
    "    \n",
    "    # Seperate the images and bounding boxes in different lists.\n",
    "    images = [val[0] for val in train_data.values()]\n",
    "    bounding_boxes = [val[1] for val in train_data.values()]\n",
    "    \n",
    "    # Initialize object detector Options\n",
    "    options = dlib.simple_object_detector_training_options()\n",
    "    options.add_left_right_image_flips = False\n",
    "    options.C = 5\n",
    "    \n",
    "    # Train the model\n",
    "    detector = dlib.train_simple_object_detector(images, bounding_boxes, options)\n",
    "    \n",
    "    # Check results\n",
    "    results = dlib.test_simple_object_detector(images, bounding_boxes, detector)\n",
    "    print(f'Training Results: {results}')\n",
    "    \n",
    "    # Save model\n",
    "    detector.save(filename)\n",
    "    print(f'Saved the model to {filename}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'det_data' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "File \u001b[0;32m<timed eval>:2\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'det_data' is not defined"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Train the Model\n",
    "train_detector(det_data, 'detector.svm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Binarize():\n",
    "    '''PyTorch Transforms Object'''\n",
    "    def __init__(self):\n",
    "        '''Converts Grayscale to Binary (except white every other color is zeroed)'''\n",
    "        pass\n",
    "    \n",
    "    def __call__(self, img_tensor):\n",
    "        '''\n",
    "        Args:\n",
    "            img_tensor (tensor): 0-1 scaled tensor with 1 channel\n",
    "        Returns:\n",
    "            tensor\n",
    "        '''\n",
    "        return (img_tensor > 0.95).float()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the images\n",
    "shapes_data = ImageFolder('data/shapes_data/', transform=tfms.Compose([tfms.Grayscale(), \n",
    "                                                                       tfms.Resize((40, 40)), \n",
    "                                                                       tfms.ToTensor(),\n",
    "                                                                       Binarize()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'circle': 0, 'rectangle': 1}"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shapes_data.class_to_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the dataloader\n",
    "shapes_loader = DataLoader(shapes_data, batch_size=20, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ShapeClassifier(nn.Module):\n",
    "    '''Simple CNN based Image Classifier for Shapes (circle | rectangle)'''\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.conv = nn.Sequential(nn.Conv2d(1, 28, 3, stride=2, padding=1),\n",
    "                                  nn.BatchNorm2d(28),\n",
    "                                  nn.Conv2d(28, 28, 3, stride=2, padding=1),\n",
    "                                  nn.BatchNorm2d(28),\n",
    "                                  nn.Conv2d(28, 28, 3, stride=2, padding=1),\n",
    "                                  nn.BatchNorm2d(28))\n",
    "        self.fc = nn.Linear(700, 1)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        '''Forward Pass'''\n",
    "        # batch_size (N)\n",
    "        N = x.size()[0]\n",
    "        # Extract features with CNN\n",
    "        x = self.conv(x)\n",
    "        # Classifier head\n",
    "        x = self.fc(x.reshape(N, -1))\n",
    "        \n",
    "        return x\n",
    "    \n",
    "    def train_classifier(self, train_loader, lr=0.0001, epochs=10, filename='classifier.pth', device=None):\n",
    "        '''Train the shape classifier'''\n",
    "        # Automatically set device if not provided\n",
    "        if device is None:\n",
    "            device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "        \n",
    "        # Mount to device\n",
    "        self.to(device)\n",
    "        \n",
    "        # Create optimizer and loss function\n",
    "        optimizer = torch.optim.Adam(self.parameters(), lr=lr)\n",
    "        criterion = nn.BCEWithLogitsLoss()\n",
    "        \n",
    "        self.train()\n",
    "        # Start Training\n",
    "        for epoch in range(epochs):\n",
    "            pbar = tqdm(total=len(train_loader), desc='Epoch {}'.format(epoch+1))\n",
    "            losses = []\n",
    "            \n",
    "            for i, (image, label) in enumerate(train_loader):\n",
    "                # Mount to device\n",
    "                image, label = image.to(device).float(), label.to(device)\n",
    "                \n",
    "                # Forward prop\n",
    "                out = self(image)\n",
    "                \n",
    "                # Loss\n",
    "                loss = criterion(out.squeeze(1), label.float())\n",
    "                \n",
    "                # Backprop and Optimization\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "                \n",
    "                # Verbose\n",
    "                losses.append(loss.item())\n",
    "                pbar.update(1)\n",
    "                pbar.set_postfix({'Loss': loss.item()})\n",
    "            \n",
    "            print(f'Epoch {epoch+1}: Mean Loss = {sum(losses)/len(losses)}')\n",
    "            pbar.close()\n",
    "            \n",
    "        # Save model\n",
    "        torch.save(self.state_dict(), filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model\n",
    "shape_classifier = ShapeClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train\n",
    "shape_classifier.train_classifier(shapes_loader, lr=0.0001, epochs=50, filename='classifier.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PerceptionPipe():\n",
    "    '''\n",
    "    Full Perception Pipeline i.e.\n",
    "    detector -> attribute extraction -> structural scene representation\n",
    "    '''\n",
    "    def __init__(self, detector_file, classifer_file, device='cpu'):\n",
    "        # Object detector\n",
    "        self.detector = dlib.simple_object_detector(detector_file)\n",
    "        \n",
    "        # Shape Classifier\n",
    "        self.classifier = ShapeClassifier().to(device)\n",
    "        self.classifier.load_state_dict(torch.load(classifer_file))\n",
    "        self.device = device\n",
    "        \n",
    "        self.colors = np.array([[0,0,255], [0,255,0], [255,0,0], \n",
    "                               [0,156,255], [128,128,128], [0,255,255]])\n",
    "        \n",
    "        self.idx2color = {0: 'red', 1: 'green', 2: 'blue', 3: 'orange', 4: 'gray', 5: 'yellow'}\n",
    "        self.preproc = tfms.Compose([tfms.Grayscale(),\n",
    "                                     tfms.Resize((40, 40)),\n",
    "                                     tfms.ToTensor(),\n",
    "                                     Binarize()])\n",
    "    \n",
    "    \n",
    "    def detect(self, img):\n",
    "        '''Detects and Returns Objects and its centers'''\n",
    "        # Detect\n",
    "        detections = self.detector(img)\n",
    "        objects = []\n",
    "        \n",
    "        for detection in detections:\n",
    "            # Get the bbox coords\n",
    "            x1, y1 = int(detection.left()), int(detection.top())\n",
    "            x2, y2 = int(detection.right()), int(detection.bottom())\n",
    "            \n",
    "            # Clip negative values to zero\n",
    "            x1, y1, x2, y2 = np.array([x1, y1, x2, y2]).clip(min=0).tolist()\n",
    "\n",
    "            # Find the center\n",
    "            center = (int((x1+x2)/2), int((y1+y2)/2))\n",
    "\n",
    "            # Crop the individual object\n",
    "            obj = img[y1:y2, x1:x2]\n",
    "\n",
    "            objects.append((obj, center))\n",
    "            \n",
    "        return objects\n",
    "    \n",
    "    \n",
    "    def extract_attributes(self, x_img, prob=0.5, debug=False):\n",
    "        '''Returns the shape and color of a given object'''\n",
    "        # Load image as PIL instance (color image)\n",
    "        image = Image.fromarray(cv2.cvtColor(x_img, cv2.COLOR_BGR2RGB))\n",
    "        # Preprocess (binarized image)\n",
    "        img = self.preproc(image).unsqueeze(0).to(self.device)\n",
    "        \n",
    "        # Predict Shape\n",
    "        with torch.no_grad():\n",
    "            out = torch.sigmoid(self.classifier(img)).squeeze()\n",
    "            if debug:\n",
    "                print(out)\n",
    "        if out < prob:\n",
    "            shape = 'circle'\n",
    "        else:\n",
    "            shape = 'rectangle'\n",
    "            \n",
    "        # Extract Color\n",
    "        center_pixel = (x_img[20, 20, :]).astype('int')\n",
    "        \n",
    "        color_id = cosine_similarity(center_pixel.reshape(1, -1), self.colors).argmax()\n",
    "        color = self.idx2color[color_id]\n",
    "        \n",
    "#         print(center_pixel)\n",
    "#         print(color_id)\n",
    "        \n",
    "        return shape, color\n",
    "    \n",
    "    def scene_repr(self, img, prob=0.5, debug=False):\n",
    "        '''Returns a structured scene representation as a dataframe'''\n",
    "        # Perform object detection and get the objects\n",
    "        objects = self.detect(img)\n",
    "        \n",
    "        # Init Scene representation\n",
    "        scene_df = pd.DataFrame(columns=['shape', 'color', 'position'])\n",
    "        \n",
    "        for obj, center in objects:\n",
    "            shape, color = self.extract_attributes(obj, prob, debug)\n",
    "            scene_df = scene_df.append({'shape': shape, \n",
    "                                        'color': color, \n",
    "                                        'position': center}, ignore_index=True)\n",
    "        \n",
    "        return scene_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the Model\n",
    "perceiver = PerceptionPipe('detector.svm', 'classifier.pth', device='cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Inference with Detector\n",
    "objects = perceiver.detect(img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(objects[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape = circle\n",
      "Color = blue\n"
     ]
    }
   ],
   "source": [
    "# Inference with Attribute Extractor (Shape Classifier, Color Extractor)\n",
    "shape, color = perceiver.extract_attributes(objects[3][0])\n",
    "print(f'Shape = {shape}\\nColor = {color}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_8862/1678006832.py:88: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  scene_df = scene_df.append({'shape': shape,\n",
      "/tmp/ipykernel_8862/1678006832.py:88: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  scene_df = scene_df.append({'shape': shape,\n",
      "/tmp/ipykernel_8862/1678006832.py:88: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  scene_df = scene_df.append({'shape': shape,\n",
      "/tmp/ipykernel_8862/1678006832.py:88: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  scene_df = scene_df.append({'shape': shape,\n",
      "/tmp/ipykernel_8862/1678006832.py:88: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  scene_df = scene_df.append({'shape': shape,\n",
      "/tmp/ipykernel_8862/1678006832.py:88: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  scene_df = scene_df.append({'shape': shape,\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>shape</th>\n",
       "      <th>color</th>\n",
       "      <th>position</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>rectangle</td>\n",
       "      <td>orange</td>\n",
       "      <td>(171, 172)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>circle</td>\n",
       "      <td>red</td>\n",
       "      <td>(23, 140)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>circle</td>\n",
       "      <td>gray</td>\n",
       "      <td>(127, 20)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>circle</td>\n",
       "      <td>blue</td>\n",
       "      <td>(127, 136)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>rectangle</td>\n",
       "      <td>green</td>\n",
       "      <td>(31, 80)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>circle</td>\n",
       "      <td>yellow</td>\n",
       "      <td>(87, 17)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       shape   color    position\n",
       "0  rectangle  orange  (171, 172)\n",
       "1     circle     red   (23, 140)\n",
       "2     circle    gray   (127, 20)\n",
       "3     circle    blue  (127, 136)\n",
       "4  rectangle   green    (31, 80)\n",
       "5     circle  yellow    (87, 17)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scene = perceiver.scene_repr(img)\n",
    "scene"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fa0ff56e5c0>"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd0AAAHVCAYAAAC0biEDAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAqeUlEQVR4nO3df3BU9b3/8ddGyPIr2ZhAstkSIFDLj/LjKmrM2FIpKRApyiVtBWmFygW1AStRy80dFeHeueHCvepoKfQPAe8oapnhx0BvuRN+hNRLQAQzFKoZwkR+lCRYmOySUJaEfL5/+M3WJYEksPvZ3fB8zJyZ7DmfPXnvye557fucsxuHMcYIAACEXVykCwAA4HZB6AIAYAmhCwCAJYQuAACWELoAAFhC6AIAYAmhCwCAJYQuAACWELoAAFhC6AIAYEnEQnfVqlUaNGiQevTooaysLH388ceRKgUAACsiEroffvihCgoKtGTJEh0+fFhjxozRpEmTdO7cuUiUAwCAFY5I/MODrKws3Xffffr1r38tSWpublZGRoYWLlyof/7nf273/s3NzTp79qwSEhLkcDjCXS4AADdkjNHFixfl8XgUF3f9frabxZokSVeuXNGhQ4dUWFgYmBcXF6ecnByVlZW1eR+/3y+/3x+4/Ze//EUjRowIe60AAHTG6dOn1b9//+sutx66f/3rX3X16lWlpaUFzU9LS9Pnn3/e5n2Kioq0dOnSVvNPnz6txMTEsNQJBHOFYZ3eMKwTQCT4fD5lZGQoISHhhuOsh+7NKCwsVEFBQeB2y4NLTEwkdBHDeO4CXU17pzyth27fvn11xx13qLa2Nmh+bW2t3G53m/dxOp1yOp02ygO+JtzXC1y7fuuXVwCwzPrVy/Hx8Ro7dqx27doVmNfc3Kxdu3YpOzvbdjkAAFgTkcPLBQUFmj17tu69917df//9euONN9TQ0KCf//znkSgH+P8ifSU8nS/Q1UUkdB977DF9+eWXeuWVV1RTU6N/+Id/0I4dO1pdXAUAQFcSkc/p3iqfzyeXyyWv18uFVAiBSHe47Ym5lyhw2+loLvHdy7iNORT9gSvFRo0AOoLQBQDAkpj4nC4QWrHYObbUzKFmIJbR6QIAYAmhCwCAJYQuAACWcE4Xt5FYPJd7ra5zbrfln5gsWbIkwpUA9tDpAgBgCV+OgdtIV+h0W0T3y7atf8V5K+iGEe34cgwAAKIMnS5uA12pw71WdLx8Q93ZtofOF9GGThcAgChDp4vbAJ1uqNnubNtD54tIo9MFACDK0OniNkCnGyrR1uFei44XkdLRXCJ0cRsgdG9VtIfttQhf2MbhZQAAogydLm4DdLq3Ita63K+j44UtdLoAAEQZQhddXFfucqWvHl9Xf4xA10HoAgBgCed0cRvoyp1g+F6+sXwu91qc20W4cU4XAIAoQ+gCAGAJoQsAgCWc08VtgHO6ndGVzuV+Hed1EU6c0wUAIMoQugAAWELoAgBgCaELAIAlhC4AAJZ0i3QBQPi1XOHbla5ijrkPHQAQnS4AANYQugAAWMKXY+A2FMuHmfkHB7eKL8lAOPDlGAAARJmQh25RUZHuu+8+JSQkKDU1VdOmTVNFRUXQmIceekgOhyNoevrpp0NdCoBOWLJkSZfuArv640NsCHno7t27V/n5+dq/f7+Ki4vV2NioiRMnqqGhIWjcvHnzVF1dHZhWrFgR6lIAAIgqIf/I0I4dO4Jur1+/XqmpqTp06JDGjRsXmN+rVy+53e5Q/3qgA2LxI0Qxd+kFgDaE/Zyu1+uVJCUnJwfNf++999S3b1+NHDlShYWFunTp0nXX4ff75fP5giYAAGJNWL8co7m5Wc8995wefPBBjRw5MjD/8ccf18CBA+XxeHTkyBEtXrxYFRUV2rRpU5vrKSoqum2urIRNsdLx0uUCXUVYQzc/P19Hjx7VRx99FDR//vz5gZ9HjRql9PR0TZgwQSdOnNCQIUNaraewsFAFBQWB2z6fTxkZGeErHACAMAhb6C5YsEDbt29XaWmp+vfvf8OxWVlZkqTKyso2Q9fpdMrpdIalTiB6O146XKCrCXnoGmO0cOFCbd68WSUlJcrMzGz3PuXl5ZKk9PT0UJcDAEDUCHno5ufna8OGDdq6dasSEhJUU1MjSXK5XOrZs6dOnDihDRs26OGHH1ZKSoqOHDmiRYsWady4cRo9enSoywE64drO0nbnS2cLdHUhD93Vq1dL+uoLML5u3bp1mjNnjuLj47Vz50698cYbamhoUEZGhvLy8vTSSy+FuhQAAKIK370MdEg4ut7oful1pU8M8E1UCDe+exkAgCjDP7EHOqSjXWlLRxzdXSyAyODwMoAbiuXDzBxWhi0cXgYAIMoQugAAWELoAgBgCed0AbQr1s7rci4XtnFOFwCAKEOnC6DDor3jpcNFpNDpAgAQZeh0AXRatHW8dLiINDpdAACiDJ0ugFtmu/Ols0W0odMFACDK0OkCCLlQd750toh2dLoAAEQZOl0AEdHSDdPFoivoaC4RugAA3CIOLwMAEGUIXQAALCF0AQCwhNAFAMASQhcAAEsIXQAALCF0AQCwhNAFAMASQhcAAEsIXQAALCF0AQCwhNAFAMASQhcAAEsIXQAALCF0AQCwhNAFAMASQhcAAEsIXQAALAl56L766qtyOBxB07BhwwLLL1++rPz8fKWkpKhPnz7Ky8tTbW1tqMsAACDqhKXT/fa3v63q6urA9NFHHwWWLVq0SNu2bdPGjRu1d+9enT17VtOnTw9HGQAARJVuYVlpt25yu92t5nu9Xr399tvasGGDvv/970uS1q1bp+HDh2v//v164IEHwlEOAABRISyd7vHjx+XxeDR48GDNmjVLp06dkiQdOnRIjY2NysnJCYwdNmyYBgwYoLKysuuuz+/3y+fzBU0AAMSakIduVlaW1q9frx07dmj16tWqqqrSd7/7XV28eFE1NTWKj49XUlJS0H3S0tJUU1Nz3XUWFRXJ5XIFpoyMjFCXDQBA2IX88HJubm7g59GjRysrK0sDBw7U7373O/Xs2fOm1llYWKiCgoLAbZ/PR/ACAGJO2D8ylJSUpG9961uqrKyU2+3WlStXVFdXFzSmtra2zXPALZxOpxITE4MmAABiTdhDt76+XidOnFB6errGjh2r7t27a9euXYHlFRUVOnXqlLKzs8NdCgAAERXyw8svvPCCpk6dqoEDB+rs2bNasmSJ7rjjDs2cOVMul0tz585VQUGBkpOTlZiYqIULFyo7O5srlwEAXV7IQ/fMmTOaOXOmzp8/r379+uk73/mO9u/fr379+kmSXn/9dcXFxSkvL09+v1+TJk3Sb37zm1CXAQBA1HEYY0yki+gsn88nl8slr9fL+V0AQMR1NJf47mUAACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsKRbpAuIFQ45Il2CNUYm0iUAQJdEpwsAgCWELgAAlhC6AABYQugCAGAJoQsAgCWELgAAlhC6AABYQugCAGAJoQsAgCWELgAAlhC6AABYQugCAGAJoQsAgCWELgAAlhC6AABYQugCAGAJoQsAgCUhD91BgwbJ4XC0mvLz8yVJDz30UKtlTz/9dKjLAAAg6nQL9QoPHjyoq1evBm4fPXpUP/jBD/TjH/84MG/evHlatmxZ4HavXr1CXQYAAFEn5KHbr1+/oNvLly/XkCFD9L3vfS8wr1evXnK73aH+1QAARLWwntO9cuWK3n33XT355JNyOByB+e+995769u2rkSNHqrCwUJcuXbrhevx+v3w+X9AEAECsCXmn+3VbtmxRXV2d5syZE5j3+OOPa+DAgfJ4PDpy5IgWL16siooKbdq06brrKSoq0tKlS8NZKgAAYecwxphwrXzSpEmKj4/Xtm3brjtm9+7dmjBhgiorKzVkyJA2x/j9fvn9/sBtn8+njIwMeb1eJSYmhrzutjjkaH9QF2EUtqcEAHRJPp9PLper3VwKW6d78uRJ7dy584YdrCRlZWVJ0g1D1+l0yul0hrxGAABsCts53XXr1ik1NVVTpky54bjy8nJJUnp6erhKAQAgKoSl021ubta6des0e/Zsdev2919x4sQJbdiwQQ8//LBSUlJ05MgRLVq0SOPGjdPo0aPDUQoAAFEjLKG7c+dOnTp1Sk8++WTQ/Pj4eO3cuVNvvPGGGhoalJGRoby8PL300kvhKAMAgKgS1gupwqWjJ6xDiQupAADX09Fc4ruXAQCwhNAFAMASQhcAAEsIXQAALCF0AQCwhNAFAMASQhcAAEsIXQAALCF0AQCwhNAFAMASQhcAAEsIXQAALCF0AQCwhNAFAMASQhcAAEsIXQAALCF0AQCwhNAFAMASQhcAAEsIXQAALCF0AQCwhNAFAMASQhcAAEsIXQAALCF0AQCwhNAFAMASQhcAAEsIXQAALCF0AQCwpFukC4gVRibSJQAAYhydLgAAlhC6AABYQugCAGAJoQsAgCWELgAAlhC6AABY0unQLS0t1dSpU+XxeORwOLRly5ag5cYYvfLKK0pPT1fPnj2Vk5Oj48ePB425cOGCZs2apcTERCUlJWnu3Lmqr6+/pQcCAEC063ToNjQ0aMyYMVq1alWby1esWKE333xTa9as0YEDB9S7d29NmjRJly9fDoyZNWuWjh07puLiYm3fvl2lpaWaP3/+zT8KAABigMMYc9Pf+uBwOLR582ZNmzZN0lddrsfj0fPPP68XXnhBkuT1epWWlqb169drxowZ+uyzzzRixAgdPHhQ9957ryRpx44devjhh3XmzBl5PJ52f6/P55PL5ZLX61ViYuLNlg8AQEh0NJdCek63qqpKNTU1ysnJCcxzuVzKyspSWVmZJKmsrExJSUmBwJWknJwcxcXF6cCBA22u1+/3y+fzBU0AAMSakIZuTU2NJCktLS1oflpaWmBZTU2NUlNTg5Z369ZNycnJgTHXKioqksvlCkwZGRmhLBsAACti4urlwsJCeb3ewHT69OlIlwQAQKeFNHTdbrckqba2Nmh+bW1tYJnb7da5c+eCljc1NenChQuBMddyOp1KTEwMmgAAiDUhDd3MzEy53W7t2rUrMM/n8+nAgQPKzs6WJGVnZ6uurk6HDh0KjNm9e7eam5uVlZUVynIAAIgqnf7XfvX19aqsrAzcrqqqUnl5uZKTkzVgwAA999xz+rd/+zfdddddyszM1MsvvyyPxxO4wnn48OGaPHmy5s2bpzVr1qixsVELFizQjBkzOnTlMgAAsarTofvJJ59o/PjxgdsFBQWSpNmzZ2v9+vX61a9+pYaGBs2fP191dXX6zne+ox07dqhHjx6B+7z33ntasGCBJkyYoLi4OOXl5enNN98MwcMBACB63dLndCOFz+kCAKJJRD6nCwAAro/QBQDAEkIXAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AEeFwfDUBt5NOh25paammTp0qj8cjh8OhLVu2BJY1NjZq8eLFGjVqlHr37i2Px6MnnnhCZ8+eDVrHoEGD5HA4gqbly5ff8oMBACCadTp0GxoaNGbMGK1atarVskuXLunw4cN6+eWXdfjwYW3atEkVFRV65JFHWo1dtmyZqqurA9PChQtv7hEAiDotXeyNppsZC8S6bp29Q25urnJzc9tc5nK5VFxcHDTv17/+te6//36dOnVKAwYMCMxPSEiQ2+3u7K+PbS17D2MiWwcQYuEOxmvXz0sIsSrs53S9Xq8cDoeSkpKC5i9fvlwpKSm6++67tXLlSjU1NV13HX6/Xz6fL2gCACDWdLrT7YzLly9r8eLFmjlzphITEwPzn332Wd1zzz1KTk7Wvn37VFhYqOrqar322mttrqeoqEhLly4NZ6m3rjNv9Ts6lrfziFKRPuRL54tY5TDm5p+uDodDmzdv1rRp01ota2xsVF5ens6cOaOSkpKg0L3W2rVr9dRTT6m+vl5Op7PVcr/fL7/fH7jt8/mUkZEhr9d7w/VaFY69EHsSRKlIh+61eKkg0nw+n1wuV7u5FJZOt7GxUT/5yU908uRJ7d69u91gzMrKUlNTk7744gsNHTq01XKn09lmGEeUjb3O138HexVEgWgL2xZcLoFYEfLQbQnc48ePa8+ePUpJSWn3PuXl5YqLi1NqamqoywEAIGp0OnTr6+tVWVkZuF1VVaXy8nIlJycrPT1dP/rRj3T48GFt375dV69eVU1NjSQpOTlZ8fHxKisr04EDBzR+/HglJCSorKxMixYt0k9/+lPdeeedoXtkoRbpt/icxEIERfrp31F0vIh2nT6nW1JSovHjx7eaP3v2bL366qvKzMxs83579uzRQw89pMOHD+sXv/iFPv/8c/n9fmVmZupnP/uZCgoKOnwIuaPHzkMq2vY67FVgUbQ9/dvDywO2dTSXbulCqkixGrrRvreJvT8fYky0vwRuhJcHbOloLvHdywAAWBLWz+nGtFh5e89JLACIGXS6AABYQqfblljpcr+OjhchFosvg2vxskC0odMFAMASQhcAAEsIXQAALOGc7tdxEgvoEi+DazkcvCQQHeh0AQCwhNAFAMASQlf66thTVzum1tUeDwB0AYQuAACWELoAAFhC6AIAYAmhCwCAJYQuAACW8OUYACR1/Qve+d4YRAM6XQAALKHTBSDp7x1gV+146XARDeh0AQCw5PbudLvqW/oWnMQCgKhCpwsAgCW3d6fLSSwAgEV0ugAAWELoAgBgCaELAIAlhC4AAJYQugCCGNP1rsHrao8HsYvQBQDAEkIXAABLCF0AACy5vb8co0VX/JIMTmLhFnWFlwUvA0QbOl0AACwhdAEAsITQBQDAEs7pfh0nsYBWjIm9lwQvA0SrTne6paWlmjp1qjwejxwOh7Zs2RK0fM6cOXI4HEHT5MmTg8ZcuHBBs2bNUmJiopKSkjR37lzV19ff0gMBACDadTp0GxoaNGbMGK1ateq6YyZPnqzq6urA9P777wctnzVrlo4dO6bi4mJt375dpaWlmj9/fuerB2BFrHxLVazUidtXpw8v5+bmKjc394ZjnE6n3G53m8s+++wz7dixQwcPHtS9994rSXrrrbf08MMP6z//8z/l8Xg6WxIAADEhLBdSlZSUKDU1VUOHDtUzzzyj8+fPB5aVlZUpKSkpELiSlJOTo7i4OB04cKDN9fn9fvl8vqAprGLxrTJv8WFBtD7NorUu4FohD93Jkyfrv//7v7Vr1y79x3/8h/bu3avc3FxdvXpVklRTU6PU1NSg+3Tr1k3Jycmqqalpc51FRUVyuVyBKSMjI9RltxYrr+JYqRNdSsvTLlJPv0j/fuBmhfzq5RkzZgR+HjVqlEaPHq0hQ4aopKREEyZMuKl1FhYWqqCgIHDb5/PZCV4AAEIo7J/THTx4sPr27avKykpJktvt1rlz54LGNDU16cKFC9c9D+x0OpWYmBg0WROtb6WjtS7clsLdedLZoqsIe+ieOXNG58+fV3p6uiQpOztbdXV1OnToUGDM7t271dzcrKysrHCXAwBAxHT68HJ9fX2ga5WkqqoqlZeXKzk5WcnJyVq6dKny8vLkdrt14sQJ/epXv9I3v/lNTZo0SZI0fPhwTZ48WfPmzdOaNWvU2NioBQsWaMaMGdF95fK1b69tf1sAb+8RQzrydG15CfHUxu3EYUznnvIlJSUaP358q/mzZ8/W6tWrNW3aNH366aeqq6uTx+PRxIkT9a//+q9KS0sLjL1w4YIWLFigbdu2KS4uTnl5eXrzzTfVp0+fDtXg8/nkcrnk9XrtHmr+OkIXuCWELrqSjuZSp0M3GkRF6F4rHCEce38aALgtdTSX+IcHAABYwj88CBVOYgEA2kGnCwCAJXS6NtHhAsBtjU4XAABL6HQBIFTetvxRwkiZy1G7m0WnCwCAJYQuAACWELoAAFhC6AIAYAmhCwCAJYQuAACWELoAAFhC6AIAYAmhCwCAJYQuAACWELoAAFhC6AIAYAmhCwCAJYQuAACWELoAAFhC6AIAYAmhCwCAJYQuAACWELoAAFhC6AIAYAmhCwCAJYQuAACWELoAAFhC6AIAYAmhCwCAJYQuAACWELoAAFhC6AIAYAmhCwCAJYQuAACWdDp0S0tLNXXqVHk8HjkcDm3ZsiVoucPhaHNauXJlYMygQYNaLV++fPktPxgAAKJZp0O3oaFBY8aM0apVq9pcXl1dHTStXbtWDodDeXl5QeOWLVsWNG7hwoU39wgAAIgR3Tp7h9zcXOXm5l53udvtDrq9detWjR8/XoMHDw6an5CQ0GosAABdWVjP6dbW1ur3v/+95s6d22rZ8uXLlZKSorvvvlsrV65UU1PTddfj9/vl8/mCJgAAYk2nO93OeOedd5SQkKDp06cHzX/22Wd1zz33KDk5Wfv27VNhYaGqq6v12muvtbmeoqIiLV26NJylAgAQdg5jjLnpOzsc2rx5s6ZNm9bm8mHDhukHP/iB3nrrrRuuZ+3atXrqqadUX18vp9PZarnf75ff7w/c9vl8ysjIkNfrVWJi4s2WDwCh9bYj0hXYMfemY6PL8vl8crlc7eZS2DrdP/7xj6qoqNCHH37Y7tisrCw1NTXpiy++0NChQ1stdzqdbYYxAACxJGzndN9++22NHTtWY8aMaXdseXm54uLilJqaGq5yAACIuE53uvX19aqsrAzcrqqqUnl5uZKTkzVgwABJX7XZGzdu1H/913+1un9ZWZkOHDig8ePHKyEhQWVlZVq0aJF++tOf6s4777yFhwIAQHTrdOh+8sknGj9+fOB2QUGBJGn27Nlav369JOmDDz6QMUYzZ85sdX+n06kPPvhAr776qvx+vzIzM7Vo0aLAegAA6Kpu6UKqSOnoCWsAsIoLqW5bHc0lvnsZAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsITQBQDAEkIXAABLCF0AACwhdAEAsITQBQDAkm6RLgAAugz+uTvaQacLAIAlhC4AAJYQugAAWELoAgBgCaELAIAlhC4AAJYQugAAWELoAgBgCaELAIAlhC4AAJYQugAAWELoAgBgCaELAIAlhC4AAJYQugAAWELoAgBgCaELAIAlhC4AAJYQugAAWNKp0C0qKtJ9992nhIQEpaamatq0aaqoqAgac/nyZeXn5yslJUV9+vRRXl6eamtrg8acOnVKU6ZMUa9evZSamqoXX3xRTU1Nt/5oAACIYp0K3b179yo/P1/79+9XcXGxGhsbNXHiRDU0NATGLFq0SNu2bdPGjRu1d+9enT17VtOnTw8sv3r1qqZMmaIrV65o3759euedd7R+/Xq98soroXtUAABEIYcxxtzsnb/88kulpqZq7969GjdunLxer/r166cNGzboRz/6kSTp888/1/Dhw1VWVqYHHnhAf/jDH/TDH/5QZ8+eVVpamiRpzZo1Wrx4sb788kvFx8e3+3t9Pp9cLpe8Xq8SExNvtnwAAEKio7l0S+d0vV6vJCk5OVmSdOjQITU2NionJycwZtiwYRowYIDKysokSWVlZRo1alQgcCVp0qRJ8vl8OnbsWJu/x+/3y+fzBU0AAMSamw7d5uZmPffcc3rwwQc1cuRISVJNTY3i4+OVlJQUNDYtLU01NTWBMV8P3JblLcvaUlRUJJfLFZgyMjJutmwAACLmpkM3Pz9fR48e1QcffBDKetpUWFgor9cbmE6fPh323wkAQKh1u5k7LViwQNu3b1dpaan69+8fmO92u3XlyhXV1dUFdbu1tbVyu92BMR9//HHQ+lqubm4Zcy2n0ymn03kzpQIAEDU61ekaY7RgwQJt3rxZu3fvVmZmZtDysWPHqnv37tq1a1dgXkVFhU6dOqXs7GxJUnZ2tv70pz/p3LlzgTHFxcVKTEzUiBEjbuWxAAAQ1TrV6ebn52vDhg3aunWrEhISAudgXS6XevbsKZfLpblz56qgoEDJyclKTEzUwoULlZ2drQceeECSNHHiRI0YMUI/+9nPtGLFCtXU1Oill15Sfn4+3SwAoEvr1EeGHA5Hm/PXrVunOXPmSPrqyzGef/55vf/++/L7/Zo0aZJ+85vfBB06PnnypJ555hmVlJSod+/emj17tpYvX65u3Tr2HoCPDAEAoklHc+mWPqcbKYQuACCaWPmcLgAA6DhCFwAASwhdAAAsIXQBALCE0AUAwBJCFwAASwhdAAAsIXQBALCE0AUAwBJCFwAASwhdAAAsIXQBALCE0AUAwBJCFwAASwhdAAAsIXQBALCE0AUAwBJCFwAASwhdAAAsIXQBALCE0AUAwBJCFwAASwhdAAAsIXQBALCE0AUAwBJCFwAASwhdAAAsIXQBALCE0AUAwBJCFwAASwhdAAAsIXQBALCE0AUAwBJCFwAASwhdAAAsIXQBALCkW6QLuBnGGEmSz+eLcCUAAPw9j1ry6XpiMnQvXrwoScrIyIhwJQAA/N3Fixflcrmuu9xh2ovlKNTc3KyKigqNGDFCp0+fVmJiYqRL6hSfz6eMjAxqt4zaI4PaI4Pa7TLG6OLFi/J4PIqLu/6Z25jsdOPi4vSNb3xDkpSYmBgzf5RrUXtkUHtkUHtkULs9N+pwW3AhFQAAlhC6AABYErOh63Q6tWTJEjmdzkiX0mnUHhnUHhnUHhnUHp1i8kIqAABiUcx2ugAAxBpCFwAASwhdAAAsIXQBALCE0AUAwJKYDd1Vq1Zp0KBB6tGjh7KysvTxxx9HuqQgRUVFuu+++5SQkKDU1FRNmzZNFRUVQWMeeughORyOoOnpp5+OUMV/9+qrr7aqa9iwYYHlly9fVn5+vlJSUtSnTx/l5eWptrY2ghUHGzRoUKv6HQ6H8vPzJUXXdi8tLdXUqVPl8XjkcDi0ZcuWoOXGGL3yyitKT09Xz549lZOTo+PHjweNuXDhgmbNmqXExEQlJSVp7ty5qq+vj2jtjY2NWrx4sUaNGqXevXvL4/HoiSee0NmzZ4PW0dbfavny5RGtXZLmzJnTqq7JkycHjYnG7S6pzee+w+HQypUrA2Misd07sk/syL7l1KlTmjJlinr16qXU1FS9+OKLampqCmvtoRSTofvhhx+qoKBAS5Ys0eHDhzVmzBhNmjRJ586di3RpAXv37lV+fr7279+v4uJiNTY2auLEiWpoaAgaN2/ePFVXVwemFStWRKjiYN/+9reD6vroo48CyxYtWqRt27Zp48aN2rt3r86ePavp06dHsNpgBw8eDKq9uLhYkvTjH/84MCZatntDQ4PGjBmjVatWtbl8xYoVevPNN7VmzRodOHBAvXv31qRJk3T58uXAmFmzZunYsWMqLi7W9u3bVVpaqvnz50e09kuXLunw4cN6+eWXdfjwYW3atEkVFRV65JFHWo1dtmxZ0N9i4cKFEa29xeTJk4Pqev/994OWR+N2lxRUc3V1tdauXSuHw6G8vLygcba3e0f2ie3tW65evaopU6boypUr2rdvn9555x2tX79er7zySlhrDykTg+6//36Tn58fuH316lXj8XhMUVFRBKu6sXPnzhlJZu/evYF53/ve98wvf/nLyBV1HUuWLDFjxoxpc1ldXZ3p3r272bhxY2DeZ599ZiSZsrIySxV2zi9/+UszZMgQ09zcbIyJ3u0uyWzevDlwu7m52bjdbrNy5crAvLq6OuN0Os37779vjDHmz3/+s5FkDh48GBjzhz/8wTgcDvOXv/wlYrW35eOPPzaSzMmTJwPzBg4caF5//fXwFteOtmqfPXu2efTRR697n1ja7o8++qj5/ve/HzQvGrb7tfvEjuxb/ud//sfExcWZmpqawJjVq1ebxMRE4/f77T6AmxRzne6VK1d06NAh5eTkBObFxcUpJydHZWVlEazsxrxeryQpOTk5aP57772nvn37auTIkSosLNSlS5ciUV4rx48fl8fj0eDBgzVr1iydOnVKknTo0CE1NjYGbf9hw4ZpwIABUbn9r1y5onfffVdPPvmkHA5HYH60bvevq6qqUk1NTdC2drlcysrKCmzrsrIyJSUl6d577w2MycnJUVxcnA4cOGC95hvxer1yOBxKSkoKmr98+XKlpKTo7rvv1sqVK6PmUGFJSYlSU1M1dOhQPfPMMzp//nxgWaxs99raWv3+97/X3LlzWy2L9Ha/dp/YkX1LWVmZRo0apbS0tMCYSZMmyefz6dixYxarv3kx91+G/vrXv+rq1atBG12S0tLS9Pnnn0eoqhtrbm7Wc889pwcffFAjR44MzH/88cc1cOBAeTweHTlyRIsXL1ZFRYU2bdoUwWqlrKwsrV+/XkOHDlV1dbWWLl2q7373uzp69KhqamoUHx/faseZlpammpqayBR8A1u2bFFdXZ3mzJkTmBet2/1aLduzred6y7KamhqlpqYGLe/WrZuSk5Oj6u9x+fJlLV68WDNnzgz6rzHPPvus7rnnHiUnJ2vfvn0qLCxUdXW1XnvttQhW+9Wh5enTpyszM1MnTpzQv/zLvyg3N1dlZWW64447Yma7v/POO0pISGh1+ifS272tfWJH9i01NTVtvh5alsWCmAvdWJSfn6+jR48GnReVFHT+Z9SoUUpPT9eECRN04sQJDRkyxHaZAbm5uYGfR48eraysLA0cOFC/+93v1LNnz4jVdTPefvtt5ebmyuPxBOZF63bvqhobG/WTn/xExhitXr06aFlBQUHg59GjRys+Pl5PPfWUioqKIvq9uzNmzAj8PGrUKI0ePVpDhgxRSUmJJkyYELG6Omvt2rWaNWuWevToETQ/0tv9evvE20HMHV7u27ev7rjjjlZXtNXW1srtdkeoqutbsGCBtm/frj179qh///43HJuVlSVJqqystFFahyUlJelb3/qWKisr5Xa7deXKFdXV1QWNicbtf/LkSe3cuVP/9E//dMNx0brdW7bnjZ7rbre71QWETU1NunDhQlT8PVoC9+TJkyouLm73f6NmZWWpqalJX3zxhZ0CO2jw4MHq27dv4DkS7dtdkv74xz+qoqKi3ee/ZHe7X2+f2JF9i9vtbvP10LIsFsRc6MbHx2vs2LHatWtXYF5zc7N27dql7OzsCFYWzBijBQsWaPPmzdq9e7cyMzPbvU95ebkkKT09PczVdU59fb1OnDih9PR0jR07Vt27dw/a/hUVFTp16lRUbX9JWrdunVJTUzVlypQbjovW7Z6ZmSm32x20rX0+nw4cOBDY1tnZ2aqrq9OhQ4cCY3bv3q3m5ubAm4lIaQnc48ePa+fOnUpJSWn3PuXl5YqLi2t16DbSzpw5o/PnzweeI9G83Vu8/fbbGjt2rMaMGdPuWBvbvb19Ykf2LdnZ2frTn/4U9Ian5c3ciBEjwlZ7SEX4Qq6b8sEHHxin02nWr19v/vznP5v58+ebpKSkoCvaIu2ZZ54xLpfLlJSUmOrq6sB06dIlY4wxlZWVZtmyZeaTTz4xVVVVZuvWrWbw4MFm3LhxEa7cmOeff96UlJSYqqoq83//938mJyfH9O3b15w7d84YY8zTTz9tBgwYYHbv3m0++eQTk52dbbKzsyNcdbCrV6+aAQMGmMWLFwfNj7btfvHiRfPpp5+aTz/91Egyr732mvn0008DV/guX77cJCUlma1bt5ojR46YRx991GRmZpq//e1vgXVMnjzZ3H333ebAgQPmo48+MnfddZeZOXNmRGu/cuWKeeSRR0z//v1NeXl50Gug5SrTffv2mddff92Ul5ebEydOmHfffdf069fPPPHEExGt/eLFi+aFF14wZWVlpqqqyuzcudPcc8895q677jKXL18OrCMat3sLr9drevXqZVavXt3q/pHa7u3tE41pf9/S1NRkRo4caSZOnGjKy8vNjh07TL9+/UxhYWFYaw+lmAxdY4x56623zIABA0x8fLy5//77zf79+yNdUhBJbU7r1q0zxhhz6tQpM27cOJOcnGycTqf55je/aV588UXj9XojW7gx5rHHHjPp6ekmPj7efOMb3zCPPfaYqaysDCz/29/+Zn7xi1+YO++80/Tq1cv84z/+o6muro5gxa397//+r5FkKioqguZH23bfs2dPm8+T2bNnG2O++tjQyy+/bNLS0ozT6TQTJkxo9ZjOnz9vZs6cafr06WMSExPNz3/+c3Px4sWI1l5VVXXd18CePXuMMcYcOnTIZGVlGZfLZXr06GGGDx9u/v3f/z0o2CJR+6VLl8zEiRNNv379TPfu3c3AgQPNvHnzWr2pj8bt3uK3v/2t6dmzp6mrq2t1/0ht9/b2icZ0bN/yxRdfmNzcXNOzZ0/Tt29f8/zzz5vGxsaw1h5K/D9dAAAsiblzugAAxCpCFwAASwhdAAAsIXQBALCE0AUAwBJCFwAASwhdAAAsIXQBALCE0AUAwBJCFwAASwhdAAAs+X8t9ZhRyBExZgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchtext.data import BucketIterator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import os\n",
    "import torch\n",
    "from torch import nn\n",
    "from torchtext.data import Field, TabularDataset, BucketIterator\n",
    "import spacy\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "class Preprocessor():\n",
    "    '''Preprocessor for preparing Queries and Programs for Seq2Seq'''\n",
    "    def __init__(self, train_csv):\n",
    "        self.spacy_en = spacy.load(\"en_core_web_sm\")\n",
    "        \n",
    "        # Create fields\n",
    "        self.que_f = Field(tokenize=self.tokenizer, use_vocab=True, init_token=\"<sos>\", eos_token=\"<eos>\", lower=True)\n",
    "        self.prog_f = Field(tokenize=self.tokenizer, use_vocab=True, init_token=\"<sos>\", eos_token=\"<eos>\", lower=True)\n",
    "        \n",
    "        # Preprocess\n",
    "        self.train_data = self.preprocess(train_csv)\n",
    "        \n",
    "    def tokenizer(self, text):\n",
    "        tokens = [tok.text.lower() for tok in self.spacy_en.tokenizer(text)]\n",
    "\n",
    "        updated_tokens = []\n",
    "        for i, tok in enumerate(tokens):\n",
    "            if tok == '<':\n",
    "                updated_tokens.append(''.join(tokens[i:i+3]))\n",
    "            elif tok in ['nxt', '>']:\n",
    "                continue\n",
    "            else:\n",
    "                updated_tokens.append(tok)\n",
    "\n",
    "        return updated_tokens\n",
    "    \n",
    "    def preprocess(self, train_csv):\n",
    "        '''Returns the Dataset'''\n",
    "        \n",
    "        # Create the fields\n",
    "        self.fields = {'query_text': ('query', self.que_f), 'program_text': ('program', self.prog_f)}\n",
    "        \n",
    "        # Create dataset object\n",
    "        train_data = TabularDataset.splits(path=\"./\", \n",
    "                                           train=train_csv, \n",
    "                                           format=\"csv\", \n",
    "                                           fields=self.fields)[0]\n",
    "        \n",
    "        # Build vocabulary\n",
    "        self.que_f.build_vocab(train_data, max_size=100, min_freq=1)\n",
    "        self.prog_f.build_vocab(train_data, max_size=100, min_freq=1, specials=['<nxt>'])\n",
    "        \n",
    "        return train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the preprocessor object and preprocess\n",
    "preproc = Preprocessor('data/train/train_q2p.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the dataset object\n",
    "train_data = preproc.train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torchtext.data.dataset.TabularDataset"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(<bound method Vocab._default_unk_index of <torchtext.vocab.Vocab object at 0x7fa08b6a7e80>>,\n",
       "            {'<unk>': 0,\n",
       "             '<pad>': 1,\n",
       "             '<sos>': 2,\n",
       "             '<eos>': 3,\n",
       "             '<nxt>': 4,\n",
       "             'filter': 5,\n",
       "             'query': 6,\n",
       "             'shape': 7,\n",
       "             'position': 8,\n",
       "             'relate': 9,\n",
       "             'orange': 10,\n",
       "             'gray': 11,\n",
       "             'count': 12,\n",
       "             'isleft': 13,\n",
       "             'istop': 14,\n",
       "             'closest': 15,\n",
       "             'red': 16,\n",
       "             'yellow': 17,\n",
       "             'furthest': 18,\n",
       "             'blue': 19,\n",
       "             'green': 20})"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Looking at the Vocabulary\n",
    "preproc.prog_f.vocab.stoi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(<bound method Vocab._default_unk_index of <torchtext.vocab.Vocab object at 0x7fa08b6a7fa0>>,\n",
       "            {'<unk>': 0,\n",
       "             '<pad>': 1,\n",
       "             '<sos>': 2,\n",
       "             '<eos>': 3,\n",
       "             'the': 4,\n",
       "             '?': 5,\n",
       "             'object': 6,\n",
       "             'is': 7,\n",
       "             'shape': 8,\n",
       "             'there': 9,\n",
       "             'what': 10,\n",
       "             'a': 11,\n",
       "             'on': 12,\n",
       "             'of': 13,\n",
       "             'orange': 14,\n",
       "             'gray': 15,\n",
       "             'are': 16,\n",
       "             'as': 17,\n",
       "             'how': 18,\n",
       "             'many': 19,\n",
       "             'objects': 20,\n",
       "             'same': 21,\n",
       "             'left': 22,\n",
       "             'top': 23,\n",
       "             'closest': 24,\n",
       "             'to': 25,\n",
       "             'red': 26,\n",
       "             'yellow': 27,\n",
       "             'from': 28,\n",
       "             'furthest': 29,\n",
       "             'blue': 30,\n",
       "             'green': 31})"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Looking at the Vocabulary\n",
    "preproc.que_f.vocab.stoi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training hyperparameters\n",
    "num_epochs = 3\n",
    "learning_rate = 3e-4\n",
    "batch_size = 8\n",
    "num_steps = len(train_data) / batch_size\n",
    "\n",
    "# Model hyperparameters\n",
    "config = {\n",
    "    'que_vocab_size': len(preproc.que_f.vocab),\n",
    "    'prog_vocab_size': len(preproc.prog_f.vocab),\n",
    "    'embedding_dim': 256,\n",
    "    'num_heads': 8,\n",
    "    'num_encoder_layers': 3,\n",
    "    'num_decoder_layers': 3,\n",
    "    'dropout': 0.10,\n",
    "    'max_len': 20,\n",
    "    'forward_expansion': 4,\n",
    "    'que_pad_idx': preproc.que_f.vocab.stoi[\"<pad>\"]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the config as a json file\n",
    "import json\n",
    "with open('config.json', 'w') as f:\n",
    "    json.dump(config, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Training Generator\n",
    "train_loader = BucketIterator.splits((train_data,),\n",
    "                                     batch_size=batch_size,\n",
    "                                     sort_within_batch=True,\n",
    "                                     sort_key=lambda x: len(x.query),\n",
    "                                     device=device)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Seq2Seq(nn.Module):\n",
    "    '''Sequence to Sequence Model using Transformers'''\n",
    "    def __init__(self,\n",
    "                 config,\n",
    "                 device=None):\n",
    "        super(Seq2Seq, self).__init__()\n",
    "        '''Initialize the model'''\n",
    "        \n",
    "        # Create word embedding layers\n",
    "        self.src_word_embedding = nn.Embedding(config['que_vocab_size'], config['embedding_dim'])\n",
    "        self.trg_word_embedding = nn.Embedding(config['prog_vocab_size'], config['embedding_dim'])\n",
    "        \n",
    "        # Create positional embedding layers\n",
    "        self.src_position_embedding = nn.Embedding(config['max_len'], config['embedding_dim'])\n",
    "        self.trg_position_embedding = nn.Embedding(config['max_len'], config['embedding_dim'])\n",
    "        \n",
    "        # Create transformer\n",
    "        self.transformer = nn.Transformer(config['embedding_dim'],\n",
    "                                          config['num_heads'],\n",
    "                                          config['num_encoder_layers'],\n",
    "                                          config['num_decoder_layers'],\n",
    "                                          config['forward_expansion'],\n",
    "                                          config['dropout'])\n",
    "        \n",
    "        # Feedforward for Logits over vocabulary\n",
    "        self.fc_out = nn.Linear(config['embedding_dim'], config['prog_vocab_size'])\n",
    "        self.dropout = nn.Dropout(config['dropout'])\n",
    "        \n",
    "        self.src_pad_idx = config['que_pad_idx']\n",
    "        \n",
    "        if device is None:\n",
    "            device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "        self.device = device\n",
    "        \n",
    "        # Mount to device\n",
    "        self.to(device)\n",
    "        \n",
    "    def make_src_mask(self, src):\n",
    "        '''Create padding mask for src sequence'''\n",
    "        src_mask = src.transpose(0, 1) == self.src_pad_idx\n",
    "        # (N, src_len)\n",
    "        return src_mask.to(self.device)\n",
    "    \n",
    "    def forward(self, src, trg):\n",
    "        '''Forward pass'''\n",
    "        src_seq_length, N = src.shape\n",
    "        trg_seq_length, N = trg.shape\n",
    "        \n",
    "        # Create positions\n",
    "        src_positions = torch.arange(0, src_seq_length).unsqueeze(1).expand(src_seq_length, N).to(self.device)\n",
    "        trg_positions = torch.arange(0, trg_seq_length).unsqueeze(1).expand(trg_seq_length, N).to(self.device)\n",
    "        \n",
    "        # Get embeddings\n",
    "        src_embeds = self.dropout(\n",
    "            (self.src_word_embedding(src) + self.src_position_embedding(src_positions))\n",
    "        )\n",
    "        trg_embeds = self.dropout(\n",
    "            (self.trg_word_embedding(trg) + self.trg_position_embedding(trg_positions))\n",
    "        )\n",
    "        \n",
    "        # Create masks\n",
    "        src_padding_mask = self.make_src_mask(src)\n",
    "        trg_mask = self.transformer.generate_square_subsequent_mask(trg_seq_length).to(self.device)\n",
    "        \n",
    "        # Pass everything through the transformer\n",
    "        out = self.transformer(src_embeds, \n",
    "                               trg_embeds, \n",
    "                               src_key_padding_mask=src_padding_mask,\n",
    "                               tgt_mask=trg_mask)\n",
    "        \n",
    "        # Get logits over the vocabulary with a linear layer\n",
    "        out = self.fc_out(out)\n",
    "        # output shape (trg_seq_len, N, trg_vocab_size)\n",
    "        return out\n",
    "    \n",
    "    def train_model(self, \n",
    "                    train_loader,\n",
    "                    num_epochs,\n",
    "                    num_steps,\n",
    "                    lr=3e-4,\n",
    "                    filename='semantic_parser.pth'):\n",
    "        \n",
    "        # Create optimizer and loss function\n",
    "        optimizer = torch.optim.Adam(self.parameters(), lr=lr)\n",
    "        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, \n",
    "                                                               factor=0.1, \n",
    "                                                               patience=10,\n",
    "                                                               verbose=True)\n",
    "        criterion = nn.CrossEntropyLoss(ignore_index=self.src_pad_idx)\n",
    "        \n",
    "        for epoch in range(num_epochs):\n",
    "            pbar = tqdm(total=num_steps, desc='Epoch {}'.format(epoch+1))\n",
    "            \n",
    "            self.train()\n",
    "            losses = []\n",
    "            \n",
    "            for i, batch in enumerate(train_loader):\n",
    "                # Get the inputs and targets\n",
    "                inp_seq, target = batch.query, batch.program\n",
    "\n",
    "                # Forward pass\n",
    "                output = self(inp_seq, target[:-1, :])\n",
    "\n",
    "                # Reshape the output and targets for criterion\n",
    "                output = output.reshape(-1, output.shape[2]) # (trg_seq_len * N, trg_vocab_size)\n",
    "                target = target[1:].reshape(-1)\n",
    "\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "                # Calculate Loss\n",
    "                loss = criterion(output, target)\n",
    "                losses.append(loss.item())\n",
    "\n",
    "                # Backprop and Optimize\n",
    "                loss.backward()\n",
    "                nn.utils.clip_grad_norm_(self.parameters(), max_norm=1)\n",
    "                optimizer.step()\n",
    "                \n",
    "                # Metrics\n",
    "                pbar.update(1)\n",
    "                pbar.set_postfix({'Loss': loss.item()})\n",
    "\n",
    "            mean_loss = sum(losses) / len(losses)\n",
    "            scheduler.step(mean_loss)\n",
    "\n",
    "            print(f'Epoch {epoch+1}: Mean Loss = {mean_loss}\\n')\n",
    "            pbar.close()\n",
    "        \n",
    "        # Save Model\n",
    "        torch.save(self.state_dict(), filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Model\n",
    "seq2seq = Seq2Seq(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c6b334b0b9ba4241a1e417ba3a43f71d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1:   0%|          | 0/250.0 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1: Mean Loss = 0.0014343057175719878\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "220e84e27fba45eb8c155fe3725dcf1b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2:   0%|          | 0/250.0 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2: Mean Loss = 3.0090658765402622e-05\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "403135e3466b46dda56ec8c6f8bac964",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3:   0%|          | 0/250.0 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3: Mean Loss = 1.5092090870894026e-05\n",
      "\n",
      "CPU times: user 57.7 s, sys: 193 ms, total: 57.9 s\n",
      "Wall time: 23.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Train Model\n",
    "seq2seq.train_model(train_loader, num_epochs, num_steps, filename='semantic_parser.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SemanticParser():\n",
    "    '''Full Pipeline for Semantic Parsing from Query -> Program'''\n",
    "    def __init__(self, preprocessor, config, filename='models/semantic_parser.pth', device=None, max_len=20):\n",
    "        # Device\n",
    "        if device is None:\n",
    "            device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "        self.device = device\n",
    "        \n",
    "        # Fields (includes the vocabulary)\n",
    "        self.que_f = preprocessor.que_f\n",
    "        self.prog_f = preprocessor.prog_f\n",
    "        self.preproc = preprocessor\n",
    "        \n",
    "        # Maximum length of program\n",
    "        self.max_len = max_len\n",
    "        \n",
    "        # Load Model\n",
    "        self.model = Seq2Seq(config, device)\n",
    "        self.model.load_state_dict(torch.load(filename))\n",
    "        \n",
    "        \n",
    "    def predict(self, query):\n",
    "        '''Predicts the Program, given a query'''\n",
    "        # Tokenize\n",
    "        tokens = self.preproc.tokenizer(query)\n",
    "\n",
    "        # Add <sos> and <eos> in beginning and end respectively\n",
    "        tokens.insert(0, self.que_f.init_token)\n",
    "        tokens.append(self.que_f.eos_token)\n",
    "\n",
    "        # Convert the tokenized sequence into integers\n",
    "        query_indices = [self.que_f.vocab.stoi[tok] for tok in tokens]\n",
    "\n",
    "        # Convert to Tensor\n",
    "        query_tensor = torch.LongTensor(query_indices).unsqueeze(1).to(self.device)\n",
    "        \n",
    "        # Init the program sequence with <sos>\n",
    "        outputs = [self.prog_f.vocab.stoi[\"<sos>\"]]\n",
    "        \n",
    "        # Generating the program\n",
    "        for i in range(self.max_len):\n",
    "            \n",
    "            # Create program output tensor\n",
    "            program_tensor = torch.LongTensor(outputs).unsqueeze(1).to(self.device)\n",
    "            \n",
    "            # Predict next token\n",
    "            with torch.no_grad():\n",
    "                output = self.model(query_tensor, program_tensor)\n",
    "\n",
    "            # Get the word with highest probability\n",
    "            word_idx = output.argmax(2)[-1, :].item()\n",
    "            # Append to outputs\n",
    "            outputs.append(word_idx)\n",
    "\n",
    "            if word_idx == self.prog_f.vocab.stoi[\"<eos>\"]:\n",
    "                break\n",
    "\n",
    "        # Decode to english\n",
    "        program = [self.prog_f.vocab.itos[idx] for idx in outputs][1:-1]\n",
    "        # Convert to list of instructions\n",
    "        program_ = ' '.join(program).split(' <nxt> ')\n",
    "\n",
    "        return program_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sem_parser = SemanticParser(preproc, config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['filter red', 'query shape', 'filter', 'count']"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "program = sem_parser.predict('How many red circles are there?')\n",
    "program"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "class DSL():\n",
    "    '''Domain Specific Language consisting of functions and relations'''\n",
    "    def __init__(self):\n",
    "        # Value to Attribute Converter\n",
    "        self.val2attr = {'shape': ['circle', 'rectangle'], 'color': ['red', 'green', 'blue', 'orange', 'gray', 'yellow']}\n",
    "        self.get_attr = lambda x: [k for k, v in self.val2attr.items() if x in v][0]\n",
    "        \n",
    "        # String to Function\n",
    "        self.str2func = {'filter': self.filter_, \n",
    "                         'query': self.query, \n",
    "                         'count': self.count, \n",
    "                         'relate': self.relate, \n",
    "                         'isleft': self.isLeft, \n",
    "                         'istop': self.isTop}\n",
    "        \n",
    "    def filter_(self, param):\n",
    "        '''Returns a subset of the scene based on the param'''    \n",
    "        # Filter Object(s) for scene\n",
    "        attr = self.get_attr(param)\n",
    "        filtered_objects = self.scene[self.scene[attr] == param]\n",
    "\n",
    "        return filtered_objects\n",
    "    \n",
    "    def query(self, obj, attr):\n",
    "        '''Returns the column value of object(s)'''\n",
    "        i = obj.index[0]\n",
    "        result = obj[attr][i]\n",
    "\n",
    "        return result\n",
    "    \n",
    "    def relate(self, obj, param):\n",
    "        '''Returns object which is either closest or furthest from all other objects of the scene'''\n",
    "        obj_pos = self.query(obj, 'position')\n",
    "        scene_pos = self.scene['position']\n",
    "\n",
    "        # Calculate distances\n",
    "        distances = np.array([np.linalg.norm(np.array(obj_pos) - np.array(pos)) for pos in scene_pos])\n",
    "        \n",
    "        sorted_dists = distances.argsort()\n",
    "            \n",
    "        if param == 'closest':\n",
    "            idx = sorted_dists[0] if distances[sorted_dists[0]] != 0 else sorted_dists[1]\n",
    "        elif param == 'furthest':\n",
    "            idx = sorted_dists[-1] if distances[sorted_dists[-1]] != 0 else sorted_dists[-2]\n",
    "        \n",
    "#         print(sorted_dists, distances, idx)\n",
    "\n",
    "        # Get the object from the scene of that index\n",
    "        req_obj = self.scene[self.scene.index == idx]\n",
    "\n",
    "        return req_obj\n",
    "    \n",
    "    def count(self, objects):\n",
    "        '''Counts the objects'''\n",
    "        return len(objects)\n",
    "    \n",
    "    def isLeft(self, pos):\n",
    "        '''Checks if a position is on the left half or not'''\n",
    "        return 'yes' if pos[0] < 112 else 'no'\n",
    "    \n",
    "    def isTop(self, pos):\n",
    "        '''Checks if a position is on the top half or not'''\n",
    "        return 'yes' if pos[1] < 112 else 'no'\n",
    "\n",
    "class ProgramExecutor(DSL):\n",
    "    '''Executes a given program'''\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        pass\n",
    "    \n",
    "    def func_executor(self, func, param, prev_out):\n",
    "        '''Executes a given function with or without a parameter'''\n",
    "        # 0-1 arg functions\n",
    "        if func in ['filter']:\n",
    "            prev_out = self.filter_(param) if param != None else self.filter_(prev_out)\n",
    "\n",
    "        # Two arg functions\n",
    "        elif func in ['query', 'relate']:\n",
    "            prev_out = self.str2func[func](prev_out, param)\n",
    "\n",
    "        # One arg functions\n",
    "        elif func in ['count', 'isleft', 'istop']:\n",
    "            prev_out = self.str2func[func](prev_out)\n",
    "\n",
    "        return prev_out\n",
    "    \n",
    "    def __call__(self, scene, program):\n",
    "        '''Executes a program on the scene'''\n",
    "        self.scene = scene\n",
    "        \n",
    "        prev_out = None\n",
    "        for seq in program:\n",
    "            args = seq.split()\n",
    "            # print(args)\n",
    "#             try:\n",
    "            if len(args) < 2:\n",
    "                prev_out = self.func_executor(args[0], None, prev_out)\n",
    "            else:\n",
    "                prev_out = self.func_executor(args[0], args[1], prev_out)\n",
    "            # print(prev_out, '\\n')\n",
    "#             except:\n",
    "#                 prev_out = 'Failed'\n",
    "\n",
    "        return prev_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "executor = ProgramExecutor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['What is the closest shape to the orange object?',\n",
       " 'What is the furthest shape from the yellow object?',\n",
       " 'What is the furthest shape from the yellow object?',\n",
       " 'What is the furthest shape from the green object?',\n",
       " 'What is the furthest shape from the green object?',\n",
       " 'What is the furthest shape from the gray object?',\n",
       " 'How many objects of the same shape as the red object are there?',\n",
       " 'What is the closest shape to the red object?',\n",
       " 'What is the closest shape to the green object?',\n",
       " 'How many objects of the same shape as the blue object are there?',\n",
       " 'What is the shape of the yellow object?',\n",
       " 'Is there a red object on the top?',\n",
       " 'Is there a orange object on the left?',\n",
       " 'What is the shape of the red object?',\n",
       " 'Is there a yellow object on the left?',\n",
       " 'Is there a red object on the left?',\n",
       " 'Is there a green object on the top?',\n",
       " 'Is there a blue object on the left?',\n",
       " 'Is there a orange object on the left?',\n",
       " 'Is there a yellow object on the left?']"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "queries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run the program executor with on a full sample of 20 queries\n",
    "pred_ans = []\n",
    "for que in queries:\n",
    "    program = sem_parser.predict(que)\n",
    "    pred_ans.append(executor(scene, program))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('circle', 'circle'),\n",
       " ('rectangle', 'rectangle'),\n",
       " ('rectangle', 'rectangle'),\n",
       " ('rectangle', 'rectangle'),\n",
       " ('rectangle', 'rectangle'),\n",
       " ('circle', 'circle'),\n",
       " (4, '4'),\n",
       " ('rectangle', 'rectangle'),\n",
       " ('circle', 'circle'),\n",
       " (4, '4'),\n",
       " ('circle', 'circle'),\n",
       " ('no', 'no'),\n",
       " ('no', 'no'),\n",
       " ('circle', 'circle'),\n",
       " ('yes', 'yes'),\n",
       " ('yes', 'yes'),\n",
       " ('yes', 'yes'),\n",
       " ('no', 'no'),\n",
       " ('no', 'no'),\n",
       " ('yes', 'yes')]"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(zip(pred_ans, answers))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from program_executor import *\n",
    "from perception import *\n",
    "from semantic_parser import *\n",
    "import torch\n",
    "from skimage.io import imshow\n",
    "import cv2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the config file for transformers\n",
    "import json\n",
    "with open('config.json', 'r') as f:\n",
    "    config = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NSAIPipeline():\n",
    "    '''End-to-End Pipeline of Neuro-Symbolic AI on Sort-of-CLEVR dataset'''\n",
    "    def __init__(self, \n",
    "                 config,\n",
    "                 detector='models/detector.svm',\n",
    "                 classifier='models/classifier.pth',\n",
    "                 sem_parser='models/semantic_parser.pth',\n",
    "                 train_csv='data/train/train_q2p.csv',\n",
    "                 device=None):\n",
    "        \n",
    "        if device is None:\n",
    "            device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "        self.device = device\n",
    "        \n",
    "        # Perception Module\n",
    "        self.perceiver = PerceptionPipe(detector, classifier, self.device)\n",
    "        \n",
    "        # Semantic Parser\n",
    "        self.preproc = Preprocessor(train_csv)\n",
    "        self.sem_parser = SemanticParser(self.preproc, config, filename=sem_parser, device=self.device)\n",
    "        \n",
    "        # Program Executor\n",
    "        self.executor = ProgramExecutor()\n",
    "        \n",
    "    def predict(self, img, query):\n",
    "        '''\n",
    "        Make Prediction on a single image and question pair\n",
    "        \n",
    "        Args:\n",
    "            img (str/array): pixel values should be in 0-255 range\n",
    "                             of dtype uint8 in BGR color format or\n",
    "                             file path of the image\n",
    "            query (str): question about the image\n",
    "            \n",
    "        Returns:\n",
    "            str: answer of the query\n",
    "        '''\n",
    "        # Load img if it's a path\n",
    "        if type(img) == str:\n",
    "            img = cv2.imread(img)\n",
    "        \n",
    "        # Structured Scene Representation\n",
    "        scene = self.perceiver.scene_repr(img)\n",
    "        # Synthesize Program from Query\n",
    "        program = self.sem_parser.predict(query)\n",
    "        # Execute Program\n",
    "        answer = self.executor(scene, program)\n",
    "        \n",
    "        return answer, program\n",
    "    \n",
    "    def evaluate(self, csv, img_dir, debug=True):\n",
    "        '''\n",
    "        Evaluate the model on a dataset\n",
    "        \n",
    "        Args:\n",
    "            csv (str): path of the csv containing image filename, answer, query and program\n",
    "            img_dir (str): directory containing the images\n",
    "            debug (bool): View the data points which were wrong\n",
    "        Returns:\n",
    "            int: accuracy of the model\n",
    "        '''\n",
    "        data = pd.read_csv(csv).values\n",
    "        \n",
    "        correct = []\n",
    "        for filename, answer, query, program in tqdm(data):\n",
    "            # Load Image\n",
    "            img_path = os.path.join(img_dir, filename)\n",
    "            img = cv2.imread(img_path)\n",
    "            \n",
    "            # Make prediction\n",
    "            pred, pred_prog = self.predict(img, query)\n",
    "            \n",
    "            # Verify answer\n",
    "            if str(pred) == answer:\n",
    "                correct.append(1)\n",
    "            else:\n",
    "                if debug:\n",
    "                    print(filename, answer, pred, query, pred_prog)\n",
    "                correct.append(0)\n",
    "        \n",
    "        acc = (sum(correct) / len(correct)) * 100\n",
    "        \n",
    "        return acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nsai = NSAIPipeline(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/shr/Documents/projects/nsvqa/perception.py:208: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  scene_df = scene_df.append({'shape': shape,\n",
      "/home/shr/Documents/projects/nsvqa/perception.py:208: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  scene_df = scene_df.append({'shape': shape,\n",
      "/home/shr/Documents/projects/nsvqa/perception.py:208: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  scene_df = scene_df.append({'shape': shape,\n",
      "/home/shr/Documents/projects/nsvqa/perception.py:208: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  scene_df = scene_df.append({'shape': shape,\n",
      "/home/shr/Documents/projects/nsvqa/perception.py:208: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  scene_df = scene_df.append({'shape': shape,\n",
      "/home/shr/Documents/projects/nsvqa/perception.py:208: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  scene_df = scene_df.append({'shape': shape,\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(6, ['filter gray', 'query shape', 'filter', 'count'])"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nsai.predict('data/test/images/1.jpg', 'How many objects of the same shape as the gray object are there?')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluate on the test and get the accuracy\n",
    "nsai.evaluate('data/test/test_q2p.csv', 'data/test/images/', debug=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fa0876ad510>"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd0AAAHVCAYAAAC0biEDAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABVUklEQVR4nO3de5gU1Z0//vep7ulmgLk44DCMDheN1yhEUUceE6OBCOhqXMlFJAlGFqMBo5ALy/68Qfa7w+qu5klidPd5jLiraOLz9fJoVvMgCMR1RMXwJZrICouigQEDMs1terq7zu+POqe663RV9wzMVHfPvF8+bXdVnao+VT306U+dm5BSShAREVG/s0qdASIiosGChS4REVFIWOgSERGFhIUuERFRSFjoEhERhYSFLhERUUhY6BIREYWEhS4REVFIWOgSERGFhIUuERFRSEpW6D7wwAMYN24chgwZgtbWVrzxxhulygoREVEoSlLo/vrXv8aiRYtw11134e2338bEiRMxbdo07NmzpxTZISIiCoUoxYQHra2tOP/88/GLX/wCAGDbNlpaWnDLLbfg7//+74vub9s2du7ciZqaGggh+ju7REREBUkpceDAATQ3N8OyguPZaIh5AgB0d3dj48aNWLJkibvOsixMnToV7e3tvvskk0kkk0l3+S9/+QvOPPPMfs8rERFRb3z00Uc48cQTA7eHXuj+9a9/RSaTwahRozzrR40ahffee893n7a2NixdujRv/UcffYTa2tp+yScREVFPJRIJtLS0oKampmC60Avdo7FkyRIsWrTIXdYnV1tby0KXiIjKRrEqz9AL3ZEjRyISiWD37t2e9bt370ZTU5PvPvF4HPF4PIzsERER9ZvQWy/HYjFMmjQJq1evdtfZto3Vq1dj8uTJYWeHiIgoNCW5vbxo0SLMmTMH5513Hi644AL89Kc/xaFDh/Cd73ynFNkhIiIKRUkK3W984xv45JNPcOedd6KjowOf+9zn8NJLL+U1riIiIhpIStJP91glEgnU1dWhs7OTDamIiKjkeloucexlIiKikLDQJSIiCgkLXSIiopCw0CUiIgoJC10iIqKQsNAlIiIKCQtdIiKikLDQJSIiCgkLXSIiopCw0CUiIgoJC10iIqKQsNAlIiIKCQtdIiKikLDQJSIiCgkLXSIiopCw0CUiIgoJC10iIqKQsNAlIiIKCQtdIiKikERLnQEiokIkbIi8tUcTL9jHsG8pHE1+7YD1lXLOAx8/CSIiopAw0iWiElFRmfT/7S+FrVLZsCDVWh3zFo4XhJkcNvKiwID37TciKAoNkpPezKv0LsLSaW1PAqkugPN/xljlgJ8CERFRSBjpElFpBEWanugUyA/rgom8F7ks77NOY0bFfsu9SRu4rN/frKv1LkufulxhHi+QN8LNO2cqOX4SREREIWGkS0SlYUZtRoQrVAILgHA3Bn1lmXWaKp6QOZGeEf26QagovtybtIWWPXlzc2zGPvmxkKXPS+jjmmls9T6RwGNQeeAnQ0REFBJGukRUGma9a149bG7L2z74qspr0RyeQm9p1uz2ROBNAip7jHSJiIhCwkiXiEpCqvAsv61u8Ra3Iq8Vby9a6ap9hdFvtlDgXSwoLxq0SyswPNX5iMBkBbzO5b16Im99sf0pbPwkiIiIQsJIl4hC1fNet1l5ke2xVGLqY4U9IlWeHo6TXPCC6T7HvR3tikqFhS4RhcosL7O3VY1uP7InX0/efWRgEyML7i3YMml1ZN5eNwtbvdSj7OofEEEFtOjpgai/lfqnHhER0aDBSJeIyoRxi1TkTIhQpLtPfoBXxtP4GRFu0dvtIieRCDivoOEnqeyU4V8kERHRwNTnhW5bWxvOP/981NTUoLGxEVdffTW2bNniSXPJJZdACOF53HTTTX2dFSIqQ1I90uqRUQ8JSz1UGikhpfSPbnUiVVcrEFWPiHpUqYdalhaE9DbIkiE+PGwJ2BKW+i9b3+ykFrDVQ526hBPhCj09oQ2JlHqkIZFGRgAZ4dQTS5FzSP1gfW7Z6PNCd926dZg/fz5ef/11rFq1CqlUCpdddhkOHTrkSTdv3jzs2rXLfdxzzz19nRUiIqKy0ud1ui+99JJnecWKFWhsbMTGjRtx8cUXu+uHDh2Kpqamvn57IqoQZvWjrV5Y7mD+7hbACqjDzKvrDWjFmzs9n0tPrOB/qNzlY5rRz82De4Kerdk0eoIH1RLb1vnLaXpsnINZLxxUk81At3z0e51uZ2cnAKChocGz/vHHH8fIkSNx1llnYcmSJTh8+HDgMZLJJBKJhOdBRERUafq19bJt27jttttw0UUX4ayzznLXX3fddRg7diyam5uxefNmLF68GFu2bMHTTz/te5y2tjYsXbq0P7NKRCFxoy6phj9Uka3IG6wio9Jl3J2k6twqVGfbbEBrDOkofCJeI/o105gTxecuF9rWk2UIG7C84XkmIObRfY0tS/VgtnOHkAzqa+w/7CMj3PIjpJTmjZg+c/PNN+PFF1/Eq6++ihNPPDEw3Zo1azBlyhRs3boVJ598ct72ZDKJZDLpLicSCbS0tKCzsxO1tbX9knci6l8ZVejq28lubxi3pFCFLmxAOAVQjwtd8zaz9Gz0PvfkXvHR3lf2dPXx5jGdF/Oo66HO20JOoRuQd32FpHGr3GKhG7pEIoG6urqi5VK/RboLFizACy+8gPXr1xcscAGgtbUVAAIL3Xg8jng83i/5JKLSiBiRpqrChJU38n8V7LRKo9ZkdBdet1CzPNstM6r1aQStR4TSLZoLLfcmrd8yYLl57E6pbXHvOegfClFVfx3TIzxK293XLNSz0wJanuXs3QRzBZVanxe6UkrccssteOaZZ7B27VqMHz++6D6bNm0CAIwePbqvs0NERFQ2+rzQnT9/PlauXInnnnsONTU16OjoAADU1dWhuroa27Ztw8qVK3H55ZdjxIgR2Lx5MxYuXIiLL74YEyZM6OvsEFG5Us2VbRXiWhF1C9mtGHWWMzaQVpFu0+g2AEC6+0xnW+Z451nX+erRnoRuCawOJe2c6NcbSocT6WbPsypeBQDoznR58lGFmJM76x0AwO5P5wEAYlUWIn63yeEdWTp32XcuBUa7ZaHPC90HH3wQgDMARq5HHnkE119/PWKxGF5++WX89Kc/xaFDh9DS0oKZM2fi9ttv7+usEBERlZV+ub1cSEtLC9atW9fXb0tElUY1htIRrm5JZUsdpjlfTxkAXaoe1M6cBgDoTp4LAEhnmgEAUtd5ug2N1LF0aChzQj+p3i/kqf101rpTKmyPqrzpMB5DnCdV15tSraSqqhAYpXKI5crDsZeJiIhCwlmGiKgkdPCZHYDKeaFbNWf0yEzSQlTX2apmy5bR4jfvBpvMtusFACGz9biyRLGG281J593WddhGOttJF80N3o1E2RGovGNQuXW7rL8tWyx0iagkREDZZ46WaIlsIaILLlsXwkEHNwaTKAf6trlQX7vS/dXhvd2tq+jcHxYF7yF7f1y4yyx0y1b5/EUSERENcIx0iag85I3pr26ZSriBnK3GYJKqoZTMnegeCAx9y6PBkRuuq2VvntPqRVSFuG6qAlFrtqsQ46dKwU+KiIgoJIx0iai0jEjOHbmwQJ2mLYxE5RHKHh1jnGbpG9kGTdrn3dccHSO7mvFVueAnQUREFBIWukRERCFhoUtERBQSFrpEREQhYaFLREQUEha6REREIWGhS0REFBIWukRERCFhoUtERBQSFrpEREQhYaFLREQUEha6REREIWGhS0REFBIWukRERCFhoUtERBQSFrpEREQhYaFLREQUEha6REREIWGhS0REFBIWukRERCFhoUtERBQSFrpEREQhYaFLREQUEha6REREIWGhS0REFBIWukRERCGJljoDRDRIiR6ulrkLlv82Yatl/zgi95jSN0UJSP9F6XtdvOeVl0Sfv7n6KLJF/YuFLhGVCaPgVKWQFPkFkSU9ewQSxnPuXv6FW/8R6rx03qXOB/Sys912l7Pc3xZ5K/QxMt73clPyZma54SdCREQUkj4vdO+++24IITyP008/3d3e1dWF+fPnY8SIERg+fDhmzpyJ3bt393U2iKjMSQTc6hW2esDn/qgd8ND7+u1jvK9P5ByOgDxrZt7d5QLxvLQAafkc2XIeajuVj375ND772c9i165d7uPVV191ty1cuBDPP/88nnrqKaxbtw47d+7ENddc0x/ZICIiKiv9UqcbjUbR1NSUt76zsxMPP/wwVq5ciS996UsAgEceeQRnnHEGXn/9dVx44YX9kR0iqghGROdWyFo5r9PqOeOf1iDdUDrn2DryCzXatSGFkxm3DtfNW+HYx6n71ecb8d3Hctf37JhUOv3yybz//vtobm7GSSedhNmzZ2PHjh0AgI0bNyKVSmHq1Klu2tNPPx1jxoxBe3t74PGSySQSiYTnQUREVGn6vNBtbW3FihUr8NJLL+HBBx/E9u3b8YUvfAEHDhxAR0cHYrEY6uvrPfuMGjUKHR0dgcdsa2tDXV2d+2hpaenrbBNRyPKqX6XlffRu74J0Pa5vXa5Zuey33Ju0Qcsqy9J8mLuo9bZ6+NZ8S+Oh63DdulxzO5WLPr+9PGPGDPf1hAkT0NrairFjx+I3v/kNqqurj+qYS5YswaJFi9zlRCLBgpeIiCpOv/fTra+vx6mnnoqtW7fiy1/+Mrq7u7F//35PtLt7927fOmAtHo8jHo/3d1aJqBTcSs6AgS1yozXpfGUJWz2rfWTeTTtVh6sjW79jSyNJgeVC23q0nPu2fisLsCFguednnEexYJ+jY5Sdfq9tP3jwILZt24bRo0dj0qRJqKqqwurVq93tW7ZswY4dOzB58uT+zgoREVFJ9Xmk+8Mf/hBXXnklxo4di507d+Kuu+5CJBLBrFmzUFdXh7lz52LRokVoaGhAbW0tbrnlFkyePJktl4kGGzPSM5d9IkIhzYhPeNMERXY5wySKHgwVaS4X2taTZUgrJw/+42Rl9/ULgQtHuPnDQur39X0LKqE+L3Q//vhjzJo1C3v37sXxxx+Pz3/+83j99ddx/PHHAwDuv/9+WJaFmTNnIplMYtq0afjlL3/Z19kgIiIqO0JKWXG/hRKJBOrq6tDZ2Yna2tpSZ4eIesU2nhVVX+tGZyq4S6eBri7nddOopwEA3ckLnG2ZEzy7ZEM+s89vztecrlQNc6Sm3MbWQr3ISHcTkD2H+JDXAAD7Ehc5y1UpWKjypA28K5BHXwf22+1vPS2X+EkQERGFhLMMEVFJBFbDGtW02f6qyKkXdbeq5wqIH6S3Tle4z3qt7XnOPaPiU/mZ55/2yUAFXKNBgIUuEYVKd+/JQN9edYoUyyhZcotX946sOxyiLqC8ha9Ut4yFeg/pzrObc/CSTABgA9LJe0Q4X7sWvLeX9blEVOFs61Ot8rt3HDCvsBZ4H5pKjT99iIiIQsJIl4hKQke4aRUBRvUk72bvGJGNgqui+5wX0b+ojcbE90YcUWgKP/PGdKHl3qT1WwZs97wyaX3rN6LSqOhcTeIQsfaqZ3UOtmB4NIDwoyQiIgoJI10iCpVuKATpRHwxVWErVNSqAl9EVKhn20BUzVxnYQsAIB5z0mbsD6C3ODubcUQ2As52K1J1purZUht0Yy2/5ULberMMAEKdS3XVECfLaeeEMxkV8Vf9r+cMIlYkr9VZsbFA8uMpxlflgp8EERFRSBjpElFJRISerEDFbXqSAF35qZarrOwU7rv3/NTZpFbYxYaS9BkTw2WOSlFouTdpfZZtka3ndbOhqnYjaoVuoW25UX1xwREv46lyxU+GiIgoJIx0iShk3pbGMq36q+omyhkd8go3mR7FMaq+sWx1iIg+pI4Wg7qlSp8eqyFHutLNrEPX97r1vnpXtZxtvQwINzwKGELTfT/vUJo6uucMf+WDkS4REVFIGOkSUUlZURWHud1XjSa6ACz1TZW2vcvudrVz/rR4wk1RyhhDIlsvLaFHptLDPTp5tFXcrkeisjxDdKmW3erZVudpec6PKgE/KSIiopAw0iWiUOlYNBv5OWuiUedZj9gUjcQAAOmMjUhUjdqk6kV1pOdWZUrvV5kwZix1xmDOwF84Y1JFdR21rok2mlNbqjLXUpW5mVRaLefW6XrPS79LpCTjSdPR4CdFREQUksqOdCUKT6Ih8n9tHtV7AJXd/G8gnAMNWLae8k5FgjqqlXpWnmgkf0Yi449ZmK2Xhbk9grwWv64Qmi/Dyr6URhqVaWmrmZPUckQ31RY2fHr5qqMGfLfx33rZYqRLREQUksqOdIH8X43usDbSHbZGuo0h1a9koy7FHKnGv6+fN2oOa5bKYtNi5o2yI70RgcP/F74M+M3lOX/+YqY+ZtRsIqIqavMnas/+fUaK/CGKvBd+ShxjuBPnmq2z1djT5vRKrux6s62ynjc46Lz5z7f8VHahK2XebaTc0skskKR568fdbuf8H4j4/eOU/n/c/XlDquA/mKCNuf+g03pEAbNxh1fgTXj+i6V+1PPB+gfon2KvTsp7TQbk9RgkeHuZiIgoJJUd6Voyp7GU7jyub69WuckyZlcBNVm0jij1EfTtZymyt22Cutqbt32F8FkutK0Xy85KdX7u7XTnOaUbmwjvhNiAlR1kwM2rf7Ru5Q0pZ3u2ep+JiOho8ZuUiIgoJBUd6dqeQd/UhNeq+0EkJ1oVOXs4pLFeGMvhNZTqLeFG9KpRmIi6W4Dc8/f5PZVXEWTW8XqvDxER9S1GukRERCGp6EhXQsDOa9WX01I3rTvX6/pdy92kEjvbjRb8fi0Dg+bGNlv/5y4X2tabZSfP3rm6RMT56HS3i+wp5VwPv7phT2qT328w/i4jIuor/EYlIiIKSUVHuhacwdABwFJ1mBHRrbZ+CkSPqNc60tWTYhuzSavIT/isy9b/enuzmtFwoeXepPVbdt4224vYUaPSVjvP0jnHvG7LgNEauRA9a3axzBAR0dFgpEtERBSSio50BYCqiLflscAB9er/AdirXsd89szl1xtXr+s2toXdb1VHqbqv8TD13KSeTwAACDHSeZZDAHjrlPPxtxYRUSnw25eIiCgkFR3pSpmBldfmeD8AoLNzFaJV25x0tnOaQo/a5Aa2ekox79jLub9FpJXyvKewzfrgQpNYH+2E19llqfImVJ1uKl3nLMuzAAB1NV9wljFc7eNEuplMGlE95nLASFSBAls9ExHRsWCkS0REFJKKjnSFZw46HR12AgCs2J8Qi23WKdWzannsTppdfL48aaQRUk+irRPoKNLOXy60rYfLtnqd0eF5dBQAIN2lI+5T1XOzenYi3mihT7ank9qL3GidiIiOFb9RiYiIQlLRka73N4N+nQQARKy9iEQ+Vuv0JNlmf1VzDGZNGCl8trgb+rdO11KRrmXkRFr71SvdF9mYSUkip39ub39b5c7cxN9lRER9pc+/UceNGwchRN5j/vz5AIBLLrkkb9tNN910lO9muQ8JXQ7G1MOCgFNIWlLNAgj/B6QwHhL6gEJanodLSPWw1cNvudC2Hi6rjOgzjUj1sKOI2FE4PygicAYAqQJk1HkAyJ6E3YsBMoiIqL/0eaT75ptvIpPJRl3vvPMOvvzlL+NrX/uau27evHlYtmyZuzx06NC+zgYREVHZ6fNC9/jjj/csL1++HCeffDK++MUvuuuGDh2KpqYmc9ejYk5Er5sXCQkI88ZxsQZEPW1g5LuT33Khbb1cVi8tPeFB3u1t46ZFgXOQPT4/3lomIupL/fqt2t3djcceeww33HADRM7wSI8//jhGjhyJs846C0uWLMHhw4cLHieZTCKRSHgeRERElaZfG1I9++yz2L9/P66//np33XXXXYexY8eiubkZmzdvxuLFi7FlyxY8/fTTgcdpa2vD0qVLe/iuBeoujanzjNUVOrh/0PkezcgWrPclIupPQkrZb+MOTZs2DbFYDM8//3xgmjVr1mDKlCnYunUrTj75ZN80yWQSyWTSXU4kEmhpaUFnZydqamsB5N5efhMAcKTrVlQPed1ZafR/zZ6war2cd79Vwuzb63IbJIU8XJN6OzvTAgDoPnIRAGBIja4rv0glcPrxwjOSlr7pbuUeyoffbEu8xUxEVEwikUBdXR06OztRq8olP/0W6X744Yd4+eWXC0awANDa2goABQvdeDyOeDyev0H6Badpz/ZC8iaLd9OLnKjYG/2VLBg2+irJvMLfNtLlKlZw6sJWD8qR82dRkdE/EVF56rcw5pFHHkFjYyOuuOKKguk2bdoEABg9enR/ZYWIiKgs9Euka9s2HnnkEcyZMwfRnPEIt23bhpUrV+Lyyy/HiBEjsHnzZixcuBAXX3wxJkyY0Ps3ynbOhSV0hOt0V8ofCMO7m+8aHfLK3KEogo4Tdgjo5MhWb2tb+nxt77MboQfnL7vF2Je3komI+lW/FLovv/wyduzYgRtuuMGzPhaL4eWXX8ZPf/pTHDp0CC0tLZg5cyZuv/32/sgGERFRWemXQveyyy6DX/uslpYWrFu3rm/fLGCkJatgfW7hyl7/fqxH1Ym3zxRr/JTPYvxKRFRm+H1MREQUksqe8EDkvtSnkjvJvK6r9XYVEsZzXvqc6FEUHTUqHHl5leZIVGa+LHdL/nma0XFl/xkQEVUKRrpEREQhqewQx7c+N3cmoIDdzBVGCCwgs3Fj4PBVpVXs15JAqWJyIiIKwkiXiIgoJCx0iYiIQsJCl4iIKCQsdImIiELCQpeIiCgkLHSJiIhCwkKXiIgoJCx0iYiIQsJCl4iIKCQsdImIiELCQpeIiCgkLHSJiIhCwkKXiIgoJCx0iYiIQsJCl4iIKCQsdImIiELCQpeIiCgkLHSJiIhCwkKXiIgoJCx0iYiIQsJCl4iIKCQsdImIiELCQpeIiCgkLHSJiIhCwkKXiIgoJNFSZ+DYWAj63WCL4L0knI0CMmB7jgLHCZN/TnPZxrNf1vW2lDpoRj2n1fqMkT4NyJSRkSp1cOO6C7Uew9Vzzp+WkZHgc7GN5PxNSEQDC7/ViIiIQlLRka5EftQkdJwUiWa3SfVKCHc/ndr5v/coEtmY0PKkDD/w1TnT+ZHqXCJuRlKeFFJFq775tJPOc+aQWqGfP1XPR5wnod8tif+38t8AABOvnaUyUKvS6MhWRcfWCLX+M2r9cDcTUh8u4j0XTaqzFG6k6yxbOf8nIhoI+I1GREQUkoqOdP2jTidKciJCFcn2tE4xJyA2I9xS0e/v5ifqfGRHDjkRbkyHj27UrthHAPuw2tmIbGUCAPDHJx4CAAwRTqQr7APOZhVNx6MZjLL3AwA+WXknACCZHuYcI1rtvI2Kig/JegDAGbNvV9kZBdhRlWddz+ut7xXqelvqA9IRsCj5VSci6h+MdImIiEJS0ZEuYENAt7zV8auq47QlhI4CpTe2FcJbd2iGvgI50XFeWBxyFKbrRVVG0inn/KriQ1QC/btpqPOUUsuRJBDpUDtvAwB8+H9/4ex74C8AgBbpRL6xmHM9Mkiq93LetMqOICKc9xMqLO1WhxfSeRGRTtpMxIl8P/6PrzvLw1sw9mu3qryNU/k42XlK1Tj7Rqv0yTnLVjR3kYhowOl1pLt+/XpceeWVaG5uhhACzz77rGe7lBJ33nknRo8ejerqakydOhXvv/++J82+ffswe/Zs1NbWor6+HnPnzsXBgweP6USIiIjKXa8j3UOHDmHixIm44YYbcM011+Rtv+eee/Czn/0Mjz76KMaPH4877rgD06ZNw5/+9CcMGeJEZ7Nnz8auXbuwatUqpFIpfOc738GNN96IlStX9i4zuc2XzXpbKXNaxQYJDGeLh1vmrn7Lhbb1ZhmApVbq+lYd8cZ0wKuiVIhO59n+AJuf/EcAQI39v85zZicA4DjL+YETkd3OMdXNAt1LN2o575Xulm4r6YzKyzAVnKbVPlFdpawqZPUdgmRyLz58/A4AwKfyJADA52YvVcc/Ue3ToHaKg4hoMBBSyqO+myeEwDPPPIOrr74agFMgNDc34wc/+AF++MMfAgA6OzsxatQorFixAtdeey3+/Oc/48wzz8Sbb76J8847DwDw0ksv4fLLL8fHH3+M5ubmou+bSCRQV1eHzr/uR22t7sKivu2jGwAAR7oWonrIBt/9pVEMC1uXGN6uReqkjJPWx/As+i4fa1nrzaVzUyKTHA0ASHdfBACI11yptp/mPKnC+OMnl2B491YAwNCYKlxTTgOquOUsZ9Sd+IgqSDOq1I3o87cjkLb64RJzStfutLNvrFr9CDjsbHe7NMUjankIMnAK0yN2HQDgQKYRADD2m3epNzpX7TXSeVIleMDvKCKisuWWS52d2XLJR582pNq+fTs6OjowdepUd11dXR1aW1vR3t4OAGhvb0d9fb1b4ALA1KlTYVkWNmzwLySTySQSiYTnQUREVGn6tCFVR4fTcGfUqFGe9aNGjXK3dXR0oLGx0ZuJaBQNDQ1uGlNbWxuWLl2av8ES2VEi3IBUdUuxz0KyW71UkaxUjX6E5YR2OrCNuDdWVYOinCEOhTG8oixRg2+pziEWcW7NJrvHAwDiUPeX5S4AwJYnfg4AaLH/B0PFJwAAu0udg+WcsBvh6k9fXwd9q7hb9+UBRJVq3JR2bl9XqdNPqwhX7xKJOekytkqQ6cIQ6Qy2EVfPNVXOMT58zPksx875uXof5/OwM053JBHh7WYiGpgqosvQkiVL0NnZ6T4++uijUmeJiIio1/o00m1qagIA7N69G6NHj3bX7969G5/73OfcNHv27PHsl06nsW/fPnd/UzweRzyeH/1kRHZiA6l7yqAFAFA99HvIDnNoDuJonrY3mq2ClbPOnARAx3Zh/V4x8+F0zakdUa+W/woA+GDljwEAJ9hOS3GR3APEVBLh5NmydJ2pOqZtdLfSiyq6dVpLpXMP4db7RvUdBnVMdKdVLp3rEo1G3bpyYTuhtezaCwBoqHbOYfMTdwMAJsy+S+XvVJUBRrpENDD1ackxfvx4NDU1YfXq1e66RCKBDRs2YPLkyQCAyZMnY//+/di4caObZs2aNbBtG62trX2ZHSIiorLS60j34MGD2Lp1q7u8fft2bNq0CQ0NDRgzZgxuu+02/OM//iNOOeUUt8tQc3Oz28L5jDPOwPTp0zFv3jw89NBDSKVSWLBgAa699toetVz2sABpVOlm4LQai8jPZdMJo07XbBdrLObGtpaO9Nw14Y4nkm07raJTmVT5UZMTpJ168KFJp053OHY766tsdxc9zKKuF9Z3Byx9e8BWkyToU9PhbM7MiXrSArfeV7VqRkqFx+o9YjFnh3S6G5Y7rqOzUySaUVlTEW/kf1WCD9WbtKjlerUfiIgGlF6XIG+99RYuvfRSd3nRokUAgDlz5mDFihX48Y9/jEOHDuHGG2/E/v378fnPfx4vvfSS20cXAB5//HEsWLAAU6ZMgWVZmDlzJn72s5/1wekQERGVr2Pqp1sqfv2h8vp25s4f18N5+cxp9IDST3yg86Q/JUvsd14knbrb//nN/wEANKWd2/W1wol47e4MrKjw7KzvCqQsb49gtwWyt2pbUWndPszefJm9i+2cCgs1giSsqPODS6oRNdJVTmVzlwqbP4l8FgBw0rVPqCONc/aviGZ+REQl6qdLREREwSp7woOcGF34rMtuNKdNN35r+AxE5e4qvWnC/pkizBcpNcwjdgAAajJO/XpEOtPyZVR9bCQWydbNGqyAMS51JOxGmHbuBfEfMlNfH2lcO0tmpyGUKVUPrSY0sFS/3ZjqDzwkquqhbedZRNQIVXJITkUzEVHlY6RLREQUksoOI4SNbAWk8ftBICfCTRk76inl3JDOszWSezwzKAy7BlxPOKD6ukajXQCADx//FwBAo+VM02epSDcS05MVZNwrYhnVstFMwEQPbn2tbt6cE+kb0yCadee6RXgkN72qw3WPEVFRuJo9QbeErlNjQW9+bBkAYMK3/496/3FwWzITEQ0AjHSJiIhCUtGRroSd32pZeLf7CW6J7NduOUDIU/tFLT2nnhMVWklnXOXqKqeON61aKHer8aZjVch2OM6L1oPCdTUhfc6p65bNIu8Y5lyK5nuJnAHA9KxFuuOwWp/RKQ8DAGojaiKL9F/VfifwZyERDSgVXejasGCrb+WIsS0DG9lvbEv9P+CWsW8prAd98LltnXMIUWC50LaeLxt5lk53myFqUoBM0tkQrVJXQA3PKJPpbNbNgiuvAFXDQ6odMlZ2eEhh69vG6rhG9yv3Dr7Rpci5ve99Q/U2EGp4ykzSmw3L7W9U0X+WRESBGEcQERGFpMJDCivwV4MFC3ZQIyst79605VkNAKJYdyOYjbFylwtt6+myeq1uKwNqij11y9ZSbcTSXc692qhqSCUlIPRkBHmTNhiMVlG5t5KtYg3HzPvOuXcN1Omk9F1lFYV3HXL2GTZcDcupziGa6VLZVSFwxL96gIioUjHSJSIiCklFR7oR+AxeoQgrPw7OGyIysEVVdr9sMOyNRs3BOISw8pcLbevxsp5xXg2KkXIGkMiknGkLhWpfFVXPUN1xrFgUdto7LZ/IbeTkecOMZ20sk1sxaw6GUWwsTV33K92xNfT7S5W3YdVOfXT6sBo0I+JE8XEcUDscVM9mVy8iosrGSJeIiCgkFR3pFhyoQua3Bg5Oq3575NXf9uK9Cy33Jq3fMuCGoW89+zQAYFxGT1eoNuseRXqSeaRhRVUdtTHhQWDEm+doRgKRea90T6GYzqTOu85rLKZy4+Tnjyv/EwBw9vWcX5mIBhZGukRERCGp7EjX0xfUZ1v+S4f5U8NovexNH/C7JLS5/lR0aNcDAM6bOQ8A8Ol/rnayYfR1jVapSeSlDaGbHuvWw3okR/XCrZ+VVbmHQCbi1KVKkY2Ks62Y/Sc8yK7Iea2O795AkHquP2d9lfrrS6rK34Tt1PWePee7agez9zURUWVjpEtERBSSyo50c/Um8jyqKLVUv0+c97VFXC1VA3CrRYFqFQ2mvK2r02kbMTOiVfImMXBH33JTGNuzs/zpiDeo1XjuiFTSdjZWRdUQVCnV11i1YkaVPjd1hyE2XL2ZirytgfPnSUQEMNIlIiIKDUOJCiFyu84CSOn5+nRzZXeDE7UOiUfdqfUCGyHrY0a8rZsjaV2XakHqClmhR6vy1hNnm0TrZekuVqlMZ9QsDJGYE6XLjLMsVd4z7tupCFcMcd+fiGgg4bcaERFRSBjpVghzBCw76tR/pm0nerS6u7zpMun8CeelscKo287W9aowVkbzh/EqNh1h7mpVpxtR/YUzKZXHqNEqWQ2ndTil6n5ltdrA1stENLAw0iUiIgoJI91KoZsrq8lou606AMChdA0AYFjc+Shl2hm/WESdeZYAQKSMSl3zp5Y5EJcKeaXVnR2tK2hfo6Ourg62AUR1YK1mF4qotGlV1xxVVbdplS5VdZzzIqKe4e0/TERU6Vjoljtdprmz/jmF7vhrvgcA2PlUGwAgkv4AAFCtutmk7bQ7zGLR2xlG9x892bwtpFtwm0lhtp8yDmkBsFRjL7s740kj1F9dt2qEdVg6pe+E2TepNx7qPEf550lEAwtvLxMREYWEoUS50yM1qlkDhBjmrKg6HQDwqT0WADBc7gUARKQzLZ7I5IwtkfLv1pP/XmrYRrU54knm7VZkRsd6AI5o7j7qeJZOpH7iZdQt8sPSiWi7os1qh1rnKVqn9mdDKiIaWBjpEhERhYSRboUQlg4T42pNEwDgs9f9EADwyX86dbwysweAmlMgrff279eTnerPf1xMZ8IDs99RD0kAUjf+UuvUYgrOORyqOhEAcNI3FjobrJFqX9V1CJEQJ5YgIup/jHSJiIhCwki3UuiuO3qIRD12ohgBAOi0GgAAcel0IRoe6YaVUc2X3RbQzgs9qbw77KPq0mOnnQ1pS68HLHUINScBpIqe9YiN6W43gwCAaJUTpWbsFCJ6hEq35bXzlFSTNhyMnKQ2jFOHqFfL+txARDSgMNIlIiIKCSPdSqNb9OqhFDP1AIDPfPP/AwB89JulzuqudzFc7APgzqDnRrg6kNRzJURUCGqp4Rqz6e2cYSWdJ8tYjlarQTnU4B1JNX2fZTmRMgB0qWhYDhkNIBvhnjZT1+Xq1st6ogN9rvmnT0RUyRjpEhERhYSRbtnzTq1nS91f1wkjhe7TKicCAFq+8a8AgA8fuxVxbFe7dgIALCuljuWEkFVRb+Vu9yGnwjY2xPmzsDI5o1Hpl1W6Mtc5VneX2meo8/tN1w9bluVOeIBqp6X1B/gsAOCMa+9W+Rir3t8Z9tFWoXFuQM5gl4gGEka6REREIWGkW2Es1YzYHVRKtWaW6lmoKHbsrGX48Il/AgAMsT5VzwkAgN3tjF41JOJUtka7jwAAYsNV/9iUE70Ky8q+UbWKcI+oCtqYE5XGqp33SyWdSt5kxIm8u+1hsG0nT51WCwDgjG/9ROV5nPOsWysLp99uxuzXyziXiAaYXke669evx5VXXonm5mYIIfDss8+621KpFBYvXoyzzz4bw4YNQ3NzM7797W9j586dnmOMGzcOQgjPY/ny5cd8MkREROWs15HuoUOHMHHiRNxwww245pprPNsOHz6Mt99+G3fccQcmTpyITz/9FLfeeiuuuuoqvPXWW560y5Ytw7x589zlmpqaozyFgU5FtvpZNUG23H67zlNGRaSWUJPaV03E2Ot+qdI4ke6mx/4ZANA05C8AgCrbebbizjFlOukk9xk/2dYdclXAa6umz3qsZRl1Pr8EnHrazujJOOPaHwAARlqj1LHUlH16/GjhHEyqvFcJc45BC6wBIaKBpNeF7owZMzBjxgzfbXV1dVi1apVn3S9+8QtccMEF2LFjB8aMGeOur6mpQVNTU2/fftCz9HCQunxSz1FV+km3281QwNI/ZJwBND73rX90FuVWAMD7K+8DAAyLHgYARCLOs6UK1AgkbD0ahuqiJC3V2EnfdlaT4abVEI4nXPv3AIBmayxgjVFZVPlQPxT0LfKUGowjGjWGp9STO1gscIloYOn3b7XOzk4IIVBfX+9Zv3z5cowYMQLnnHMO7r33Xndicz/JZBKJRMLzICIiqjT92pCqq6sLixcvxqxZs1BbW+uu//73v49zzz0XDQ0NeO2117BkyRLs2rUL9913n+9x2trasHTp0v7MatnTbYz0z6S0igarot4xFjMZ58eLFYnDiuqJA4arJDpKdj6LU771iNqufvDoWe/dBkxH8PoTDwEALpx1rWdfRKo9u8Ia4n0WwwB9q9v4M8vo4FnlPa1OLqrHmnQjXEa6RDSw9Fuhm0ql8PWvfx1SSjz44IOebYsWLXJfT5gwAbFYDN/97nfR1taGeDxuHgpLlizx7JNIJNDS0tJfWSciIuoX/VLo6gL3ww8/xJo1azxRrp/W1lak02l88MEHOO200/K2x+Nx38J4MNHBnzt3gYoSbbVGx4TRiBqWEcJtXCVklTqGasiknqU7t59Qy55FCHThwtn6DoPeqAaycBtseTfbdkblT7gNpPTx3LnsjQBWd3PK2E4IHLEGwuT12VsT/hMrli/f/Pb2JDyN8Y41R0QDR58XurrAff/99/HKK69gxIgRRffZtGkTLMtCY2NjX2eHiIiobPS60D148CC2bt3qLm/fvh2bNm1CQ0MDRo8eja9+9at4++238cILLyCTyaCjowMA0NDQgFgshvb2dmzYsAGXXnopampq0N7ejoULF+Kb3/wmjjvuuL47swFKBw2Wu2yuyaaL6DpS4d1m625HljCW9cAbOqqNAaLeyEDc592yrGg2ShVmhCP810fUBuGGzeFwz1MReRkGJLzdmMwUMu+6e5uVS1ieYS29+3qJoA39TXjf1mg+4O1CZlyP7CbLmzY3MnZf+3UJIxpchDS/eYpYu3YtLr300rz1c+bMwd13343x48f77vfKK6/gkksuwdtvv43vfe97eO+995BMJjF+/Hh861vfwqJFi3p8CzmRSKCurg6dnZ1Fb10PVPl3+3r+hWYWskGFrhASQNLYW39GxvGP4R5qqW6/hlPoRgdWoSuOotAN2JeFLg0kPS2Xel3olgMWur2TLUSF73Kx9JUo9xwymYxnWyTiX2esu61Fo9kbQPmFrlkk6cFLvNuzBVi2YHHfVQ8oYlzevKvtGy36bCu2HLTNpAdaUYv+ka46REABykKXBquelkv8qyciIgoJJzwYBMyItacRbyXT52TbthvZ6vM1z1vfXs+NcPOOp19I/9vq7iUUulV5lmWkdY8ZcF/dXZ1z21eI4G3FloO2uYwA1PKpizeDXdv4vZ73693vT8q8dgPvz46oKEa6REREIWGkSwOejmSDIvqgyD+3B7Su2zX7HAdFqX1BBrzu7XLQtqA65EIzK5q1snlbhM/v+IprNULUfxjpEhERhYSR7iBm1mlq1gCb3afY+eRdB73sPQoAIOJGdM6T2bpZp/OOiK3fyEwb0ALYWHbSmUfszXKRtHqxQEis8xQJSAqj1XaWlX93gGgQG1jfrkRERGWMke4gEhTRLlu2DEB+/9V0Ol2xUa+un41Go0gmnQE+9ExV5jnlDQ6i1lvIjw3zo0ZvCrMaNL8ts99ymehFRBrU8Dg/8s8ZQKRYv2SiQYCF7iBiFjZ6MIiqKmf4RV046XSVWuAC2VvG6XTaHenMbEilB80wuxTlpssbRsNohRTU7cfTlSioq4xbuPkXSllWwOveLnu3yYCfBBEzf724LSxE0G3m4DHTWPjSYFK536pEREQVhpHuIGKO+KkHg9ARrjk4RAWOEOrK7QakI3qza1DQcJA9mpYuYJAMN32hW7XmvpUg6Hzc8w4agzr3lnMFnjdRH+O/AiIiopAw0h1EggaHMOs0B4JCQ1wWGyQDnijNaHSlkmQnAzDrSc1j5tRkSm/dbeAsQyHxnxQSyKScOwORqvz64MBJGgIzb0MY3aiK7kI0gDHSJSIiCgkjXaIeCOr047b0tVW9cUpNjJdJZxPntXBW3YsCWjy7R3cjY9v72txWbDlom86Wro/VrbktlZGUek5nANXCXVjOM6JVuadSeGhJ9b7CiPSJBiNGukRERCFhpEuk+VRpu8MfBkxs4E6EkE45K/YfdJ6T3dmUZovmou8fUqRrVsjq5e4u5zmmWnfLKNzf58OGq23qfIfEnOeIs13F+bDcaN4uPjsDI18aRBjpEhERhYSRLpHm9kXtyW9RW/3feY6oSPfRL14CADjxwGF1qGxtsF1gtCYAiARMci+F97W5rdhysbS66jYa1UNZOvFqSvXfjsVr8Kn6fd5RXw8AWPDyKmenKicaFm6XZ+8oV1buhAdmxTgjXBqEGOkSERGFhJEukcHsi9ojKSfSHZdIAABO/zShNuSGdyo6tnrWH1ons4X3tbmt2HLgNninNLRUxbWwnXNxx96WR7Cv2qnLjerT6VL1vtWqFXMseFxnNwjmFH9EjHSJiIjCwkiX6Bi4v1ozTghY3+3Ugx5/5BAAHdzp0K7wlH5HFWH3AUvForZ06nJ1/aweoMy2u6FnhayvHuK86E7pjSqxk3lL5M9c5PbhNc+Pdbo0CDHSJSIiCgkjXSKDyK1zLNq3Vv0TsnP6pQKQET0KU+7xvJWa0gj9rNDHvvbOuuT2PdatmlWn21jUQkb9PE/pGFa1dIalwmKhWz473JbYwmdaXqNbMNFgwkKXyJQ7iETgJPXG+Id6tS7A3Of8NG7avJ2PKrfHwFsc6t8AtlqtZz7M2DayXaTMffVi8E2z7E+NoGnsiQYP/tUTERGFhJEuUR6f36JBk9TDu+xOl6fS596qzpsWr8y6zuj8WD7tvfQ24Z5YwM3hAveMBX/jE/FfARERUVgY6RIFEUB+Nx/LJ00OFc76RrF59Z6FuxCVipl3KbIrs+N6ZAd69BM41R/RIMdIl4iIKCQsdImIiELCQpeIiCgkLHSJiIhCwkKXiIgoJCx0iYiIQtLrQnf9+vW48sor0dzcDCEEnn32Wc/266+/HkIIz2P69OmeNPv27cPs2bNRW1uL+vp6zJ07FwcPHjymEyEiIip3vS50Dx06hIkTJ+KBBx4ITDN9+nTs2rXLfTzxxBOe7bNnz8a7776LVatW4YUXXsD69etx44039j73REREFaTXg2PMmDEDM2bMKJgmHo+jqanJd9uf//xnvPTSS3jzzTdx3nnnAQB+/vOf4/LLL8e//Mu/oLm5ubdZIiIiqgj9Uqe7du1aNDY24rTTTsPNN9+MvXv3utva29tRX1/vFrgAMHXqVFiWhQ0bNvgeL5lMIpFIeB5ERESVps8L3enTp+M//uM/sHr1avzzP/8z1q1bhxkzZiCTcSbn7OjoQGNjo2efaDSKhoYGdHR0+B6zra0NdXV17qOlpaWvs01ERNTv+nzs5WuvvdZ9ffbZZ2PChAk4+eSTsXbtWkyZMuWojrlkyRIsWrTIXU4kEix4iYio4vR7l6GTTjoJI0eOxNatWwEATU1N2LNnjydNOp3Gvn37AuuB4/E4amtrPQ8iIqJK0++F7scff4y9e/di9OjRAIDJkydj//792Lhxo5tmzZo1sG0bra2t/Z0dIiKikun17eWDBw+6USsAbN++HZs2bUJDQwMaGhqwdOlSzJw5E01NTdi2bRt+/OMf4zOf+QymTZsGADjjjDMwffp0zJs3Dw899BBSqRQWLFiAa6+9li2XiYhoQOt1pPvWW2/hnHPOwTnnnAMAWLRoEc455xzceeediEQi2Lx5M6666iqceuqpmDt3LiZNmoTf//73iMfj7jEef/xxnH766ZgyZQouv/xyfP7zn8e///u/991ZERERlaFeR7qXXHIJpPSbodvxu9/9rugxGhoasHLlyt6+NRERUUXj2MtEREQhYaFLREQUEha6REREIWGhS0REFBIWukRERCFhoUtERBQSFrpEREQhYaFLREQUEha6REREIenzqf2IBiSpfp+KYziGsL3HKhPSOCcRPOAcbDdt4XNwk0nkXTNppil3udejWKbNa1cxJ0lhYaE7COhhO4UQvsua3/ZCQ36WM53vSCSCTCbjWVfsOni/Y43C1iwxjMsjhaVWWyqZnf/F6xa+PT6dfqELW9vInyV13h1C2jk76cLWPCnjnISZPuf99PsEHKm/5RX67go7PzEAwAosTIv+gMhJUMqPm2V/+Sivn9xEREQDGCPdQcCM5ExBEaBt27Csyvxdps8lk8kEnoN5XWzbiXREoXPOi1qD0pVHNNsXzNvP3o3Gcl60mH8tSxXhFqWicqnOwZO/gMz2JmovdrOk0PLR7kvlpzK/UYmIiCoQI91BwIxkg+pydaSnI0MhhLuu0uRGutGo98/cjWiN6+EbEefV91me1TrWETo6kvrZ7xhqD/VsmQ2Y8i61zo9tvDa3FVv236bzap61UOly63x19GeZdbcGvdpWR7Ws7LXIa6BVonAsry5XL6oNGeOKRISds49/nGKr6xLxqcs230sYYWmPlnuTNvf9GPKWHUa6REREIWGkOwjktc4t0opXR4JSyoqt09XnEIvFkEqlAARH+mY0b4lC52z7LrkRjk9/m4J1oiV0bPnSoZVaVNfM9m71pgm4a1AyZpTYB9Ghewi/c3QvSt/drSi6zIi37FTmNyoREVEFYqQ7CBSr07377rvDzlJo/Fpg6367kUgEQEBdLqCajeo6TPUs9JO5j67rNfv8RnIiHOeF5QaJ3qg5P/K0A173drlYWp0Bbxtcof5uLNgQUkeyut+yvhBmHbfRxzd3oxv9FW/h3D8C3l96vwbdJU8LdP86Wl0vbxuRpl+VtzDP3/28e7Dcm7SeZY3xVbngJ0FERBQSRrqDgBnZmpFeUB1vJdfp6nOwLCvvfPWzmTZ3HwBGtOBfRyaL/m7N7WVpVm6a9XDlIXuKOuK1YJnDVmkqeuvZGZTXebqC7jDouxvSKlonmr1H4D3H3JbQlnEXIH/vniz3Ji2VI35CREREIWGkOwiYkawZ6QX12y02klU5y817ULTu1y9ZbfFJXez3acBgzH7j+cqAelBZ2khQGq+yEa/tNu11++nqrb3Kc6kje+9n6J6vUR8qzHP0tB429s3j3eJ3xsXvjvS9yv2XPPAw0iUiIgoJI91BoOisOgXWV3K0C/ifQ17dbUH+U/od05DKgVP8lTYSdEei6lE9oTcajATuiwItaUv0m1/PouSOQKVW584MBcDSddo5I0K5hOcJ+S2jvZ+lxfiGFBa6g0hQAdrb9ZXE7xyKTgCR8wVZrHNLdnhA9cPF7feT0/DKHSLSfJ8g4XahcQf4qHKqHTIppxiKCmfZlrbbRcrSJyEznmPo28z5XakQ2Ogs9L+u3LvFPvmQqtYlo74WpVtgSljGYBf5hW0Gfir/XxD1Nf78IiIiCgkjXSJDbnRS7Iav2yQtWgUA+KC2Ni+N5bZIUlMmuqFVecRBlhrDMtndDQCIxeIAgHQ67SxHq/BplXN+Hw0b6uwUH+I8C30F1JXSF8rsJYXca+i95Rp+xKtyYunoHZ58uOOgWD75C5zwXlXhwGik6LnjcSyZpoGCkS4REVFIGOkSmXK6wehIJRIUpeik8eEAgKs2veksp7vU9ijy4mX3WCVuUKRJJ6JFxBziUYV1yW43kv+8inhRpSLdqmp1DHifNZGtGw2+XxDS+avoVEYzKjfORBiWjKnNznlXGXW/EoBtDPuYf5dCDTRjBW2XPhMehEeISPFEFApGukRERCFhpEtk8hvnolgfoSr1+zVep1YMV/tFkO12FNQWurRdhfSpZdSJ62cbTgQ8BIBwB8PQT+qrI2gS9ZxAL9sFxyvsKs78xsv6c8kb29PzLAR86qj9+5CZgW42VW56o4V7f07tR2WHnwwREVFIGOkSGbITkSM/wjUnBc/72Rr1PEuRG3sYrWHz6j9LE/G6fXCNaNV2p7yTSGdU/WdU1V26rXXdgzhPqurQbwIAETQYSEghr1s/Dz1NoVO3m9J1u+r6u6OkZpwXQlgIqsoVxpSPsL0Db3jvbgTV4XPCg8GEnxAREVFIGOkSBZDIiVjMCNecgF5v1pMZiGyqox8ysi/q9Art65AZ78QPOsM64JNCIBKNeY6ho0QB/1ax7ihXue9nRrohV+qmVR31YXSp5yMAgINIAACO4BAAIKojfz31JSKIwjn/jDvylHOsiB5tTL+J0KNZOXIHy8yZgqMvTqdHhsJpZd6ABgxFdWjvS8F6/emvX78eV155JZqbmyGEwLPPPuvZLoTwfdx7771umnHjxuVtX758+TGfDBERUTnrdaR76NAhTJw4ETfccAOuueaavO27du3yLL/44ouYO3cuZs6c6Vm/bNkyzJs3z12uqanpbVaI+l1enaW7Qf9eTXvWZ4T3d6yuPXTSmHW1RgtgYy//5T6sD9R1uO5qlb+M5UkuAdjCifAiaicB77JZX6n75kpkx23OO8+QI90jOAwA2AXnO+r29tsBAFvqtwMAukY6249kDgAAIupy1Ig6qCptpFQfX2npSD/teY9IRke6ahxrfQ0FYOmBsEKssz/xry0AgEcnrMBQnBja+1KwXhe6M2bMwIwZMwK3NzU1eZafe+45XHrppTjppJM862tqavLSEhERDWT9Wqe7e/du/Pa3v8Wjjz6at2358uX4yU9+gjFjxuC6667DwoULEY36ZyeZTCKZTLrLiUSi3/JMVIinZbNnRfA+gZvKbSxeHc5F4t71QkIYHXLd9rx5LbC9dZwSVvA1C1lKRaWd+BQAsKu+AwDwydhPAAAd8R1OQqOaWqQ+RsRyRuJK6/A3b6J7kzEmtWef8Ptj6/prKr1+LXQfffRR1NTU5N2G/v73v49zzz0XDQ0NeO2117BkyRLs2rUL9913n+9x2trasHTp0v7MKhERUb/r10L3V7/6FWbPno0hQ4Z41i9atMh9PWHCBMRiMXz3u99FW1sb4vG4eRgsWbLEs08ikUBLS0v/ZZwGNd85YQMT+9Xh5h7Lb22ZcCNPlT81vrIZvVsQbr/cvBa45jEC3qL4yjCoWZ7Uc0ZVsqb1ONlD3Q66nr1ElYWM7GmU6nbY9dlWmhHHqLz0W6H7+9//Hlu2bMGvf/3romlbW1uRTqfxwQcf4LTTTsvbHo/HfQtjotLrzcTsZV74Fhisomh3l3K7Ve7DvCEs9cAVQjcO042/vPfBZW7aPMVOnAUtefXbt8DDDz+MSZMmYeLEiUXTbtq0CZZlobGxsb+yQ0REVHK9jnQPHjyIrVu3usvbt2/Hpk2b0NDQgDFjxgBwbv8+9dRT+Nd//de8/dvb27FhwwZceumlqKmpQXt7OxYuXIhvfvObOO64447hVIiIiMpbrwvdt956C5deeqm7rOta58yZgxUrVgAAnnzySUgpMWvWrLz94/E4nnzySdx9991IJpMYP348Fi5c6KmzJSIiGoh6XehecsklkLJwm/8bb7wRN954o++2c889F6+//npv35aIiKjilWnLDiIiooGHhS4REVFIWOgSERGFhIUuERFRSFjoEhERhYSFLhERUUhY6BIREYWEhS4REVFIWOgSERGFhIUuERFRSFjoEhERhYSFLhERUUhY6BIREYWEhS4REVFIWOgSERGFhIUuERFRSFjoEhERhYSFLhERUUhY6BIREYWEhS4REVFIWOgSERGFhIUuERFRSKKlzgARUbicWEPInsYcds5r4XssbxqiYIx0iYiIQsJIl4gGBUtFqcKIdIUsvJ/0XdIRrzduEcgYqc3IOFx2ad+efDDSJSIiCgkjXSIaFMxaV5m3xYxiRV7KIG79sPAeq+dHoMGChS4RDRJO0ZdRhay00p6tlnSKSFvdAJQidz//Ijv41rTffd3wbyxaNm9mlht+IkRERCFhpEtEg4pUjZ1sfStYPeuo1VKxiBvbCgnp3iA2Q1snlaVuL2eEtwtR7o1q6R6Rsc5gxk+fiIgoJIx0iWhQ0DGq29RJhbZu7KqWdQDsxqwFut1Y+mgBaaSnQZVOFOJAGoKDdpQbRrpEREQhYaRLRIOCOZyFkAHhqehN3WtvOgOZXZJoMGKkS0REFBJGukQ0qEQQAZBtcawHtsjW7eoXtrkij52X1j+O8R9KMgwclqPc9CrSbWtrw/nnn4+amho0Njbi6quvxpYtWzxpurq6MH/+fIwYMQLDhw/HzJkzsXv3bk+aHTt24IorrsDQoUPR2NiIH/3oR0invR3ViYiIBppeFbrr1q3D/Pnz8frrr2PVqlVIpVK47LLLcOjQITfNwoUL8fzzz+Opp57CunXrsHPnTlxzzTXu9kwmgyuuuALd3d147bXX8Oijj2LFihW48847++6siIgMFgQsCPV/C5YdhWVHIYUFKSzYQqiH7XkQ9SUhpTzq+w+ffPIJGhsbsW7dOlx88cXo7OzE8ccfj5UrV+KrX/0qAOC9997DGWecgfb2dlx44YV48cUX8Td/8zfYuXMnRo0aBQB46KGHsHjxYnzyySeIxWJF3zeRSKCurg6dnZ2ora092uwT0SCyD/sAAP+DbQCARe8tAgC8f4Jzt25vzV8D9oy4A2qYt2uF8Urmza9b2tu7p+48BQDwfPNvcSpOKWleBrqelkvH1JCqs7MTANDQ0AAA2LhxI1KpFKZOneqmOf300zFmzBi0t7cDANrb23H22We7BS4ATJs2DYlEAu+++67v+ySTSSQSCc+DiKh3LAAWIuo/mQJkCm7kK+EUkVJK5MYiUuZWfQnkFrXSeJjv5U0vQn/o6J3Kx1EXurZt47bbbsNFF12Es846CwDQ0dGBWCyG+vp6T9pRo0aho6PDTZNb4OrtepuftrY21NXVuY+WlpajzTYREVHJHHWhO3/+fLzzzjt48skn+zI/vpYsWYLOzk738dFHH/X7exLRQGMDsGFDwoZENB5DNB5DdyaN7kxONCssQFjZiFc68bGFCOA+nEg2G1MG/ReBUOn1GnOPniwf7b5COA8qH0fVZWjBggV44YUXsH79epx44onu+qamJnR3d2P//v2eaHf37t1oampy07zxxhue4+nWzTqNKR6PIx6PH01WiYiIykavIl0pJRYsWIBnnnkGa9aswfjx4z3bJ02ahKqqKqxevdpdt2XLFuzYsQOTJ08GAEyePBl//OMfsWfPHjfNqlWrUFtbizPPPPNYzoWIKJCOcFNIIoUkjsiDOCIPwrIs54EoLEQB23YeStyKQ2YykJkMhJQQUkJHlNJ9OP9Z0vuAfkA4E/TKnDrX3iwf5b4i4zyofPQq0p0/fz5WrlyJ5557DjU1NW4dbF1dHaqrq1FXV4e5c+di0aJFaGhoQG1tLW655RZMnjwZF154IQDgsssuw5lnnolvfetbuOeee9DR0YHbb78d8+fPZzRLRP3GUg2a4qgCANSmhgMARnY1qO1qknvVTSgScQbRQFoCtrPNEs46PYKkbTmtmi3VikoPtGEKGnGyv9UeqQEAxNQ5U+n1qtB98MEHAQCXXHKJZ/0jjzyC66+/HgBw//33w7IszJw5E8lkEtOmTcMvf/lLN20kEsELL7yAm2++GZMnT8awYcMwZ84cLFu27NjOhIiIqMwdUz/dUmE/XSLqrQM4CAD4GE5DzJ/8/icAgF1DdgIAUnV6cnsnfcp2liPSRkRNTi/URp1GGoNn5Ee6tkoHlGKo+6ZPnZ4hD174IJowqkhqOhah9NMlIiKinuOEB0Q0KFSpiQ5GwKnDXfaFpQCAgzgAADiEIwAAC86oeNkpDNLu8BYRI05JGxPSC7Vdp5c60vXJj06r0xRa7k3a3OVaOPXWQ1HtkwMqBUa6REREIWGkS0SDQpVqwVuHOgDAcAwDAKQxAgByRleOqvU6asy4W2MqWtaxbErFsNnp6Z31uiW07YmEvTGObk1tq7SFlnuTNnc5qs6FkW75YKRLREQUEka6RDQoRNTXXUSqrz3dcUMOcZ4ttV4FpzKSXbR01Gu2TtatmL2LLpkT6QrGOARGukRERKFhpEtEg4swXgjjazDiTeZM0Fc4PgkacIrRLZn4F0FERBQSRrpENDj1cDxkEbhA1HuMdImIiELCQpeIiCgkLHSJiIhCwkKXiIgoJCx0iYiIQsJCl4iIKCQsdImIiELCQpeIiCgkLHSJiIhCwkKXiIgoJCx0iYiIQsJCl4iIKCQsdImIiELCQpeIiCgkLHSJiIhCwkKXiIgoJCx0iYiIQsJCl4iIKCQsdImIiELCQpeIiCgkLHSJiIhCwkKXiIgoJCx0iYiIQsJCl4iIKCQsdImIiELCQpeIiCgk0VJn4GhIKQEAiUSixDkhIiLKlke6fApSkYXugQMHAAAtLS0lzgkREVHWgQMHUFdXF7hdyGLFchmybRtbtmzBmWeeiY8++gi1tbWlzlKvJBIJtLS0MO8hY95Lg3kvDeY9XFJKHDhwAM3NzbCs4Jrbiox0LcvCCSecAACora2tmA/FxLyXBvNeGsx7aTDv4SkU4WpsSEVERBQSFrpEREQhqdhCNx6P46677kI8Hi91VnqNeS8N5r00mPfSYN7LU0U2pCIiIqpEFRvpEhERVRoWukRERCFhoUtERBQSFrpEREQhYaFLREQUkootdB944AGMGzcOQ4YMQWtrK954441SZ8mjra0N559/PmpqatDY2Iirr74aW7Zs8aS55JJLIITwPG666aYS5Tjr7rvvzsvX6aef7m7v6urC/PnzMWLECAwfPhwzZ87E7t27S5hjr3HjxuXlXwiB+fPnAyiv675+/XpceeWVaG5uhhACzz77rGe7lBJ33nknRo8ejerqakydOhXvv/++J82+ffswe/Zs1NbWor6+HnPnzsXBgwdLmvdUKoXFixfj7LPPxrBhw9Dc3Ixvf/vb2Llzp+cYfp/V8uXLS5p3ALj++uvz8jV9+nRPmnK87gB8//aFELj33nvdNKW47j35TuzJd8uOHTtwxRVXYOjQoWhsbMSPfvQjpNPpfs17X6rIQvfXv/41Fi1ahLvuugtvv/02Jk6ciGnTpmHPnj2lzppr3bp1mD9/Pl5//XWsWrUKqVQKl112GQ4dOuRJN2/ePOzatct93HPPPSXKsddnP/tZT75effVVd9vChQvx/PPP46mnnsK6deuwc+dOXHPNNSXMrdebb77pyfuqVasAAF/72tfcNOVy3Q8dOoSJEyfigQce8N1+zz334Gc/+xkeeughbNiwAcOGDcO0adPQ1dXlppk9ezbeffddrFq1Ci+88ALWr1+PG2+8saR5P3z4MN5++23ccccdePvtt/H0009jy5YtuOqqq/LSLlu2zPNZ3HLLLSXNuzZ9+nRPvp544gnP9nK87gA8ed61axd+9atfQQiBmTNnetKFfd178p1Y7Lslk8ngiiuuQHd3N1577TU8+uijWLFiBe68885+zXufkhXoggsukPPnz3eXM5mMbG5ulm1tbSXMVWF79uyRAOS6devcdV/84hflrbfeWrpMBbjrrrvkxIkTfbft379fVlVVyaeeespd9+c//1kCkO3t7SHlsHduvfVWefLJJ0vbtqWU5XvdAchnnnnGXbZtWzY1Ncl7773XXbd//34Zj8flE088IaWU8k9/+pMEIN988003zYsvviiFEPIvf/lLyfLu54033pAA5IcffuiuGzt2rLz//vv7N3NF+OV9zpw58itf+UrgPpV03b/yla/IL33pS5515XDdze/Enny3/Nd//Ze0LEt2dHS4aR588EFZW1srk8lkuCdwlCou0u3u7sbGjRsxdepUd51lWZg6dSra29tLmLPCOjs7AQANDQ2e9Y8//jhGjhyJs846C0uWLMHhw4dLkb0877//Ppqbm3HSSSdh9uzZ2LFjBwBg48aNSKVSnut/+umnY8yYMWV5/bu7u/HYY4/hhhtugBDCXV+u1z3X9u3b0dHR4bnWdXV1aG1tda91e3s76uvrcd5557lppk6dCsuysGHDhtDzXEhnZyeEEKivr/esX758OUaMGIFzzjkH9957b9ncKly7di0aGxtx2mmn4eabb8bevXvdbZVy3Xfv3o3f/va3mDt3bt62Ul938zuxJ98t7e3tOPvsszFq1Cg3zbRp05BIJPDuu++GmPujV3GzDP31r39FJpPxXHQAGDVqFN57770S5aow27Zx22234aKLLsJZZ53lrr/uuuswduxYNDc3Y/PmzVi8eDG2bNmCp59+uoS5BVpbW7FixQqcdtpp2LVrF5YuXYovfOELeOedd9DR0YFYLJb3xTlq1Ch0dHSUJsMFPPvss9i/fz+uv/56d125XneTvp5+f+t6W0dHBxobGz3bo9EoGhoayurz6OrqwuLFizFr1izPrDHf//73ce6556KhoQGvvfYalixZgl27duG+++4rYW6dW8vXXHMNxo8fj23btuEf/uEfMGPGDLS3tyMSiVTMdX/00UdRU1OTV/1T6uvu953Yk++Wjo4O338PelslqLhCtxLNnz8f77zzjqdeFICn/ufss8/G6NGjMWXKFGzbtg0nn3xy2Nl0zZgxw309YcIEtLa2YuzYsfjNb36D6urqkuXraDz88MOYMWMGmpub3XXlet0HqlQqha9//euQUuLBBx/0bFu0aJH7esKECYjFYvjud7+Ltra2ko67e+2117qvzz77bEyYMAEnn3wy1q5diylTppQsX731q1/9CrNnz8aQIUM860t93YO+EweDiru9PHLkSEQikbwWbbt370ZTU1OJchVswYIFeOGFF/DKK6/gxBNPLJi2tbUVALB169YwstZj9fX1OPXUU7F161Y0NTWhu7sb+/fv96Qpx+v/4Ycf4uWXX8bf/d3fFUxXrtddX89Cf+tNTU15DQjT6TT27dtXFp+HLnA//PBDrFq1qujcqK2trUin0/jggw/CyWAPnXTSSRg5cqT7N1Lu1x0Afv/732PLli1F//6BcK970HdiT75bmpqafP896G2VoOIK3VgshkmTJmH16tXuOtu2sXr1akyePLmEOfOSUmLBggV45plnsGbNGowfP77oPps2bQIAjB49up9z1zsHDx7Etm3bMHr0aEyaNAlVVVWe679lyxbs2LGjrK4/ADzyyCNobGzEFVdcUTBduV738ePHo6mpyXOtE4kENmzY4F7ryZMnY//+/di4caObZs2aNbBt2/0xUSq6wH3//ffx8ssvY8SIEUX32bRpEyzLyrt1W2off/wx9u7d6/6NlPN11x5++GFMmjQJEydOLJo2jOte7DuxJ98tkydPxh//+EfPDx79Y+7MM8/st7z3qRI35DoqTz75pIzH43LFihXyT3/6k7zxxhtlfX29p0Vbqd18882yrq5Orl27Vu7atct9HD58WEop5datW+WyZcvkW2+9Jbdv3y6fe+45edJJJ8mLL764xDmX8gc/+IFcu3at3L59u/zv//5vOXXqVDly5Ei5Z88eKaWUN910kxwzZoxcs2aNfOutt+TkyZPl5MmTS5xrr0wmI8eMGSMXL17sWV9u1/3AgQPyD3/4g/zDH/4gAcj77rtP/uEPf3Bb+C5fvlzW19fL5557Tm7evFl+5StfkePHj5dHjhxxjzF9+nR5zjnnyA0bNshXX31VnnLKKXLWrFklzXt3d7e86qqr5Iknnig3bdrk+TegW5m+9tpr8v7775ebNm2S27Ztk4899pg8/vjj5be//e2S5v3AgQPyhz/8oWxvb5fbt2+XL7/8sjz33HPlKaecIru6utxjlON11zo7O+XQoUPlgw8+mLd/qa57se9EKYt/t6TTaXnWWWfJyy67TG7atEm+9NJL8vjjj5dLlizp17z3pYosdKWU8uc//7kcM2aMjMVi8oILLpCvv/56qbPkAcD38cgjj0gppdyxY4e8+OKLZUNDg4zH4/Izn/mM/NGPfiQ7OztLm3Ep5Te+8Q05evRoGYvF5AknnCC/8Y1vyK1bt7rbjxw5Ir/3ve/J4447Tg4dOlT+7d/+rdy1a1cJc5zvd7/7nQQgt2zZ4llfbtf9lVde8f07mTNnjpTS6TZ0xx13yFGjRsl4PC6nTJmSd0579+6Vs2bNksOHD5e1tbXyO9/5jjxw4EBJ8759+/bAfwOvvPKKlFLKjRs3ytbWVllXVyeHDBkizzjjDPlP//RPnoKtFHk/fPiwvOyyy+Txxx8vq6qq5NixY+W8efPyftSX43XX/u3f/k1WV1fL/fv35+1fqute7DtRyp59t3zwwQdyxowZsrq6Wo4cOVL+4Ac/kKlUql/z3pc4ny4REVFIKq5Ol4iIqFKx0CUiIgoJC10iIqKQsNAlIiIKCQtdIiKikLDQJSIiCgkLXSIiopCw0CUiIgoJC10iIqKQsNAlIiIKCQtdIiKikPz/I4lza+Iv9LQAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "img2 = cv2.imread('data/train/images/3.jpg')\n",
    "imshow(cv2.cvtColor(img2, cv2.COLOR_BGR2RGB))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/shr/Documents/projects/nsvqa/perception.py:208: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  scene_df = scene_df.append({'shape': shape,\n",
      "/home/shr/Documents/projects/nsvqa/perception.py:208: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  scene_df = scene_df.append({'shape': shape,\n",
      "/home/shr/Documents/projects/nsvqa/perception.py:208: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  scene_df = scene_df.append({'shape': shape,\n",
      "/home/shr/Documents/projects/nsvqa/perception.py:208: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  scene_df = scene_df.append({'shape': shape,\n",
      "/home/shr/Documents/projects/nsvqa/perception.py:208: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  scene_df = scene_df.append({'shape': shape,\n",
      "/home/shr/Documents/projects/nsvqa/perception.py:208: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  scene_df = scene_df.append({'shape': shape,\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>shape</th>\n",
       "      <th>color</th>\n",
       "      <th>position</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>rectangle</td>\n",
       "      <td>green</td>\n",
       "      <td>(159, 188)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>rectangle</td>\n",
       "      <td>red</td>\n",
       "      <td>(103, 152)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>rectangle</td>\n",
       "      <td>yellow</td>\n",
       "      <td>(19, 84)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>rectangle</td>\n",
       "      <td>blue</td>\n",
       "      <td>(151, 44)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>rectangle</td>\n",
       "      <td>gray</td>\n",
       "      <td>(59, 148)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>circle</td>\n",
       "      <td>orange</td>\n",
       "      <td>(47, 112)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       shape   color    position\n",
       "0  rectangle   green  (159, 188)\n",
       "1  rectangle     red  (103, 152)\n",
       "2  rectangle  yellow    (19, 84)\n",
       "3  rectangle    blue   (151, 44)\n",
       "4  rectangle    gray   (59, 148)\n",
       "5     circle  orange   (47, 112)"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nsai.executor.scene = nsai.perceiver.scene_repr(img2)\n",
    "nsai.executor.scene"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['filter green', 'query shape', 'filter', 'count']"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "program = nsai.sem_parser.predict('How many objects of same shape of green colour object?')\n",
    "program"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "string=''\n",
    "for i in program:\n",
    "    if len(i.split(' ')) == 2:\n",
    "        string = string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nsai.executor.count(nsai.executor.filter_(nsai.executor.query(nsai.executor.filter_('green'), 'shape')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nsai.executor.count(nsai.executor.filter_(nsai.executor.query(nsai.executor.filter_('orange'), 'shape')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'circle'"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nsai.executor.query(nsai.executor.relate(nsai.executor.filter_('green'), 'furthest'), 'shape')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}