eval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from model import *\n",
    "from data import *\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "import yaml\n",
    "from utils import DotDict, adjust_learning_rate, accuracy\n",
    "import torch\n",
    "import traceback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "losses = {'attn_lstm_vocab_10k': 10165.64429010069,\n",
    " 'attn_lstm_vocab_1k': 15595.033411789249,\n",
    " 'attn_lstm_vocab_50k': 6836.341175179966,\n",
    " 'pointer_vocab_10k': 21672.131244335622,\n",
    " 'pointer_vocab_1k': 43398.31753725069,\n",
    " 'pointer_vocab_50k': 7057.6959175422135,\n",
    " 'simple_lstm_vocab_10k': 20159.74126170967,\n",
    " 'simple_lstm_vocab_1k': 109026.57322798869,\n",
    " 'simple_lstm_vocab_50k': 13842.316331750844}\n",
    "\n",
    "\n",
    "epoch = {'attn_lstm_vocab_10k': 8,\n",
    " 'attn_lstm_vocab_1k': 5,\n",
    " 'attn_lstm_vocab_50k': 1,\n",
    " 'pointer_vocab_10k': 8,\n",
    " 'pointer_vocab_1k': 5,\n",
    " 'pointer_vocab_50k': 1,\n",
    " 'simple_lstm_vocab_10k': 8,\n",
    " 'simple_lstm_vocab_1k': 5,\n",
    " 'simple_lstm_vocab_50k': 1}\n",
    "\n",
    "acces = {'attn_lstm_vocab_10k': 0.6577360621117113,\n",
    " 'attn_lstm_vocab_1k': 0.6494665779492861,\n",
    " 'attn_lstm_vocab_50k': 0.6315070176565867,\n",
    " 'pointer_vocab_10k': 0.6705065943604063,\n",
    " 'pointer_vocab_1k': 0.6662379230942852,\n",
    " 'pointer_vocab_50k': 0.634914800961635,\n",
    " 'simple_lstm_vocab_10k': 0.657081022143478,\n",
    " 'simple_lstm_vocab_1k': 0.6633081460420689,\n",
    " 'simple_lstm_vocab_50k': 0.6168174382832627}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "started label_smoothing_pointer_10k\n",
      "started pointer_vocab_10k\n",
      "already calculated\n",
      "started simple_lstm_vocab_10k\n",
      "already calculated\n",
      "started attn_lstm_vocab_50k\n",
      "already calculated\n",
      "started simple_lstm_vocab_1k\n",
      "reading data from  ./pickle_data/PY_non_terminal_small.pickle\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"<ipython-input-15-71a36957bcc8>\", line 13, in <module>\n",
      "    last_cpk = sorted(os.listdir(checkpoint_folder), key=lambda x: int(x[6:-4]), reverse=True)[0]\n",
      "FileNotFoundError: [Errno 2] No such file or directory: 'checkpoints/label_smoothing_pointer_10k'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the vocab_sizeN is 330 (not including the eof)\n",
      "the number of training data is 100000\n",
      "the number of test data is 50000\n",
      "\n",
      "reading data from  ./pickle_data/PY_terminal_1k_whole.pickle\n",
      "the vocab_sizeT is 1000 (not including the unk and eof)\n",
      "the attn_size is 50\n",
      "the number of training data is 100000\n",
      "the number of test data is 50000\n",
      "Finish reading data and take 13.77\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 50000/50000 [00:33<00:00, 1501.39it/s]\n",
      "100%|██████████| 619/619 [04:29<00:00,  2.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss: 109026.57322798869 acc: 0.6633081460420689\n",
      "started attn_lstm_vocab_10k\n",
      "already calculated\n",
      "started pointer_vocab_50k\n",
      "already calculated\n",
      "started pointer_vocab_1k\n",
      "already calculated\n",
      "started attn_lstm_vocab_1k\n",
      "already calculated\n",
      "started simple_lstm_vocab_50k\n",
      "already calculated\n"
     ]
    }
   ],
   "source": [
    "for config in os.listdir('configs'):\n",
    "    if not config.endswith('yml'):\n",
    "        continue\n",
    "    config = os.path.join('configs', config)\n",
    "    with open(config, 'r') as f:\n",
    "        config = DotDict(yaml.safe_load(f))\n",
    "    print('started', config.name)\n",
    "    if config.name in acces:\n",
    "        print('already calculated')\n",
    "        continue\n",
    "    checkpoint_folder = os.path.join('checkpoints', config.name)\n",
    "    try:\n",
    "        last_cpk = sorted(os.listdir(checkpoint_folder), key=lambda x: int(x[6:-4]), reverse=True)[0]\n",
    "    except:\n",
    "        traceback.print_exc()\n",
    "        continue\n",
    "    checkpoint_path = os.path.join(checkpoint_folder, last_cpk)\n",
    "    \n",
    "    device = config.train.device\n",
    "\n",
    "    data_val = MainDataset(\n",
    "        N_filename = config.data.N_filename,\n",
    "        T_filename = config.data.T_filename,\n",
    "        is_train=False,\n",
    "        truncate_size=config.data.truncate_size\n",
    "    )\n",
    "\n",
    "    test_loader = torch.utils.data.DataLoader(\n",
    "        data_val,\n",
    "        batch_size=config.train.batch_size,\n",
    "        shuffle=False,\n",
    "        num_workers=config.train.num_workers,\n",
    "        collate_fn=data_val.collate_fn\n",
    "    )\n",
    "    \n",
    "    ignored_index = data_val.vocab_sizeT - 1\n",
    "    unk_index = data_val.vocab_sizeT - 2\n",
    "    \n",
    "    model = MixtureAttention(\n",
    "        hidden_size = config.model.hidden_size,\n",
    "        vocab_sizeT = data_val.vocab_sizeT,\n",
    "        vocab_sizeN = data_val.vocab_sizeN,\n",
    "        attn_size = data_val.attn_size,\n",
    "        embedding_sizeT = config.model.embedding_sizeT,\n",
    "        embedding_sizeN = config.model.embedding_sizeN,\n",
    "        num_layers = 1,\n",
    "        dropout = config.model.dropout,\n",
    "        label_smoothing = config.model.label_smoothing,\n",
    "        pointer = config.model.pointer,\n",
    "        attn = config.model.attn,\n",
    "        device = device\n",
    "    )\n",
    "    cpk = torch.load(checkpoint_path)\n",
    "    model.load_state_dict(cpk['model'])\n",
    "    model = model.to(device)\n",
    "    with torch.no_grad():\n",
    "        model = model.eval()\n",
    "        acc = 0.\n",
    "        loss_eval = 0.\n",
    "        for i, (n, t, p) in enumerate(tqdm(test_loader)):\n",
    "            n, t, p = n.to(device), t.to(device), p.to(device)\n",
    "            loss, ans = model(n, t, p)\n",
    "            loss_eval += loss.item()\n",
    "            acc += accuracy(ans.cpu().numpy().flatten(), t.cpu().numpy().flatten(), ignored_index, unk_index)\n",
    "        acc /= len(test_loader)\n",
    "        loss_eval /= len(test_loader)\n",
    "        losses[config.name] = loss_eval\n",
    "        acces[config.name] = acc\n",
    "    print('loss:', losses[config.name], 'acc:', acces[config.name])\n",
    "    torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/data/anaconda/envs/py35/lib/python3.5/importlib/_bootstrap.py:222: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n",
      "/data/anaconda/envs/py35/lib/python3.5/importlib/_bootstrap.py:222: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_table(dic, m = 1):\n",
    "    data = pd.DataFrame(columns=['1k', '10k', '50k'], index=['simple_lstm', 'attn_lstm', 'pointer'])\n",
    "    for item in dic:\n",
    "        id_ = int([i for i, j in enumerate(item.split('_')) if j == 'vocab'][0])\n",
    "        name = '_'.join(item.split('_')[:id_])\n",
    "        vocab = item.split('_')[-1]\n",
    "        data.loc[name, vocab] = dic[item]\n",
    "    return data * m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'attn_lstm_vocab_10k': 0.6577360621117113,\n",
       " 'attn_lstm_vocab_1k': 0.6494665779492861,\n",
       " 'attn_lstm_vocab_50k': 0.6315070176565867,\n",
       " 'pointer_vocab_10k': 0.6705065943604063,\n",
       " 'pointer_vocab_1k': 0.6662379230942852,\n",
       " 'pointer_vocab_50k': 0.634914800961635,\n",
       " 'simple_lstm_vocab_10k': 0.657081022143478,\n",
       " 'simple_lstm_vocab_1k': 0.6633081460420689,\n",
       " 'simple_lstm_vocab_50k': 0.6168174382832627}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'attn_lstm_vocab_10k': 10165.64429010069,\n",
       " 'attn_lstm_vocab_1k': 15595.033411789249,\n",
       " 'attn_lstm_vocab_50k': 6836.341175179966,\n",
       " 'pointer_vocab_10k': 21672.131244335622,\n",
       " 'pointer_vocab_1k': 43398.31753725069,\n",
       " 'pointer_vocab_50k': 7057.6959175422135,\n",
       " 'simple_lstm_vocab_10k': 20159.74126170967,\n",
       " 'simple_lstm_vocab_1k': 109026.57322798869,\n",
       " 'simple_lstm_vocab_50k': 13842.316331750844}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "losses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1k</th>\n",
       "      <th>10k</th>\n",
       "      <th>50k</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>simple_lstm</th>\n",
       "      <td>66.3308</td>\n",
       "      <td>65.7081</td>\n",
       "      <td>61.6817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>attn_lstm</th>\n",
       "      <td>64.9467</td>\n",
       "      <td>65.7736</td>\n",
       "      <td>63.1507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pointer</th>\n",
       "      <td>66.6238</td>\n",
       "      <td>67.0507</td>\n",
       "      <td>63.4915</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  1k      10k      50k\n",
       "simple_lstm  66.3308  65.7081  61.6817\n",
       "attn_lstm    64.9467  65.7736  63.1507\n",
       "pointer      66.6238  67.0507  63.4915"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "make_table(acces, m = 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1k</th>\n",
       "      <th>10k</th>\n",
       "      <th>50k</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>simple_lstm</th>\n",
       "      <td>109027</td>\n",
       "      <td>20159.7</td>\n",
       "      <td>13842.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>attn_lstm</th>\n",
       "      <td>15595</td>\n",
       "      <td>10165.6</td>\n",
       "      <td>6836.34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pointer</th>\n",
       "      <td>43398.3</td>\n",
       "      <td>21672.1</td>\n",
       "      <td>7057.7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  1k      10k      50k\n",
       "simple_lstm   109027  20159.7  13842.3\n",
       "attn_lstm      15595  10165.6  6836.34\n",
       "pointer      43398.3  21672.1   7057.7"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "make_table(losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1k</th>\n",
       "      <th>10k</th>\n",
       "      <th>50k</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>simple_lstm</th>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>attn_lstm</th>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pointer</th>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             1k  10k  50k\n",
       "simple_lstm   5    8    1\n",
       "attn_lstm     5    8    1\n",
       "pointer       5    8    1"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "make_table(epoch)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Eval from an example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from model import *\n",
    "from data import *\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "import yaml\n",
    "from utils import DotDict, adjust_learning_rate, accuracy\n",
    "import torch\n",
    "import traceback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ['CUDA_VISIBLE_DEVICES'] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "started pointer_vocab_10k\n",
      "reading data from  ./pickle_data/PY_non_terminal_small.pickle\n",
      "the vocab_sizeN is 330 (not including the eof)\n",
      "the number of training data is 100000\n",
      "the number of test data is 50000\n",
      "\n",
      "reading data from  ./pickle_data/PY_terminal_10k_whole.pickle\n",
      "the vocab_sizeT is 10000 (not including the unk and eof)\n",
      "the attn_size is 50\n",
      "the number of training data is 100000\n",
      "the number of test data is 50000\n",
      "Finish reading data and take 13.11\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 50000/50000 [00:33<00:00, 1504.67it/s]\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "Attempting to deserialize object on a CUDA device but torch.cuda.is_available() is False. If you are running on a CPU-only machine, please use torch.load with map_location=torch.device('cpu') to map your storages to the CPU.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-16-8fb5cfa1896d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     41\u001b[0m     \u001b[0mdevice\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     42\u001b[0m )\n\u001b[0;32m---> 43\u001b[0;31m \u001b[0mcpk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcheckpoint_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     44\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_state_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcpk\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'model'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     45\u001b[0m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/anaconda/envs/py35/lib/python3.5/site-packages/torch/serialization.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(f, map_location, pickle_module, **pickle_load_args)\u001b[0m\n\u001b[1;32m    384\u001b[0m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    385\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 386\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_load\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmap_location\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpickle_module\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mpickle_load_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    387\u001b[0m     \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    388\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mnew_fd\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/anaconda/envs/py35/lib/python3.5/site-packages/torch/serialization.py\u001b[0m in \u001b[0;36m_load\u001b[0;34m(f, map_location, pickle_module, **pickle_load_args)\u001b[0m\n\u001b[1;32m    571\u001b[0m     \u001b[0munpickler\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle_module\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mUnpickler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mpickle_load_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    572\u001b[0m     \u001b[0munpickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpersistent_load\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpersistent_load\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 573\u001b[0;31m     \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0munpickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    574\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    575\u001b[0m     \u001b[0mdeserialized_storage_keys\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle_module\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mpickle_load_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/anaconda/envs/py35/lib/python3.5/site-packages/torch/serialization.py\u001b[0m in \u001b[0;36mpersistent_load\u001b[0;34m(saved_id)\u001b[0m\n\u001b[1;32m    534\u001b[0m                 \u001b[0mobj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    535\u001b[0m                 \u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_torch_load_uninitialized\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 536\u001b[0;31m                 \u001b[0mdeserialized_objects\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot_key\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrestore_location\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlocation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    537\u001b[0m             \u001b[0mstorage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdeserialized_objects\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot_key\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    538\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mview_metadata\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/anaconda/envs/py35/lib/python3.5/site-packages/torch/serialization.py\u001b[0m in \u001b[0;36mdefault_restore_location\u001b[0;34m(storage, location)\u001b[0m\n\u001b[1;32m    117\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mdefault_restore_location\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstorage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlocation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    118\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m \u001b[0;32min\u001b[0m \u001b[0m_package_registry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 119\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstorage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlocation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    120\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    121\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/anaconda/envs/py35/lib/python3.5/site-packages/torch/serialization.py\u001b[0m in \u001b[0;36m_cuda_deserialize\u001b[0;34m(obj, location)\u001b[0m\n\u001b[1;32m     93\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_cuda_deserialize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlocation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mlocation\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstartswith\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'cuda'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 95\u001b[0;31m         \u001b[0mdevice\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvalidate_cuda_device\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlocation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     96\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"_torch_load_uninitialized\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     97\u001b[0m             \u001b[0mstorage_type\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/anaconda/envs/py35/lib/python3.5/site-packages/torch/serialization.py\u001b[0m in \u001b[0;36mvalidate_cuda_device\u001b[0;34m(location)\u001b[0m\n\u001b[1;32m     77\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     78\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_available\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 79\u001b[0;31m         raise RuntimeError('Attempting to deserialize object on a CUDA '\n\u001b[0m\u001b[1;32m     80\u001b[0m                            \u001b[0;34m'device but torch.cuda.is_available() is False. '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     81\u001b[0m                            \u001b[0;34m'If you are running on a CPU-only machine, '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: Attempting to deserialize object on a CUDA device but torch.cuda.is_available() is False. If you are running on a CPU-only machine, please use torch.load with map_location=torch.device('cpu') to map your storages to the CPU."
     ]
    }
   ],
   "source": [
    "config = 'configs/pointer_vocab_10k.yml'\n",
    "with open(config, 'r') as f:\n",
    "    config = DotDict(yaml.safe_load(f))\n",
    "print('started', config.name)\n",
    "checkpoint_folder = os.path.join('checkpoints', config.name)\n",
    "last_cpk = sorted(os.listdir(checkpoint_folder), key=lambda x: int(x[6:-4]), reverse=True)[0]\n",
    "checkpoint_path = os.path.join(checkpoint_folder, last_cpk)\n",
    "\n",
    "device = 'cpu'\n",
    "\n",
    "data_val = MainDataset(\n",
    "    N_filename = config.data.N_filename,\n",
    "    T_filename = config.data.T_filename,\n",
    "    is_train=False,\n",
    "    truncate_size=config.data.truncate_size\n",
    ")\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "    data_val,\n",
    "    batch_size=config.train.batch_size,\n",
    "    shuffle=False,\n",
    "    num_workers=config.train.num_workers,\n",
    "    collate_fn=data_val.collate_fn\n",
    ")\n",
    "\n",
    "ignored_index = data_val.vocab_sizeT - 1\n",
    "unk_index = data_val.vocab_sizeT - 2\n",
    "\n",
    "model = MixtureAttention(\n",
    "    hidden_size = config.model.hidden_size,\n",
    "    vocab_sizeT = data_val.vocab_sizeT,\n",
    "    vocab_sizeN = data_val.vocab_sizeN,\n",
    "    attn_size = data_val.attn_size,\n",
    "    embedding_sizeT = config.model.embedding_sizeT,\n",
    "    embedding_sizeN = config.model.embedding_sizeN,\n",
    "    num_layers = 1,\n",
    "    dropout = config.model.dropout,\n",
    "    label_smoothing = config.model.label_smoothing,\n",
    "    pointer = config.model.pointer,\n",
    "    attn = config.model.attn,\n",
    "    device = device\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "cpk = torch.load(checkpoint_path, map_location=torch.device('cpu'))\n",
    "model.load_state_dict(cpk['model'])\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reading data from  ./pickle_data/PY_non_terminal_small.pickle\n",
      "the vocab_sizeN is 330 (not including the eof)\n",
      "the number of training data is 100000\n",
      "the number of test data is 50000\n",
      "\n",
      "reading data from  ./pickle_data/PY_terminal_10k_whole.pickle\n",
      "the vocab_sizeT is 10000 (not including the unk and eof)\n",
      "the attn_size is 50\n",
      "the number of training data is 100000\n",
      "the number of test data is 50000\n",
      "Finish reading data and take 9.66\n",
      "\n"
     ]
    }
   ],
   "source": [
    "train_dataN, test_dataN, vocab_sizeN, train_dataT, test_dataT, vocab_sizeT, attn_size, train_dataP, test_dataP = input_data(\n",
    "    config.data.N_filename, config.data.T_filename\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open('pickle_data/terminal_dict_10k_PY.pickle', 'rb')\n",
    "t_dict = pickle.load(f)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10000"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(t_dict['terminal_dict'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "t_reversed = {val: key for key, val in t_dict['terminal_dict'].items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "def decode(arr):\n",
    "    return [t_reversed[item.item()] if item.item() < 10000 else item.item() - 10000 for item in arr]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_n = torch.tensor(test_dataN[5:6])\n",
    "sample_t = torch.tensor(test_dataT[5:6])\n",
    "sample_p = torch.tensor(test_dataP[5:6])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss, ans = model(sample_n, sample_t, sample_p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['<empty>',\n",
       " 'enum',\n",
       " 0,\n",
       " 'component',\n",
       " 'Component',\n",
       " 'object',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 8,\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 0,\n",
       " '0',\n",
       " '<empty>',\n",
       " 0,\n",
       " '1',\n",
       " '<empty>',\n",
       " 0,\n",
       " '2',\n",
       " '<empty>',\n",
       " 0,\n",
       " '3',\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 18,\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 18,\n",
       " '0',\n",
       " '<empty>',\n",
       " 0,\n",
       " '1',\n",
       " '<empty>',\n",
       " 0,\n",
       " '2',\n",
       " '<empty>',\n",
       " 0,\n",
       " '3',\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'Component',\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 'enabled',\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " 'bool',\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'materials',\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " 'bool',\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " 'bool',\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " 'material',\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 'self',\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 'self',\n",
       " 'materials',\n",
       " '<empty>',\n",
       " '0',\n",
       " '<empty>',\n",
       " 'property',\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 0,\n",
       " '0',\n",
       " '<empty>',\n",
       " 0,\n",
       " '1',\n",
       " '<empty>',\n",
       " 0,\n",
       " '2',\n",
       " '<empty>',\n",
       " 0,\n",
       " '3',\n",
       " '<empty>',\n",
       " 0,\n",
       " '4',\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 21,\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 0,\n",
       " '0',\n",
       " '<empty>',\n",
       " 0,\n",
       " '1',\n",
       " '<empty>',\n",
       " 0,\n",
       " '2',\n",
       " '<empty>',\n",
       " 0,\n",
       " '3',\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'Component',\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 36,\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 36,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 36,\n",
       " '<empty>',\n",
       " 36,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 36,\n",
       " '<empty>',\n",
       " 36,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 36,\n",
       " '<empty>',\n",
       " 'mesh',\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>']"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "decode(sample_t[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['<empty>',\n",
       " '<empty>',\n",
       " 'Enum',\n",
       " 0,\n",
       " 0,\n",
       " '<empty>',\n",
       " 'Object',\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 'object',\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 0,\n",
       " '1',\n",
       " '<empty>',\n",
       " 0,\n",
       " '2',\n",
       " '<empty>',\n",
       " 0,\n",
       " '3',\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 9,\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 15,\n",
       " '2',\n",
       " '<empty>',\n",
       " 24,\n",
       " '1',\n",
       " '<empty>',\n",
       " 0,\n",
       " '2',\n",
       " '<empty>',\n",
       " 0,\n",
       " '3',\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 9,\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 15,\n",
       " '1',\n",
       " '<empty>',\n",
       " 'mandatory',\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 'self',\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 'self',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'property',\n",
       " 0,\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 'code',\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 0,\n",
       " '1',\n",
       " '<empty>',\n",
       " 0,\n",
       " '2',\n",
       " '<empty>',\n",
       " 0,\n",
       " '3',\n",
       " '<empty>',\n",
       " 0,\n",
       " '4',\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 0,\n",
       " '1',\n",
       " '<empty>',\n",
       " 0,\n",
       " '1',\n",
       " '<empty>',\n",
       " 0,\n",
       " '2',\n",
       " '<empty>',\n",
       " 0,\n",
       " '3',\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " '_object',\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 47,\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 0,\n",
       " '1',\n",
       " '<empty>',\n",
       " 'type',\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 48,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 'type',\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>',\n",
       " 0,\n",
       " '<empty>',\n",
       " 'field',\n",
       " 0,\n",
       " '<empty>']"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "decode(ans[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}