text_model_64_dim_target.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Summary -\n",
    "# - There are totally 59K touch gestures.\n",
    "# - Only ~22K (37%) touch gestures was clicked on a leaf element which had text content.\n",
    "# - Created text dataset in the following format\n",
    "#   - [[e11, e21, ... e1(MAX_TOKEN), e21, e22, ... e2(MAX_TOKEN), TARGET_TEXT],\n",
    "#      ...\n",
    "#     ]\n",
    "#   - Vectorized the dataset.\n",
    "# - Tried a simple classification model with a single hidden layer(1024).\n",
    "#   - Accuracy on train data : 48%\n",
    "#   - Accuracy on validation (20%) : 50%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import statements.\n",
    "import datetime\n",
    "import glob\n",
    "import json\n",
    "import numpy as np\n",
    "import os\n",
    "import PIL\n",
    "import tensorflow as tf\n",
    "import re\n",
    "import string\n",
    "from sklearn.model_selection import train_test_split\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Constants.\n",
    "LONG_TOUCH_THRESHOLD = 5\n",
    "DIM_X = 1440\n",
    "DIM_Y = 2560\n",
    "MAX_TOKEN = 64\n",
    "BATCH_SIZE = 100\n",
    "BUFFER_SIZE = 100\n",
    "VOCAB_SIZE = 100\n",
    "TRAIN_SIZE = 0.8\n",
    "VAL_SIZE = 0.1\n",
    "TEST_SIZE = 0.1\n",
    "TRACES_PATH = 'filtered_traces/*/*'\n",
    "NEGATIVE_SAMPLE_TARGET = '[null]'\n",
    "PLACEHOLDER_TEXT = 'n/a'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Gets all leaf nodes for a given element.\n",
    "def get_leaf_nodes(element, leaf_nodes):\n",
    "    if not element:\n",
    "        return leaf_nodes\n",
    "    if 'children' not in element:\n",
    "        leaf_nodes.append(element)\n",
    "        return leaf_nodes\n",
    "    for child in element['children']:\n",
    "        get_leaf_nodes(child, leaf_nodes)\n",
    "    return leaf_nodes\n",
    "\n",
    "\n",
    "def get_all_leaf_nodes(view_hierarchy_json):\n",
    "    activity = view_hierarchy_json.get('activity')\n",
    "    if not activity:\n",
    "        return dataset\n",
    "    root = activity.get('root')\n",
    "    return get_leaf_nodes(root, [])\n",
    "\n",
    "\n",
    "def get_target_text(leaf_nodes, x, y):\n",
    "    target_text = None\n",
    "    for leaf_node in leaf_nodes:\n",
    "        bounds = leaf_node['bounds']\n",
    "        if bounds[0] <= x and bounds[2] >= x and bounds[1] <= y and bounds[3] >= y:\n",
    "            if 'text' in leaf_node:\n",
    "                target_text = leaf_node['text'] or leaf_node.get('text-hint')\n",
    "    return target_text\n",
    "\n",
    "\n",
    "def get_leaf_node_features(leaf_nodes):\n",
    "    i = 0\n",
    "    element_features = []\n",
    "    for leaf_node in leaf_nodes:\n",
    "        if 'text' in leaf_node:\n",
    "            text = leaf_node['text'] or leaf_node.get('text-hint')\n",
    "            _class = leaf_node['class']\n",
    "            element_features.append(str(text))\n",
    "#             element_features.append(str(_class))\n",
    "            i += 1\n",
    "            if i == MAX_TOKEN:\n",
    "                break\n",
    "    return element_features\n",
    "\n",
    "\n",
    "\n",
    "# Identifies if a given gesture is a TOUCH gesture.\n",
    "# In this task, we will only be focussing on TOUCH gestures.\n",
    "def is_touch_gesture(gesture):\n",
    "    if len(gesture) <= LONG_TOUCH_THRESHOLD:\n",
    "        return True\n",
    "    return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of touch gestures  59602\n",
      "Number of non-touch gestures  6659\n"
     ]
    }
   ],
   "source": [
    "dirs = glob.glob(TRACES_PATH)\n",
    "touch_gesture_count = 0\n",
    "non_touch_gesture_count = 0\n",
    "for d in dirs:\n",
    "  with open(f'{d}/gestures.json') as f:\n",
    "    gestures = json.load(f)\n",
    "    gestures = [gestures[x] for x in sorted(gestures, key=lambda x: int(x))]\n",
    "    for gesture in gestures:\n",
    "        if is_touch_gesture(gesture):\n",
    "            touch_gesture_count += 1\n",
    "        else:\n",
    "            non_touch_gesture_count += 1\n",
    "print('Number of touch gestures ', touch_gesture_count)\n",
    "print('Number of non-touch gestures ', non_touch_gesture_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Processes view hierarchies to construct dataset.\n",
    "# Extract texts from MAX_TOKEN elements from both view hierarchies.\n",
    "# Construct the dataset in the following format -\n",
    "# [[e11, e21, ... e1(MAX_TOKEN), e21, e22, ... e2(MAX_TOKEN), TARGET_TEXT], ...]\n",
    "\n",
    "y_true = []\n",
    "\n",
    "def get_features(leaf_nodes1, leaf_nodes2):\n",
    "    screen1_features = get_leaf_node_features(leaf_nodes1)\n",
    "    screen2_features = get_leaf_node_features(leaf_nodes2)\n",
    "\n",
    "    for i in range(len(screen1_features), MAX_TOKEN):\n",
    "        screen1_features.append(PLACEHOLDER_TEXT)  # Element Text.\n",
    "#         screen1_features.append(PLACEHOLDER_TEXT)  # Element Class.\n",
    "    for i in range(len(screen2_features), MAX_TOKEN):\n",
    "        screen2_features.append(PLACEHOLDER_TEXT)  # Element Text.\n",
    "#         screen1_features.append(PLACEHOLDER_TEXT)  # Element Class.\n",
    "    \n",
    "    return screen1_features, screen2_features\n",
    "\n",
    "\n",
    "def process_view_hierarchy(view_hierarchy1, view_hierarchy2, dataset, is_positive_sample = True):\n",
    "    if not view_hierarchy1 or not view_hierarchy2:\n",
    "        return dataset\n",
    "    \n",
    "    trace_path = view_hierarchy1.split('view_hierarchies')[0]\n",
    "    gesture_path = f'{trace_path}/gestures.json'\n",
    "    with open(gesture_path) as file:\n",
    "        gestures = json.load(file)\n",
    "\n",
    "    with open(view_hierarchy1) as file:\n",
    "        view_hierarchy1_json = json.load(file)\n",
    "    with open(view_hierarchy2) as file:\n",
    "        view_hierarchy2_json = json.load(file)\n",
    "    \n",
    "    if not view_hierarchy1_json or not view_hierarchy2_json:\n",
    "        return dataset\n",
    "\n",
    "    ui_number = view_hierarchy1.split('/')[-1].split('.')[0]\n",
    "    gesture = gestures[ui_number]\n",
    "    if not is_touch_gesture(gesture):\n",
    "        return dataset\n",
    "    \n",
    "    if not len(gesture):\n",
    "        return dataset\n",
    "    x_cord = gesture[0][0]\n",
    "    y_cord = gesture[0][1]\n",
    "    x = x_cord * DIM_X\n",
    "    y = y_cord * DIM_Y\n",
    "\n",
    "    leaf_nodes1 = get_all_leaf_nodes(view_hierarchy1_json)\n",
    "    leaf_nodes2 = get_all_leaf_nodes(view_hierarchy2_json)\n",
    "\n",
    "    target_text = get_target_text(leaf_nodes1, x, y)\n",
    "    if not target_text:\n",
    "        return dataset\n",
    "    \n",
    "    screen1_features, screen2_features = get_features(leaf_nodes1, leaf_nodes2)\n",
    "\n",
    "    if is_positive_sample:\n",
    "        target = None\n",
    "        for i, element in enumerate(screen1_features):\n",
    "            if element == target_text:\n",
    "                target = tf.keras.utils.to_categorical(i, MAX_TOKEN+1)\n",
    "                break\n",
    "        if target is None:\n",
    "            target = tf.keras.utils.to_categorical(MAX_TOKEN, MAX_TOKEN+1)  # Null element.\n",
    "        dataset.append(screen1_features + screen2_features)\n",
    "    else:\n",
    "        target = tf.keras.utils.to_categorical(MAX_TOKEN, MAX_TOKEN+1)  # Null element.\n",
    "        dataset.append(screen1_features + screen2_features)\n",
    "    y_true.append(target)\n",
    "    return dataset\n",
    "        \n",
    "\n",
    "def process_trace(trace_path, dataset):\n",
    "    view_hierarchies_path = f'{trace_path}/view_hierarchies/*'\n",
    "    view_hierarchies = sorted(glob.glob(view_hierarchies_path))\n",
    "    for i in range(len(view_hierarchies) - 1):\n",
    "        dataset = process_view_hierarchy(view_hierarchies[i], view_hierarchies[i+1], dataset)\n",
    "\n",
    "\n",
    "def add_negative_samples(dataset):\n",
    "    traces = sorted(glob.glob(TRACES_PATH))\n",
    "    total_positive_samples = len(dataset)\n",
    "    negative_samples_threshold = 0.1 * total_positive_samples\n",
    "    negative_samples_counter = 0\n",
    "    for i in range(len(traces) - 1):\n",
    "        trace_path1 = traces[i]\n",
    "        trace_path2 = traces[i+1]\n",
    "        view_hierarchies1_path = sorted(glob.glob(f'{trace_path1}/view_hierarchies/*'))\n",
    "        view_hierarchies2_path = sorted(glob.glob(f'{trace_path2}/view_hierarchies/*'))\n",
    "        for (view_hierarchy1, view_hierarchy2) in zip(view_hierarchies1_path, view_hierarchies2_path):\n",
    "            dataset = process_view_hierarchy(view_hierarchy1, view_hierarchy2, dataset, False)\n",
    "            negative_samples_counter += 1\n",
    "            if negative_samples_counter >= negative_samples_threshold:\n",
    "                break\n",
    "    return dataset\n",
    "\n",
    "\n",
    "dataset = []\n",
    "for trace_path in sorted(glob.glob(TRACES_PATH)):\n",
    "    process_trace(trace_path, dataset)\n",
    "\n",
    "dataset = add_negative_samples(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2609.6000000000004"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_positive_samples = len(dataset)\n",
    "negative_samples_threshold = 0.1 * total_positive_samples\n",
    "negative_samples_threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "targets = []\n",
    "for row in dataset:\n",
    "    targets.append(re.sub('[%s]' % re.escape(string.punctuation), '', row[-1].lower()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unique_words =  193239\n"
     ]
    }
   ],
   "source": [
    "# We create a custom standardization function to lowercase the text and \n",
    "# remove punctuation.\n",
    "def custom_standardization(input_data):\n",
    "  lowercase = tf.strings.lower(input_data)\n",
    "  lowercase = tf.strings.regex_replace(lowercase, '\\s', '')\n",
    "  return tf.strings.regex_replace(lowercase,\n",
    "                                  '[%s]' % re.escape(string.punctuation), '')\n",
    "\n",
    "# Define the number of words in a sequence.\n",
    "sequence_length = 1\n",
    "\n",
    "# Use the text vectorization layer to normalize, split, and map strings to\n",
    "# integers. Set output_sequence_length length to pad all samples to same length.\n",
    "vectorize_layer = tf.keras.layers.experimental.preprocessing.TextVectorization(\n",
    "    standardize=custom_standardization,\n",
    "    output_mode='int',\n",
    "    output_sequence_length=sequence_length)\n",
    "\n",
    "all_words = []\n",
    "for row in dataset:\n",
    "    for word in row:\n",
    "        all_words.append(str(word))\n",
    "unique_words = set(all_words)\n",
    "print('unique_words = ',len(unique_words))\n",
    "vectorize_layer.adapt(list(unique_words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['', '[UNK]', '1', '0', '10', '000', '00', '11', '100', '2', '12', '20', '15', '4', '3', 'signup', '5', '21', '25', '14']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "172487"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Save the created vocabulary for reference.\n",
    "inverse_vocab = vectorize_layer.get_vocabulary()\n",
    "print(inverse_vocab[:20])\n",
    "len(inverse_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Vectorize the data.\n",
    "text_ds = tf.data.Dataset.from_tensor_slices(dataset)\n",
    "text_vector_ds = text_ds.map(vectorize_layer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[    64 114488 122169    295 114490 122170    295    218  92638    295\n",
      " 143584  58939  51636  54039    295    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297   4951  34586  61880  61904  46059  99818\n",
      "  46060  99819  46061  99820    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297] => ['settings', 'fancylist', 'displaychapterslistusingcardviewwidget', 'none', 'fancydetails', 'displaychapterdetailsusingcardviewwidget', 'none', 'language', 'languageusedtodisplayapplicationdata', 'none', 'atrás', 'saltar', 'siguiente', 'settingsactivity', 'none', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'supportdevelopment', 'unabletoconnecttobillingservices', 'removeadvertisinginthisapp', 'removeadbannersandinterstitialadsinthisapplicationandenjoythecontent', 'supportappdevelopmentsmall', 'ifyouenjoythisapplicationyoucansupportusbybuyingthissmallpack', 'supportappdevelopmentmedium', 'ifyouenjoythisapplicationyoucansupportusbybuyingthismediumpack', 'supportappdevelopmentlarge', 'ifyouenjoythisapplicationyoucansupportusbybuyingthislargepack', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na']\n",
      "[ 68210 128328  70765 111244  78775  60511  59925  67315  93522  59926\n",
      " 133506 104803 122191 105950 105949   9678     64   4951    236    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297  90421    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297] => ['pocketphysics', 'constantcircularmotion', 'periodfrequencyangularvelocitycentripetalaccelerationcentripetalforce', 'force', 'newtonslawsweightforcenormalforcefriction', 'rigidbody', 'rotarymotionangulardisplacementangularvelocityangularaccelerationuniformlyvariablemotionmomentofinertiamomentofforcemomentofmomentumprincipleofconservationofmomentum', 'powerworkenergy', 'kineticenergypotentialenergywork', 'rotarymotion', 'centerofthemassangularpositionangularvelocitymomentofinertia', 'harmonicmotion', 'displacementangularfrequencyvelocityacceleration', 'gravity', 'gravityforcepotentialenergykepplerslaws', 'chapters', 'settings', 'supportdevelopment', 'about', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'linearmotion', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na']\n",
      "[ 24451    295   6857  52317 105516 103137  35919    801  84918 133252\n",
      "      2    295     36     24    174     47     94    169    225    309\n",
      "    459    449   2705   2688    443    215    427    294    292    637\n",
      "    208    617    374    571    364   1773    265   1710    552    552\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297 103162  89131   1219     89  35754    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297] => ['wordguessgame', 'none', 'newgame', 'showresult', 'guessthewordusingthehintbelow', 'hintलडाका', 'truuet', 'answer', 'melanoma', 'chancesleft', '1', 'none', 'next', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'z', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'hindienglishdictionarypoweredbyhinkhojcom', 'logintosaveyourwordsontheserver', 'loginwith', 'or', 'trywithoutlogin', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na']\n",
      "[  3041    454   1432    126 111828  61321  51012 141841  24222    229\n",
      "    426    200     64     26    295    295  84027    295    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297   3041    454   1432 138465 138421    126\n",
      " 111828  61321  51012 141841  24222    229    426    200    295    295\n",
      "  84027    295    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297] => ['dev', 'home', 'bookaflight', 'checkin', 'flightstatusschedules', 'reservationticket', 'skypass', 'baggageinformation', 'worldwideairports', 'contactus', 'news', 'termsandconditions', 'settings', 'login', 'none', 'none', 'mileageavailableforuse', 'none', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'dev', 'home', 'bookaflight', 'bookdomesticflightkorea', 'bookinternationalflight', 'checkin', 'flightstatusschedules', 'reservationticket', 'skypass', 'baggageinformation', 'worldwideairports', 'contactus', 'news', 'termsandconditions', 'none', 'none', 'mileageavailableforuse', 'none', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na']\n",
      "[165256 159478 155613 164785 159285    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297   1377    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297    297    297\n",
      "    297    297    297    297    297    297    297    297] => ['1monthgraph', '3monthgraph', '6monthgraph', '1yeargraph', '3yeargraph', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'currency', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na', 'na']\n"
     ]
    }
   ],
   "source": [
    "sequences = list(text_vector_ds.as_numpy_iterator())\n",
    "sequences = np.squeeze(sequences)\n",
    "\n",
    "for seq in sequences[:5]:\n",
    "  print(f\"{seq} => {[inverse_vocab[i] for i in seq]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def map_to_dataset(sequences, inverse_vocab):\n",
    "    num_ns = len(inverse_vocab)\n",
    "    labels = []\n",
    "    input_data = []\n",
    "    for input_instance in sequences:\n",
    "        y = input_instance[-1:]\n",
    "        labels.append(y)\n",
    "        input_data.append(input_instance[:-1])\n",
    "    categorized_labels = tf.keras.utils.to_categorical(labels, num_ns)\n",
    "    return tf.data.Dataset.from_tensor_slices((input_data, categorized_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true_reshaped = np.array(y_true).reshape(len(sequences), -1)\n",
    "sequences_stack = np.hstack((sequences, y_true_reshaped))\n",
    "train, test = train_test_split(sequences_stack, test_size=TEST_SIZE)\n",
    "train, val = train_test_split(sequences_stack, test_size=VAL_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "train, test = train_test_split(sequences, test_size=TEST_SIZE)\n",
    "train, val = train_test_split(sequences, test_size=VAL_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "ename": "MemoryError",
     "evalue": "Unable to allocate 15.1 GiB for an array with shape (23486, 172487) and data type float32",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-36-32314e6c72e6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtrain\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmap_to_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minverse_vocab\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBATCH_SIZE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmap_to_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minverse_vocab\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBATCH_SIZE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mtest\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmap_to_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minverse_vocab\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBATCH_SIZE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-33-3fc050c1165e>\u001b[0m in \u001b[0;36mmap_to_dataset\u001b[0;34m(sequences, inverse_vocab)\u001b[0m\n\u001b[1;32m      7\u001b[0m         \u001b[0mlabels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m         \u001b[0minput_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_instance\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m     \u001b[0mcategorized_labels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_categorical\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_ns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_tensor_slices\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcategorized_labels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/utils/np_utils.py\u001b[0m in \u001b[0;36mto_categorical\u001b[0;34m(y, num_classes, dtype)\u001b[0m\n\u001b[1;32m     75\u001b[0m     \u001b[0mnum_classes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     76\u001b[0m   \u001b[0mn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 77\u001b[0;31m   \u001b[0mcategorical\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_classes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     78\u001b[0m   \u001b[0mcategorical\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     79\u001b[0m   \u001b[0moutput_shape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput_shape\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mnum_classes\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mMemoryError\u001b[0m: Unable to allocate 15.1 GiB for an array with shape (23486, 172487) and data type float32"
     ]
    }
   ],
   "source": [
    "train = map_to_dataset(train, inverse_vocab).batch(BATCH_SIZE)\n",
    "val = map_to_dataset(val, inverse_vocab).batch(BATCH_SIZE)\n",
    "test = map_to_dataset(test, inverse_vocab).batch(BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.keras.Sequential([\n",
    "    tf.keras.layers.Flatten(input_shape=(MAX_TOKEN*2, )),\n",
    "    tf.keras.layers.Embedding(\n",
    "        input_dim=len(inverse_vocab),\n",
    "        output_dim=64,\n",
    "        # Use masking to handle the variable sequence lengths\n",
    "        mask_zero=True),\n",
    "    tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64)),\n",
    "    tf.keras.layers.Dense(64, activation='relu'),\n",
    "    tf.keras.layers.Dense(MAX_TOKEN + 1, activation='softmax')\n",
    "])\n",
    "\n",
    "log_dir = \"logs/fit/\" + datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n",
    "tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)\n",
    "\n",
    "model.compile(optimizer='adam',\n",
    "              loss=tf.keras.losses.CategoricalCrossentropy(from_logits=True),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "history = model.fit(train, validation_data=val, epochs=10, callbacks=[tensorboard_callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def plot_graphs(history, metric):\n",
    "  plt.plot(history.history[metric])\n",
    "  plt.plot(history.history['val_'+metric], '')\n",
    "  plt.xlabel(\"Epochs\")\n",
    "  plt.ylabel(metric)\n",
    "  plt.legend([metric, 'val_'+metric])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(20,8))\n",
    "plt.subplot(1,2,1)\n",
    "plot_graphs(history, 'accuracy')\n",
    "plt.ylim(0,0.3)\n",
    "plt.subplot(1,2,2)\n",
    "plot_graphs(history, 'loss')\n",
    "plt.ylim(15, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "environment": {
   "name": "tf2-2-3-gpu.2-3.m59",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/tf2-2-3-gpu.2-3:m59"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}