Data-augmenting.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Augmentation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculate Unlabeled Data Similarities:\n",
    "\n",
    "Unlabeled data: Spanish - English\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "from utils import save_embed, load_embed\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import nltk\n",
    "import re"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read Unlabeled Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>spanish</th>\n",
       "      <th>english</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Por qué no se ha podido procesar mi pago por r...</td>\n",
       "      <td>Why my payment could not be processed due to s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tengo una duda Tengo una pregunta sobre mis cu...</td>\n",
       "      <td>I have a question I have a question about my c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Por favor necesito saber si mi compra todavía ...</td>\n",
       "      <td>Please I need to know if my purchase is still ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Cómo cancelo un pedido Si cancelo el pedido re...</td>\n",
       "      <td>How do I cancel an order? If I cancel the orde...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Cómo puedo recibir un reembolso mediante tarje...</td>\n",
       "      <td>How can I receive a refund by card How can I r...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             spanish  \\\n",
       "0  Por qué no se ha podido procesar mi pago por r...   \n",
       "1  tengo una duda Tengo una pregunta sobre mis cu...   \n",
       "2  Por favor necesito saber si mi compra todavía ...   \n",
       "3  Cómo cancelo un pedido Si cancelo el pedido re...   \n",
       "4  Cómo puedo recibir un reembolso mediante tarje...   \n",
       "\n",
       "                                             english  \n",
       "0  Why my payment could not be processed due to s...  \n",
       "1  I have a question I have a question about my c...  \n",
       "2  Please I need to know if my purchase is still ...  \n",
       "3  How do I cancel an order? If I cancel the orde...  \n",
       "4  How can I receive a refund by card How can I r...  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "un = pd.read_csv('input/cikm_unlabel_spanish_train_20180516.txt',sep='\t', header=None, error_bad_lines=False)\n",
    "un.columns = ['spanish','english']\n",
    "un.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(55669, 2)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "un.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read Labeled Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training data\n",
    "train_en_sp = pd.read_csv('./input/cikm_english_train_20180516.txt', sep='\t', header=None, error_bad_lines=False)\n",
    "train_sp_en = pd.read_csv('./input/cikm_spanish_train_20180516.txt', sep='\t', header=None, error_bad_lines=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_en_sp.columns = ['english1', 'spanish1', 'english2', 'spanish2', 'result']\n",
    "train_sp_en.columns = ['spanish3', 'english3', 'spanish4', 'english4', 'result']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read Test Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_sp_sp = pd.read_csv('./input/cikm_test_a_20180516.txt', sep='\t', header=None, error_bad_lines=False)\n",
    "test_sp_sp.columns = ['spanish5', 'spanish6']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Get all English sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(42800, 1)\n"
     ]
    }
   ],
   "source": [
    "all_en = pd.DataFrame(pd.concat([train_en_sp['english1'], train_en_sp['english2'], \n",
    "                                   train_sp_en['english3'], train_sp_en['english4']], axis=0))\n",
    "all_en.columns = ['english']\n",
    "all_en = all_en.reset_index()\n",
    "all_en = all_en.drop(columns='index')\n",
    "print(all_en.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>english</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>hello, i click in product received</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Hello! I have closed the dispute on may 21, 20...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>l ordered from spain to spain now they send th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Do I need to pay custom duty</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I didn't receive my order?</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             english\n",
       "0                 hello, i click in product received\n",
       "1  Hello! I have closed the dispute on may 21, 20...\n",
       "2  l ordered from spain to spain now they send th...\n",
       "3                       Do I need to pay custom duty\n",
       "4                         I didn't receive my order?"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_en.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Get all Spanish sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(52800, 1)\n"
     ]
    }
   ],
   "source": [
    "all_sp = pd.DataFrame(pd.concat([train_en_sp['spanish1'], train_en_sp['spanish2'], train_sp_en['spanish3'], \n",
    "                                   train_sp_en['spanish4'],test_sp_sp['spanish5'], test_sp_sp['spanish6']], axis=0))\n",
    "all_sp.columns = ['spanish']\n",
    "all_sp = all_sp.reset_index()\n",
    "all_sp = all_sp.drop(columns='index')\n",
    "print(all_sp.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data preprocess"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Clean English & Spanish: Stopwords, Punctuations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Lower and clean punctuations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_sent(sent):\n",
    "    sent = sent.lower()\n",
    "    sent = re.sub(u\"[_'\\-;%()|+&=*%.,!?:#$@\\[\\]/]\",' ',sent)\n",
    "    sent = re.sub('¡',' ',sent)\n",
    "    sent = re.sub('¿',' ',sent)\n",
    "    sent = re.sub('Á','á',sent)\n",
    "    sent = re.sub('Ó','ó',sent)\n",
    "    sent = re.sub('Ú','ú',sent)\n",
    "    sent = re.sub('É','é',sent)\n",
    "    sent = re.sub('Í','í',sent)\n",
    "    return sent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-11-83045b7d9295>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m all_sp['spanish'] = all_sp.spanish.map(lambda x: ' '.join([word for word in\n\u001b[1;32m      2\u001b[0m                                                      nltk.word_tokenize(clean_sent(x))]))\n\u001b[0;32m----> 3\u001b[0;31m all_en['english'] = all_en.english.map(lambda x: ' '.join([word for word in\n\u001b[0m\u001b[1;32m      4\u001b[0m                                                      nltk.word_tokenize(clean_sent(x))]))\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\u001b[0m in \u001b[0;36mmap\u001b[0;34m(self, arg, na_action)\u001b[0m\n\u001b[1;32m   2994\u001b[0m         \"\"\"\n\u001b[1;32m   2995\u001b[0m         new_values = super(Series, self)._map_values(\n\u001b[0;32m-> 2996\u001b[0;31m             arg, na_action=na_action)\n\u001b[0m\u001b[1;32m   2997\u001b[0m         return self._constructor(new_values,\n\u001b[1;32m   2998\u001b[0m                                  index=self.index).__finalize__(self)\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/pandas/core/base.py\u001b[0m in \u001b[0;36m_map_values\u001b[0;34m(self, mapper, na_action)\u001b[0m\n\u001b[1;32m   1002\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1003\u001b[0m         \u001b[0;31m# mapper is a function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1004\u001b[0;31m         \u001b[0mnew_values\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmap_f\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmapper\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1005\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1006\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mnew_values\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/src/inference.pyx\u001b[0m in \u001b[0;36mpandas._libs.lib.map_infer\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m<ipython-input-11-83045b7d9295>\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m      2\u001b[0m                                                      nltk.word_tokenize(clean_sent(x))]))\n\u001b[1;32m      3\u001b[0m all_en['english'] = all_en.english.map(lambda x: ' '.join([word for word in\n\u001b[0;32m----> 4\u001b[0;31m                                                      nltk.word_tokenize(clean_sent(x))]))\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/nltk/tokenize/__init__.py\u001b[0m in \u001b[0;36mword_tokenize\u001b[0;34m(text, language, preserve_line)\u001b[0m\n\u001b[1;32m    127\u001b[0m     \"\"\"\n\u001b[1;32m    128\u001b[0m     \u001b[0msentences\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mpreserve_line\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0msent_tokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlanguage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 129\u001b[0;31m     return [token for sent in sentences\n\u001b[0m\u001b[1;32m    130\u001b[0m             for token in _treebank_word_tokenizer.tokenize(sent)]\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/nltk/tokenize/__init__.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    128\u001b[0m     \u001b[0msentences\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mpreserve_line\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0msent_tokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlanguage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    129\u001b[0m     return [token for sent in sentences\n\u001b[0;32m--> 130\u001b[0;31m             for token in _treebank_word_tokenizer.tokenize(sent)]\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/nltk/tokenize/treebank.py\u001b[0m in \u001b[0;36mtokenize\u001b[0;34m(self, text, convert_parentheses, return_str)\u001b[0m\n\u001b[1;32m    113\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    114\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mregexp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msubstitution\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPUNCTUATION\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 115\u001b[0;31m             \u001b[0mtext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mregexp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msub\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msubstitution\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    116\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    117\u001b[0m         \u001b[0;31m# Handles parentheses.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/re.py\u001b[0m in \u001b[0;36m_subx\u001b[0;34m(pattern, template)\u001b[0m\n\u001b[1;32m    322\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0msre_parse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpand_template\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtemplate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    323\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 324\u001b[0;31m \u001b[0;32mdef\u001b[0m \u001b[0m_subx\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpattern\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtemplate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    325\u001b[0m     \u001b[0;31m# internal: pattern.sub/subn implementation helper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    326\u001b[0m     \u001b[0mtemplate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_compile_repl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtemplate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpattern\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "all_sp['spanish'] = all_sp.spanish.map(lambda x: ' '.join([word for word in\n",
    "                                                     nltk.word_tokenize(clean_sent(x))]))\n",
    "all_en['english'] = all_en.english.map(lambda x: ' '.join([word for word in\n",
    "                                                     nltk.word_tokenize(clean_sent(x))]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_sp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_en.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Remove stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "sp_stop = set(stopwords.words(\"spanish\"))\n",
    "en_stop = set(stopwords.words(\"english\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_sp['spanish'] = all_sp.spanish.map(lambda x: ' '.join([word for word in nltk.word_tokenize(x) if word not in sp_stop]))\n",
    "all_en['english'] = all_en.english.map(lambda x: ' '.join([word for word in nltk.word_tokenize(x) if word not in en_stop]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_sp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_en.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_en.replace('', np.nan, inplace=True)\n",
    "dirty_data = all_en[all_en.isnull().any(axis=1)]\n",
    "print('Before clean:', len(all_en))\n",
    "print('English dirty sample count:', dirty_data.shape[0])\n",
    "all_en = all_en.dropna()\n",
    "print('After clean:', len(all_en))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_sp.replace('', np.nan, inplace=True)\n",
    "dirty_data = all_en[all_sp.isnull().any(axis=1)]\n",
    "print('Before clean:', len(all_sp))\n",
    "print('English dirty sample count:', dirty_data.shape[0])\n",
    "all_sp = all_sp.dropna()\n",
    "print('After clean:', len(all_sp))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_en.to_csv(\"input/all_en.csv\", index=False)\n",
    "all_sp.to_csv(\"input/all_sp.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_en = pd.read_csv(\"input/all_en.csv\")\n",
    "all_sp = pd.read_csv(\"input/all_sp.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_dict(sents):\n",
    "    counter = Counter()\n",
    "    for sent in sents:\n",
    "        counter.update(sent.split())\n",
    "    dic = counter.items()\n",
    "    return dict(dic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "sp_dict = make_dict(all_sp['spanish'])\n",
    "en_dict = make_dict(all_en['english'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unlabeled English Vocab Count: 2683 \n",
      "Unlabeled Spanish Vocab Count: 5763\n"
     ]
    }
   ],
   "source": [
    "print('Unlabeled English Vocab Count:',len(en_dict), '\\nUnlabeled Spanish Vocab Count:', len(sp_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Vocab(object):\n",
    "    def __init__(self, all_sents, max_size=None, sos_token=None, eos_token=None, unk_token=None):\n",
    "        \"\"\"Initialize the vocabulary.\n",
    "        Args:\n",
    "            iter: An iterable which produces sequences of tokens used to update\n",
    "                the vocabulary.\n",
    "            max_size: (Optional) Maximum number of tokens in the vocabulary.\n",
    "            sos_token: (Optional) Token denoting the start of a sequence.\n",
    "            eos_token: (Optional) Token denoting the end of a sequence.\n",
    "            unk_token: (Optional) Token denoting an unknown element in a\n",
    "                sequence.\n",
    "        \"\"\"\n",
    "        self.max_size = max_size\n",
    "        self.pad_token = '<pad>'\n",
    "        self.sos_token = sos_token\n",
    "        self.eos_token = eos_token\n",
    "        self.unk_token = unk_token\n",
    "\n",
    "        # Add special tokens.\n",
    "        id2word = [self.pad_token]\n",
    "        if sos_token is not None:\n",
    "            id2word.append(self.sos_token)\n",
    "        if eos_token is not None:\n",
    "            id2word.append(self.eos_token)\n",
    "        if unk_token is not None:\n",
    "            id2word.append(self.unk_token)\n",
    "\n",
    "        # Update counter with token counts.\n",
    "        counter = Counter()\n",
    "        for x in all_sents:\n",
    "            counter.update(x.split())\n",
    "\n",
    "        # Extract lookup tables.\n",
    "        if max_size is not None:\n",
    "            counts = counter.most_common(max_size)\n",
    "        else:\n",
    "            counts = counter.items()\n",
    "            counts = sorted(counts, key=lambda x: x[1], reverse=True)\n",
    "        words = [x[0] for x in counts]\n",
    "        id2word.extend(words)\n",
    "        word2id = {x: i for i, x in enumerate(id2word)}\n",
    "\n",
    "        self._id2word = id2word\n",
    "        self._word2id = word2id\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self._id2word)\n",
    "\n",
    "    def word2id(self, word):\n",
    "        \"\"\"Map a word in the vocabulary to its unique integer id.\n",
    "        Args:\n",
    "            word: Word to lookup.\n",
    "        Returns:\n",
    "            id: The integer id of the word being looked up.\n",
    "        \"\"\"\n",
    "        if word in self._word2id:\n",
    "            return self._word2id[word]\n",
    "        elif self.unk_token is not None:\n",
    "            return self._word2id[self.unk_token]\n",
    "        else:\n",
    "            raise KeyError('Word \"%s\" not in vocabulary.' % word)\n",
    "\n",
    "    def id2word(self, id):\n",
    "        \"\"\"Map an integer id to its corresponding word in the vocabulary.\n",
    "        Args:\n",
    "            id: Integer id of the word being looked up.\n",
    "        Returns:\n",
    "            word: The corresponding word.\n",
    "        \"\"\"\n",
    "        return self._id2word[id]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "en_vocab = Vocab(all_en['english'].tolist())\n",
    "sp_vocab = Vocab(all_sp['spanish'].tolist())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get Pre-trained Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "en_embed_path = 'input/wiki.en.vec'\n",
    "sp_embed_path = 'input/wiki.es.vec'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import get_embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 2608/2684 words with embedding vectors\n",
      "Missing Ratio: 2.83%\n",
      "Filled missing words' embeddings.\n",
      "Embedding Matrix Size:  2684\n"
     ]
    }
   ],
   "source": [
    "en_embed = get_embedding(en_vocab._id2word, en_embed_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 5142/5764 words with embedding vectors\n",
      "Missing Ratio: 10.79%\n",
      "Filled missing words' embeddings.\n",
      "Embedding Matrix Size:  5764\n"
     ]
    }
   ],
   "source": [
    "sp_embed = get_embedding(sp_vocab._id2word, sp_embed_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Embedding saved\n",
      "Embedding saved\n",
      "Embedding saved!\n"
     ]
    }
   ],
   "source": [
    "save_embed(en_embed,'input/en_embed.pkl')\n",
    "save_embed(sp_embed,'input/sp_embed.pkl')\n",
    "print('Embedding saved!')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Second Start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_en = pd.read_csv(\"input/all_en.csv\")\n",
    "all_sp = pd.read_csv(\"input/all_sp.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "en_embed = load_embed('input/en_embed.pkl')\n",
    "sp_embed = load_embed('input/sp_embed.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Utilize the unlabeled data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Takes in one column of Spanish or English sentences list, corresponding embeddings.  \n",
    "\n",
    "2. Calculate every sentence's embedding.\n",
    "    How to represent sentences: word embeddings average?\n",
    "\n",
    "3. Calculate every sentence pairs cosine distance. \n",
    "    E.g. s1-s2, s1-s3 ... s1-s1000, get the first 5% and last 5% score to be positive or negative sentence pairs.\n",
    "    \n",
    "4. Concat all results to be added into train data."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clean up unlabeled data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unlabeled Sentences Count: 55669\n"
     ]
    }
   ],
   "source": [
    "un = pd.read_csv('input/cikm_unlabel_spanish_train_20180516.txt',sep='\t', header=None, error_bad_lines=False)\n",
    "un.columns = ['spanish','english']\n",
    "print('Unlabeled Sentences Count:',len(un['spanish']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "sp_stop = set(stopwords.words(\"spanish\"))\n",
    "en_stop = set(stopwords.words(\"english\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_sent(sent):\n",
    "    sent = sent.lower()\n",
    "    sent = re.sub(u\"[_'\\-;%()|+&=*%.,!?:#$@\\[\\]/]\",' ',sent)\n",
    "    sent = re.sub('¡',' ',sent)\n",
    "    sent = re.sub('¿',' ',sent)\n",
    "    sent = re.sub('Á','á',sent)\n",
    "    sent = re.sub('Ó','ó',sent)\n",
    "    sent = re.sub('Ú','ú',sent)\n",
    "    sent = re.sub('É','é',sent)\n",
    "    sent = re.sub('Í','í',sent)\n",
    "    return sent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-28-c285595af17f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m un['spanish'] = un.spanish.map(lambda x: ' '.join([word for word in\n\u001b[1;32m      2\u001b[0m                                                      nltk.word_tokenize(clean_sent(x))]))\n\u001b[0;32m----> 3\u001b[0;31m un['english'] = un.english.map(lambda x: ' '.join([word for word in\n\u001b[0m\u001b[1;32m      4\u001b[0m                                                      nltk.word_tokenize(clean_sent(x))]))\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\u001b[0m in \u001b[0;36mmap\u001b[0;34m(self, arg, na_action)\u001b[0m\n\u001b[1;32m   2994\u001b[0m         \"\"\"\n\u001b[1;32m   2995\u001b[0m         new_values = super(Series, self)._map_values(\n\u001b[0;32m-> 2996\u001b[0;31m             arg, na_action=na_action)\n\u001b[0m\u001b[1;32m   2997\u001b[0m         return self._constructor(new_values,\n\u001b[1;32m   2998\u001b[0m                                  index=self.index).__finalize__(self)\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/pandas/core/base.py\u001b[0m in \u001b[0;36m_map_values\u001b[0;34m(self, mapper, na_action)\u001b[0m\n\u001b[1;32m   1002\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1003\u001b[0m         \u001b[0;31m# mapper is a function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1004\u001b[0;31m         \u001b[0mnew_values\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmap_f\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmapper\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1005\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1006\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mnew_values\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/src/inference.pyx\u001b[0m in \u001b[0;36mpandas._libs.lib.map_infer\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m<ipython-input-28-c285595af17f>\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m      2\u001b[0m                                                      nltk.word_tokenize(clean_sent(x))]))\n\u001b[1;32m      3\u001b[0m un['english'] = un.english.map(lambda x: ' '.join([word for word in\n\u001b[0;32m----> 4\u001b[0;31m                                                      nltk.word_tokenize(clean_sent(x))]))\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/nltk/tokenize/__init__.py\u001b[0m in \u001b[0;36mword_tokenize\u001b[0;34m(text, language, preserve_line)\u001b[0m\n\u001b[1;32m    127\u001b[0m     \"\"\"\n\u001b[1;32m    128\u001b[0m     \u001b[0msentences\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mpreserve_line\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0msent_tokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlanguage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 129\u001b[0;31m     return [token for sent in sentences\n\u001b[0m\u001b[1;32m    130\u001b[0m             for token in _treebank_word_tokenizer.tokenize(sent)]\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/nltk/tokenize/__init__.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    128\u001b[0m     \u001b[0msentences\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mpreserve_line\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0msent_tokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlanguage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    129\u001b[0m     return [token for sent in sentences\n\u001b[0;32m--> 130\u001b[0;31m             for token in _treebank_word_tokenizer.tokenize(sent)]\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/nltk/tokenize/treebank.py\u001b[0m in \u001b[0;36mtokenize\u001b[0;34m(self, text, convert_parentheses, return_str)\u001b[0m\n\u001b[1;32m    131\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    132\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mregexp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msubstitution\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mENDING_QUOTES\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 133\u001b[0;31m             \u001b[0mtext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mregexp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msub\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msubstitution\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    134\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    135\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mregexp\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCONTRACTIONS2\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/re.py\u001b[0m in \u001b[0;36m_subx\u001b[0;34m(pattern, template)\u001b[0m\n\u001b[1;32m    322\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0msre_parse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpand_template\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtemplate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    323\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 324\u001b[0;31m \u001b[0;32mdef\u001b[0m \u001b[0m_subx\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpattern\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtemplate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    325\u001b[0m     \u001b[0;31m# internal: pattern.sub/subn implementation helper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    326\u001b[0m     \u001b[0mtemplate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_compile_repl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtemplate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpattern\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "un['spanish'] = un.spanish.map(lambda x: ' '.join([word for word in\n",
    "                                                     nltk.word_tokenize(clean_sent(x))]))\n",
    "un['english'] = un.english.map(lambda x: ' '.join([word for word in\n",
    "                                                     nltk.word_tokenize(clean_sent(x))]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "un['spanish'] = un.spanish.map(lambda x: ' '.join([word for word in nltk.word_tokenize(x) if word not in sp_stop]))\n",
    "un['english'] = un.english.map(lambda x: ' '.join([word for word in nltk.word_tokenize(x) if word not in en_stop]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "un.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "un.replace('', np.nan, inplace=True)\n",
    "dirty_data = un[un.isnull().any(axis=1)]\n",
    "print('Before clean:', len(un))\n",
    "print('dirty sample count:', dirty_data.shape[0])\n",
    "un = un.dropna()\n",
    "print('After clean:', len(un))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Drop duplicated English and Spanish Sentences respectively"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "un_sp = pd.DataFrame(un['spanish'])\n",
    "un_en = pd.DataFrame(un['english'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Spanish sentences count:', len(un_sp))\n",
    "print('current duplicated sentences:',un_sp.duplicated().sum())\n",
    "un_sp = un_sp.drop_duplicates()\n",
    "print('After drop duplicated:', len(un_sp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Spanish sentences count:', len(un_en))\n",
    "print('current duplicated sentences:',un_en.duplicated().sum())\n",
    "un_en = un_en.drop_duplicates()\n",
    "print('After drop duplicated:', len(un_en))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "un_sp.to_csv(\"input/cleaned_unlabeled_Spanish.csv\", index=False)\n",
    "un_en.to_csv(\"input/cleaned_unlabeled_English.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "un.to_csv(\"input/cleaned_unlabeled.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Second Start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "un_sp = pd.read_csv(\"input/cleaned_unlabeled_Spanish.csv\")\n",
    "un_en = pd.read_csv(\"input/cleaned_unlabeled_English.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "en_embed = load_embed('input/en_embed.pkl')\n",
    "sp_embed = load_embed('input/sp_embed.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Drop sentences that less than 5 words to get better cosine results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sent length < 5: 12694\n"
     ]
    }
   ],
   "source": [
    "sents = un_en['english'].tolist()\n",
    "for i in range(len(sents)):\n",
    "    if len(sents[i].split()) <= 5:\n",
    "        un_en['english'].iloc[i] = np.nan\n",
    "    \n",
    "print('sent length < 5:',un_en['english'].isnull().sum())\n",
    "un_en = un_en.dropna()\n",
    "un_en = un_en.reset_index()\n",
    "un_en = un_en.drop(['index'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sent length < 5: 8380\n"
     ]
    }
   ],
   "source": [
    "sents = un_sp['spanish'].tolist()\n",
    "for i in range(len(sents)):\n",
    "    if len(sents[i].split()) <= 5:\n",
    "        un_sp['spanish'].iloc[i] = np.nan\n",
    "    \n",
    "print('sent length < 5:',un_sp['spanish'].isnull().sum())\n",
    "un_sp = un_sp.dropna()\n",
    "un_sp = un_sp.reset_index()\n",
    "un_sp = un_sp.drop(['index'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Spanish sentences count: 44820 \n",
      "English sentences count: 38299\n"
     ]
    }
   ],
   "source": [
    "print('Spanish sentences count:',un_sp.shape[0],'\\nEnglish sentences count:', un_en.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cosine_similarity(s1,s2):\n",
    "    return np.linalg.norm(s1-s2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def WMD_core(s1,s2,embedding):\n",
    "    s1_vect = []\n",
    "    # if can't find corresponding word embeddings, add single word embeddings\n",
    "    # 分词后的词语若无对应词向量，则添加该单字向量\n",
    "    \n",
    "    for word in s1.split():\n",
    "        if word in embedding:\n",
    "            s1_vect.append(embedding[word])\n",
    "        else: continue\n",
    "\n",
    "    s2_vect = []\n",
    "    for word in s2.split():\n",
    "        if word in embedding:\n",
    "            s2_vect.append(embedding[word])\n",
    "        else: continue\n",
    "\n",
    "#     print(len(s1_vect), len(s2_vect))\n",
    "    total_min = []\n",
    "    sum_min = 0.0\n",
    "    \n",
    "    # find every words' nearest\n",
    "    for w1 in s1_vect: \n",
    "        cur_min = 1000.0\n",
    "        min_dis = []\n",
    "        # for every word in s1, find the nearest in s2\n",
    "        for w2 in s2_vect:\n",
    "            temp = cosine_similarity(w1,w2)\n",
    "            if temp < cur_min:\n",
    "                cur_min = temp\n",
    "#         min_dis.append(cur_min)\n",
    "        sum_min += cur_min\n",
    "#         total_min.extend(min_dis)\n",
    "#     print(total_min)\n",
    "#     return round(sum_min/len(s1),6)\n",
    "    return sum_min/len(s1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def qselectmin(A,k):\n",
    "    if len(A)<k:return A\n",
    "    pivot = A[-1]\n",
    "    right = [pivot] + [x for x in A[:-1] if x<pivot]\n",
    "    rlen = len(right)\n",
    "    if rlen==k:\n",
    "        return right\n",
    "    if rlen>k:\n",
    "        return qselectmin(right, k)\n",
    "    else:\n",
    "        left = [x for x in A[:-1] if x>=pivot]\n",
    "        return qselectmin(left, k-rlen) + right\n",
    "\n",
    "def qselectmax(A,k):\n",
    "    if len(A)<k:return A\n",
    "    pivot = A[-1]\n",
    "    right = [pivot] + [x for x in A[:-1] if x>pivot]\n",
    "    rlen = len(right)\n",
    "    if rlen==k:\n",
    "        return right\n",
    "    if rlen>k:\n",
    "        return qselectmax(right, k)\n",
    "    else:\n",
    "        left = [x for x in A[:-1] if x<=pivot]\n",
    "        return qselectmax(left, k-rlen) + right\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Input: sentence; embedding\n",
    "Output: This sentence's embeddings\n",
    "\n",
    "Using words' embedding averages as sentence embeddings\n",
    "option: WMD?\n",
    "\n",
    "\"\"\"\n",
    "def sent_embed(sent,embedding):\n",
    "    sent_len = len(sent)\n",
    "    sent_vec = np.zeros(embed_size)\n",
    "    for w in sent.split():\n",
    "        sent_len = len(sent)\n",
    "        if w in embedding:\n",
    "            sent_vec += embedding[w]\n",
    "        else: continue\n",
    "    sent_vec = sent_vec/sent_len \n",
    "    return sent_vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.31838501826110932"
      ]
     },
     "execution_count": 245,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cosine_similarity(sent_embed(sents[0], embedding) , sent_embed(sents[1], embedding))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "sents1 = un_en['english'][100:200]\n",
    "sents1 = sents1.reset_index()\n",
    "sents1 = sents1.drop(['index'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "en1 = un_en['english'][0:100]\n",
    "en2 = un_en['english'][100:200]\n",
    "en2 = en2.reset_index()\n",
    "en2 = en2.drop(['index'],axis=1)['english']\n",
    "\n",
    "sp1 = un_sp['spanish'][0:50]\n",
    "sp2 = un_sp['spanish'][500:1000]\n",
    "sp2 = sp2.reset_index()\n",
    "sp2 = sp2.drop(['index'],axis=1)['spanish']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "embed_size = 300"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "i: 0.0 / 0.5\n"
     ]
    },
    {
     "ename": "KeyError",
     "evalue": "51",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-58-8e23eecd4038>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0maug_sp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0maug_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msp1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msp2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msp_embed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-54-dd021acc09b5>\u001b[0m in \u001b[0;36maug_data\u001b[0;34m(sents1, sents2, embedding, embed_size)\u001b[0m\n\u001b[1;32m     23\u001b[0m             \u001b[0;31m# neg pairs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0ms\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mneg_score\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m                 \u001b[0mpair\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0morigin_sent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msents1\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mscore\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     26\u001b[0m                 \u001b[0mpairs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpair\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m    764\u001b[0m         \u001b[0mkey\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply_if_callable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    765\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 766\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_value\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    767\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    768\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_scalar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_value\u001b[0;34m(self, series, key)\u001b[0m\n\u001b[1;32m   3101\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3102\u001b[0m             return self._engine.get_value(s, k,\n\u001b[0;32m-> 3103\u001b[0;31m                                           tz=getattr(series.dtype, 'tz', None))\n\u001b[0m\u001b[1;32m   3104\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3105\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minferred_type\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'integer'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'boolean'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_value\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_value\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.Int64HashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.Int64HashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 51"
     ]
    }
   ],
   "source": [
    "aug_sp = aug_data(sp1, sp2, sp_embed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "def aug_data(sents1, sents2, embedding, embed_size=300):\n",
    "    sum_pairs = pd.DataFrame()\n",
    "    score = {}\n",
    "    for i in range(len(sents1)):\n",
    "        if(i%100==0): print('i:', i/100,'/',len(sents1)/100)\n",
    "        origin_sent = sents1[i]\n",
    "        origin_sent_embed = sent_embed(sents1[i], embedding)\n",
    "        score[i] = []\n",
    "\n",
    "        for j in range(len(sents2)):\n",
    "    #         if(j%10000==0): print(j,'/',len(sents))\n",
    "            score[i].append(cosine_similarity(origin_sent_embed, sent_embed(sents2[j], embedding)))\n",
    "\n",
    "    #         WMD Score\n",
    "    #         score[i].append(WMD_core(sents[i], sents[j], embedding))        \n",
    "\n",
    "            # max distances to be negative scores\n",
    "            neg_score = qselectmax(score[i], 10)\n",
    "            # min distances to be positive scores\n",
    "            pos_score = qselectmin(score[i], 10)\n",
    "            pairs = []\n",
    "\n",
    "            # neg pairs\n",
    "            for s in neg_score:\n",
    "                pair = [origin_sent, sents1[score[i].index(s)],0]\n",
    "                pairs.append(pair)\n",
    "\n",
    "            # pos pairs\n",
    "            for s in pos_score:\n",
    "                if s == 0: continue\n",
    "                pair = [origin_sent, sents1[score[i].index(s)],1]\n",
    "                pairs.append(pair)\n",
    "\n",
    "            pairs = pd.DataFrame(pairs)\n",
    "            pairs = pairs.drop_duplicates()\n",
    "        sum_pairs = sum_pairs.append(pairs)\n",
    "    print('Done')\n",
    "    sum_pairs.columns = ['s1','s2','label']\n",
    "    sum_pairs = sum_pairs.sort_values('label')\n",
    "    print('Augmented Count:',len(sum_pairs))\n",
    "    print('Positive Count:',sum_pairs[sum_pairs['label']==1].shape[0])\n",
    "    return sum_pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 408,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>s1</th>\n",
       "      <th>s2</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>product received correspond description paid c...</td>\n",
       "      <td>product received correspond description satisf...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>hello good night concern request already count...</td>\n",
       "      <td>use debit card mastercard debit card</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>receive refund pay credit card credit card pay...</td>\n",
       "      <td>fuck male want talk human need human</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>able open dispute request refund product arriv...</td>\n",
       "      <td>pay mercado pago pay mercado pago</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>receive refund credit card receive refund cred...</td>\n",
       "      <td>receive refund pay credit card paid credit car...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  s1  \\\n",
       "5  product received correspond description paid c...   \n",
       "1  hello good night concern request already count...   \n",
       "3  receive refund pay credit card credit card pay...   \n",
       "4  able open dispute request refund product arriv...   \n",
       "7  receive refund credit card receive refund cred...   \n",
       "\n",
       "                                                  s2  label  \n",
       "5  product received correspond description satisf...      1  \n",
       "1               use debit card mastercard debit card      0  \n",
       "3               fuck male want talk human need human      0  \n",
       "4                  pay mercado pago pay mercado pago      0  \n",
       "7  receive refund pay credit card paid credit car...      1  "
      ]
     },
     "execution_count": 408,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.utils import shuffle\n",
    "sum_pairs = shuffle(sum_pairs)\n",
    "sum_pairs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "i: 0.0 / 10.0\n",
      "i: 1.0 / 10.0\n",
      "i: 2.0 / 10.0\n",
      "i: 3.0 / 10.0\n",
      "i: 4.0 / 10.0\n",
      "i: 5.0 / 10.0\n",
      "i: 6.0 / 10.0\n",
      "i: 7.0 / 10.0\n",
      "i: 8.0 / 10.0\n",
      "i: 9.0 / 10.0\n",
      "Done\n",
      "Augmented Count: 18996\n",
      "Positive Count: 9071\n"
     ]
    }
   ],
   "source": [
    "sents = un_sp['spanish'][0:1000]\n",
    "embedding = en_embed\n",
    "embed_size = 300\n",
    "sum_pairs = pd.DataFrame()\n",
    "score = {}\n",
    "for i in range(len(sents)):\n",
    "    if(i%100==0): print('i:', i/100,'/',len(sents)/100)\n",
    "    origin_sent = sents[i]\n",
    "    origin_sent_embed = sent_embed(sents[i], embedding)\n",
    "    score[i] = []\n",
    "\n",
    "    for j in range(len(sents)):\n",
    "#         if(j%10000==0): print(j,'/',len(sents))\n",
    "        score[i].append(cosine_similarity(origin_sent_embed, sent_embed(sents[j], embedding)))\n",
    "\n",
    "#         WMD Score\n",
    "#         score[i].append(WMD_core(sents[i], sents[j], embedding))        \n",
    "\n",
    "        # max distances to be negative scores\n",
    "        neg_score = qselectmax(score[i], 10)\n",
    "        # min distances to be positive scores\n",
    "        pos_score = qselectmin(score[i], 30)\n",
    "        pairs = []\n",
    "\n",
    "        # neg pairs\n",
    "        for s in neg_score:\n",
    "            pair = [origin_sent, sents[score[i].index(s)],0]\n",
    "            pairs.append(pair)\n",
    "\n",
    "        # pos pairs\n",
    "        for s in pos_score:\n",
    "            if s == 0: continue\n",
    "            pair = [origin_sent, sents[score[i].index(s)],1]\n",
    "            pairs.append(pair)\n",
    "\n",
    "        pairs = pd.DataFrame(pairs)\n",
    "        pairs = pairs.drop_duplicates()\n",
    "    sum_pairs = sum_pairs.append(pairs)\n",
    "print('Done')\n",
    "sum_pairs.columns = ['s1','s2','label']\n",
    "sum_pairs = sum_pairs.sort_values('label')\n",
    "print('Augmented Count:',len(sum_pairs))\n",
    "print('Positive Count:',sum_pairs[sum_pairs['label']==1].shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "i: 0.0 / 12.0\n",
      "i: 1.0 / 12.0\n",
      "i: 2.0 / 12.0\n",
      "i: 3.0 / 12.0\n",
      "i: 4.0 / 12.0\n",
      "i: 5.0 / 12.0\n",
      "i: 6.0 / 12.0\n",
      "i: 7.0 / 12.0\n",
      "i: 8.0 / 12.0\n",
      "i: 9.0 / 12.0\n"
     ]
    }
   ],
   "source": [
    "sents = un_sp['spanish'][0:1200]\n",
    "embedding = en_embed\n",
    "embed_size = 300\n",
    "sum_pairs_wmd = pd.DataFrame()\n",
    "score = {}\n",
    "for i in range(len(sents)):\n",
    "    if(i%100==0): print('i:', i/100,'/',len(sents)/100)\n",
    "    origin_sent = sents[i]\n",
    "    origin_sent_embed = sent_embed(sents[i], embedding)\n",
    "    score[i] = []\n",
    "\n",
    "    for j in range(len(sents)):\n",
    "#         if(j%10000==0): print(j,'/',len(sents))\n",
    "#         score[i].append(cosine_similarity(origin_sent_embed, sent_embed(sents[j], embedding)))\n",
    "\n",
    "#         WMD Score\n",
    "        score[i].append(WMD_core(sents[i], sents[j], embedding))        \n",
    "\n",
    "        # max distances to be negative scores\n",
    "        neg_score = qselectmax(score[i], 10)\n",
    "        # min distances to be positive scores\n",
    "        pos_score = qselectmin(score[i], 30)\n",
    "        pairs = []\n",
    "\n",
    "        # neg pairs\n",
    "        for s in neg_score:\n",
    "            pair = [origin_sent, sents[score[i].index(s)],0]\n",
    "            pairs.append(pair)\n",
    "\n",
    "        # pos pairs\n",
    "        for s in pos_score:\n",
    "            if s == 0: continue\n",
    "            pair = [origin_sent, sents[score[i].index(s)],1]\n",
    "            pairs.append(pair)\n",
    "\n",
    "        pairs = pd.DataFrame(pairs)\n",
    "        pairs = pairs.drop_duplicates()\n",
    "        sum_pairs_wmd = sum_pairs_wmd.append(pairs)\n",
    "print('Done')\n",
    "sum_pairs_wmd.columns = ['s1','s2','label']\n",
    "sum_pairs_wmd = sum_pairs_wmd.sort_values('label')\n",
    "print('Augmented Count:',len(sum_pairs_wmd))\n",
    "print('Positive Count:',sum_pairs_wmd[sum_pairs_wmd['label']==1].shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "sum_pairs.to_csv('input/aug_sp_1000_less.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 415,
   "metadata": {},
   "outputs": [],
   "source": [
    "aug_sp = pd.read_csv('aug_sp.csv')\n",
    "aug_en = pd.read_csv('aug_en.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 380,
   "metadata": {},
   "outputs": [],
   "source": [
    "# max distances to be negative scores\n",
    "neg_score = qselectmax(score[i], 10)\n",
    "# min distances to be positive scores\n",
    "pos_score = qselectmin(score[i], 10)\n",
    "pairs = []\n",
    "\n",
    "# neg pairs\n",
    "for s in neg_score:\n",
    "    pair = [origin_sent, sents[score[i].index(s)],0,s]\n",
    "    pairs.append(pair)\n",
    "\n",
    "# pos pairs\n",
    "for s in pos_score:\n",
    "    if s == 0: continue\n",
    "    pair = [origin_sent, sents[score[i].index(s)],1,s]\n",
    "    pairs.append(pair)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['payment could processed due security reasons order closed security reasons payment still charged',\n",
       " 'payment problem presented order closed security reasons payment still charged',\n",
       " 1,\n",
       " 0.078975945574712936]"
      ]
     },
     "execution_count": 376,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pairs.iloc[6,:].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total: 10 \n",
      "Pos: 0 \n",
      "Neg: 10\n"
     ]
    }
   ],
   "source": [
    "pos_score = qselectmax(score, 10)\n",
    "neg_score = qselectmin(score, 10)\n",
    "pairs = []\n",
    "\n",
    "# neg pairs\n",
    "for s in pos_score:\n",
    "    pair = [origin_sent, sents[score.index(s)],0]\n",
    "    pairs.append(pair)\n",
    "\n",
    "# pos pairs\n",
    "for s in neg_score:\n",
    "    if s == 0: continue\n",
    "    pair = [origin_sent, sents[score.index(s)],1]\n",
    "    pairs.append(pair)\n",
    "\n",
    "pairs = pd.DataFrame(pairs)\n",
    "pairs = pairs.drop_duplicates()\n",
    "pairs.columns = ['s1','s2','label']\n",
    "print('Total:', pairs.shape[0], '\\nPos:', pairs[pairs['label'] == 1].shape[0],\n",
    "      '\\nNeg:', pairs[pairs['label'] == 0].shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['payment could processed due security reasons order closed security reasons payment still charged',\n",
       " 'hello status order closed would like know done correctly order closed security reasons payment still charged',\n",
       " 1,\n",
       " 0.095889463288334739]"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pairs.iloc[3,:].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>spanish1</th>\n",
       "      <th>spanish2</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   spanish1  spanish2  label\n",
       "0         5         6      1\n",
       "1         1         2      0"
      ]
     },
     "execution_count": 357,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame([[5,6,1],[1,2,0]],columns= ['spanish1','spanish2','label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 313,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max score index: 277\n",
      "Original Sent: podido procesar pago razones seguridad si pedido cerrado razones seguridad aún cobra pago\n",
      "Similar Sent: hola hola\n"
     ]
    }
   ],
   "source": [
    "print('Max score index:', score[0].index(max(score[0])))\n",
    "print('Original Sent:', un_sp[0])\n",
    "print('Similar Sent:',un_sp[score[0].index(max(score[0]))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Input: sentences list, embeddings.\n",
    "Output: labeled data in pd forms.\n",
    "\"\"\"\n",
    "def aug_data(sents,embedding):\n",
    "    score = {}\n",
    "    sum_pairs = []\n",
    "    for i in range(len(sents)):\n",
    "        if(i%1000==0): print('i:', i/1000)\n",
    "        origin_sent = sents[i]\n",
    "        score[i] = []\n",
    "\n",
    "        for j in range(len(sents)):\n",
    "\n",
    "            score[i].append(WMD_core(sents[i], sents[j], embedding))\n",
    "            # max distances to be negative scores\n",
    "            neg_score = qselectmax(score[i], 5)\n",
    "            # min distances to be positive scores\n",
    "            pos_score = qselectmin(score[i], 5)\n",
    "            pairs = []\n",
    "\n",
    "            # neg pairs\n",
    "            for s in neg_score:\n",
    "                pair = [origin_sent, sents[score[i].index(s)],0]\n",
    "                pairs.append(pair)\n",
    "\n",
    "            # pos pairs\n",
    "            for s in pos_score:\n",
    "                if s == 0: continue\n",
    "                pair = [origin_sent, sents[score[i].index(s)],1]\n",
    "                pairs.append(pair)\n",
    "\n",
    "            pairs = pd.DataFrame(pairs)\n",
    "            pairs = pairs.drop_duplicates()\n",
    "\n",
    "        sum_pairs = sum_pairs.append(pairs)\n",
    "    return sum_pairs.drop_duplicates()\n",
    "            \n",
    "    \n",
    "        \n",
    "        \n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "dlopen(/Users/liushijing/anaconda3/lib/python3.6/site-packages/torch/_C.cpython-36m-darwin.so, 9): no suitable image found.  Did find:\n\t/Users/liushijing/anaconda3/lib/python3.6/site-packages/torch/_C.cpython-36m-darwin.so: truncated mach-o error: segment __TEXT extends to 12820480 which is past end of file 7979008\n\t/Users/liushijing/anaconda3/lib/python3.6/site-packages/torch/_C.cpython-36m-darwin.so: truncated mach-o error: segment __TEXT extends to 12820480 which is past end of file 7979008",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-245-7111213e236e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mtorch\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0membed_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m300\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0msp_vocab_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msp_embed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0men_vocab_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0men_embed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;31m# initialize nn embedding\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/torch/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     76\u001b[0m     \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     77\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 78\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_C\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     79\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     80\u001b[0m __all__ += [name for name in dir(_C)\n",
      "\u001b[0;31mImportError\u001b[0m: dlopen(/Users/liushijing/anaconda3/lib/python3.6/site-packages/torch/_C.cpython-36m-darwin.so, 9): no suitable image found.  Did find:\n\t/Users/liushijing/anaconda3/lib/python3.6/site-packages/torch/_C.cpython-36m-darwin.so: truncated mach-o error: segment __TEXT extends to 12820480 which is past end of file 7979008\n\t/Users/liushijing/anaconda3/lib/python3.6/site-packages/torch/_C.cpython-36m-darwin.so: truncated mach-o error: segment __TEXT extends to 12820480 which is past end of file 7979008"
     ]
    }
   ],
   "source": [
    "from torch import nn\n",
    "embed_size = 300\n",
    "sp_vocab_size = len(sp_embed)\n",
    "en_vocab_size = len(en_embed)\n",
    "# initialize nn embedding\n",
    "sp_embedding = nn.Embedding(sp_vocab_size, embed_size)\n",
    "en_embedding = nn.Embedding(en_vocab_size, embed_size)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}