Paragraph_Matching.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Paragraph Matching.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "f620df5b96464bbb98883577738c5bdb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4dbfc23c32464745abc44c95944543a5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6b2fcc3224f34a61bc378fa6f2437b19",
              "IPY_MODEL_d452a3da8afc45ca8da0db1efd54bac2"
            ]
          }
        },
        "4dbfc23c32464745abc44c95944543a5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6b2fcc3224f34a61bc378fa6f2437b19": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d148cf203e824ce6a62313260d0442b0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 4023,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 4023,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a24080dcbab34602a925dac734e7a317"
          }
        },
        "d452a3da8afc45ca8da0db1efd54bac2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_eae1ed12bd5349cda0584bd1f240471f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100% 4023/4023 [00:32&lt;00:00, 122.04it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d90ceec9d91949b0a75ebeff2875f3d3"
          }
        },
        "d148cf203e824ce6a62313260d0442b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a24080dcbab34602a925dac734e7a317": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "eae1ed12bd5349cda0584bd1f240471f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d90ceec9d91949b0a75ebeff2875f3d3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "543539d606614353b154a7403d20ae01": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c1d5b000ad0d4199989ca61a86a0cd0d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_cbbc1ff94d554410990122c35056a1a0",
              "IPY_MODEL_e3a65e40d201408baa2e3d6f12bc547e"
            ]
          }
        },
        "c1d5b000ad0d4199989ca61a86a0cd0d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cbbc1ff94d554410990122c35056a1a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a57d1248b9c241d6a03bea865c5209db",
            "_dom_classes": [],
            "description": "",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 4023,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 4023,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_029957210366452091fcde805af42145"
          }
        },
        "e3a65e40d201408baa2e3d6f12bc547e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c466f8b9135d42efbb6bc79c2589d3fe",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100% 4023/4023 [00:29&lt;00:00, 137.85it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3143f4cb5665480d84239b9f670d998a"
          }
        },
        "a57d1248b9c241d6a03bea865c5209db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "029957210366452091fcde805af42145": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c466f8b9135d42efbb6bc79c2589d3fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3143f4cb5665480d84239b9f670d998a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "DI7X0oZkzEXL",
        "colab_type": "code",
        "outputId": "253b1aa3-dc45-4c20-ab04-044eb3d8a86a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount(\"/content/drive\")"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nvLWf-w-p98m",
        "colab_type": "text"
      },
      "source": [
        "Necessary imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0uNDVBNx8mDQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 79
        },
        "outputId": "d3d838f0-1232-421d-e4dd-3b599bcc8bc6"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import os\n",
        "import sys\n",
        "from tqdm import tqdm_notebook\n",
        "import tensorflow as tf\n",
        "import tensorflow_hub as hub\n",
        "from nltk import sent_tokenize\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import pickle\n",
        "import math\n",
        "import keras\n",
        "import nltk\n",
        "%matplotlib inline"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E0omYQH4qjV3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Loading the dataset from google drive\n",
        "df = pd.read_csv(\"/content/drive/Shared drives/ChiragSaini SharedDrive/Precily Assignment/Text_Similarity_Dataset.csv\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l0ZQczPAq-bu",
        "colab_type": "code",
        "outputId": "8ab83a43-5878-4273-c45a-c80c2702fcff",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        }
      },
      "source": [
        "df.head()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unique_ID</th>\n",
              "      <th>text1</th>\n",
              "      <th>text2</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>savvy searchers fail to spot ads internet sear...</td>\n",
              "      <td>newcastle 2-1 bolton kieron dyer smashed home ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>millions to miss out on the net by 2025  40% o...</td>\n",
              "      <td>nasdaq planning $100m share sale the owner of ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>young debut cut short by ginepri fifteen-year-...</td>\n",
              "      <td>ruddock backs yapp s credentials wales coach m...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>diageo to buy us wine firm diageo  the world s...</td>\n",
              "      <td>mci shares climb on takeover bid shares in us ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>be careful how you code a new european directi...</td>\n",
              "      <td>media gadgets get moving pocket-sized devices ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unique_ID  ...                                              text2\n",
              "0          0  ...  newcastle 2-1 bolton kieron dyer smashed home ...\n",
              "1          1  ...  nasdaq planning $100m share sale the owner of ...\n",
              "2          2  ...  ruddock backs yapp s credentials wales coach m...\n",
              "3          3  ...  mci shares climb on takeover bid shares in us ...\n",
              "4          4  ...  media gadgets get moving pocket-sized devices ...\n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JuI1O6hLq_gb",
        "colab_type": "code",
        "outputId": "b7e0385a-e2cd-4ea1-e74f-6a4327782744",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 151
        }
      },
      "source": [
        "df.info()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 4023 entries, 0 to 4022\n",
            "Data columns (total 3 columns):\n",
            "Unique_ID    4023 non-null int64\n",
            "text1        4023 non-null object\n",
            "text2        4023 non-null object\n",
            "dtypes: int64(1), object(2)\n",
            "memory usage: 94.4+ KB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I6mllen5rGF8",
        "colab_type": "text"
      },
      "source": [
        "So in our Dataset, there are 4023 observation i.e. 4023 pairs of paragraphs"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nq57Kmz1rVca",
        "colab_type": "text"
      },
      "source": [
        "Preprocessing Data:\n",
        "    Cleaning text and removing unwanted words and symbols"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-LiU9E31rdk9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import re\n",
        "def clean_paragraph(text_list):\n",
        "    clean_list = []\n",
        "    for sent in text_list:\n",
        "        sent = re.sub('[%s]' % re.escape(\"\"\"!\"#$%&'()*+,-.:;<=>?@[\\]^`{|}~\"\"\"), '',sent)\n",
        "        sent = sent.replace('[]','')\n",
        "        sent = re.sub('\\d+',' ',sent)\n",
        "        sent = sent.lower()\n",
        "        clean_list.append(sent)\n",
        "    return clean_list"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JCNyRrUerq7y",
        "colab_type": "text"
      },
      "source": [
        "Embedding using Google universal encoder\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fcT_Uwfar1Ty",
        "colab_type": "code",
        "outputId": "974ffd28-1c65-4d62-a144-d015ab209aec",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 198,
          "referenced_widgets": [
            "f620df5b96464bbb98883577738c5bdb",
            "4dbfc23c32464745abc44c95944543a5",
            "6b2fcc3224f34a61bc378fa6f2437b19",
            "d452a3da8afc45ca8da0db1efd54bac2",
            "d148cf203e824ce6a62313260d0442b0",
            "a24080dcbab34602a925dac734e7a317",
            "eae1ed12bd5349cda0584bd1f240471f",
            "d90ceec9d91949b0a75ebeff2875f3d3",
            "543539d606614353b154a7403d20ae01",
            "c1d5b000ad0d4199989ca61a86a0cd0d",
            "cbbc1ff94d554410990122c35056a1a0",
            "e3a65e40d201408baa2e3d6f12bc547e",
            "a57d1248b9c241d6a03bea865c5209db",
            "029957210366452091fcde805af42145",
            "c466f8b9135d42efbb6bc79c2589d3fe",
            "3143f4cb5665480d84239b9f670d998a"
          ]
        }
      },
      "source": [
        "# In this problem, we cant use word encodings.\n",
        "# we need to use Sentence encoding as many sentences are similar in textual context,\n",
        "# but not in semantic or logical context\n",
        "# for example: \"Lion killed Deer\" and \"Deer killer Lion\" are similar in textual context but different in logical context\n",
        "nltk.download(\"punkt\")\n",
        "X=[]\n",
        "with tf.Graph().as_default():\n",
        "    #loading model\n",
        "    embed = hub.Module(\"https://tfhub.dev/google/universal-sentence-encoder/2\")\n",
        "    print(\"embed\",embed)\n",
        "    messages = tf.placeholder(dtype=tf.string, shape=[None])\n",
        "    output = embed(messages)\n",
        "    #tensorflow session\n",
        "    with tf.Session() as session:\n",
        "        session.run([tf.global_variables_initializer(), tf.tables_initializer()])\n",
        "        for text in [\"text1\",\"text2\"]:\n",
        "            phrases_emb_list = []\n",
        "            for phrases in tqdm_notebook(df[text]):\n",
        "                #breaking paragraph into sentences  \n",
        "                sent_list = sent_tokenize(phrases)\n",
        "                #cleaning the sentence\n",
        "                clean_sent_list = clean_paragraph(sent_list)\n",
        "                #feeding sentence in the encoder\n",
        "                sent_embed = session.run(output, feed_dict={messages: clean_sent_list})\n",
        "                #appending the encodings in a list\n",
        "                phrases_emb_list.append(sent_embed.mean(axis=0).reshape(1,512))\n",
        "            X.append(phrases_emb_list)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "embed <tensorflow_hub.module.Module object at 0x7fb97bb1efd0>\n",
            "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f620df5b96464bbb98883577738c5bdb",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, max=4023), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "543539d606614353b154a7403d20ae01",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, max=4023), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UkQwSyWCs-av",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# saving the encodings for future use\n",
        "encodings = open(\"/content/drive/Shared drives/ChiragSaini SharedDrive/Precily Assignment/enc.pkl\", \"wb\")\n",
        "pickle.dump(X, encodings)\n",
        "encodings.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PPgHZWLKsrHs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Converting the encoding into numpy arrays\n",
        "text1 = np.array(X[0])\n",
        "text2 = np.array(X[1])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CCyylZVPuHcf",
        "colab_type": "code",
        "outputId": "075a20d1-c158-4f5e-aa77-b150bd0a3750",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "text1.shape"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(4023, 1, 512)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gKC-cPj5uJDf",
        "colab_type": "code",
        "outputId": "4c07e4a6-1bac-470d-a2fd-6efe60f0616a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "text2.shape"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(4023, 1, 512)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oA9Zhs1nuLc_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#converting data to numpy aaray\n",
        "X_data=np.array(X)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "igrRtqhquRE2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_cosine(vec1, vec2):\n",
        "     \n",
        "     numerator = sum(vec1 * vec2 )\n",
        "\n",
        "     sum1 = sum([vec1[x]**2 for x in range(len(vec1))])\n",
        "     sum2 = sum([vec2[x]**2 for x in range(len(vec2))])\n",
        "     denominator = math.sqrt(sum1) * math.sqrt(sum2)\n",
        "\n",
        "     if not denominator:\n",
        "        return 0.0\n",
        "     else:\n",
        "        return float(numerator) / denominator"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vwqJjqCouXPe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df[\"similarity_score\"]=0"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RkCHdxF7ws5y",
        "colab_type": "code",
        "outputId": "bf3b4736-32c4-4261-c84c-901ab25f5884",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        }
      },
      "source": [
        "df.head()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unique_ID</th>\n",
              "      <th>text1</th>\n",
              "      <th>text2</th>\n",
              "      <th>similarity_score</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>savvy searchers fail to spot ads internet sear...</td>\n",
              "      <td>newcastle 2-1 bolton kieron dyer smashed home ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>millions to miss out on the net by 2025  40% o...</td>\n",
              "      <td>nasdaq planning $100m share sale the owner of ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>young debut cut short by ginepri fifteen-year-...</td>\n",
              "      <td>ruddock backs yapp s credentials wales coach m...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>diageo to buy us wine firm diageo  the world s...</td>\n",
              "      <td>mci shares climb on takeover bid shares in us ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>be careful how you code a new european directi...</td>\n",
              "      <td>media gadgets get moving pocket-sized devices ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unique_ID  ... similarity_score\n",
              "0          0  ...                0\n",
              "1          1  ...                0\n",
              "2          2  ...                0\n",
              "3          3  ...                0\n",
              "4          4  ...                0\n",
              "\n",
              "[5 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a7rMASEFyDzb",
        "colab_type": "code",
        "outputId": "553a32c4-ab44-46bf-ec59-557e7b2999f4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "text2[0][0].shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(512,)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 79
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LQnP5iEUuhkf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "values = []\n",
        "for i in range(len(df)):\n",
        "    cosine=get_cosine(text1[i][0], text2[i][0])\n",
        "    values.append(cosine)\n",
        "df[\"similarity_score\"] = values"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GQeTHhljRhxL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "outputId": "949cc015-0387-4235-965b-2ca862dfbc5c"
      },
      "source": [
        "df.head()"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unique_ID</th>\n",
              "      <th>text1</th>\n",
              "      <th>text2</th>\n",
              "      <th>similarity_score</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>savvy searchers fail to spot ads internet sear...</td>\n",
              "      <td>newcastle 2-1 bolton kieron dyer smashed home ...</td>\n",
              "      <td>0.166281</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>millions to miss out on the net by 2025  40% o...</td>\n",
              "      <td>nasdaq planning $100m share sale the owner of ...</td>\n",
              "      <td>0.640406</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>young debut cut short by ginepri fifteen-year-...</td>\n",
              "      <td>ruddock backs yapp s credentials wales coach m...</td>\n",
              "      <td>0.737810</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>diageo to buy us wine firm diageo  the world s...</td>\n",
              "      <td>mci shares climb on takeover bid shares in us ...</td>\n",
              "      <td>0.693244</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>be careful how you code a new european directi...</td>\n",
              "      <td>media gadgets get moving pocket-sized devices ...</td>\n",
              "      <td>0.636458</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unique_ID  ... similarity_score\n",
              "0          0  ...         0.166281\n",
              "1          1  ...         0.640406\n",
              "2          2  ...         0.737810\n",
              "3          3  ...         0.693244\n",
              "4          4  ...         0.636458\n",
              "\n",
              "[5 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PJucIDsoumbc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#saving the pandas data frame to csv\n",
        "df.to_csv('/content/drive/Shared drives/ChiragSaini SharedDrive/Precily Assignment/results.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "riOaUPZLuzNv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Saving the data Frame as a pickle file\n",
        "pickle_out = open(\"/content/drive/Shared drives/ChiragSaini SharedDrive/Precily Assignment/data.pkl\",\"wb\")\n",
        "pickle.dump(df, pickle_out)\n",
        "pickle_out.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hoM56zQ7u941",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}