pos_tagging.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copy of Sentence_POS_tagging.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "CJR0w3PPdu62",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b1d9acde-cd40-4567-8d25-4f8fd59157d1"
      },
      "source": [
        "from google.colab import drive\n",
        "gdrive_dir = \"/content/gdrive\"\n",
        "drive.mount(gdrive_dir, force_remount=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RBlaV7U0lBic"
      },
      "source": [
        "import os\n",
        "import os.path as osp"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nCLoDK7qlBlh"
      },
      "source": [
        "os.chdir(\"/content/gdrive/My Drive/BT4221-Chi/\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KYhLRwIrlBoe"
      },
      "source": [
        "from googleapiclient.discovery import build\n",
        "from google_auth_oauthlib.flow import InstalledAppFlow\n",
        "from collections import Mapping\n",
        "from itertools import chain\n",
        "from operator import add\n",
        "import pandas as pd\n",
        "import pickle\n",
        "from tqdm import tqdm \n",
        "from tqdm import trange\n",
        "# import demoji\n",
        "# from langdetect import detect\n",
        "import re"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FdgLlDpRlBsW"
      },
      "source": [
        "import en_core_web_sm\n",
        "nlp = en_core_web_sm.load()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kVERfvgWlK-N"
      },
      "source": [
        "import spacy\n",
        "from spacy import displacy"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0nYfQ48AlLBi",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0d5c8717-897e-4c0f-afa7-cc285a963e55"
      },
      "source": [
        "os.listdir()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['.ipynb_checkpoints',\n",
              " 'Homework',\n",
              " 'TextFiles',\n",
              " 'ExtractInfo_ChannelsLoop.ipynb',\n",
              " 'peterrabbit (1).gdoc',\n",
              " 'peterrabbit.gdoc',\n",
              " 'videoTitleSentiments_df.csv',\n",
              " 'videos_df.csv',\n",
              " 'channel_df.csv',\n",
              " '06-POS-Assessment-Solutions.ipynb',\n",
              " 'pos_tags.sav',\n",
              " 'pos_tags_sentence.sav',\n",
              " 'pos_tags_sentence_string.sav',\n",
              " 'df.sav',\n",
              " 'sentence_pos_all.sav',\n",
              " 'Tagging_sequence.csv',\n",
              " 'Untitled0.ipynb',\n",
              " 'sentence_pos.sav',\n",
              " 'common_longest_substring_list.sav',\n",
              " 'Sentence_POS_tagging.ipynb',\n",
              " 'Untitled1.ipynb',\n",
              " 'Copy of Sentence_POS_tagging.ipynb']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o9TA75jAlLGL",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 413
        },
        "outputId": "ff0583a0-048c-4b55-8d5d-1bc3fe628e80"
      },
      "source": [
        "df = pd.read_csv(\"videoTitleSentiments_df.csv\")\n",
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>videoId</th>\n",
              "      <th>sentence</th>\n",
              "      <th>sentiment</th>\n",
              "      <th>polarity</th>\n",
              "      <th>Sentiment_Type</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>NXX338WY_Lw</td>\n",
              "      <td>PREVIEW: Attempting 200mph in the Jaguar XJ220...</td>\n",
              "      <td>0.50</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>Positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>dtHcdU2c71Y</td>\n",
              "      <td>Which car will win Top Gear Speed Week 2020? (...</td>\n",
              "      <td>0.60</td>\n",
              "      <td>0.466667</td>\n",
              "      <td>Positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>vnrtWe-RAzg</td>\n",
              "      <td>Chris Harris on... the Ferrari SF90 Stradale |...</td>\n",
              "      <td>0.50</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>Positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Ra1F0TsOCPs</td>\n",
              "      <td>Chris Harris vs 2020’s Best Performance Cars |...</td>\n",
              "      <td>0.75</td>\n",
              "      <td>0.400000</td>\n",
              "      <td>Positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>fXysipmTxcQ</td>\n",
              "      <td>FASTEST TOP GEAR LAP? Ferrari SF90 Stiglap | T...</td>\n",
              "      <td>0.50</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>Positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>408685</th>\n",
              "      <td>DaoL1-H1ReE</td>\n",
              "      <td>Rainbow crepes with chocolate brownies</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>Neutral</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>408686</th>\n",
              "      <td>ERMJ5xE7-6A</td>\n",
              "      <td>The \"world's best pizzeria'\" is in London</td>\n",
              "      <td>1.00</td>\n",
              "      <td>0.300000</td>\n",
              "      <td>Positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>408687</th>\n",
              "      <td>scdq2xfCE34</td>\n",
              "      <td>Ice cream-filled donut cones in Prague</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>Neutral</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>408688</th>\n",
              "      <td>HgJI5ybtk5Y</td>\n",
              "      <td>Inside London's dim sum chain Ping Pong</td>\n",
              "      <td>0.10</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>Positive</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>408689</th>\n",
              "      <td>zg2yqK-5_U0</td>\n",
              "      <td>Pop-up restaurant only serves avocado food and...</td>\n",
              "      <td>0.00</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>Neutral</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>408690 rows × 5 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "            videoId  ... Sentiment_Type\n",
              "0       NXX338WY_Lw  ...       Positive\n",
              "1       dtHcdU2c71Y  ...       Positive\n",
              "2       vnrtWe-RAzg  ...       Positive\n",
              "3       Ra1F0TsOCPs  ...       Positive\n",
              "4       fXysipmTxcQ  ...       Positive\n",
              "...             ...  ...            ...\n",
              "408685  DaoL1-H1ReE  ...        Neutral\n",
              "408686  ERMJ5xE7-6A  ...       Positive\n",
              "408687  scdq2xfCE34  ...        Neutral\n",
              "408688  HgJI5ybtk5Y  ...       Positive\n",
              "408689  zg2yqK-5_U0  ...        Neutral\n",
              "\n",
              "[408690 rows x 5 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bnxpoiGvgNC3"
      },
      "source": [
        "def pos_process(sentence): \n",
        "  doc = nlp(sentence)\n",
        "  item = []\n",
        "  item_sent = []\n",
        "  for sent in list(doc.sents):\n",
        "    for token in sent: \n",
        "      item.append(token.pos_)\n",
        "  return \"-\".join(item)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "urZtFOQWg27y"
      },
      "source": [
        "df[\"title_structure\"] = df[\"sentence\"].apply(lambda x: pos_process(x))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4nVQgBs_-cIh"
      },
      "source": [
        "df.to_csv(\"parse_tree_df.csv\", index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SUNYKS1vlLIq"
      },
      "source": [
        "# sentence_list = df[\"sentence\"].values.tolist()\n",
        "# sentence_list_meta = \". \".join(df[\"sentence\"].values.tolist())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IawhOsORl5ev"
      },
      "source": [
        "# sentence_pos = []\n",
        "# pos_tags = []\n",
        "# pos_tags_sentence = []\n",
        "# sentence_pos_all = []\n",
        "# for sentence in sentence_list: \n",
        "#   doc = nlp(sentence)\n",
        "#   item = []\n",
        "#   item_sent = []\n",
        "#   for sent in list(doc.sents):\n",
        "#     for token in sent: \n",
        "#       sentence_pos.append({\"text\": token.text, \"pos_tag\": token.pos_, \"tag\": token.tag_, \"tag_explain\": spacy.explain(token.tag_)})\n",
        "#       item.append(token.pos_)\n",
        "#       item_sent.append(sentence_pos)\n",
        "#       pos_tags.append(token.pos_)\n",
        "#   pos_tags_sentence.append(item)\n",
        "#   sentence_pos_all.append(item_sent)\n",
        "#   item = []\n",
        "#   item_sent = []"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8cZ6LkDoub2s"
      },
      "source": [
        "# pos_tags_sentence_string = []\n",
        "# for i in pos_tags_sentence: \n",
        "#   pos_tags_sentence_string.append(\"-\".join(i))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1oVm_-7anTE-"
      },
      "source": [
        "# import pickle\n",
        "# pickle.dump(sentence_pos, open(f\"sentence_pos.sav\", \"wb\"))\n",
        "# pickle.dump(pos_tags, open(f\"pos_tags.sav\", \"wb\"))\n",
        "# pickle.dump(pos_tags_sentence, open(f\"pos_tags_sentence.sav\", \"wb\"))\n",
        "# pickle.dump(pos_tags_sentence_string, open(f\"pos_tags_sentence_string.sav\", \"wb\"))\n",
        "# pickle.dump(sentence_pos_all, open(f\"sentence_pos_all.sav\", \"wb\"))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x9l0RFOzlLLE"
      },
      "source": [
        "# from collections import Counter\n",
        "# count_dict = dict(Counter(pos_tags_sentence_string))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tqw1XzoQVy6U"
      },
      "source": [
        "# df = pd.DataFrame.from_dict(count_dict, orient='index', columns=['count'])\n",
        "# df.sort_values([\"count\"], ascending=False)\n",
        "# df.to_csv(\"Tagging_sequence.csv\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jJ8f4uqDXY1G"
      },
      "source": [
        "# pickle.dump(df, open(f\"df.sav\", \"wb\"))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fwGjvyOPuGGk"
      },
      "source": [
        "df = pd.read_csv(\"Tagging_sequence.csv\")\n",
        "df.columns = [\"sent_structure\", \"count\"]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "twjecz11u4t-"
      },
      "source": [
        "df[\"len\"] = df[\"sent_structure\"].apply(lambda x: len(x.split(\"-\")))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yAEHimLXVC3N",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 413
        },
        "outputId": "2bb1b927-679e-4c75-d1e0-eb65ab6dace9"
      },
      "source": [
        "df.sort_values([\"len\"], ascending=False)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sent_structure</th>\n",
              "      <th>count</th>\n",
              "      <th>len</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>20043</th>\n",
              "      <td>NOUN-NOUN-NOUN-NOUN-NOUN-NOUN-NOUN-NOUN-NOUN-N...</td>\n",
              "      <td>1</td>\n",
              "      <td>57</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21175</th>\n",
              "      <td>ADJ-PROPN-NUM-PUNCT-PUNCT-PUNCT-PUNCT-PUNCT-PU...</td>\n",
              "      <td>1</td>\n",
              "      <td>43</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20632</th>\n",
              "      <td>PROPN-NOUN-NOUN-PROPN-PROPN-PROPN-PUNCT-PUNCT-...</td>\n",
              "      <td>1</td>\n",
              "      <td>42</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20574</th>\n",
              "      <td>NOUN-NOUN-NOUN-NOUN-NOUN-NOUN-NOUN-NOUN-NOUN-N...</td>\n",
              "      <td>1</td>\n",
              "      <td>38</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20557</th>\n",
              "      <td>PROPN-VERB-NOUN-PRON-ADV-VERB-PUNCT-PUNCT-PUNC...</td>\n",
              "      <td>1</td>\n",
              "      <td>36</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>107099</th>\n",
              "      <td>ADJ</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18556</th>\n",
              "      <td>PROPN</td>\n",
              "      <td>112</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>107447</th>\n",
              "      <td>ADP</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>57561</th>\n",
              "      <td>X</td>\n",
              "      <td>21</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11690</th>\n",
              "      <td>NOUN</td>\n",
              "      <td>35</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>310495 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                           sent_structure  count  len\n",
              "20043   NOUN-NOUN-NOUN-NOUN-NOUN-NOUN-NOUN-NOUN-NOUN-N...      1   57\n",
              "21175   ADJ-PROPN-NUM-PUNCT-PUNCT-PUNCT-PUNCT-PUNCT-PU...      1   43\n",
              "20632   PROPN-NOUN-NOUN-PROPN-PROPN-PROPN-PUNCT-PUNCT-...      1   42\n",
              "20574   NOUN-NOUN-NOUN-NOUN-NOUN-NOUN-NOUN-NOUN-NOUN-N...      1   38\n",
              "20557   PROPN-VERB-NOUN-PRON-ADV-VERB-PUNCT-PUNCT-PUNC...      1   36\n",
              "...                                                   ...    ...  ...\n",
              "107099                                                ADJ      2    1\n",
              "18556                                               PROPN    112    1\n",
              "107447                                                ADP      1    1\n",
              "57561                                                   X     21    1\n",
              "11690                                                NOUN     35    1\n",
              "\n",
              "[310495 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DBRbriu7vKM2"
      },
      "source": [
        "# quantile_list = [i/100 for i in range(0,101, 10)]\n",
        "# df[\"len\"].quantile(quantile_list)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SYj94U9UVZL6"
      },
      "source": [
        "# df.groupby(\"len\").count()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hpkG_Rxpvm_E"
      },
      "source": [
        "df = df[(df[\"len\"] > 9) & (df[\"count\"] > 5)]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DZeuN4HWDsvu",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 443
        },
        "outputId": "c2300159-102f-4116-9ffa-eb3827751c98"
      },
      "source": [
        "df = df.sort_values(\"len\", ascending=False).reset_index()\n",
        "df_count = df[[\"count\", \"len\", \"sent_structure\"]].set_index(\"sent_structure\")\n",
        "df_count"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>count</th>\n",
              "      <th>len</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>sent_structure</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>PUNCT-PROPN-PUNCT-NOUN-PROPN-PUNCT-INTJ-PROPN-PUNCT-PUNCT-PROPN-PROPN-PROPN-PUNCT-PROPN-PROPN-PUNCT-NUM-SYM-NUM-PUNCT</th>\n",
              "      <td>6</td>\n",
              "      <td>21</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>PROPN-CCONJ-DET-PROPN-PUNCT-PROPN-PROPN-PROPN-PROPN-PUNCT-PROPN-VERB-ADJ-NOUN-PROPN-X-PROPN-PROPN-PROPN-NOUN-NOUN</th>\n",
              "      <td>6</td>\n",
              "      <td>21</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>VERB-PROPN-AUX-PROPN-NUM-PROPN-PROPN-PUNCT-PROPN-PUNCT-PROPN-PROPN-PROPN-PROPN-CCONJ-PROPN-VERB-ADP-ADJ-NOUN</th>\n",
              "      <td>7</td>\n",
              "      <td>20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>PROPN-PROPN-CCONJ-PROPN-PROPN-ADP-PROPN-PROPN-CCONJ-PROPN-PROPN-PUNCT-PROPN-NUM-NOUN-PUNCT-PROPN-NUM-PUNCT-NUM</th>\n",
              "      <td>9</td>\n",
              "      <td>20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>PROPN-PROPN-PROPN-PROPN-PROPN-PROPN-PROPN-PROPN-PROPN-PROPN-PROPN-PROPN-PROPN-PROPN-PROPN-PROPN-PROPN-PROPN-PROPN-PROPN</th>\n",
              "      <td>8</td>\n",
              "      <td>20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>VERB-PROPN-PROPN-PUNCT-DET-PROPN-PUNCT-PROPN-ADP-PROPN</th>\n",
              "      <td>10</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>PROPN-PROPN-PROPN-PROPN-PROPN-PROPN-PROPN-PART-PROPN-NOUN</th>\n",
              "      <td>7</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>ADJ-PROPN-PROPN-PROPN-PROPN-PROPN-PROPN-PROPN-PROPN-PROPN</th>\n",
              "      <td>17</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>VERB-PROPN-PROPN-PROPN-PUNCT-NOUN-NOUN-VERB-PROPN-PROPN</th>\n",
              "      <td>8</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>PROPN-PROPN-PROPN-PROPN-PROPN-PROPN-ADP-PROPN-PROPN-NUM</th>\n",
              "      <td>38</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1216 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    count  len\n",
              "sent_structure                                                \n",
              "PUNCT-PROPN-PUNCT-NOUN-PROPN-PUNCT-INTJ-PROPN-P...      6   21\n",
              "PROPN-CCONJ-DET-PROPN-PUNCT-PROPN-PROPN-PROPN-P...      6   21\n",
              "VERB-PROPN-AUX-PROPN-NUM-PROPN-PROPN-PUNCT-PROP...      7   20\n",
              "PROPN-PROPN-CCONJ-PROPN-PROPN-ADP-PROPN-PROPN-C...      9   20\n",
              "PROPN-PROPN-PROPN-PROPN-PROPN-PROPN-PROPN-PROPN...      8   20\n",
              "...                                                   ...  ...\n",
              "VERB-PROPN-PROPN-PUNCT-DET-PROPN-PUNCT-PROPN-AD...     10   10\n",
              "PROPN-PROPN-PROPN-PROPN-PROPN-PROPN-PROPN-PART-...      7   10\n",
              "ADJ-PROPN-PROPN-PROPN-PROPN-PROPN-PROPN-PROPN-P...     17   10\n",
              "VERB-PROPN-PROPN-PROPN-PUNCT-NOUN-NOUN-VERB-PRO...      8   10\n",
              "PROPN-PROPN-PROPN-PROPN-PROPN-PROPN-ADP-PROPN-P...     38   10\n",
              "\n",
              "[1216 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 73
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KWAucMG1t8h3"
      },
      "source": [
        "# sentence_pos = pickle.load(open(f\"sentence_pos.sav\", \"rb\"))\n",
        "pos_tags = pickle.load(open(f\"pos_tags.sav\", \"rb\"))\n",
        "pos_tags_sentence = pickle.load(open(f\"pos_tags_sentence.sav\", \"rb\"))\n",
        "pos_tags_sentence_string = pickle.load(open(f\"pos_tags_sentence_string.sav\", \"rb\"))\n",
        "sentence_pos_all = pickle.load(open(f\"sentence_pos_all.sav\", \"rb\"))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yBFKQzncASSM"
      },
      "source": [
        "pos_tags_sentence_rev = sorted(pos_tags_sentence, key=len)[::-1]\n",
        "# pos_tags_sentence_rev"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MqEm1cRCBmKB"
      },
      "source": [
        "# pos_tags_sentence_rev[0]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HsDmJKv99axF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b48ef2ee-5282-45d3-e814-c6214d6d7571"
      },
      "source": [
        "from difflib import SequenceMatcher\n",
        "sm = SequenceMatcher(a=pos1, b=pos2)\n",
        "for diff in sm.get_opcodes():\n",
        "    op, f1_from, f1_to, f2_from, f2_to = diff\n",
        "    if op == 'equal':\n",
        "        print(\"{} {}\".format(f1_to-f1_from, list(pos1[f1_from:f1_to])))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2 ['PUNCT', 'PUNCT']\n",
            "4 ['PUNCT', 'PROPN', 'PROPN', 'PUNCT']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZzkLxO_7uVYf"
      },
      "source": [
        "common_longest_substring_list = []\n",
        "pos_tag_list = df_count.index.tolist()\n",
        "for i in range(len(pos_tag_list)-1): \n",
        "  for j in range(i+1, len(pos_tag_list)): \n",
        "    pos1 = pos_tag_list[i].split(\"-\")\n",
        "    pos2 = pos_tag_list[j].split(\"-\")\n",
        "    sm = SequenceMatcher(a=pos1, b=pos2)\n",
        "    res = list()\n",
        "    for diff in sm.get_opcodes():\n",
        "      op, f1_from, f1_to, f2_from, f2_to = diff\n",
        "      if op == 'equal':\n",
        "        res.append(list(pos1[f1_from:f1_to]))\n",
        "    if len(res) > 0: \n",
        "      res = sorted(res, key=len)[::-1]\n",
        "      common_longest_substring_list.append({\n",
        "          \"sequence_1\": pos_tag_list[i], \n",
        "          \"sequence_2\": pos_tag_list[j], \n",
        "          \"common_sequence\": \"-\".join(res[0]), \n",
        "          \"count\": min(df_count.loc[pos_tag_list[i]][\"count\"], df_count.loc[pos_tag_list[j]][\"count\"])})\n",
        "      pickle.dump(common_longest_substring_list, open(f\"common_longest_substring_list.sav\", \"wb\"))"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}