Gab_Hate.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Gab Hate",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZxMtn3WzW1Jc"
      },
      "source": [
        "Mount Google Drive so we can use our files."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wojnKiKbQpio",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "905035ea-58ea-47b1-e80b-d2b521ff6104"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Pn0kh50RPDjp"
      },
      "source": [
        "First, we simply read in our corpus."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "tl-0QB3pAb3u",
        "outputId": "f9f6d207-fa7a-4f77-bc87-4d1915ce6f9e"
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "tsk = pd.read_csv('/content/drive/My Drive/gab data/GabHateCorpus_annotations.tsv', sep='\\t')\n",
        "\n",
        "# HD = assaults on human dignity\n",
        "# CV = calls to violence\n",
        "# VO = vulgar or offensive language\n",
        "\n",
        "tsk"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ID</th>\n",
              "      <th>Annotator</th>\n",
              "      <th>Text</th>\n",
              "      <th>Hate</th>\n",
              "      <th>HD</th>\n",
              "      <th>CV</th>\n",
              "      <th>VO</th>\n",
              "      <th>REL</th>\n",
              "      <th>RAE</th>\n",
              "      <th>SXO</th>\n",
              "      <th>GEN</th>\n",
              "      <th>IDL</th>\n",
              "      <th>NAT</th>\n",
              "      <th>POL</th>\n",
              "      <th>MPH</th>\n",
              "      <th>EX</th>\n",
              "      <th>IM</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>27044</td>\n",
              "      <td>4</td>\n",
              "      <td>Ah the PSYOPS antifa crew is back. That’s how ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>27044</td>\n",
              "      <td>15</td>\n",
              "      <td>Ah the PSYOPS antifa crew is back. That’s how ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>27044</td>\n",
              "      <td>10</td>\n",
              "      <td>Ah the PSYOPS antifa crew is back. That’s how ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>27044</td>\n",
              "      <td>8</td>\n",
              "      <td>Ah the PSYOPS antifa crew is back. That’s how ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>27045</td>\n",
              "      <td>4</td>\n",
              "      <td>Get the new Android app update released today ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>86524</th>\n",
              "      <td>9188</td>\n",
              "      <td>6</td>\n",
              "      <td>He thinks only peons own guns, you're supposed...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>86525</th>\n",
              "      <td>9188</td>\n",
              "      <td>3</td>\n",
              "      <td>He thinks only peons own guns, you're supposed...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>86526</th>\n",
              "      <td>9222</td>\n",
              "      <td>11</td>\n",
              "      <td>America must stop funding the United Nations.</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>86527</th>\n",
              "      <td>9222</td>\n",
              "      <td>3</td>\n",
              "      <td>America must stop funding the United Nations.</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>86528</th>\n",
              "      <td>9222</td>\n",
              "      <td>6</td>\n",
              "      <td>America must stop funding the United Nations.</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>86529 rows × 17 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "          ID  Annotator  ...   EX   IM\n",
              "0      27044          4  ...  NaN  NaN\n",
              "1      27044         15  ...  NaN  NaN\n",
              "2      27044         10  ...  NaN  NaN\n",
              "3      27044          8  ...  NaN  NaN\n",
              "4      27045          4  ...  NaN  NaN\n",
              "...      ...        ...  ...  ...  ...\n",
              "86524   9188          6  ...  1.0  0.0\n",
              "86525   9188          3  ...  NaN  NaN\n",
              "86526   9222         11  ...  NaN  NaN\n",
              "86527   9222          3  ...  NaN  NaN\n",
              "86528   9222          6  ...  NaN  NaN\n",
              "\n",
              "[86529 rows x 17 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DZUqoy2EPLQp"
      },
      "source": [
        "Now we'll take 80% of the data and make it our \"training set\", and the other 20% will be our \"test set\"."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bSHJahYVKWO_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f7a705aa-d8a5-4206-ffd5-5c2cd1ff61f3"
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "x_train, x_test, y_train, y_test = train_test_split(tsk['Text'], tsk['Hate'], test_size=0.20, random_state=240)\n",
        "\n",
        "y_test.values"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 0, 0, ..., 0, 0, 0])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yzFxg5NSHQLp",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 481
        },
        "outputId": "4ff8f316-80da-4a2d-c528-d34ea6761b85"
      },
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from keras.layers.core import Dense, Dropout\n",
        "from nltk.corpus import stopwords\n",
        "import nltk\n",
        "from tensorflow.python.keras import models, layers\n",
        "\n",
        "nltk.download(\"stopwords\")\n",
        "\n",
        "features = 5000\n",
        "\n",
        "vectorizer = TfidfVectorizer(stop_words=stopwords.words('english'), token_pattern=r'(?u)\\b[a-zA-Z][a-zA-Z]+\\b', max_features=features)\n",
        "\n",
        "vectorizer.fit(x_train.values)\n",
        "tfidf = vectorizer.transform(x_train.values)\n",
        "\n",
        "tfidf = pd.DataFrame(tfidf.toarray(), columns=vectorizer.get_feature_names())\n",
        "tfidf\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>abandoned</th>\n",
              "      <th>abc</th>\n",
              "      <th>abiding</th>\n",
              "      <th>ability</th>\n",
              "      <th>able</th>\n",
              "      <th>aborted</th>\n",
              "      <th>abortion</th>\n",
              "      <th>abortions</th>\n",
              "      <th>absolute</th>\n",
              "      <th>absolutely</th>\n",
              "      <th>abuse</th>\n",
              "      <th>abused</th>\n",
              "      <th>accept</th>\n",
              "      <th>acceptable</th>\n",
              "      <th>accepted</th>\n",
              "      <th>access</th>\n",
              "      <th>accident</th>\n",
              "      <th>accomplish</th>\n",
              "      <th>accomplished</th>\n",
              "      <th>according</th>\n",
              "      <th>account</th>\n",
              "      <th>accountable</th>\n",
              "      <th>accounts</th>\n",
              "      <th>acct</th>\n",
              "      <th>accurate</th>\n",
              "      <th>accusations</th>\n",
              "      <th>accuse</th>\n",
              "      <th>accused</th>\n",
              "      <th>accuses</th>\n",
              "      <th>acosta</th>\n",
              "      <th>across</th>\n",
              "      <th>act</th>\n",
              "      <th>acting</th>\n",
              "      <th>action</th>\n",
              "      <th>actions</th>\n",
              "      <th>active</th>\n",
              "      <th>actively</th>\n",
              "      <th>activist</th>\n",
              "      <th>activists</th>\n",
              "      <th>activity</th>\n",
              "      <th>...</th>\n",
              "      <th>wtf</th>\n",
              "      <th>wwii</th>\n",
              "      <th>www</th>\n",
              "      <th>xd</th>\n",
              "      <th>xi</th>\n",
              "      <th>ya</th>\n",
              "      <th>yahoo</th>\n",
              "      <th>yard</th>\n",
              "      <th>ye</th>\n",
              "      <th>yea</th>\n",
              "      <th>yeah</th>\n",
              "      <th>year</th>\n",
              "      <th>years</th>\n",
              "      <th>yellow</th>\n",
              "      <th>yep</th>\n",
              "      <th>yes</th>\n",
              "      <th>yesterday</th>\n",
              "      <th>yet</th>\n",
              "      <th>yo</th>\n",
              "      <th>york</th>\n",
              "      <th>young</th>\n",
              "      <th>younger</th>\n",
              "      <th>youre</th>\n",
              "      <th>youth</th>\n",
              "      <th>youtu</th>\n",
              "      <th>youtube</th>\n",
              "      <th>yr</th>\n",
              "      <th>yrs</th>\n",
              "      <th>yup</th>\n",
              "      <th>zealand</th>\n",
              "      <th>zero</th>\n",
              "      <th>zerohedge</th>\n",
              "      <th>zimbabwe</th>\n",
              "      <th>zionist</th>\n",
              "      <th>zionists</th>\n",
              "      <th>zog</th>\n",
              "      <th>zombie</th>\n",
              "      <th>zone</th>\n",
              "      <th>zones</th>\n",
              "      <th>zuckerberg</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.125925</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.293743</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69218</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69219</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69220</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69221</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.201636</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69222</th>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>69223 rows × 5000 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       abandoned  abc  abiding  ability  ...  zombie  zone  zones  zuckerberg\n",
              "0            0.0  0.0      0.0      0.0  ...     0.0   0.0    0.0         0.0\n",
              "1            0.0  0.0      0.0      0.0  ...     0.0   0.0    0.0         0.0\n",
              "2            0.0  0.0      0.0      0.0  ...     0.0   0.0    0.0         0.0\n",
              "3            0.0  0.0      0.0      0.0  ...     0.0   0.0    0.0         0.0\n",
              "4            0.0  0.0      0.0      0.0  ...     0.0   0.0    0.0         0.0\n",
              "...          ...  ...      ...      ...  ...     ...   ...    ...         ...\n",
              "69218        0.0  0.0      0.0      0.0  ...     0.0   0.0    0.0         0.0\n",
              "69219        0.0  0.0      0.0      0.0  ...     0.0   0.0    0.0         0.0\n",
              "69220        0.0  0.0      0.0      0.0  ...     0.0   0.0    0.0         0.0\n",
              "69221        0.0  0.0      0.0      0.0  ...     0.0   0.0    0.0         0.0\n",
              "69222        0.0  0.0      0.0      0.0  ...     0.0   0.0    0.0         0.0\n",
              "\n",
              "[69223 rows x 5000 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p04iNjIJfYdd",
        "outputId": "22a36057-3c38-4d68-f019-b4f84b1721d8"
      },
      "source": [
        "model = models.Sequential()\n",
        "\n",
        "model.add(layers.Dense(1024, activation='relu', input_shape=(features,)))\n",
        "model.add(layers.Dense(512, activation='relu'))\n",
        "model.add(layers.Dense(256, activation='relu'))\n",
        "model.add(layers.Dense(1, activation='sigmoid'))\n",
        "\n",
        "model.compile(optimizer=\"adam\", loss='binary_crossentropy', metrics=['accuracy'])\n",
        "\n",
        "model.fit(tfidf.values, y_train.values, epochs=2, batch_size=350)\n",
        "\n",
        "results = model.evaluate(vectorizer.transform(x_test.values).toarray(), y_test.values)\n",
        "\n",
        "results\n",
        "\n",
        "# for new data, vectorize the data the same way\n",
        "# model.predict()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/2\n",
            "198/198 [==============================] - 34s 168ms/step - loss: 0.3554 - accuracy: 0.8663\n",
            "Epoch 2/2\n",
            "198/198 [==============================] - 34s 170ms/step - loss: 0.2957 - accuracy: 0.8787\n",
            "541/541 [==============================] - 7s 13ms/step - loss: 0.3365 - accuracy: 0.8735\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.3365194499492645, 0.8734542727470398]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xpxMgTlf_m_P",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 965
        },
        "outputId": "917420d4-c22e-4086-cee0-4f0a1f28b5b4"
      },
      "source": [
        "# convert the data into something useful (currently json)\n",
        "import json\n",
        "import pandas as pd\n",
        "\n",
        "ohbaby = pd.read_json('/content/drive/My Drive/gab data/maga_after_10_01_2020', lines=True)\n",
        "\n",
        "ohbaby.drop_duplicates(subset='content')\n",
        "\n",
        "ohbaby"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>created_at</th>\n",
              "      <th>revised_at</th>\n",
              "      <th>in_reply_to_id</th>\n",
              "      <th>in_reply_to_account_id</th>\n",
              "      <th>sensitive</th>\n",
              "      <th>spoiler_text</th>\n",
              "      <th>visibility</th>\n",
              "      <th>language</th>\n",
              "      <th>uri</th>\n",
              "      <th>url</th>\n",
              "      <th>replies_count</th>\n",
              "      <th>reblogs_count</th>\n",
              "      <th>pinnable</th>\n",
              "      <th>pinnable_by_group</th>\n",
              "      <th>favourites_count</th>\n",
              "      <th>quote_of_id</th>\n",
              "      <th>expires_at</th>\n",
              "      <th>has_quote</th>\n",
              "      <th>bookmark_collection_id</th>\n",
              "      <th>favourited</th>\n",
              "      <th>reblogged</th>\n",
              "      <th>content</th>\n",
              "      <th>rich_content</th>\n",
              "      <th>plain_markdown</th>\n",
              "      <th>reblog</th>\n",
              "      <th>quote</th>\n",
              "      <th>account</th>\n",
              "      <th>group</th>\n",
              "      <th>media_attachments</th>\n",
              "      <th>mentions</th>\n",
              "      <th>tags</th>\n",
              "      <th>emojis</th>\n",
              "      <th>card</th>\n",
              "      <th>poll</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>106027103000843760</td>\n",
              "      <td>2021-04-08 01:26:39.598000+00:00</td>\n",
              "      <td>NaT</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>False</td>\n",
              "      <td></td>\n",
              "      <td>public</td>\n",
              "      <td>en</td>\n",
              "      <td>/QueueMeInToo/posts/106027103000843767</td>\n",
              "      <td>https://gab.com/QueueMeInToo/posts/10602710300...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaT</td>\n",
              "      <td>False</td>\n",
              "      <td>NaN</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>Daniel Hale&lt;br /&gt;&lt;br /&gt;&lt;a href=\"https://stateo...</td>\n",
              "      <td></td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>{'id': '3039669', 'username': 'QueueMeInToo', ...</td>\n",
              "      <td>None</td>\n",
              "      <td>[{'id': '70980926', 'type': 'image', 'url': 'h...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[{'name': 'wwg1wga', 'url': '/tags/wwg1wga'}, ...</td>\n",
              "      <td>[]</td>\n",
              "      <td>{'id': 10884380, 'url': 'https://stateofthenat...</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>106027089360096544</td>\n",
              "      <td>2021-04-08 01:23:11.464000+00:00</td>\n",
              "      <td>NaT</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>False</td>\n",
              "      <td></td>\n",
              "      <td>public</td>\n",
              "      <td>en</td>\n",
              "      <td>/FaithSpiritual1/posts/106027089360096537</td>\n",
              "      <td>https://gab.com/FaithSpiritual1/posts/10602708...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaT</td>\n",
              "      <td>False</td>\n",
              "      <td>NaN</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>Been cleaning out my store building the last f...</td>\n",
              "      <td></td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>{'id': '2145853', 'username': 'FaithSpiritual1...</td>\n",
              "      <td>None</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[{'name': 'resistjoebiden', 'url': '/tags/resi...</td>\n",
              "      <td>[]</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>106027086542845600</td>\n",
              "      <td>2021-04-08 01:22:28.486000+00:00</td>\n",
              "      <td>NaT</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>False</td>\n",
              "      <td></td>\n",
              "      <td>public</td>\n",
              "      <td>en</td>\n",
              "      <td>/myerikd/posts/106027086542845606</td>\n",
              "      <td>https://gab.com/myerikd/posts/106027086542845606</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>False</td>\n",
              "      <td>True</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaT</td>\n",
              "      <td>False</td>\n",
              "      <td>NaN</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>Asa Hutchinson’s Trans-Bill Veto Draws Scrutin...</td>\n",
              "      <td></td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>{'id': '185312', 'username': 'myerikd', 'acct'...</td>\n",
              "      <td>{'id': '692', 'title': 'News', 'description': ...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[{'id': '1614939', 'username': 'Aliatra', 'url...</td>\n",
              "      <td>[{'name': 'stopthesteal', 'url': '/tags/stopth...</td>\n",
              "      <td>[]</td>\n",
              "      <td>{'id': 10881349, 'url': 'https://www.breitbart...</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>106027085727875488</td>\n",
              "      <td>2021-04-08 01:22:16.035000+00:00</td>\n",
              "      <td>NaT</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>False</td>\n",
              "      <td></td>\n",
              "      <td>public</td>\n",
              "      <td>en</td>\n",
              "      <td>/VIEWZUU/posts/106027085727875480</td>\n",
              "      <td>https://gab.com/VIEWZUU/posts/106027085727875480</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>False</td>\n",
              "      <td>True</td>\n",
              "      <td>4</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaT</td>\n",
              "      <td>False</td>\n",
              "      <td>NaN</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>They tried and failed, Joe.&lt;br /&gt;You and your ...</td>\n",
              "      <td></td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>{'id': '851590', 'username': 'VIEWZUU', 'acct'...</td>\n",
              "      <td>{'id': '22', 'title': '/g/The_Donald', 'descri...</td>\n",
              "      <td>[{'id': '70980691', 'type': 'image', 'url': 'h...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[{'name': 'joebidenisnotmypresident', 'url': '...</td>\n",
              "      <td>[]</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>106027081437902496</td>\n",
              "      <td>2021-04-08 01:21:10.575000+00:00</td>\n",
              "      <td>NaT</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>False</td>\n",
              "      <td></td>\n",
              "      <td>public</td>\n",
              "      <td>None</td>\n",
              "      <td>/QueueMeInToo/posts/106027081437902503</td>\n",
              "      <td>https://gab.com/QueueMeInToo/posts/10602708143...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaT</td>\n",
              "      <td>False</td>\n",
              "      <td>NaN</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>&lt;a href=\"/tags/MAGA\" class=\"mention hashtag\" r...</td>\n",
              "      <td></td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>{'id': '3039669', 'username': 'QueueMeInToo', ...</td>\n",
              "      <td>None</td>\n",
              "      <td>[{'id': '70980628', 'type': 'image', 'url': 'h...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[{'name': 'wwg1wga', 'url': '/tags/wwg1wga'}, ...</td>\n",
              "      <td>[]</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>60595</th>\n",
              "      <td>105638959438612416</td>\n",
              "      <td>2021-01-29 12:16:38.847000+00:00</td>\n",
              "      <td>NaT</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>False</td>\n",
              "      <td></td>\n",
              "      <td>public</td>\n",
              "      <td>en</td>\n",
              "      <td>/OpenQuotes/posts/105638959438612411</td>\n",
              "      <td>https://gab.com/OpenQuotes/posts/1056389594386...</td>\n",
              "      <td>7</td>\n",
              "      <td>4</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>7</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaT</td>\n",
              "      <td>False</td>\n",
              "      <td>NaN</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>What's the definition of Racism again?&lt;br /&gt;&lt;b...</td>\n",
              "      <td></td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>{'id': '98146', 'username': 'OpenQuotes', 'acc...</td>\n",
              "      <td>None</td>\n",
              "      <td>[{'id': '63670901', 'type': 'image', 'url': 'h...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[{'name': 'teamtrump', 'url': '/tags/teamtrump...</td>\n",
              "      <td>[]</td>\n",
              "      <td>{'id': 8359995, 'url': 'https://reddit.com/r/w...</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>60596</th>\n",
              "      <td>105638954869564768</td>\n",
              "      <td>2021-01-29 12:15:29.128000+00:00</td>\n",
              "      <td>NaT</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>False</td>\n",
              "      <td></td>\n",
              "      <td>public</td>\n",
              "      <td>de</td>\n",
              "      <td>/lucrum-sanguinem/posts/105638954869564760</td>\n",
              "      <td>https://gab.com/lucrum-sanguinem/posts/1056389...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaT</td>\n",
              "      <td>False</td>\n",
              "      <td>NaN</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>🎺✨🎺 &lt;a href=\"/tags/USA\" class=\"mention hashtag...</td>\n",
              "      <td></td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>{'id': '110116', 'username': 'lucrum-sanguinem...</td>\n",
              "      <td>None</td>\n",
              "      <td>[{'id': '63670789', 'type': 'image', 'url': 'h...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[{'name': 'trusttheplan', 'url': '/tags/trustt...</td>\n",
              "      <td>[]</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>60597</th>\n",
              "      <td>105638954702671504</td>\n",
              "      <td>2021-01-29 12:15:26.954000+00:00</td>\n",
              "      <td>NaT</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>False</td>\n",
              "      <td></td>\n",
              "      <td>public</td>\n",
              "      <td>en</td>\n",
              "      <td>/follownoone/posts/105638954702671503</td>\n",
              "      <td>https://gab.com/follownoone/posts/105638954702...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaT</td>\n",
              "      <td>False</td>\n",
              "      <td>NaN</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>🇺🇸 🇺🇸 RISE AND SHINE PATRIOTS 🇺🇸 🇺🇸&lt;br /&gt;&lt;br /...</td>\n",
              "      <td></td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>{'id': '865126', 'username': 'follownoone', 'a...</td>\n",
              "      <td>None</td>\n",
              "      <td>[{'id': '63658014', 'type': 'image', 'url': 'h...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[{'name': 'goarmy', 'url': '/tags/goarmy'}, {'...</td>\n",
              "      <td>[]</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>60598</th>\n",
              "      <td>105638946236500304</td>\n",
              "      <td>2021-01-29 12:13:17.394000+00:00</td>\n",
              "      <td>NaT</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>False</td>\n",
              "      <td></td>\n",
              "      <td>public</td>\n",
              "      <td>en</td>\n",
              "      <td>/lucrum-sanguinem/posts/105638946236500297</td>\n",
              "      <td>https://gab.com/lucrum-sanguinem/posts/1056389...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaT</td>\n",
              "      <td>False</td>\n",
              "      <td>NaN</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>🎺✨🎺 &lt;a href=\"/tags/USA\" class=\"mention hashtag...</td>\n",
              "      <td></td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>{'id': '110116', 'username': 'lucrum-sanguinem...</td>\n",
              "      <td>None</td>\n",
              "      <td>[{'id': '63670524', 'type': 'image', 'url': 'h...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[{'name': 'trusttheplan', 'url': '/tags/trustt...</td>\n",
              "      <td>[]</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>60599</th>\n",
              "      <td>105638935630829520</td>\n",
              "      <td>2021-01-29 12:10:35.579000+00:00</td>\n",
              "      <td>NaT</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>False</td>\n",
              "      <td></td>\n",
              "      <td>public</td>\n",
              "      <td>en</td>\n",
              "      <td>/lucrum-sanguinem/posts/105638935630829523</td>\n",
              "      <td>https://gab.com/lucrum-sanguinem/posts/1056389...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaT</td>\n",
              "      <td>False</td>\n",
              "      <td>NaN</td>\n",
              "      <td>False</td>\n",
              "      <td>False</td>\n",
              "      <td>Today, Friday, 29 January 2021&lt;br /&gt;&lt;br /&gt;⚡🙏✝⚡...</td>\n",
              "      <td></td>\n",
              "      <td>None</td>\n",
              "      <td>NaN</td>\n",
              "      <td>None</td>\n",
              "      <td>{'id': '110116', 'username': 'lucrum-sanguinem...</td>\n",
              "      <td>None</td>\n",
              "      <td>[]</td>\n",
              "      <td>[]</td>\n",
              "      <td>[{'name': 'trusttheplan', 'url': '/tags/trustt...</td>\n",
              "      <td>[]</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>60600 rows × 35 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                       id  ...  poll\n",
              "0      106027103000843760  ...  None\n",
              "1      106027089360096544  ...  None\n",
              "2      106027086542845600  ...  None\n",
              "3      106027085727875488  ...  None\n",
              "4      106027081437902496  ...  None\n",
              "...                   ...  ...   ...\n",
              "60595  105638959438612416  ...  None\n",
              "60596  105638954869564768  ...  None\n",
              "60597  105638954702671504  ...  None\n",
              "60598  105638946236500304  ...  None\n",
              "60599  105638935630829520  ...  None\n",
              "\n",
              "[60600 rows x 35 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DDrsIZBHaAXA",
        "outputId": "bddd4154-dcb3-46a4-f4de-4a46b0035c36"
      },
      "source": [
        "# make the text all pretty and nice\n",
        "import re\n",
        "!pip install emoji\n",
        "import emoji\n",
        "\n",
        "def give_emoji_free_text(text):\n",
        "    allchars = [str for str in text]\n",
        "    emoji_list = [c for c in allchars if c in emoji.UNICODE_EMOJI]\n",
        "    clean_text = ' '.join([str for str in text.split() if not any(i in str for i in emoji_list)])\n",
        "    return clean_text\n",
        "\n",
        "def cleanhtml(raw_html):\n",
        "  cleanr = re.compile('<.*?>')\n",
        "  cleantext = re.sub(cleanr, '', raw_html)\n",
        "  cleantext = give_emoji_free_text(cleantext)\n",
        "  return cleantext\n",
        "\n",
        "\n",
        "clean = [cleanhtml(x) for x in ohbaby['content'].values]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting emoji\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/24/fa/b3368f41b95a286f8d300e323449ab4e86b85334c2e0b477e94422b8ed0f/emoji-1.2.0-py3-none-any.whl (131kB)\n",
            "\r\u001b[K     |██▌                             | 10kB 13.2MB/s eta 0:00:01\r\u001b[K     |█████                           | 20kB 8.1MB/s eta 0:00:01\r\u001b[K     |███████▌                        | 30kB 4.8MB/s eta 0:00:01\r\u001b[K     |██████████                      | 40kB 4.7MB/s eta 0:00:01\r\u001b[K     |████████████▌                   | 51kB 2.9MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 61kB 3.3MB/s eta 0:00:01\r\u001b[K     |█████████████████▌              | 71kB 3.5MB/s eta 0:00:01\r\u001b[K     |████████████████████            | 81kB 3.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████▌         | 92kB 3.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 102kB 3.7MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▌    | 112kB 3.7MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 122kB 3.7MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 133kB 3.7MB/s \n",
            "\u001b[?25hInstalling collected packages: emoji\n",
            "Successfully installed emoji-1.2.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MxKdiVnrZcPQ",
        "outputId": "28d7f0f4-b84f-4045-8673-f3407372ba4a"
      },
      "source": [
        "ridf = vectorizer.transform(clean)\n",
        "\n",
        "ridf = pd.DataFrame(ridf.toarray(), columns=vectorizer.get_feature_names())\n",
        "\n",
        "res = model.predict(ridf.values)\n",
        "\n",
        "res"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.01144329],\n",
              "       [0.02641803],\n",
              "       [0.08204904],\n",
              "       ...,\n",
              "       [0.00671065],\n",
              "       [0.28727356],\n",
              "       [0.14241445]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5FdGRCXQeSZx"
      },
      "source": [
        "# res now contains our scores.\n",
        "#NOTE: ohbaby['created_at'] contains the times of each post, ohbaby['content'] has the text and res contains the score.\n",
        "\n",
        "# import numpy as np \n",
        "# import matplotlib.pyplot as plt\n",
        "# from matplotlib.pyplot import figure\n",
        "\n",
        "# figure(figsize=(8, 6), dpi=600)\n",
        "\n",
        "# plt.title(\"Line graph\") \n",
        "# plt.xlabel(\"Date\") \n",
        "# plt.ylabel(\"Likelihood of it being hate\") \n",
        "# plt.plot(ohbaby['created_at'], res, color =\"red\") \n",
        "# plt.show()\n",
        "\n",
        "# import numpy\n",
        "\n",
        "# a = np.array([ohbaby['content'].tolist(), ohbaby['created_at'].tolist(), res.tolist()])\n",
        "# numpy.savetxt(\"output.log\", a, delimiter='\\t', header=\"Text,Date,Score\", comments=\"\", fmt=\"%s\")\n",
        "\n",
        "\n",
        "df = pd.DataFrame({\"Text\" : ohbaby['content'], \"Date\" : ohbaby['created_at'], \"Score\" : res.flatten()})\n",
        "df.to_csv(\"submission2.csv\", index=False)\n",
        "\n",
        "# pd.DataFrame(a).to_csv('new_out.tsv', sep='\\t')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 264
        },
        "id": "wBeMQvQnGo72",
        "outputId": "fd6b3a7d-f226-405c-be99-07bf8753b2f8"
      },
      "source": [
        "print(\"Max:\", res.max())\n",
        "print(\"Min:\", res.min())\n",
        "print(\"Mean:\", res.mean())\n",
        "print(\"Median:\", res.median())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Max: 0.6038654\n",
            "Min: 4.0496573e-07\n",
            "Mean: 0.090283535\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-11-43f1855fee3c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Min:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mres\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Mean:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mres\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Median:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mres\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmedian\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m: 'numpy.ndarray' object has no attribute 'median'"
          ]
        }
      ]
    }
  ]
}