examples/notebooks/Getting_Started_with_Text_on_Hub.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "adjusted-visit",
   "metadata": {
    "id": "continental-chosen"
   },
   "source": [
    "## Getting started"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "finished-property",
   "metadata": {
    "id": "according-personality"
   },
   "source": [
    "#### Standard imports and installations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "surrounded-makeup",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "xKDwaemyJF-R",
    "outputId": "3b8db4cf-40a3-4190-d97b-c181a6d858d5"
   },
   "outputs": [],
   "source": [
    "!pip3 install hub pandas numpy tqdm sklearn"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "nuclear-diving",
   "metadata": {},
   "source": [
    "#### To be able to create datasets or download from hub, please create an account by visiting this link:  \n",
    "#### https://app.activeloop.ai/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "running-thesis",
   "metadata": {
    "id": "KB9q8bHps86T"
   },
   "outputs": [],
   "source": [
    "# Use the username & password used to register on hub here to login\n",
    "!hub login -u <username> -p <password>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "defensive-smile",
   "metadata": {
    "id": "baking-authentication"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import hub\n",
    "from hub.schema import Text, ClassLabel\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import TfidfTransformer, CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "industrial-google",
   "metadata": {
    "id": "BApS3hkD9hkY"
   },
   "outputs": [],
   "source": [
    "!wget https://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz --quiet\n",
    "!tar -xf aclImdb_v1.tar.gz"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "funky-continuity",
   "metadata": {
    "id": "9J7u1Ysso-q8"
   },
   "source": [
    "Reading one sample review"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "peaceful-completion",
   "metadata": {
    "id": "pregnant-shoot"
   },
   "outputs": [],
   "source": [
    "filename = \"aclImdb/train/pos/0_9.txt\"\n",
    "with open(filename, \"r\") as fin:\n",
    "    line = fin.readline()\n",
    "fin.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "union-enhancement",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 122
    },
    "id": "offensive-diameter",
    "outputId": "bdf04d56-c0b7-4a99-f6c0-dc502a4a046e"
   },
   "outputs": [],
   "source": [
    "line"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "driving-reynolds",
   "metadata": {
    "id": "automated-welding"
   },
   "source": [
    "#### Collecting all filenames for processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "verbal-spray",
   "metadata": {
    "id": "premier-deficit"
   },
   "outputs": [],
   "source": [
    "file_names = []\n",
    "reviews_df = pd.DataFrame(columns=[\"Review\", \"Label\"])\n",
    "for root, dirs, files in os.walk(\"aclImdb/train/pos\"):\n",
    "    file_names.append(files)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "searching-array",
   "metadata": {
    "id": "civilian-wings"
   },
   "source": [
    "#### Appending all positive reviews to the DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "rotary-homeless",
   "metadata": {
    "id": "special-anxiety"
   },
   "outputs": [],
   "source": [
    "root_dir = \"aclImdb/train/pos/\"\n",
    "count = 0\n",
    "for i in file_names[0]:\n",
    "    with open(root_dir + i, \"r\") as fin:\n",
    "        reviews_df = reviews_df.append(\n",
    "            {\"Review\": fin.readline(), \"Label\": 1}, ignore_index=True\n",
    "        )\n",
    "        count += 1\n",
    "fin.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "invalid-dylan",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 424
    },
    "id": "statistical-negative",
    "outputId": "22bdc236-3142-4428-c88a-1a16d541ee68"
   },
   "outputs": [],
   "source": [
    "reviews_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "assured-uniform",
   "metadata": {
    "id": "running-exemption"
   },
   "source": [
    "#### Appending all negative reviews to the DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "mighty-integral",
   "metadata": {
    "id": "alternative-necessity"
   },
   "outputs": [],
   "source": [
    "file_names = []\n",
    "for root, dirs, files in os.walk(\"aclImdb/train/neg\"):\n",
    "    file_names.append(files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "artistic-powell",
   "metadata": {
    "id": "lovely-minute"
   },
   "outputs": [],
   "source": [
    "root_dir = \"aclImdb/train/neg/\"\n",
    "count = 0\n",
    "for i in file_names[0]:\n",
    "    with open(root_dir + i, \"r\") as fin:\n",
    "        reviews_df = reviews_df.append(\n",
    "            {\"Review\": fin.readline(), \"Label\": 0}, ignore_index=True\n",
    "        )\n",
    "        count += 1\n",
    "fin.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "challenging-qatar",
   "metadata": {
    "id": "complimentary-memphis"
   },
   "outputs": [],
   "source": [
    "max_length = 0\n",
    "for i in reviews_df[\"Review\"]:\n",
    "    if len(i) > max_length:\n",
    "        max_length = len(i)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "varied-subdivision",
   "metadata": {
    "id": "flexible-lexington"
   },
   "source": [
    "### Uploading the DataFrame to Hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "collaborative-determination",
   "metadata": {
    "id": "american-affiliate"
   },
   "outputs": [],
   "source": [
    "# Please run this cell only once. Once you have uploaded the dataset, you can simply fetch it by running\n",
    "# hub.Dataset(url)\n",
    "\n",
    "# Replace url with your username and dataset name. for example, if your name is Akash and your dataset is\n",
    "# FlipkartReviews, then\n",
    "# url = Akash/FlipkartReviews\n",
    "# Before you can upload datasets, please login into Hub. Run the first cell.\n",
    "\n",
    "url = \"<your username>/IMDB-MovieReviews\"\n",
    "\n",
    "# Uncomment the following lines if you\"re uploading *this* dataset for the first time.\n",
    "my_schema = {\"Review\": Text(shape=(None, ), max_shape=(max_length, )),\n",
    "             \"Label\": ClassLabel(num_classes=2)}\n",
    "\n",
    "ds = hub.Dataset(url, shape=(25000,), schema=my_schema)\n",
    "for i in tqdm(range(len(ds))):\n",
    "    ds[\"Review\", i] = reviews_df[\"Review\"][i]\n",
    "    ds[\"Label\", i] = reviews_df[\"Label\"][i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "general-modern",
   "metadata": {
    "id": "3Jarp3GHtcis"
   },
   "outputs": [],
   "source": [
    "# Comment out the following line if you\"re uploading the dataset for the first time.\n",
    "ds = hub.Dataset(url)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "intermediate-lewis",
   "metadata": {
    "id": "wrong-finding"
   },
   "source": [
    "#### Flushing dataset to disk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "spare-mailman",
   "metadata": {
    "id": "large-shanghai"
   },
   "outputs": [],
   "source": [
    "# If you\"ve gone ahead and uploaded your own dataset into Hub, run this command.\n",
    "# This command saves all changes to the cloud. You can also view this dataset at\n",
    "# https://app.activeloop.ai\n",
    "\n",
    "ds.flush()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "earlier-museum",
   "metadata": {
    "id": "DMs9LyWwq6vw"
   },
   "source": [
    "## Fetching data from Hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "forward-spanking",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "floral-sociology",
    "outputId": "91845f74-60d4-4791-f980-f88701a05be5"
   },
   "outputs": [],
   "source": [
    "print(type(ds))\n",
    "print(ds.schema)\n",
    "\n",
    "print(ds[\"Review\", 4].compute())\n",
    "print(ds[\"Label\", 4].compute())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cutting-catch",
   "metadata": {
    "id": "OKx156WSqGMT"
   },
   "source": [
    "## Training a model with our dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bearing-supplier",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "id": "CeopDtlKf4x0",
    "outputId": "942475d1-7385-4128-93a9-bd8edcdc4385"
   },
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "\n",
    "def preprocessor(text):\n",
    "    text = re.sub(\"<[^>]*>\", \"\", text)\n",
    "    emoticons = re.findall(\"(?::|;|=)(?:-)?(?:\\)|\\(|D|P)\", text)\n",
    "    text = re.sub(\"[\\W]+\", \" \", text.lower()) + \" \".join(emoticons).replace(\"-\", \"\")\n",
    "    return text\n",
    "\n",
    "\n",
    "preprocessor(\"This is a :) test :-( !\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "daily-fighter",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "u0I0Y0Aegjk3",
    "outputId": "ff00b66c-22e1-43d0-fa3a-5d0c5ceeeb29"
   },
   "outputs": [],
   "source": [
    "from nltk.stem.porter import PorterStemmer\n",
    "\n",
    "porter = PorterStemmer()\n",
    "\n",
    "\n",
    "def tokenizer(text):\n",
    "    return text.split()\n",
    "\n",
    "\n",
    "tokenizer(\"I find it fun to use Hub\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "voluntary-colors",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ZcVnkyqLgqfz",
    "outputId": "ef664771-9239-4730-beb8-959cc91d3eee"
   },
   "outputs": [],
   "source": [
    "def tokenizer_stemmer(text):\n",
    "    return [porter.stem(word) for word in text.split()]\n",
    "\n",
    "\n",
    "tokenizer_stemmer(\"Hub is extremely easy and efficient to use\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "interpreted-canal",
   "metadata": {
    "id": "KjPjDYRpKhPe"
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "tfidf = TfidfVectorizer(\n",
    "    strip_accents=None,\n",
    "    lowercase=True,\n",
    "    preprocessor=preprocessor,\n",
    "    tokenizer=tokenizer_stemmer,\n",
    "    use_idf=True,\n",
    "    norm=\"l2\",\n",
    "    smooth_idf=True,\n",
    ")\n",
    "X = tfidf.fit_transform(\n",
    "    [item[\"Review\"].compute() for item in ds]\n",
    ")  # Our training dataset\n",
    "y = ds[\"Label\"].compute()  # Training Labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "greek-spanking",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Qdme7UlWiKng",
    "outputId": "83d64fa8-2533-4ead-a46c-2e308ead672f"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, y, random_state=1, test_size=0.5, shuffle=True\n",
    ")\n",
    "clf = LogisticRegressionCV(\n",
    "    cv=5, scoring=\"accuracy\", random_state=0, n_jobs=-1, verbose=3, max_iter=300\n",
    ").fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "express-begin",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "z_7C3lCNnQmc",
    "outputId": "05b6d62c-b61f-43b7-b30e-bba1adf4df54"
   },
   "outputs": [],
   "source": [
    "print(f\"Accuracy: {clf.score(X_test, y_test)}\")"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [
    "whole-confusion"
   ],
   "name": "Getting Started with Text on Hub.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}