nlp_learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Notebook for learning NLP tasks with NLTK and other libraries without using LLMs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import sent_tokenize, word_tokenize\n",
    "from nltk.stem import PorterStemmer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['This is a sample string to tokenize.', \"Let's see what tokens it generates\"]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string_test = \"This is a sample string to tokenize. Let's see what tokens it generates\"\n",
    "sent_tokenize(string_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['This',\n",
       " 'is',\n",
       " 'a',\n",
       " 'sample',\n",
       " 'string',\n",
       " 'to',\n",
       " 'tokenize',\n",
       " '.',\n",
       " 'Let',\n",
       " \"'s\",\n",
       " 'see',\n",
       " 'what',\n",
       " 'tokens',\n",
       " 'it',\n",
       " 'generates']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_tokenize(string_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['the',\n",
       " 'crew',\n",
       " 'of',\n",
       " 'the',\n",
       " 'uss',\n",
       " 'discoveri',\n",
       " 'discov',\n",
       " 'mani',\n",
       " 'discoveri',\n",
       " '.',\n",
       " 'discov',\n",
       " 'is',\n",
       " 'what',\n",
       " 'explor',\n",
       " 'do',\n",
       " '.']"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string_for_stemming = \"The crew of the USS Discovery discovered many discoveries. Discovering is what explorers do.\"\n",
    "words = word_tokenize(string_for_stemming)\n",
    "stemmer = PorterStemmer()\n",
    "stemmed_words = [stemmer.stem(word) for word in words]\n",
    "stemmed_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('The', 'DT'),\n",
       " ('crew', 'NN'),\n",
       " ('of', 'IN'),\n",
       " ('the', 'DT'),\n",
       " ('USS', 'NNP'),\n",
       " ('Discovery', 'NNP'),\n",
       " ('discovered', 'VBD'),\n",
       " ('many', 'JJ'),\n",
       " ('discoveries', 'NNS'),\n",
       " ('.', '.'),\n",
       " ('Discovering', 'NNP'),\n",
       " ('is', 'VBZ'),\n",
       " ('what', 'WP'),\n",
       " ('explorers', 'NNS'),\n",
       " ('do', 'VBP'),\n",
       " ('.', '.')]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.pos_tag(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.stem import WordNetLemmatizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['The',\n",
       " 'crew',\n",
       " 'of',\n",
       " 'the',\n",
       " 'USS',\n",
       " 'Discovery',\n",
       " 'discovered',\n",
       " 'many',\n",
       " 'discovery',\n",
       " '.',\n",
       " 'Discovering',\n",
       " 'is',\n",
       " 'what',\n",
       " 'explorer',\n",
       " 'do',\n",
       " '.']"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lemmatizer = WordNetLemmatizer()\n",
    "lemmatized_words = [lemmatizer.lemmatize(word) for word in words]\n",
    "lemmatized_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "lotr_quote = \"It's a dangerous business, Frodo, going out your door.\"\n",
    "words_in_lotr_quote = word_tokenize(lotr_quote)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "grammar = \"NP: {<DT>?<JJ>*<NN>}\"\n",
    "\n",
    "chunk_parser = nltk.RegexpParser(grammar)\n",
    "lotr_pos_tags = nltk.pos_tag(words_in_lotr_quote)\n",
    "tree = chunk_parser.parse(lotr_pos_tags)\n",
    "tree.draw()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}