examples/nlp/intro.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "ce734de3",
   "metadata": {},
   "source": [
    "### NLP\n",
    "\n",
    "Previous example: [/examples/keras_applications/transfer_learning.ipynb](https://github.com/serhatsoyer/py4ML/blob/main/examples/keras_applications/transfer_learning.ipynb)  \n",
    "Modified from: [NLP section of Tensorflow Udemy Course from Jose Portilla - Pierian Training](https://www.udemy.com/course/complete-tensorflow-2-and-keras-deep-learning-bootcamp/)  \n",
    "Next example: [/examples/autoencoders/intro.ipynb](https://github.com/serhatsoyer/py4ML/blob/main/examples/autoencoders/intro.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "75a12823",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.insert(0, '../../') # To be able to reach 'datasets' folder\n",
    "from pathlib import Path\n",
    "import re\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Embedding, GRU, Dense, Dropout\n",
    "from keras.losses import sparse_categorical_crossentropy\n",
    "from keras.callbacks import EarlyStopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "07bddb75",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[PosixPath('/Users/serhatsoyer/Repos/py4ML/datasets/dostoyevski/The Brothers Karamazov.txt'), PosixPath('/Users/serhatsoyer/Repos/py4ML/datasets/dostoyevski/The Idiot.txt'), PosixPath('/Users/serhatsoyer/Repos/py4ML/datasets/dostoyevski/The Possessed.txt'), PosixPath('/Users/serhatsoyer/Repos/py4ML/datasets/dostoyevski/Poor Folk.txt'), PosixPath('/Users/serhatsoyer/Repos/py4ML/datasets/dostoyevski/Crime and Punishment.txt'), PosixPath('/Users/serhatsoyer/Repos/py4ML/datasets/dostoyevski/Notes from the Underground.txt'), PosixPath('/Users/serhatsoyer/Repos/py4ML/datasets/dostoyevski/White Nights and Other Stories.txt'), PosixPath('/Users/serhatsoyer/Repos/py4ML/datasets/dostoyevski/Short Stories.txt'), PosixPath('/Users/serhatsoyer/Repos/py4ML/datasets/dostoyevski/The House of the Dead.txt'), PosixPath('/Users/serhatsoyer/Repos/py4ML/datasets/dostoyevski/The Gambler.txt')]\n"
     ]
    }
   ],
   "source": [
    "dataset_path = Path.cwd().parent.parent / 'datasets' / 'dostoyevski'\n",
    "book_names = []\n",
    "[book_names.append(item) for item in dataset_path.iterdir() if item.suffix == '.txt']\n",
    "print(book_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1b984cc4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(text) = 8474359\n",
      "text[:95] = 'The Brothers Karamazov\\n\\nPART I\\n\\n\\n\\n\\nBook I. The History Of A Family\\n\\n\\n\\n\\nChapter I.\\nFyodor Pavlov'\n"
     ]
    }
   ],
   "source": [
    "text = []\n",
    "for book_name in book_names:\n",
    "    file = open(dataset_path / book_name, 'r')\n",
    "    text.append(file.read())\n",
    "    file.close()\n",
    "\n",
    "del dataset_path, book_name, book_names, file\n",
    "text = ' '.join(text)\n",
    "def print_text(text): print(f'{len(text) = }\\n{text[:95] = }')\n",
    "print_text(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2e96c7f1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(chars) = 108\n",
      "''.join(chars) = '\\n !\"\\'()*,-.0123456789:;=?ABCDEFGHIJKLMNOPQRSTUVWXYZ[]_abcdefghijklmnopqrstuvwxyzÀÆÈÉÏàâäæçèéêëîïôöùüŒœ‐—‘’“”'\n",
      "len(chars) = 76\n",
      "''.join(chars) = '\\n !\"\\'()*,-.0123456789:;=?[]_abcdefghijklmnopqrstuvwxyzàâäæçèéêëîïôöùüœ‐—‘’“”'\n",
      "len(chars) = 62\n",
      "''.join(chars) = ' \"\\'()*,-.0=[]_abcdefghijklmnopqrstuvwxyzàâäæçèéêëîïôöùüœ‐—‘’“”'\n",
      "len(chars) = 40\n",
      "''.join(chars) = ' \"\\'()*,-.0=[]_abcdefghijklmnopqrstuvwxyz'\n",
      "len(chars) = 31\n",
      "''.join(chars) = ' *,.0abcdefghijklmnopqrstuvwxyz'\n",
      "len(text) = 8414469\n",
      "text[:95] = 'the brothers karamazov  part i     book i. the history of a family     chapter i. fyodor pavlov'\n",
      "len(text) = 8362974\n",
      "text[:95] = 'the brothers karamazov part i book i. the history of a family chapter i. fyodor pavlovitch kara'\n"
     ]
    }
   ],
   "source": [
    "def get_chars(): chars = sorted(set(text)); print(f\"{len(chars) = }\\n{''.join(chars) = }\"); return chars\n",
    "chars = get_chars()\n",
    "text = text.lower()\n",
    "chars = get_chars()\n",
    "for temp in '123456789': text = text.replace(temp, '0')\n",
    "for temp in '?!;:': text = text.replace(temp, '.')\n",
    "text = text.replace('\\n', ' ')\n",
    "chars = get_chars()\n",
    "text = text.encode('ascii', errors='ignore').decode('utf-8', errors='ignore')\n",
    "chars = get_chars()\n",
    "for temp in chars: text = text if temp in 'abcdefghijklmnopqrstuvwxyz0.,* ' else text.replace(temp, '*')\n",
    "chars = get_chars()\n",
    "char_to_idx = {char: idx for idx, char in enumerate(chars)}\n",
    "idx_to_char = np.array(chars)\n",
    "print_text(text)\n",
    "for temp in [' ', '0', '\\.', '\\,', '\\*']: text = re.sub(f'{temp}+', f'{temp if len(temp) == 1 else temp[-1]}', text)\n",
    "print_text(text)\n",
    "del temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f6d88d92",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "encoded.shape = (8362974,), encoded[:5] = array([24, 12,  9,  0,  6])\n",
      "Metal device set to: Apple M2\n",
      "\n",
      "systemMemory: 8.00 GB\n",
      "maxCacheSize: 2.67 GB\n",
      "\n",
      "Complete dataset length: len(dataset) = 64829\n",
      "Train dataset length: len(train) = 60777\n",
      "Test dataset length: len(test) = 4052\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-12-26 12:46:30.640239: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:306] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support.\n",
      "2022-12-26 12:46:30.640672: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:272] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 0 MB memory) -> physical PluggableDevice (device: 0, name: METAL, pci bus id: <undefined>)\n",
      "2022-12-26 12:46:30.705190: W tensorflow/core/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[19 19  8  6 29] [ 9 26  9 18  0] \n",
      " oodby. and alyosha ran downstairs and into the street. chapter ii. smerdyakov with a guitar he had no time to lose indeed. even \n",
      "[19  8  6 29  3] [26  9 18  0 27] \n",
      " odby. and alyosha ran downstairs and into the street. chapter ii. smerdyakov with a guitar he had no time to lose indeed. even w\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<BatchDataset element_spec=(TensorSpec(shape=(256, 128), dtype=tf.int64, name=None), TensorSpec(shape=(256, 128), dtype=tf.int64, name=None))>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seq_len = 128\n",
    "batch_size = 256\n",
    "buffer_size = 15000 # Not a critical value\n",
    "encoded = np.array([char_to_idx[char] for char in text])\n",
    "print(f'{encoded.shape = }, {encoded[:5] = }')\n",
    "dataset = tf.data.Dataset.from_tensor_slices(encoded)\n",
    "seqs = dataset.batch(seq_len + 1, drop_remainder=True)\n",
    "def get_in_and_out(seq): return seq[:-1], seq[1:]\n",
    "dataset = seqs.map(get_in_and_out)\n",
    "print(f'Complete dataset length: {len(dataset) = }')\n",
    "cutoff = round(len(dataset) / 16)\n",
    "test = dataset.take(cutoff) \n",
    "train = dataset.skip(cutoff)\n",
    "print(f'Train dataset length: {len(train) = }')\n",
    "print(f'Test dataset length: {len(test) = }')\n",
    "for input, target in train.take(1):\n",
    "    print(input.numpy()[:5], input.numpy()[-5:], '\\n', ''.join(idx_to_char[input.numpy()]))\n",
    "    print(target.numpy()[:5], target.numpy()[-5:], '\\n', ''.join(idx_to_char[target.numpy()]))\n",
    "\n",
    "train = train.shuffle(buffer_size).batch(batch_size, drop_remainder=True)\n",
    "test = test.shuffle(buffer_size).batch(batch_size, drop_remainder=True)\n",
    "del dataset, buffer_size, cutoff, encoded, input, target, seqs, text\n",
    "train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "26441eaa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " embedding (Embedding)       (256, None, 64)           1984      \n",
      "                                                                 \n",
      " gru (GRU)                   (256, None, 256)          247296    \n",
      "                                                                 \n",
      " gru_1 (GRU)                 (256, None, 256)          394752    \n",
      "                                                                 \n",
      " dense (Dense)               (256, None, 256)          65792     \n",
      "                                                                 \n",
      " dropout (Dropout)           (256, None, 256)          0         \n",
      "                                                                 \n",
      " dense_1 (Dense)             (256, None, 31)           7967      \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 717,791\n",
      "Trainable params: 717,791\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "embed_size = 64\n",
    "def sparse_cat_loss(y_true, y_pred): return sparse_categorical_crossentropy(y_true, y_pred, from_logits=True)\n",
    "def create_model(batch_size=batch_size):\n",
    "    model = Sequential()\n",
    "    model.add(Embedding(len(chars), embed_size, batch_input_shape=[batch_size, None]))\n",
    "    model.add(GRU(256, return_sequences=True, stateful=True, recurrent_initializer='glorot_uniform'))\n",
    "    model.add(GRU(256, return_sequences=True, stateful=True, recurrent_initializer='glorot_uniform'))\n",
    "    model.add(Dense(256, activation='relu'))\n",
    "    model.add(Dropout(.25))\n",
    "    model.add(Dense(len(chars)))\n",
    "    model.compile(optimizer='adam', loss=sparse_cat_loss) \n",
    "    return model\n",
    "\n",
    "\n",
    "model = create_model()\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cb29763f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input.shape = TensorShape([256, 128])\n",
      "target.shape = TensorShape([256, 128])\n",
      "pred_0.shape = TensorShape([256, 128, 31])\n",
      "pred_1.shape = TensorShape([128, 1])\n",
      "pred.shape = (128,)\n",
      "[27  0 11 16  5], [ 0 27  5 23  0]\n",
      "w glad she will be, how delighted. he muttered, but lapsed into silence again. and indeed it was not to please grushenka he was \n",
      "[ 0 11 16  5  8], [27  5 23  0 24]\n",
      " glad she will be, how delighted. he muttered, but lapsed into silence again. and indeed it was not to please grushenka he was t\n",
      "[17 15 27 16  8], [20 30  4 26 13]\n",
      "mkwldbrbhvgsiyrm0xa sh0xiltojry*km0q zdjpfujyf*kmbkkvrz nvaobzkwmmsab y0qcwfnsbmi*n,vts0sxpmtr ad,emsytn,cxowmt,xaf.pnazxlgpz0vi\n"
     ]
    }
   ],
   "source": [
    "for input, target in train.take(1):\n",
    "    pred_0 = model(input)\n",
    "    pred_1 = tf.random.categorical(pred_0[0], num_samples=1)\n",
    "    pred = tf.squeeze(pred_1, axis=-1).numpy()\n",
    "    print(f'{input.shape = }\\n{target.shape = }\\n{pred_0.shape = }\\n{pred_1.shape = }\\n{pred.shape = }')\n",
    "    print(f\"{input[0].numpy()[:5]}, {input[0].numpy()[-5:]}\\n{''.join(idx_to_char[input[0].numpy()])}\")\n",
    "    print(f\"{target[0].numpy()[:5]}, {target[0].numpy()[-5:]}\\n{''.join(idx_to_char[target[0].numpy()])}\")\n",
    "    print(f\"{pred[:5]}, {pred[-5:]}\\n{''.join(idx_to_char[pred])}\")\n",
    "\n",
    "del input, target, pred_0, pred_1, pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "78c8a757",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/128\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-12-26 12:46:34.080325: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n",
      "2022-12-26 12:46:35.328979: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n",
      "2022-12-26 12:46:35.569703: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n",
      "2022-12-26 12:46:35.929443: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n",
      "2022-12-26 12:46:36.287898: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "237/237 [==============================] - ETA: 0s - loss: 2.7759"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-12-26 12:47:26.906242: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n",
      "2022-12-26 12:47:27.236956: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n",
      "2022-12-26 12:47:27.493480: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "237/237 [==============================] - 56s 221ms/step - loss: 2.7759 - val_loss: 2.2480\n",
      "Epoch 2/128\n",
      "237/237 [==============================] - 52s 215ms/step - loss: 2.1050 - val_loss: 1.8854\n",
      "Epoch 3/128\n",
      "237/237 [==============================] - 52s 214ms/step - loss: 1.8607 - val_loss: 1.7369\n",
      "Epoch 4/128\n",
      "237/237 [==============================] - 53s 220ms/step - loss: 1.7305 - val_loss: 1.6461\n",
      "Epoch 5/128\n",
      "237/237 [==============================] - 53s 219ms/step - loss: 1.6612 - val_loss: 1.5833\n",
      "Epoch 6/128\n",
      "237/237 [==============================] - 53s 219ms/step - loss: 1.6075 - val_loss: 1.5450\n",
      "Epoch 7/128\n",
      "237/237 [==============================] - 53s 220ms/step - loss: 1.5797 - val_loss: 1.5317\n",
      "Epoch 8/128\n",
      "237/237 [==============================] - 53s 219ms/step - loss: 1.5616 - val_loss: 1.4954\n",
      "Epoch 9/128\n",
      "237/237 [==============================] - 53s 220ms/step - loss: 1.5345 - val_loss: 1.4835\n",
      "Epoch 10/128\n",
      "237/237 [==============================] - 54s 221ms/step - loss: 1.5197 - val_loss: 1.4705\n",
      "Epoch 11/128\n",
      "237/237 [==============================] - 53s 220ms/step - loss: 1.5106 - val_loss: 1.4766\n",
      "Epoch 12/128\n",
      "237/237 [==============================] - 53s 219ms/step - loss: 1.5025 - val_loss: 1.4628\n",
      "Epoch 13/128\n",
      "237/237 [==============================] - 53s 219ms/step - loss: 1.5050 - val_loss: 1.4758\n",
      "Epoch 14/128\n",
      "237/237 [==============================] - 53s 219ms/step - loss: 1.4986 - val_loss: 1.4524\n",
      "Epoch 15/128\n",
      "237/237 [==============================] - 53s 218ms/step - loss: 1.4894 - val_loss: 1.4392\n",
      "Epoch 16/128\n",
      "237/237 [==============================] - 53s 219ms/step - loss: 1.4925 - val_loss: 1.4522\n",
      "Epoch 17/128\n",
      "237/237 [==============================] - 53s 218ms/step - loss: 1.4913 - val_loss: 1.4436\n",
      "Epoch 18/128\n",
      "237/237 [==============================] - 53s 218ms/step - loss: 1.4892 - val_loss: 1.4455\n",
      "Epoch 19/128\n",
      "237/237 [==============================] - ETA: 0s - loss: 1.4914Restoring model weights from the end of the best epoch: 15.\n",
      "237/237 [==============================] - 53s 218ms/step - loss: 1.4914 - val_loss: 1.4496\n",
      "Epoch 19: early stopping\n"
     ]
    }
   ],
   "source": [
    "early_stop = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=4, restore_best_weights=True)\n",
    "model.fit(train, validation_data=test, callbacks=[early_stop], epochs=128)\n",
    "del early_stop\n",
    "model_name = 'dost1.h5'\n",
    "model.save(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "17a196b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_model = create_model(1)\n",
    "test_model.load_weights(model_name)\n",
    "test_model.build(tf.TensorShape([1, None]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "243e1286",
   "metadata": {},
   "outputs": [],
   "source": [
    "def dostoyevski_writes(model, seed):\n",
    "    input = tf.expand_dims([char_to_idx[char] for char in seed], 0)\n",
    "    output = []\n",
    "    model.reset_states()\n",
    "    for _ in range(512):\n",
    "        preds = model(input)\n",
    "        preds = tf.squeeze(preds, 0)\n",
    "        preds = preds / 0.95 # Higher prob. val means lesss surprising\n",
    "        pred_idx = tf.random.categorical(preds, num_samples=1)[-1, 0].numpy()\n",
    "        input = tf.expand_dims([pred_idx], 0)\n",
    "        output.append(idx_to_char[pred_idx])\n",
    "    \n",
    "    print(seed + ''.join(output))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2798f216",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "poormanionaless for the shamperic child thas known on the wurnst to le*t one around his way a call. he ben* nothing mart all. they all ect is work and you as, the less confition. she here.* he elets and another. i did not wronge how surpally that ou*like at such that has a last. we she dare her the room. the might there would more roublesical shar. riched dehigg means and alexanov very days, ponky with a ran at the suddenly unders her crieds delicour* brohor feet, but one will grow more would say of factioned f\n"
     ]
    }
   ],
   "source": [
    "dostoyevski_writes(test_model, 'poor')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "4d28169d",
   "metadata": {},
   "source": [
    "Previous example: [/examples/keras_applications/transfer_learning.ipynb](https://github.com/serhatsoyer/py4ML/blob/main/examples/keras_applications/transfer_learning.ipynb)  \n",
    "Modified from: [NLP section of Tensorflow Udemy Course from Jose Portilla - Pierian Training](https://www.udemy.com/course/complete-tensorflow-2-and-keras-deep-learning-bootcamp/)  \n",
    "Next example: [/examples/autoencoders/intro.ipynb](https://github.com/serhatsoyer/py4ML/blob/main/examples/autoencoders/intro.ipynb)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.6 ('study')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8 (main, Nov 24 2022, 08:08:27) [Clang 14.0.6 ]"
  },
  "vscode": {
   "interpreter": {
    "hash": "29d06b973d1ddb34db3279b24f9b5152402e688db937648b736e855ec4de60c0"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}