13_excercise_10.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow import keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_filepaths = [\"datasets/LargeMovieReviewDataset/aclImdb/test/pos/*.txt\",\n",
    "                   \"datasets/LargeMovieReviewDataset/aclImdb/test/neg/*.txt\"]\n",
    "test_pos_full = tf.data.TextLineDataset(tf.data.Dataset.list_files(\"datasets/LargeMovieReviewDataset/aclImdb/test/pos/*.txt\"))\n",
    "test_neg_full = tf.data.TextLineDataset(tf.data.Dataset.list_files(\"datasets/LargeMovieReviewDataset/aclImdb/test/neg/*.txt\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-11-27 12:27:14.086531: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)\n",
      "2022-11-27 12:27:14.104466: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2099940000 Hz\n"
     ]
    }
   ],
   "source": [
    "test_pos_full_list = list(test_pos_full.as_numpy_iterator())\n",
    "test_neg_full_list = list(test_neg_full.as_numpy_iterator())\n",
    "valid_pos, test_pos = test_pos_full_list[:7500], test_pos_full_list[7500:]\n",
    "valid_neg, test_neg = test_neg_full_list[:7500], test_neg_full_list[7500:]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_valid = tf.data.Dataset.from_tensor_slices(valid_pos + valid_neg)\n",
    "X_test = tf.data.Dataset.from_tensor_slices(test_pos + test_neg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_filepaths = [\"datasets/LargeMovieReviewDataset/aclImdb/train/pos/*.txt\",\n",
    "                   \"datasets/LargeMovieReviewDataset/aclImdb/train/neg/*.txt\"]\n",
    "train_pos_shuf = tf.data.Dataset.list_files(\"datasets/LargeMovieReviewDataset/aclImdb/train/pos/*.txt\")\n",
    "train_neg_shuf = tf.data.Dataset.list_files(train_filepaths[1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_pos_len = len(train_pos_shuf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<TextLineDatasetV2 shapes: (), types: tf.string>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_pos = tf.data.TextLineDataset(train_pos_shuf)\n",
    "train_pos\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(b'This movie is a great mocumentary. It follows the rap group, NWH, made up of Ice Cold, Tasty Taste and Tone Def through their unique path to gangster rap highs, lows and back to highs. Through trouble with women, egos, cops and whitey, this group gets to the top of the gangster rap world, as this movie goes to the top of mocumentaries. I know everybodies favorite mocumentary is This is Spinal Tap, for very good reason, however I think that if in the right mood, this movie is simply better. The laughs never end, even for someone not into the rap culture.<br /><br />I\\'m a white guy, that has no interest in rap music, culture or anything else associated with it, however I love this movie. Rusty Cundeif, who wrote the screenplay, songs and starred in it showed great potential and it is a shame that I haven\\'t seen him since Fear of a Black Hat. However, I have seen him one more time than you have, and is that, that I recommend Fear of a Black Hat to you for quick laughs.<br /><br />Remember, \"Don\\'t shoot to you see the whites!....of their eyes? No don\\'t shoot to you see the whites.\"<br /><br />FYM and enjoy the movie.', shape=(), dtype=string)\n"
     ]
    }
   ],
   "source": [
    "for item in train_pos.take(1):\n",
    "    print(item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_neg = tf.data.TextLineDataset(train_neg_shuf)\n",
    "X_train = train_pos.concatenate(train_neg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.layers."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.12 ('ml')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "976019bd7028fad645a45aaa2b51f9e68e9bcad9b4e9047d93bf7b882332665b"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}