16_spark_caching_techiniques.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "323bf5b8-f3f4-4877-a002-42667addfbc0",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "            <div>\n",
       "                <p><b>SparkSession - in-memory</b></p>\n",
       "                \n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://4c1cbc3b031c:4040\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v3.3.0</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local[*]</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>Understand Caching</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        \n",
       "            </div>\n",
       "        "
      ],
      "text/plain": [
       "<pyspark.sql.session.SparkSession at 0x7fa19823d250>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Spark Session\n",
    "from pyspark.sql import SparkSession\n",
    "\n",
    "spark = (\n",
    "    SparkSession\n",
    "    .builder\n",
    "    .appName(\"Understand Caching\")\n",
    "    .master(\"local[*]\")\n",
    "    .config(\"spark.executor.memory\", \"512M\")\n",
    "    .getOrCreate()\n",
    ")\n",
    "\n",
    "spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4dad8280-2561-4b8e-a1ad-22d68c1fb971",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Read Sales CSV Data - 752MB Size ~ 7.2M Records\n",
    "\n",
    "_schema = \"transacted_at string, trx_id string, retailer_id string, description string, amount double, city_id string\"\n",
    "\n",
    "df = spark.read.format(\"csv\").schema(_schema).option(\"header\", True).load(\"data/input/new_sales.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "affa9a57-b5d0-4103-aba6-c3b7dd681d0a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+----------+-----------+--------------------+-------+----------+\n",
      "|       transacted_at|    trx_id|retailer_id|         description| amount|   city_id|\n",
      "+--------------------+----------+-----------+--------------------+-------+----------+\n",
      "|2017-11-24T19:00:...|1734117022|  847200066|Wal-Mart  ppd id:...|1737.26|1646415505|\n",
      "|2017-11-24T19:00:...|1734117030| 1953761884|Home Depot     pp...|  384.5| 287177635|\n",
      "|2017-11-24T19:00:...|1734117153|  847200066|unkn        Kings...|2907.57|1483931123|\n",
      "|2017-11-24T19:00:...|1734117241|  486576507|              iTunes|2912.67|1663872965|\n",
      "|2017-11-24T19:00:...|2076947146|  511877722|unkn     ccd id: ...|1915.35|1698762556|\n",
      "|2017-11-24T19:00:...|2076947113| 1996661856|AutoZone  arc id:...| 1523.6|1759612211|\n",
      "|2017-11-24T19:00:...|2076946994| 1898522855|Target    ppd id:...|2589.93|2074005445|\n",
      "|2017-11-24T19:00:...|2076946121|  562903918|unkn    ccd id: 5...| 315.86|1773943669|\n",
      "|2017-11-24T19:00:...|2076946063| 1070485878|Amazon.com   arc ...| 785.27|1126623009|\n",
      "|2017-11-24T19:00:...|2076944979| 1654681099|Delhaize America ...|  303.1|1243655802|\n",
      "|2017-11-24T19:00:...|2076944941| 1157343460|unkn    ppd id: 1...|2853.33|1141716004|\n",
      "|2017-11-24T19:00:...|2076944228| 1522061472|         YUM! Brands|1737.45| 592064091|\n",
      "|2017-11-24T19:00:...|2076944195| 1070485878|unkn   ppd id: 11...|2440.55|1525790470|\n",
      "|2017-11-24T19:00:...|2076944142|  847200066|Wal-Mart  ppd id:...| 331.63|1345953582|\n",
      "|2017-11-24T19:00:...|2076944073| 2077350195|Walgreen     arc ...|  396.9|2001708947|\n",
      "|2017-11-24T19:00:...|2076943052|  103953879|Rite Aid  arc id:...| 1910.8|1998549640|\n",
      "|2017-11-24T19:00:...|2076942340|  643354906|                BJ's|  372.7| 115209716|\n",
      "|2017-11-24T19:00:...|2076942282| 1445595477|Meijer    ccd id:...|  366.9|1717498102|\n",
      "|2017-11-24T19:00:...|2076942274| 2001148981|unkn    ppd id: 2...| 333.41| 559832710|\n",
      "|2017-11-24T19:00:...|2076942246|    9225731|AT&T Wireless  pp...|  396.9| 407629665|\n",
      "+--------------------+----------+-----------+--------------------+-------+----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.where(\"amount > 300\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6a243560-b1c9-470c-a55e-43b89d2c8fc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cache DataFrame (cache or persist)\n",
    "\n",
    "df_cache = df.where(\"amount > 100\").cache()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "1b07f256-5026-4b0b-b3df-2d9fd8cc462d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2549058"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_cache.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8425d67b-9cf6-482f-9b9f-3b92262dda7f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+----------+-----------+--------------------+-------+----------+\n",
      "|       transacted_at|    trx_id|retailer_id|         description| amount|   city_id|\n",
      "+--------------------+----------+-----------+--------------------+-------+----------+\n",
      "|2017-11-24T19:00:...|1995601912| 2077350195|Walgreen       11-25| 197.23| 216510442|\n",
      "|2017-11-24T19:00:...|1734117022|  847200066|Wal-Mart  ppd id:...|1737.26|1646415505|\n",
      "|2017-11-24T19:00:...|1734117030| 1953761884|Home Depot     pp...|  384.5| 287177635|\n",
      "|2017-11-24T19:00:...|1734117089| 1898522855| Target        11-25|  66.33|1855530529|\n",
      "|2017-11-24T19:00:...|1734117117|  997626433|Sears  ppd id: 85...| 298.87| 957346984|\n",
      "|2017-11-24T19:00:...|1734117153|  847200066|unkn        Kings...|2907.57|1483931123|\n",
      "|2017-11-24T19:00:...|1734117212| 1996661856|unkn    ppd id: 4...| 140.38| 336763936|\n",
      "|2017-11-24T19:00:...|1734117241|  486576507|              iTunes|2912.67|1663872965|\n",
      "|2017-11-24T19:00:...|2076947148|  847200066|Wal-Mart         ...|  62.83|1556600840|\n",
      "|2017-11-24T19:00:...|2076947146|  511877722|unkn     ccd id: ...|1915.35|1698762556|\n",
      "|2017-11-24T19:00:...|2076947113| 1996661856|AutoZone  arc id:...| 1523.6|1759612211|\n",
      "|2017-11-24T19:00:...|2076946994| 1898522855|Target    ppd id:...|2589.93|2074005445|\n",
      "|2017-11-24T19:00:...|2076946899|  644879053|unkn     ccd id: ...|  91.91|2068475652|\n",
      "|2017-11-24T19:00:...|2076946121|  562903918|unkn    ccd id: 5...| 315.86|1773943669|\n",
      "|2017-11-24T19:00:...|2076946063| 1070485878|Amazon.com   arc ...| 785.27|1126623009|\n",
      "|2017-11-24T19:00:...|2076945932|   87529419|Albertsons  arc i...| 284.16|1813967666|\n",
      "|2017-11-24T19:00:...|2076945195|  562903918|unkn       Harris...| 165.75| 216135201|\n",
      "|2017-11-24T19:00:...|2076945156|  562903918|McDonald's  ccd i...|  119.4| 576697624|\n",
      "|2017-11-24T19:00:...|2076945132|   83040202|GameStop    arc i...| 242.75| 870108334|\n",
      "|2017-11-24T19:00:...|2076945098| 2139149619|Trader Joe's     ...| 114.17|   7441192|\n",
      "+--------------------+----------+-----------+--------------------+-------+----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.where(\"amount > 50\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "82b44adb-9c11-4029-b86d-04b6f395c967",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# MEMORY_ONLY, MEMORY_AND_DISK, MEMORY_ONLY_SER, MEMORY_AND_DISK_SER, DISK_ONLY, MEMORY_ONLY_2, MEMORY_AND_DISK_2\n",
    "import pyspark\n",
    "\n",
    "df_persist = df.persist(pyspark.StorageLevel.MEMORY_ONLY_2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e5009af8-74c1-450d-ae7f-36e8ae64ef1a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_persist.write.format(\"noop\").mode(\"overwrite\").save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "a704050a-93ef-464f-b790-5b82aba08a39",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove Cache\n",
    "\n",
    "spark.catalog.clearCache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2025bb5d-f522-490c-bf4f-8058cd35d5de",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}