movie_recommender_using_spark.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/4k5h1t/PySpark-Movie-Rec/blob/main/movie_recommender_using_spark.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8ucva9vUydNX"
      },
      "source": [
        "# Movie Recommender Systems using Spark with PySpark"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gY94IHMjyUpl"
      },
      "source": [
        "## Installing required dependencies\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M3KM7RGWsJXf",
        "outputId": "474fc97c-891b-4bd3-ce9d-ba09960ff431"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting findspark\n",
            "  Downloading findspark-2.0.1-py2.py3-none-any.whl (4.4 kB)\n",
            "Installing collected packages: findspark\n",
            "Successfully installed findspark-2.0.1\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting pyspark\n",
            "  Downloading pyspark-3.3.1.tar.gz (281.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 281.4 MB 46 kB/s \n",
            "\u001b[?25hCollecting py4j==0.10.9.5\n",
            "  Downloading py4j-0.10.9.5-py2.py3-none-any.whl (199 kB)\n",
            "\u001b[K     |████████████████████████████████| 199 kB 42.1 MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: pyspark\n",
            "  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyspark: filename=pyspark-3.3.1-py2.py3-none-any.whl size=281845512 sha256=118e6f351de951b865718db98298fb1ca47c635995ed1a29d7c1bf15d21e8c0b\n",
            "  Stored in directory: /root/.cache/pip/wheels/43/dc/11/ec201cd671da62fa9c5cc77078235e40722170ceba231d7598\n",
            "Successfully built pyspark\n",
            "Installing collected packages: py4j, pyspark\n",
            "Successfully installed py4j-0.10.9.5 pyspark-3.3.1\n"
          ]
        }
      ],
      "source": [
        "!pip install findspark\n",
        "!pip install pyspark"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N4rxFHsxyZw3"
      },
      "source": [
        "## Downloading the Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Wq1sJ8NzHCzu",
        "outputId": "de05fa93-6ed0-41cd-d3b8-4ba60c481442"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "The following package was automatically installed and is no longer required:\n",
            "  libnvidia-common-460\n",
            "Use 'apt autoremove' to remove it.\n",
            "The following additional packages will be installed:\n",
            "  libc-ares2\n",
            "The following NEW packages will be installed:\n",
            "  aria2 libc-ares2\n",
            "0 upgraded, 2 newly installed, 0 to remove and 7 not upgraded.\n",
            "Need to get 1,274 kB of archives.\n",
            "After this operation, 4,912 kB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libc-ares2 amd64 1.14.0-1ubuntu0.1 [37.5 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu bionic/universe amd64 aria2 amd64 1.33.1-1 [1,236 kB]\n",
            "Fetched 1,274 kB in 1s (1,661 kB/s)\n",
            "Selecting previously unselected package libc-ares2:amd64.\n",
            "(Reading database ... 124015 files and directories currently installed.)\n",
            "Preparing to unpack .../libc-ares2_1.14.0-1ubuntu0.1_amd64.deb ...\n",
            "Unpacking libc-ares2:amd64 (1.14.0-1ubuntu0.1) ...\n",
            "Selecting previously unselected package aria2.\n",
            "Preparing to unpack .../aria2_1.33.1-1_amd64.deb ...\n",
            "Unpacking aria2 (1.33.1-1) ...\n",
            "Setting up libc-ares2:amd64 (1.14.0-1ubuntu0.1) ...\n",
            "Setting up aria2 (1.33.1-1) ...\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
            "Processing triggers for libc-bin (2.27-3ubuntu1.6) ...\n",
            "\n",
            "12/07 04:53:39 [\u001b[1;32mNOTICE\u001b[0m] Downloading 1 item(s)\n",
            "\u001b[0m\n",
            "12/07 04:53:42 [\u001b[1;32mNOTICE\u001b[0m] Download complete: ./MovieLens//ml-25m.zip\n",
            "\n",
            "Download Results:\n",
            "gid   |stat|avg speed  |path/URI\n",
            "======+====+===========+=======================================================\n",
            "1bbff1|\u001b[1;32mOK\u001b[0m  |   123MiB/s|./MovieLens//ml-25m.zip\n",
            "\n",
            "Status Legend:\n",
            "(OK):download completed.\n"
          ]
        }
      ],
      "source": [
        "!apt-get install -y aria2\n",
        "!mkdir -p ./MovieLens/ \n",
        "!aria2c -s 16 -x 16 \"https://files.grouplens.org/datasets/movielens/ml-25m.zip\" -d ./MovieLens/"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Downloading SPARK and Java Dependencies"
      ],
      "metadata": {
        "id": "2xOXHHZ5HyLt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "\n",
        "!wget -q https://archive.apache.org/dist/spark/spark-3.2.1/spark-3.2.1-bin-hadoop3.2.tgz\n",
        "\n",
        "!tar xf spark-3.2.1-bin-hadoop3.2.tgz\n"
      ],
      "metadata": {
        "id": "TCejETh6Hx1-"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Extracting the Dataset"
      ],
      "metadata": {
        "id": "GklRjCoMHaHl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import zipfile\n",
        "with zipfile.ZipFile(\"/content/MovieLens/ml-25m.zip\", 'r') as zip_ref:\n",
        "    zip_ref.extractall(\"/content/MovieLens\")"
      ],
      "metadata": {
        "id": "rog8iPuaHZqM"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Start Setup Time"
      ],
      "metadata": {
        "id": "1uuGVTkBK-te"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "t5QmoffBehKM"
      },
      "outputs": [],
      "source": [
        "import time\n",
        "setupst = time.time()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-YrF3LKlygix"
      },
      "source": [
        "## Setting up SPARK and Java Paths"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "GgnLle5s7wyN"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "os.environ[\"SPARK_HOME\"] = \"/content/spark-3.2.1-bin-hadoop3.2\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IwcvAkwHyugv"
      },
      "source": [
        "## Importing PySpark"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "1plW0XOuydNh"
      },
      "outputs": [],
      "source": [
        "import findspark\n",
        "findspark.init('/content/spark-3.2.1-bin-hadoop3.2')\n",
        "import pyspark"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hhAjfq1SywLC"
      },
      "source": [
        "## Starting up Spark Session, Clusters etc. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "8pggfi0RydNi"
      },
      "outputs": [],
      "source": [
        "from pyspark.sql import SparkSession\n",
        "spark = SparkSession.builder.appName('recommendation').getOrCreate()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Jjivd2mUzKcU"
      },
      "source": [
        "## Importing ALS model "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "bkB5Mf6bydNj"
      },
      "outputs": [],
      "source": [
        "from pyspark.ml.recommendation import ALS"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fTVhSWL1z0Mr"
      },
      "source": [
        "## Reading loaded Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "AD24KfKMydNj"
      },
      "outputs": [],
      "source": [
        "data = spark.read.csv('MovieLens/ml-25m/ratings.csv',inferSchema=True,header=True)\n",
        "setupet = time.time()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WMcxfwChz8kj"
      },
      "source": [
        "## Describing / Showcasing Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "R_Y0ruMsydNk",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "17a50e52-6d6e-43f1-a826-b802ac6d6ee4"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Row(userId=1, movieId=296, rating=5.0, timestamp=1147880044)"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ],
      "source": [
        "data.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "zxV7rJh-ydNl",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6d39503c-e88e-4e3e-8a7c-5681a98d5268"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- userId: integer (nullable = true)\n",
            " |-- movieId: integer (nullable = true)\n",
            " |-- rating: double (nullable = true)\n",
            " |-- timestamp: integer (nullable = true)\n",
            "\n"
          ]
        }
      ],
      "source": [
        "data.printSchema()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "3DpeYGItydNn",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "93b6b7cb-142d-4adb-8517-54ea12fdfd7b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+-----------------+------------------+------------------+--------------------+\n",
            "|summary|           userId|           movieId|            rating|           timestamp|\n",
            "+-------+-----------------+------------------+------------------+--------------------+\n",
            "|  count|         25000095|          25000095|          25000095|            25000095|\n",
            "|   mean|81189.28115381162|21387.981943268616| 3.533854451353085|1.2156014431215513E9|\n",
            "| stddev|46791.71589745776| 39198.86210105973|1.0607439611423535| 2.268758080595386E8|\n",
            "|    min|                1|                 1|               0.5|           789652009|\n",
            "|    max|           162541|            209171|               5.0|          1574327703|\n",
            "+-------+-----------------+------------------+------------------+--------------------+\n",
            "\n"
          ]
        }
      ],
      "source": [
        "data.describe().show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lSmZWQ0D0Dca"
      },
      "source": [
        "## Implementing ML Algorithm and Evaluation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VVaTNWGG0Kt-"
      },
      "source": [
        "### Train Test Split"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "9z9oh3b_ydNo"
      },
      "outputs": [],
      "source": [
        "(train_data, test_data) = data.randomSplit([0.7, 0.3], seed=42)\n",
        "setupTime = setupet - setupst"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C7avO7cj0Ncj"
      },
      "source": [
        "Setting up and Training the ALS Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "wxHWI-o4ydNp"
      },
      "outputs": [],
      "source": [
        "trainst = time.time()\n",
        "als = ALS(maxIter=5, regParam=0.01, userCol=\"userId\", itemCol=\"movieId\", ratingCol=\"rating\")\n",
        "model = als.fit(train_data)\n",
        "trainet = time.time()\n",
        "trainTime = trainet - trainst"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ti0VUWjw0Wa5"
      },
      "source": [
        "### Testing trained model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "tT5gyD6WydNq"
      },
      "outputs": [],
      "source": [
        "testst = time.time()\n",
        "predictions = model.transform(test_data)\n",
        "testet = time.time()\n",
        "testTime = testet - testst"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "zvXhzCBpydNq",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d20cf012-65e8-40cc-a23a-cbff29b47190"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------+-------+------+----------+----------+\n",
            "|userId|movieId|rating| timestamp|prediction|\n",
            "+------+-------+------+----------+----------+\n",
            "|     1|    307|   5.0|1147868828|  4.044257|\n",
            "|     1|   1175|   3.5|1147868826| 3.8279672|\n",
            "|     1|   1237|   5.0|1147868839| 4.0510416|\n",
            "|     1|   1250|   4.0|1147868414| 3.5275607|\n",
            "|     1|   2012|   2.5|1147868068| 2.2809074|\n",
            "|     1|   2068|   2.5|1147869044| 3.8753283|\n",
            "|     1|   2161|   3.5|1147868609| 3.6879287|\n",
            "|     1|   2692|   5.0|1147869100| 4.2605157|\n",
            "|     1|   3448|   4.0|1147868480| 3.7469707|\n",
            "|     1|   3949|   5.0|1147868678| 4.7491083|\n",
            "|     1|   4144|   5.0|1147868898| 3.6138039|\n",
            "|     1|   4703|   4.0|1147869223| 3.8178573|\n",
            "|     1|   4973|   4.5|1147869080| 4.5596046|\n",
            "|     1|   5147|   4.0|1147877654| 3.3674011|\n",
            "|     1|   5684|   2.0|1147879797|      3.33|\n",
            "|     1|   5878|   4.0|1147868807| 3.9194965|\n",
            "|     1|   5912|   3.0|1147878698|  3.214029|\n",
            "|     1|   6377|   4.0|1147868469| 3.5168176|\n",
            "|     1|   6954|   3.5|1147869150|  4.006442|\n",
            "|     1|   7209|   4.0|1147877986| 3.0977635|\n",
            "+------+-------+------+----------+----------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "predictions.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "goYQYpw70j3Z"
      },
      "source": [
        "## Extracting information of one user"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "BOfxtFuGydNs",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fda48954-2122-4164-88ce-89286890bce9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Enter a User ID to find recommendations for: 12\n"
          ]
        }
      ],
      "source": [
        "single_user = test_data.filter(test_data['userId']==12).select(['movieId','userId'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "yYj2tyb4ydNs",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ad2dfb43-0683-4d3e-b996-b2418b9a69b8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+------+\n",
            "|movieId|userId|\n",
            "+-------+------+\n",
            "|      1|    12|\n",
            "|      2|    12|\n",
            "|     22|    12|\n",
            "|     50|    12|\n",
            "|     88|    12|\n",
            "|    101|    12|\n",
            "|    140|    12|\n",
            "|    145|    12|\n",
            "|    150|    12|\n",
            "|    163|    12|\n",
            "|    165|    12|\n",
            "|    175|    12|\n",
            "|    185|    12|\n",
            "|    203|    12|\n",
            "|    209|    12|\n",
            "|    257|    12|\n",
            "|    319|    12|\n",
            "|    351|    12|\n",
            "|    377|    12|\n",
            "|    433|    12|\n",
            "+-------+------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "single_user.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7gueNFXk0p0m"
      },
      "source": [
        "## Running Model again for one selected user (testing)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "yABzy_5qydNt"
      },
      "outputs": [],
      "source": [
        "test1st = time.time()\n",
        "reccomendations = model.transform(single_user)\n",
        "test1et = time.time()\n",
        "test1Time = test1et - test1st"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "IhhRuY5jydNt",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "53865cc7-798e-4aa4-997e-ff9ae0fee757"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+------+----------+\n",
            "|movieId|userId|prediction|\n",
            "+-------+------+----------+\n",
            "|    858|    12| 4.2775836|\n",
            "|   4973|    12|  4.250431|\n",
            "|    745|    12| 4.2312336|\n",
            "|   1207|    12| 4.1879983|\n",
            "|  40870|    12| 4.1767607|\n",
            "|   1136|    12| 4.1566386|\n",
            "|   1212|    12| 4.1313143|\n",
            "|   1219|    12| 4.1263475|\n",
            "|    926|    12|  4.109017|\n",
            "|   3435|    12| 4.1078067|\n",
            "|     50|    12|  4.094791|\n",
            "|   1197|    12| 4.0918846|\n",
            "|  63876|    12|  4.078976|\n",
            "|  74416|    12| 4.0785985|\n",
            "|    527|    12|  4.067445|\n",
            "|   1225|    12| 4.0649686|\n",
            "|   1361|    12|  4.055188|\n",
            "|   4226|    12|  4.011339|\n",
            "|   3996|    12|  4.007953|\n",
            "|  55442|    12| 3.9923105|\n",
            "+-------+------+----------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "reccomendations.orderBy('prediction',ascending=False).show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YtzjeCy40z7_"
      },
      "source": [
        "## Analysing with the help of supporting Datasets (Easier to Understand)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "id": "YUmTzUY-ydNt",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d1b44cb9-0466-4b5f-e42b-1a4c957b0f9a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+--------------------+--------------------+\n",
            "|movieId|               title|              genres|\n",
            "+-------+--------------------+--------------------+\n",
            "|      1|    Toy Story (1995)|Adventure|Animati...|\n",
            "|      2|      Jumanji (1995)|Adventure|Childre...|\n",
            "|      3|Grumpier Old Men ...|      Comedy|Romance|\n",
            "|      4|Waiting to Exhale...|Comedy|Drama|Romance|\n",
            "|      5|Father of the Bri...|              Comedy|\n",
            "|      6|         Heat (1995)|Action|Crime|Thri...|\n",
            "|      7|      Sabrina (1995)|      Comedy|Romance|\n",
            "|      8| Tom and Huck (1995)|  Adventure|Children|\n",
            "|      9| Sudden Death (1995)|              Action|\n",
            "|     10|    GoldenEye (1995)|Action|Adventure|...|\n",
            "|     11|American Presiden...|Comedy|Drama|Romance|\n",
            "|     12|Dracula: Dead and...|       Comedy|Horror|\n",
            "|     13|        Balto (1995)|Adventure|Animati...|\n",
            "|     14|        Nixon (1995)|               Drama|\n",
            "|     15|Cutthroat Island ...|Action|Adventure|...|\n",
            "|     16|       Casino (1995)|         Crime|Drama|\n",
            "|     17|Sense and Sensibi...|       Drama|Romance|\n",
            "|     18|   Four Rooms (1995)|              Comedy|\n",
            "|     19|Ace Ventura: When...|              Comedy|\n",
            "|     20|  Money Train (1995)|Action|Comedy|Cri...|\n",
            "+-------+--------------------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "moviesdf = spark.read.csv(r\"MovieLens/ml-25m/movies.csv\", inferSchema = True, header = True)  \n",
        "moviesdf.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8YXn1oCV1EK-"
      },
      "source": [
        "## Final Predictions with Movie Titles"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "id": "UdyLavaP-4bo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fb9bb22d-bf2d-4307-8626-66362d131dfc"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------+-------+--------------------+--------------------+----------+\n",
            "|userId|movieId|               title|              genres|prediction|\n",
            "+------+-------+--------------------+--------------------+----------+\n",
            "|    12|    858|Godfather, The (1...|         Crime|Drama| 4.2775836|\n",
            "|    12|   4973|Amelie (Fabuleux ...|      Comedy|Romance|  4.250431|\n",
            "|    12|    745|Wallace & Gromit:...|Animation|Childre...| 4.2312336|\n",
            "|    12|   1207|To Kill a Mocking...|               Drama| 4.1879983|\n",
            "|    12|  40870|   C.R.A.Z.Y. (2005)|               Drama| 4.1767607|\n",
            "|    12|   1136|Monty Python and ...|Adventure|Comedy|...| 4.1566386|\n",
            "|    12|   1212|Third Man, The (1...|Film-Noir|Mystery...| 4.1313143|\n",
            "|    12|   1219|       Psycho (1960)|        Crime|Horror| 4.1263475|\n",
            "|    12|    926|All About Eve (1950)|               Drama|  4.109017|\n",
            "|    12|   3435|Double Indemnity ...|Crime|Drama|Film-...| 4.1078067|\n",
            "|    12|     50|Usual Suspects, T...|Crime|Mystery|Thr...|  4.094791|\n",
            "|    12|   1197|Princess Bride, T...|Action|Adventure|...| 4.0918846|\n",
            "|    12|  63876|         Milk (2008)|               Drama|  4.078976|\n",
            "|    12|  74416|    Fish Tank (2009)|               Drama| 4.0785985|\n",
            "|    12|    527|Schindler's List ...|           Drama|War|  4.067445|\n",
            "|    12|   1225|      Amadeus (1984)|               Drama| 4.0649686|\n",
            "|    12|   1361|Paradise Lost: Th...|         Documentary|  4.055188|\n",
            "|    12|   4226|      Memento (2000)|    Mystery|Thriller|  4.011339|\n",
            "|    12|   3996|Crouching Tiger, ...|Action|Drama|Romance|  4.007953|\n",
            "|    12|  55442|   Persepolis (2007)|     Animation|Drama| 3.9923105|\n",
            "+------+-------+--------------------+--------------------+----------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ],
      "source": [
        "rec = reccomendations\n",
        "joined = moviesdf.join(rec, ['movieId'],how=\"inner\")\n",
        "joined.select('userId', 'movieId', 'title', 'genres', 'prediction').orderBy('prediction', ascending=False).show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "id": "9JCa8l7yeqsV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b34268b5-ab38-493c-abf1-62ec776229f1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Time to setup =  51.211623191833496\n",
            "Time to train =  208.77153968811035\n",
            "Time to test =  0.14310717582702637\n",
            "Time to test1 =  0.08272528648376465\n"
          ]
        }
      ],
      "source": [
        "print(\"Time to setup = \", setupTime)\n",
        "print(\"Time to train = \", trainTime)\n",
        "print(\"Time to test = \", testTime)\n",
        "print(\"Time to test1 = \", test1Time)"
      ]
    }
  ],
  "metadata": {
    "anaconda-cloud": {},
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.9"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}