RDD_Create.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "6698741f",
   "metadata": {},
   "source": [
    "# Spark RDD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "891122c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyspark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fb1cdae4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/joono/Desktop/joono/Bigdata-Analysis\n",
      "/opt/anaconda3/envs/bigdata_analysis/lib/python38.zip\n",
      "/opt/anaconda3/envs/bigdata_analysis/lib/python3.8\n",
      "/opt/anaconda3/envs/bigdata_analysis/lib/python3.8/lib-dynload\n",
      "\n",
      "/opt/anaconda3/envs/bigdata_analysis/lib/python3.8/site-packages\n",
      "/opt/anaconda3/envs/bigdata_analysis/lib/python3.8/site-packages/IPython/extensions\n",
      "/Users/joono/.ipython\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "for i in sys.path:\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7894d8fe",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "21/09/25 13:59:29 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
      "Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties\n",
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n",
      "21/09/25 13:59:30 WARN Utils: Service 'sparkDriver' could not bind on a random free port. You may check whether configuring an appropriate binding address.\n",
      "21/09/25 13:59:30 WARN Utils: Service 'sparkDriver' could not bind on a random free port. You may check whether configuring an appropriate binding address.\n",
      "21/09/25 13:59:30 WARN Utils: Service 'sparkDriver' could not bind on a random free port. You may check whether configuring an appropriate binding address.\n",
      "21/09/25 13:59:30 WARN Utils: Service 'sparkDriver' could not bind on a random free port. You may check whether configuring an appropriate binding address.\n",
      "21/09/25 13:59:30 WARN Utils: Service 'sparkDriver' could not bind on a random free port. You may check whether configuring an appropriate binding address.\n",
      "21/09/25 13:59:30 WARN Utils: Service 'sparkDriver' could not bind on a random free port. You may check whether configuring an appropriate binding address.\n",
      "21/09/25 13:59:30 WARN Utils: Service 'sparkDriver' could not bind on a random free port. You may check whether configuring an appropriate binding address.\n",
      "21/09/25 13:59:30 WARN Utils: Service 'sparkDriver' could not bind on a random free port. You may check whether configuring an appropriate binding address.\n",
      "21/09/25 13:59:30 WARN Utils: Service 'sparkDriver' could not bind on a random free port. You may check whether configuring an appropriate binding address.\n",
      "21/09/25 13:59:30 WARN Utils: Service 'sparkDriver' could not bind on a random free port. You may check whether configuring an appropriate binding address.\n",
      "21/09/25 13:59:30 WARN Utils: Service 'sparkDriver' could not bind on a random free port. You may check whether configuring an appropriate binding address.\n",
      "21/09/25 13:59:30 WARN Utils: Service 'sparkDriver' could not bind on a random free port. You may check whether configuring an appropriate binding address.\n",
      "21/09/25 13:59:30 WARN Utils: Service 'sparkDriver' could not bind on a random free port. You may check whether configuring an appropriate binding address.\n",
      "21/09/25 13:59:30 WARN Utils: Service 'sparkDriver' could not bind on a random free port. You may check whether configuring an appropriate binding address.\n",
      "21/09/25 13:59:30 WARN Utils: Service 'sparkDriver' could not bind on a random free port. You may check whether configuring an appropriate binding address.\n",
      "21/09/25 13:59:30 WARN Utils: Service 'sparkDriver' could not bind on a random free port. You may check whether configuring an appropriate binding address.\n",
      "21/09/25 13:59:30 ERROR SparkContext: Error initializing SparkContext.\n",
      "java.net.BindException: Can't assign requested address: Service 'sparkDriver' failed after 16 retries (on a random free port)! Consider explicitly setting the appropriate binding address for the service 'sparkDriver' (for example spark.driver.bindAddress for SparkDriver) to the correct binding address.\n",
      "\tat sun.nio.ch.Net.bind0(Native Method)\n",
      "\tat sun.nio.ch.Net.bind(Net.java:461)\n",
      "\tat sun.nio.ch.Net.bind(Net.java:453)\n",
      "\tat sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:222)\n",
      "\tat io.netty.channel.socket.nio.NioServerSocketChannel.doBind(NioServerSocketChannel.java:134)\n",
      "\tat io.netty.channel.AbstractChannel$AbstractUnsafe.bind(AbstractChannel.java:550)\n",
      "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.bind(DefaultChannelPipeline.java:1334)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.invokeBind(AbstractChannelHandlerContext.java:506)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.bind(AbstractChannelHandlerContext.java:491)\n",
      "\tat io.netty.channel.DefaultChannelPipeline.bind(DefaultChannelPipeline.java:973)\n",
      "\tat io.netty.channel.AbstractChannel.bind(AbstractChannel.java:248)\n",
      "\tat io.netty.bootstrap.AbstractBootstrap$2.run(AbstractBootstrap.java:356)\n",
      "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n",
      "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n",
      "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n",
      "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n",
      "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n",
      "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n",
      "\tat java.lang.Thread.run(Thread.java:748)\n"
     ]
    },
    {
     "ename": "Py4JJavaError",
     "evalue": "An error occurred while calling None.org.apache.spark.api.java.JavaSparkContext.\n: java.net.BindException: Can't assign requested address: Service 'sparkDriver' failed after 16 retries (on a random free port)! Consider explicitly setting the appropriate binding address for the service 'sparkDriver' (for example spark.driver.bindAddress for SparkDriver) to the correct binding address.\n\tat sun.nio.ch.Net.bind0(Native Method)\n\tat sun.nio.ch.Net.bind(Net.java:461)\n\tat sun.nio.ch.Net.bind(Net.java:453)\n\tat sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:222)\n\tat io.netty.channel.socket.nio.NioServerSocketChannel.doBind(NioServerSocketChannel.java:134)\n\tat io.netty.channel.AbstractChannel$AbstractUnsafe.bind(AbstractChannel.java:550)\n\tat io.netty.channel.DefaultChannelPipeline$HeadContext.bind(DefaultChannelPipeline.java:1334)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeBind(AbstractChannelHandlerContext.java:506)\n\tat io.netty.channel.AbstractChannelHandlerContext.bind(AbstractChannelHandlerContext.java:491)\n\tat io.netty.channel.DefaultChannelPipeline.bind(DefaultChannelPipeline.java:973)\n\tat io.netty.channel.AbstractChannel.bind(AbstractChannel.java:248)\n\tat io.netty.bootstrap.AbstractBootstrap$2.run(AbstractBootstrap.java:356)\n\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n\tat java.lang.Thread.run(Thread.java:748)\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mPy4JJavaError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/l4/0n86l_692hx8xxqz3fqkk6mm0000gn/T/ipykernel_3182/1728567971.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mmyConf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpyspark\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSparkConf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mspark\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpyspark\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msql\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSparkSession\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m                     \u001b[0;34m.\u001b[0m\u001b[0mbuilder\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m                     \u001b[0;34m.\u001b[0m\u001b[0mmaster\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"local\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m                     \u001b[0;34m.\u001b[0m\u001b[0mappName\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"myApp\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/envs/bigdata_analysis/lib/python3.8/site-packages/pyspark/sql/session.py\u001b[0m in \u001b[0;36mgetOrCreate\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    226\u001b[0m                             \u001b[0msparkConf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    227\u001b[0m                         \u001b[0;31m# This SparkContext may be an existing one.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 228\u001b[0;31m                         \u001b[0msc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSparkContext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetOrCreate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msparkConf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    229\u001b[0m                     \u001b[0;31m# Do not update `SparkConf` for existing `SparkContext`, as it's shared\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    230\u001b[0m                     \u001b[0;31m# by all sessions.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/envs/bigdata_analysis/lib/python3.8/site-packages/pyspark/context.py\u001b[0m in \u001b[0;36mgetOrCreate\u001b[0;34m(cls, conf)\u001b[0m\n\u001b[1;32m    382\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mSparkContext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    383\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mSparkContext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_active_spark_context\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 384\u001b[0;31m                 \u001b[0mSparkContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconf\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconf\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mSparkConf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    385\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mSparkContext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_active_spark_context\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    386\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/envs/bigdata_analysis/lib/python3.8/site-packages/pyspark/context.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, gateway, jsc, profiler_cls)\u001b[0m\n\u001b[1;32m    144\u001b[0m         \u001b[0mSparkContext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgateway\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgateway\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconf\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    145\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 146\u001b[0;31m             self._do_init(master, appName, sparkHome, pyFiles, environment, batchSize, serializer,\n\u001b[0m\u001b[1;32m    147\u001b[0m                           conf, jsc, profiler_cls)\n\u001b[1;32m    148\u001b[0m         \u001b[0;32mexcept\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/envs/bigdata_analysis/lib/python3.8/site-packages/pyspark/context.py\u001b[0m in \u001b[0;36m_do_init\u001b[0;34m(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, jsc, profiler_cls)\u001b[0m\n\u001b[1;32m    207\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    208\u001b[0m         \u001b[0;31m# Create the Java SparkContext through Py4J\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 209\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jsc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjsc\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_initialize_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_conf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jconf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    210\u001b[0m         \u001b[0;31m# Reset the SparkConf to the one actually used by the SparkContext in JVM.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    211\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_conf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSparkConf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_jconf\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jsc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/envs/bigdata_analysis/lib/python3.8/site-packages/pyspark/context.py\u001b[0m in \u001b[0;36m_initialize_context\u001b[0;34m(self, jconf)\u001b[0m\n\u001b[1;32m    319\u001b[0m         \u001b[0mInitialize\u001b[0m \u001b[0mSparkContext\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfunction\u001b[0m \u001b[0mto\u001b[0m \u001b[0mallow\u001b[0m \u001b[0msubclass\u001b[0m \u001b[0mspecific\u001b[0m \u001b[0minitialization\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    320\u001b[0m         \"\"\"\n\u001b[0;32m--> 321\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jvm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mJavaSparkContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjconf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    322\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    323\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mclassmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/envs/bigdata_analysis/lib/python3.8/site-packages/py4j/java_gateway.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m   1566\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1567\u001b[0m         \u001b[0manswer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_gateway_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_command\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1568\u001b[0;31m         return_value = get_return_value(\n\u001b[0m\u001b[1;32m   1569\u001b[0m             answer, self._gateway_client, None, self._fqn)\n\u001b[1;32m   1570\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/envs/bigdata_analysis/lib/python3.8/site-packages/py4j/protocol.py\u001b[0m in \u001b[0;36mget_return_value\u001b[0;34m(answer, gateway_client, target_id, name)\u001b[0m\n\u001b[1;32m    324\u001b[0m             \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mOUTPUT_CONVERTER\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0manswer\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgateway_client\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    325\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0manswer\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mREFERENCE_TYPE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 326\u001b[0;31m                 raise Py4JJavaError(\n\u001b[0m\u001b[1;32m    327\u001b[0m                     \u001b[0;34m\"An error occurred while calling {0}{1}{2}.\\n\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    328\u001b[0m                     format(target_id, \".\", name), value)\n",
      "\u001b[0;31mPy4JJavaError\u001b[0m: An error occurred while calling None.org.apache.spark.api.java.JavaSparkContext.\n: java.net.BindException: Can't assign requested address: Service 'sparkDriver' failed after 16 retries (on a random free port)! Consider explicitly setting the appropriate binding address for the service 'sparkDriver' (for example spark.driver.bindAddress for SparkDriver) to the correct binding address.\n\tat sun.nio.ch.Net.bind0(Native Method)\n\tat sun.nio.ch.Net.bind(Net.java:461)\n\tat sun.nio.ch.Net.bind(Net.java:453)\n\tat sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:222)\n\tat io.netty.channel.socket.nio.NioServerSocketChannel.doBind(NioServerSocketChannel.java:134)\n\tat io.netty.channel.AbstractChannel$AbstractUnsafe.bind(AbstractChannel.java:550)\n\tat io.netty.channel.DefaultChannelPipeline$HeadContext.bind(DefaultChannelPipeline.java:1334)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeBind(AbstractChannelHandlerContext.java:506)\n\tat io.netty.channel.AbstractChannelHandlerContext.bind(AbstractChannelHandlerContext.java:491)\n\tat io.netty.channel.DefaultChannelPipeline.bind(DefaultChannelPipeline.java:973)\n\tat io.netty.channel.AbstractChannel.bind(AbstractChannel.java:248)\n\tat io.netty.bootstrap.AbstractBootstrap$2.run(AbstractBootstrap.java:356)\n\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)\n\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n\tat java.lang.Thread.run(Thread.java:748)\n"
     ]
    }
   ],
   "source": [
    "myConf = pyspark.SparkConf()\n",
    "spark = pyspark.sql.SparkSession\\\n",
    "                    .builder\\\n",
    "                    .master(\"local\")\\\n",
    "                    .appName(\"myApp\")\\\n",
    "                    .config(conf=myConf)\\\n",
    "                    .getOrCreate()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "35e9e367",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'spark' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/l4/0n86l_692hx8xxqz3fqkk6mm0000gn/T/ipykernel_3182/1874492401.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m\"Spark version \\t: {}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspark\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mversion\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mprint\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m\"Spark App \\t: {}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspark\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'spark.app.name'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mprint\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m\"Spark Master \\t: {}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspark\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'spark.master'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mprint\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m\"Spark Host \\t: {}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspark\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'spark.driver.host'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'spark' is not defined"
     ]
    }
   ],
   "source": [
    "print (\"Spark version \\t: {}\".format(spark.version))\n",
    "print (\"Spark App \\t: {}\".format(spark.conf.get('spark.app.name')))\n",
    "print (\"Spark Master \\t: {}\".format(spark.conf.get('spark.master')))\n",
    "print (\"Spark Host \\t: {}\".format(spark.conf.get('spark.driver.host')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5cca3110",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'file:/Users/joono/Desktop/joono/Bigdata-Analysis/spark-warehouse'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark.conf.get('spark.sql.warehouse.dir')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a3124d53",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('spark.master', 'local'),\n",
       " ('spark.app.startTime', '1631890373641'),\n",
       " ('spark.app.id', 'local-1631890374466'),\n",
       " ('spark.driver.port', '50249'),\n",
       " ('spark.app.name', 'myApp'),\n",
       " ('spark.rdd.compress', 'True'),\n",
       " ('spark.serializer.objectStreamReset', '100'),\n",
       " ('spark.driver.host', '192.168.0.34'),\n",
       " ('spark.submit.pyFiles', ''),\n",
       " ('spark.executor.id', 'driver'),\n",
       " ('spark.submit.deployMode', 'client'),\n",
       " ('spark.sql.warehouse.dir',\n",
       "  'file:/Users/joono/Desktop/joono/Bigdata-Analysis/spark-warehouse'),\n",
       " ('spark.ui.showConsoleProgress', 'true')]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark.sparkContext.getConf().getAll()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fd4c9399",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pyspark.conf.SparkConf at 0x7fe7f8531eb0>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark.sparkContext._conf.set(\"spark.driver.bindAddress\", \"127.0.0.1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "563d6923",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pyspark.conf.SparkConf at 0x7fe7f8531eb0>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark.sparkContext._conf.set(\"spark.executor.memory\", '4g')\n",
    "spark.sparkContext._conf.set('spark.app.name', 'jsl')\n",
    "spark.sparkContext._conf.set('spark.executor.cores', '3')\n",
    "spark.sparkContext._conf.set('spark.cores.max', '3')\n",
    "spark.sparkContext._conf.set(\"spark.driver.memory\",'4g')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "610a2830",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('spark.master', 'local'),\n",
       " ('spark.app.name', 'jsl'),\n",
       " ('spark.driver.port', '50249'),\n",
       " ('spark.driver.memory', '4g'),\n",
       " ('spark.executor.memory', '4g'),\n",
       " ('spark.driver.host', '192.168.0.34'),\n",
       " ('spark.executor.id', 'driver'),\n",
       " ('spark.cores.max', '3'),\n",
       " ('spark.driver.bindAddress', '127.0.0.1'),\n",
       " ('spark.app.startTime', '1631890373641'),\n",
       " ('spark.app.id', 'local-1631890374466'),\n",
       " ('spark.rdd.compress', 'True'),\n",
       " ('spark.serializer.objectStreamReset', '100'),\n",
       " ('spark.submit.pyFiles', ''),\n",
       " ('spark.executor.cores', '3'),\n",
       " ('spark.submit.deployMode', 'client'),\n",
       " ('spark.sql.warehouse.dir',\n",
       "  'file:/Users/joono/Desktop/joono/Bigdata-Analysis/spark-warehouse'),\n",
       " ('spark.ui.showConsoleProgress', 'true')]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark.sparkContext._conf.getAll()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "91db39ab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('spark.master', 'local'),\n",
       " ('spark.app.name', 'jsl'),\n",
       " ('spark.driver.port', '50249'),\n",
       " ('spark.driver.memory', '4g'),\n",
       " ('spark.executor.memory', '4g'),\n",
       " ('spark.driver.host', '192.168.0.34'),\n",
       " ('spark.executor.id', 'driver'),\n",
       " ('spark.cores.max', '3'),\n",
       " ('spark.driver.bindAddress', '127.0.0.1'),\n",
       " ('spark.app.startTime', '1631890373641'),\n",
       " ('spark.app.id', 'local-1631890374466'),\n",
       " ('spark.rdd.compress', 'True'),\n",
       " ('spark.serializer.objectStreamReset', '100'),\n",
       " ('spark.submit.pyFiles', ''),\n",
       " ('spark.executor.cores', '3'),\n",
       " ('spark.submit.deployMode', 'client'),\n",
       " ('spark.sql.warehouse.dir',\n",
       "  'file:/Users/joono/Desktop/joono/Bigdata-Analysis/spark-warehouse'),\n",
       " ('spark.ui.showConsoleProgress', 'true')]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark.sparkContext._conf.getAll()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "81b04c5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "myList = [1, 2, 3, 4, 5, 6, 7]\n",
    "myRdd1 = spark.sparkContext.parallelize(myList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "49cd0a87",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1, 2, 3]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "myRdd1.take(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "741ea1b1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[1, 2, 3], [4, 5, 6, 7]]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark.sparkContext.parallelize([1, 2, 3, 4, 5, 6, 7], 2).glom().collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "766c295f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('file:/Users/joono/Desktop/joono/Bigdata-Analysis/data/경기도 의정부시_인구현황_20210910.csv',\n",
       "  b'\\xc7\\xe0\\xc1\\xa4\\xb1\\xe2\\xb0\\xfc,\\xc0\\xce\\xb1\\xb8\\xbc\\xf6(\\xb0\\xe8),\\xc0\\xce\\xb1\\xb8\\xbc\\xf6(\\xb3\\xb2),\\xc0\\xce\\xb1\\xb8\\xbc\\xf6(\\xbf\\xa9),\\xb1\\xb8\\xbc\\xba\\xba\\xf1(\\xb0\\xe8),\\xb1\\xb8\\xbc\\xba\\xba\\xf1(\\xb3\\xb2),\\xb1\\xb8\\xbc\\xba\\xba\\xf1(\\xbf\\xa9),\\xbc\\xba\\xba\\xf1,\\xbc\\xbc\\xb4\\xeb\\xbc\\xf6,\\xbc\\xbc\\xb4\\xeb\\xb4\\xe7\\xc0\\xce\\xb1\\xb8,\\xb0\\xfc\\xb8\\xae\\xb1\\xe2\\xb0\\xfc\\xb8\\xed,\\xb0\\xfc\\xb8\\xae\\xba\\xce\\xbc\\xad\\xb8\\xed,\\xba\\xce\\xbc\\xad\\xc0\\xfc\\xc8\\xad\\xb9\\xf8\\xc8\\xa3,\\xb5\\xa5\\xc0\\xcc\\xc5\\xcd\\xb1\\xe2\\xc1\\xd8\\xc0\\xcf\\xc0\\xda\\r\\n\\xc0\\xc7\\xc1\\xa4\\xba\\xce1\\xb5\\xbf,32292,16538,15754,6.97,3.57,3.4,104.98,19998,1.61,\\xc0\\xc7\\xc1\\xa4\\xba\\xce\\xbd\\xc3,\\xb9\\xce\\xbf\\xf8\\xbf\\xa9\\xb1\\xc7\\xb0\\xfa,031-828-2466,2021-09-10\\r\\n\\xc0\\xc7\\xc1\\xa4\\xba\\xce2\\xb5\\xbf,31380,15608,15772,6.77,3.37,3.4,98.96,16410,1.91,\\xc0\\xc7\\xc1\\xa4\\xba\\xce\\xbd\\xc3,\\xb9\\xce\\xbf\\xf8\\xbf\\xa9\\xb1\\xc7\\xb0\\xfa,031-828-2466,2021-09-10\\r\\n\\xc8\\xa3\\xbf\\xf81\\xb5\\xbf,36124,17595,18529,7.8,3.8,4,94.96,15653,2.31,\\xc0\\xc7\\xc1\\xa4\\xba\\xce\\xbd\\xc3,\\xb9\\xce\\xbf\\xf8\\xbf\\xa9\\xb1\\xc7\\xb0\\xfa,031-828-2466,2021-09-10\\r\\n\\xc8\\xa3\\xbf\\xf82\\xb5\\xbf,34957,16923,18034,7.54,3.65,3.89,93.84,13683,2.55,\\xc0\\xc7\\xc1\\xa4\\xba\\xce\\xbd\\xc3,\\xb9\\xce\\xbf\\xf8\\xbf\\xa9\\xb1\\xc7\\xb0\\xfa,031-828-2466,2021-09-10\\r\\n\\xc0\\xe5\\xbe\\xcf\\xb5\\xbf,20314,9714,10600,4.38,2.1,2.29,91.64,8604,2.36,\\xc0\\xc7\\xc1\\xa4\\xba\\xce\\xbd\\xc3,\\xb9\\xce\\xbf\\xf8\\xbf\\xa9\\xb1\\xc7\\xb0\\xfa,031-828-2466,2021-09-10\\r\\n\\xbd\\xc5\\xb0\\xee1\\xb5\\xbf,43159,21205,21954,9.31,4.58,4.74,96.59,17990,2.4,\\xc0\\xc7\\xc1\\xa4\\xba\\xce\\xbd\\xc3,\\xb9\\xce\\xbf\\xf8\\xbf\\xa9\\xb1\\xc7\\xb0\\xfa,031-828-2466,2021-09-10\\r\\n\\xbd\\xc5\\xb0\\xee2\\xb5\\xbf,47852,23232,24620,10.33,5.01,5.31,94.36,19218,2.49,\\xc0\\xc7\\xc1\\xa4\\xba\\xce\\xbd\\xc3,\\xb9\\xce\\xbf\\xf8\\xbf\\xa9\\xb1\\xc7\\xb0\\xfa,031-828-2466,2021-09-10\\r\\n\\xbc\\xdb\\xbb\\xea1\\xb5\\xbf,42817,21276,21541,9.24,4.59,4.65,98.77,18811,2.28,\\xc0\\xc7\\xc1\\xa4\\xba\\xce\\xbd\\xc3,\\xb9\\xce\\xbf\\xf8\\xbf\\xa9\\xb1\\xc7\\xb0\\xfa,031-828-2466,2021-09-10\\r\\n\\xbc\\xdb\\xbb\\xea2\\xb5\\xbf,33565,16601,16964,7.24,3.58,3.66,97.86,13216,2.54,\\xc0\\xc7\\xc1\\xa4\\xba\\xce\\xbd\\xc3,\\xb9\\xce\\xbf\\xf8\\xbf\\xa9\\xb1\\xc7\\xb0\\xfa,031-828-2466,2021-09-10\\r\\n\\xbc\\xdb\\xbb\\xea3\\xb5\\xbf,46892,22772,24120,10.12,4.91,5.21,94.41,17926,2.62,\\xc0\\xc7\\xc1\\xa4\\xba\\xce\\xbd\\xc3,\\xb9\\xce\\xbf\\xf8\\xbf\\xa9\\xb1\\xc7\\xb0\\xfa,031-828-2466,2021-09-10\\r\\n\\xc0\\xda\\xb1\\xdd\\xb5\\xbf,27087,13270,13817,5.85,2.86,2.98,96.04,11868,2.28,\\xc0\\xc7\\xc1\\xa4\\xba\\xce\\xbd\\xc3,\\xb9\\xce\\xbf\\xf8\\xbf\\xa9\\xb1\\xc7\\xb0\\xfa,031-828-2466,2021-09-10\\r\\n\\xb0\\xa1\\xb4\\xc9\\xb5\\xbf,25990,12974,13016,5.61,2.8,2.81,99.68,12492,2.08,\\xc0\\xc7\\xc1\\xa4\\xba\\xce\\xbd\\xc3,\\xb9\\xce\\xbf\\xf8\\xbf\\xa9\\xb1\\xc7\\xb0\\xfa,031-828-2466,2021-09-10\\r\\n\\xc8\\xef\\xbc\\xb1\\xb5\\xbf,19176,9769,9407,4.14,2.11,2.03,103.85,9380,2.04,\\xc0\\xc7\\xc1\\xa4\\xba\\xce\\xbd\\xc3,\\xb9\\xce\\xbf\\xf8\\xbf\\xa9\\xb1\\xc7\\xb0\\xfa,031-828-2466,2021-09-10\\r\\n\\xb3\\xec\\xbe\\xe7\\xb5\\xbf,21768,10872,10896,4.7,2.35,2.35,99.78,9556,2.28,\\xc0\\xc7\\xc1\\xa4\\xba\\xce\\xbd\\xc3,\\xb9\\xce\\xbf\\xf8\\xbf\\xa9\\xb1\\xc7\\xb0\\xfa,031-828-2466,2021-09-10\\r\\n')]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "popRdd = spark.sparkContext.binaryFiles('./data/경기도 의정부시_인구현황_20210910.csv')\n",
    "popRdd.take(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b09f7a6f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "[Stage 5:>                                                          (0 + 1) / 1]\r",
      "\r",
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['행정기관,인구수(계),인구수(남),인구수(여),구성비(계),구성비(남),구성비(여),성비,세대수,세대당인구,관리기관명,관리부서명,부서전화번호,데이터기준일자\\r\\n의정부1동,32292,16538,15754,6.97,3.57,3.4,104.98,19998,1.61,의정부시,민원여권과,031-828-2466,2021-09-10\\r\\n의정부2동,31380,15608,15772,6.77,3.37,3.4,98.96,16410,1.91,의정부시,민원여권과,031-828-2466,2021-09-10\\r\\n호원1동,36124,17595,18529,7.8,3.8,4,94.96,15653,2.31,의정부시,민원여권과,031-828-2466,2021-09-10\\r\\n호원2동,34957,16923,18034,7.54,3.65,3.89,93.84,13683,2.55,의정부시,민원여권과,031-828-2466,2021-09-10\\r\\n장암동,20314,9714,10600,4.38,2.1,2.29,91.64,8604,2.36,의정부시,민원여권과,031-828-2466,2021-09-10\\r\\n신곡1동,43159,21205,21954,9.31,4.58,4.74,96.59,17990,2.4,의정부시,민원여권과,031-828-2466,2021-09-10\\r\\n신곡2동,47852,23232,24620,10.33,5.01,5.31,94.36,19218,2.49,의정부시,민원여권과,031-828-2466,2021-09-10\\r\\n송산1동,42817,21276,21541,9.24,4.59,4.65,98.77,18811,2.28,의정부시,민원여권과,031-828-2466,2021-09-10\\r\\n송산2동,33565,16601,16964,7.24,3.58,3.66,97.86,13216,2.54,의정부시,민원여권과,031-828-2466,2021-09-10\\r\\n송산3동,46892,22772,24120,10.12,4.91,5.21,94.41,17926,2.62,의정부시,민원여권과,031-828-2466,2021-09-10\\r\\n자금동,27087,13270,13817,5.85,2.86,2.98,96.04,11868,2.28,의정부시,민원여권과,031-828-2466,2021-09-10\\r\\n가능동,25990,12974,13016,5.61,2.8,2.81,99.68,12492,2.08,의정부시,민원여권과,031-828-2466,2021-09-10\\r\\n흥선동,19176,9769,9407,4.14,2.11,2.03,103.85,9380,2.04,의정부시,민원여권과,031-828-2466,2021-09-10\\r\\n녹양동,21768,10872,10896,4.7,2.35,2.35,99.78,9556,2.28,의정부시,민원여권과,031-828-2466,2021-09-10\\r\\n']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "readData = popRdd.map(lambda x: x[1].decode('euc-kr'))\n",
    "readData.take(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "fb23cc9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "popList = readData.map(lambda x: x.split()).take(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "c7a29375",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'의정부1동,32292,16538,15754,6.97,3.57,3.4,104.98,19998,1.61,의정부시,민원여권과,031-828-2466,2021-09-10'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "popList[0][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c602547",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "bigdata_analysis",
   "language": "python",
   "name": "bigdata_analysis"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}