notebooks/contact_spark_op.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql.types import TimestampType, StructType, StringType, IntegerType, FloatType, StructField\n",
    "from pyspark.sql.functions import lit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "ss = SparkSession \\\n",
    "    .builder \\\n",
    "    .master(\"local[*]\") \\\n",
    "    .appName(\"us_import_sample\") \\\n",
    "    .getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "contact_schema = StructType([\n",
    "    StructField(\"identifier\", StringType()),\n",
    "    StructField(\"name\", StringType()),\n",
    "    StructField(\"address_1\", StringType()),\n",
    "    StructField(\"address_2\", StringType()),\n",
    "    StructField(\"address_3\", StringType()),\n",
    "    StructField(\"address_4\", StringType()),\n",
    "    StructField(\"city\", StringType()),\n",
    "    StructField(\"state_province\", StringType()),\n",
    "    StructField(\"zip_code\", StringType()),\n",
    "    StructField(\"country_code\", StringType()),\n",
    "    StructField(\"contact_name\", StringType()),\n",
    "    StructField(\"comm_number_qualifier\", StringType()),\n",
    "    StructField(\"comm_number\", StringType())\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "consignee = ss.read \\\n",
    "    .option(\"header\", True) \\\n",
    "    .option(\"escape\", '\"') \\\n",
    "    .csv(\n",
    "        './ams/2020/202001201500/ams__consignee_2020__202001201500.csv', \n",
    "        schema=contact_schema\n",
    "    )\n",
    "consignee = consignee.withColumn('contact_type', lit('consignee'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- identifier: string (nullable = true)\n",
      " |-- name: string (nullable = true)\n",
      " |-- address_1: string (nullable = true)\n",
      " |-- address_2: string (nullable = true)\n",
      " |-- address_3: string (nullable = true)\n",
      " |-- address_4: string (nullable = true)\n",
      " |-- city: string (nullable = true)\n",
      " |-- state_province: string (nullable = true)\n",
      " |-- zip_code: string (nullable = true)\n",
      " |-- country_code: string (nullable = true)\n",
      " |-- contact_name: string (nullable = true)\n",
      " |-- comm_number_qualifier: string (nullable = true)\n",
      " |-- comm_number: string (nullable = true)\n",
      " |-- contact_type: string (nullable = false)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "consignee.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "notified_party = ss.read \\\n",
    "    .option(\"header\", True) \\\n",
    "    .option(\"escape\", '\"') \\\n",
    "    .csv(\n",
    "        './ams/2020/202001201500/ams__notifyparty_2020__202001201500.csv', \n",
    "        schema=contact_schema\n",
    "    )\n",
    "notified_party = notified_party.withColumn('contact_type', lit('notified_party'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "shipper = ss.read \\\n",
    "    .option(\"header\", True) \\\n",
    "    .option(\"escape\", '\"') \\\n",
    "    .csv(\n",
    "        './ams/2020/202001201500/ams__shipper_2020__202001201500.csv', \n",
    "        schema=contact_schema\n",
    "    )\n",
    "shipper = shipper.withColumn('contact_type', lit('shipper'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "contact = consignee.union(shipper).union(notified_party)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- identifier: string (nullable = true)\n",
      " |-- name: string (nullable = true)\n",
      " |-- address_1: string (nullable = true)\n",
      " |-- address_2: string (nullable = true)\n",
      " |-- address_3: string (nullable = true)\n",
      " |-- address_4: string (nullable = true)\n",
      " |-- city: string (nullable = true)\n",
      " |-- state_province: string (nullable = true)\n",
      " |-- zip_code: string (nullable = true)\n",
      " |-- country_code: string (nullable = true)\n",
      " |-- contact_name: string (nullable = true)\n",
      " |-- comm_number_qualifier: string (nullable = true)\n",
      " |-- comm_number: string (nullable = true)\n",
      " |-- contact_type: string (nullable = false)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "contact.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}