post_request.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example post request notebook\n",
    "- 4-5 of these notebooks were run at once, each processing different chunks of 20-45"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import requests\n",
    "from joblib import Parallel, delayed\n",
    "import json\n",
    "import itertools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load in station_list and set which index to work with\n",
    "df = pd.read_csv('station_list.csv')\n",
    "submit = df['1'].to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Further break up into chunks if needed, sizes 20-45 recommend \n",
    "def break_list_into_chunks(lst, chunk_size):\n",
    "    '''\n",
    "    Breaks input list into chunks\n",
    "    '''\n",
    "    return [lst[i:i+chunk_size] for i in range(0, len(lst), chunk_size)]\n",
    "\n",
    "result = break_list_into_chunks(submit, 20)\n",
    "len(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def puller(input_list):\n",
    "    '''\n",
    "    Mimic post request, designed to be run in parallel with joblib\n",
    "    '''\n",
    "    # main url\n",
    "    url = 'https://raws.dri.edu/cgi-bin/wea_dysimts2.pl'\n",
    "    output_dict = {}\n",
    "    # post request to mimic, only change input_list (4-digit code)\n",
    "    form_data = {\n",
    "        'stn': input_list,\n",
    "        'smon': '02',\n",
    "        'sday': '01',\n",
    "        'syea': '14',\n",
    "        'emon': '05',\n",
    "        'eday': '17',\n",
    "        'eyea': '23',\n",
    "        'Submit Info': 'Submit Info',\n",
    "        'qBasic': 'ON',\n",
    "        'unit': 'E',\n",
    "        'Ofor': 'A',\n",
    "        'Datareq': 'A',\n",
    "        'qc': 'Y',\n",
    "        'miss': '08',\n",
    "        'obs': 'N',\n",
    "        'WsMon': '01',\n",
    "        'WsDay': '01',\n",
    "        'WeMon': '12',\n",
    "        'WeDay': '31',\n",
    "        '.cgifields': ['Datareq', 'qc', 'qRH', 'qPE', 'qGD4', 'qFT', 'qAT', 'qCDD', 'qWD', 'qGD5',\n",
    "                       'qBP', 'qAE', 'qPR', 'qBasic', 'unit', 'Ofor', 'qRS', 'qBT', 'obs', 'qST', 'qHDD']\n",
    "    }\n",
    "\n",
    "\n",
    "    response = requests.post(url, data=form_data)\n",
    "    output_dict[input_list] = response.text\n",
    "        \n",
    "    return output_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run all cells parallel \n",
    "- If cell fails go back and rerun at the end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    results_final = Parallel(n_jobs=-1, verbose=100)(\n",
    "    delayed(puller)(input_list) for input_list in result[0])\n",
    "except Exception as e:\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    results_final1 = Parallel(n_jobs=-1, verbose=50)(\n",
    "    delayed(puller)(input_list) for input_list in result[1])\n",
    "except Exception as e:\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    results_final2 = Parallel(n_jobs=-1, verbose=50)(\n",
    "    delayed(puller)(input_list) for input_list in result[2])\n",
    "except Exception as e:\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    results_final3 = Parallel(n_jobs=-1, verbose=50)(\n",
    "    delayed(puller)(input_list) for input_list in result[3])\n",
    "except Exception as e:\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    results_final4 = Parallel(n_jobs=-1, verbose=50)(\n",
    "    delayed(puller)(input_list) for input_list in result[4])\n",
    "except Exception as e:\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    results_final5 = Parallel(n_jobs=-1, verbose=50)(\n",
    "    delayed(puller)(input_list) for input_list in result[5])\n",
    "except Exception as e:\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    results_final6 = Parallel(n_jobs=-1, verbose=50)(\n",
    "    delayed(puller)(input_list) for input_list in result[6])\n",
    "except Exception as e:\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    results_final7 = Parallel(n_jobs=-1, verbose=50)(\n",
    "    delayed(puller)(input_list) for input_list in result[7])\n",
    "\n",
    "except Exception as e:\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    results_final8 = Parallel(n_jobs=-1, verbose=50)(\n",
    "    delayed(puller)(input_list) for input_list in result[8])\n",
    "\n",
    "except Exception as e:\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    results_final9 = Parallel(n_jobs=-1, verbose=50)(\n",
    "    delayed(puller)(input_list) for input_list in result[9])\n",
    "\n",
    "except Exception as e:\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    results_final10 = Parallel(n_jobs=-1, verbose=50)(\n",
    "    delayed(puller)(input_list) for input_list in result[10])\n",
    "\n",
    "except Exception as e:\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    results_final11 = Parallel(n_jobs=-1, verbose=50)(\n",
    "    delayed(puller)(input_list) for input_list in result[11])\n",
    "\n",
    "except Exception as e:\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    results_final12 = Parallel(n_jobs=-1, verbose=50)(\n",
    "    delayed(puller)(input_list) for input_list in result[12])\n",
    "\n",
    "except Exception as e:\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating csv and json file for each results_final\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def panda_save(lst, name, num=None):\n",
    "    '''\n",
    "    Checks over input list, creates a df to save to list, and dictionary to save to json. \n",
    "    The json file is whats used later on\n",
    "    lst : list\n",
    "    name : name to save file as\n",
    "    num : index of result should match the result index used to create lst.\n",
    "    \n",
    "    input cell : 'yes' will save, everything else will cancel\n",
    "    \n",
    "    Example:\n",
    "    \n",
    "    try:\n",
    "        results_final12 = Parallel(n_jobs=-1, verbose=50)(\n",
    "        delayed(puller)(input_list) for input_list in result[12])\n",
    "\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        \n",
    "    Uses result[12] so inputs are panda_save(results_final12, 'sl_1_20_12', 12)\n",
    "    \n",
    "    File naming system: sl / station_list index / chunck size used / index of chunk size\n",
    "    '''\n",
    "    \n",
    "    # checking file first\n",
    "    required_keys = result[num]\n",
    "    print(f\"Missing: {[r for r in required_keys if all(r not in ''.join(l.keys()) for l in lst)]}\")\n",
    "    print(f'{type(lst)}, Length: {len(lst)}\\n')\n",
    "    print(f\"{[value[:1000]for value in lst[0].values()]}\")\n",
    "    \n",
    "    def station_data(data):\n",
    "        import pandas as pd\n",
    "        conl = []\n",
    "        for station in data:\n",
    "            # pull key\n",
    "            key_name = list(station.keys())[0]\n",
    "            # Get the value of the dictionary\n",
    "            values = list(station.values())[0]\n",
    "\n",
    "            # create rows\n",
    "            rows = [line.strip().split() for line in (list(station.values())[0]).strip().split('\\n')]\n",
    "\n",
    "            header = ['date', 'year', 'day_of_year', 'day_of_run', 'total_solar_radiation_ly',\n",
    "                      'ave_mean_wind_speed_mph', 'ave_mean_wind_direction_deg', 'max_maximum_wind_gust_mph',\n",
    "                      'ave_average_air_temperature_deg_f', 'max_average_air_temperature_deg_f',\n",
    "                      'min_average_air_temperature_deg_f', 'ave_average_relative_humidity',\n",
    "                      'max_average_relative_humidity', 'min_average_relative_humidity', 'total_precipitation_in']\n",
    "\n",
    "\n",
    "            # remove header and ending rows\n",
    "            dataset = rows[7:-6]\n",
    "\n",
    "            # Creating individual dataframes\n",
    "            globals()[key_name] = pd.DataFrame(dataset, columns=header)\n",
    "\n",
    "            # Creating multi level dataframe\n",
    "            globals()[key_name].columns = pd.MultiIndex.from_product([[key_name], globals()[key_name].columns])\n",
    "            \n",
    "            # Creating list of dataframes to concat\n",
    "            conl.append(globals()[key_name])\n",
    "\n",
    "        return pd.concat(conl, axis = 1)\n",
    "    # More data checks\n",
    "    lst_df = station_data(lst)\n",
    "    display(lst_df)\n",
    "    print(f'{required_keys[0]}')\n",
    "    display(lst_df[required_keys[0]])\n",
    "    \n",
    "    # input section\n",
    "    checks = input(\"Enter (yes to save, no to cancel): \")\n",
    "    if checks == 'yes':\n",
    "#         Save to a JSON file\n",
    "        with open(f'{name}.json', 'w') as file:\n",
    "            json.dump(lst, file)\n",
    "        # save as csv\n",
    "        lst_df.to_csv(f'{name}.csv', index=False)\n",
    "            \n",
    "        print(f'\\nFile Saved')\n",
    "    \n",
    "    else:\n",
    "        print(f'\\n\\nCANCELED')\n",
    "    \n",
    "panda_save(results_final8, 'sl_1_20_8', 8)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (flatiron-env)",
   "language": "python",
   "name": "flatiron-env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}