scripts/combine_all.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "### Combine all the dataframes"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [],
   "source": [
    "import pandas as pd"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-11-09T11:45:26.670022200Z",
     "start_time": "2023-11-09T11:45:24.906391500Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "# Read MSC and Routscanner pickles\n",
    "msc = pd.read_pickle('../pickles/msc_v2_connections_combined.pickle')\n",
    "routescanner = pd.read_pickle('../pickles/routescanner_v2_connections_combined.pickle')\n",
    "maersk = pd.read_pickle('../pickles/maersk_daily/combined/combined.pickle')\n",
    "\n",
    "# Add the source to each dataframe at the beginning of the dataframe\n",
    "msc.insert(0, 'ScrapingSite', 'MSC')\n",
    "routescanner.insert(0, 'ScrapingSite', 'Routescanner')\n",
    "maersk['ScrapingSite'] = 'Maersk'"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-11-09T11:45:39.419106700Z",
     "start_time": "2023-11-09T11:45:26.648023500Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "data": {
      "text/plain": "      ScrapingSite EstimatedArrivalDatePolFormatted  \\\n11541       Maersk                             <NA>   \n11542       Maersk                             <NA>   \n11543       Maersk                             <NA>   \n11544       Maersk                             <NA>   \n11545       Maersk                             <NA>   \n11546       Maersk                             <NA>   \n11547       Maersk                             <NA>   \n11548       Maersk                             <NA>   \n11549       Maersk                             <NA>   \n11550       Maersk                             <NA>   \n\n                                        Destination ScrapingDate  \\\n11541  Amsterdam Container Terminal Vrede Amsterdam   2023-02-16   \n11542  Amsterdam Container Terminal Vrede Amsterdam   2023-02-16   \n11543  Amsterdam Container Terminal Vrede Amsterdam   2023-02-16   \n11544  Amsterdam Container Terminal Vrede Amsterdam   2023-02-16   \n11545  Amsterdam Container Terminal Vrede Amsterdam   2023-02-16   \n11546  Amsterdam Container Terminal Vrede Amsterdam   2023-02-16   \n11547  Amsterdam Container Terminal Vrede Amsterdam   2023-02-16   \n11548  Amsterdam Container Terminal Vrede Amsterdam   2023-02-16   \n11549  Amsterdam Container Terminal Vrede Amsterdam   2023-02-16   \n11550  Amsterdam Container Terminal Vrede Amsterdam   2023-02-16   \n\n      MaritimeServiceName OriginName DestinationName  RouteHeaderId  \\\n11541                <NA>        NaN             NaN           <NA>   \n11542                <NA>        NaN             NaN           <NA>   \n11543                <NA>        NaN             NaN           <NA>   \n11544                <NA>        NaN             NaN           <NA>   \n11545                <NA>        NaN             NaN           <NA>   \n11546                <NA>        NaN             NaN           <NA>   \n11547                <NA>        NaN             NaN           <NA>   \n11548                <NA>        NaN             NaN           <NA>   \n11549                <NA>        NaN             NaN           <NA>   \n11550                <NA>        NaN             NaN           <NA>   \n\n       PortOfLoadId  PortOfDischargeId  ... totalResults  TotalDistanceMeters  \\\n11541          <NA>               <NA>  ...         <NA>                  NaN   \n11542          <NA>               <NA>  ...         <NA>                  NaN   \n11543          <NA>               <NA>  ...         <NA>                  NaN   \n11544          <NA>               <NA>  ...         <NA>                  NaN   \n11545          <NA>               <NA>  ...         <NA>                  NaN   \n11546          <NA>               <NA>  ...         <NA>                  NaN   \n11547          <NA>               <NA>  ...         <NA>                  NaN   \n11548          <NA>               <NA>  ...         <NA>                  NaN   \n11549          <NA>               <NA>  ...         <NA>                  NaN   \n11550          <NA>               <NA>  ...         <NA>                  NaN   \n\n      totalCo2EmissionsInKg Departure time  Arrival time  \\\n11541                  <NA>            NaN           NaN   \n11542                  <NA>            NaN           NaN   \n11543                  <NA>            NaN           NaN   \n11544                  <NA>            NaN           NaN   \n11545                  <NA>            NaN           NaN   \n11546                  <NA>            NaN           NaN   \n11547                  <NA>            NaN           NaN   \n11548                  <NA>            NaN           NaN   \n11549                  <NA>            NaN           NaN   \n11550                  <NA>            NaN           NaN   \n\n                                                   Ports  \\\n11541  [Arica Terminal Puerto Arica (TPA), Balboa Bal...   \n11542  [Arica Terminal Puerto Arica (TPA), Amsterdam ...   \n11543  [Arica Terminal Puerto Arica (TPA), Balboa Bal...   \n11544  [Arica Terminal Puerto Arica (TPA), Amsterdam ...   \n11545  [Arica Terminal Puerto Arica (TPA), Balboa Bal...   \n11546  [Arica Terminal Puerto Arica (TPA), Amsterdam ...   \n11547  [Arica Terminal Puerto Arica (TPA), Balboa Bal...   \n11548  [Arica Terminal Puerto Arica (TPA), Amsterdam ...   \n11549  [Arica Terminal Puerto Arica (TPA), Balboa Bal...   \n11550  [Arica Terminal Puerto Arica (TPA), Amsterdam ...   \n\n                                                 Vessels  \\\n11541  [[JENS MAERSK, -, -, -, -, -], [MAERSK BUTON, ...   \n11542  [[JENS MAERSK, -, -, -, -, -], [MAERSK BUTON, ...   \n11543  [[FORT DESAIX, -, -, -, -, -], [MAERSK BRATAN,...   \n11544  [[FORT DESAIX, -, -, -, -, -], [MAERSK BRATAN,...   \n11545  [[JEPPESEN MAERSK, -, -, -, -, -], [MAERSK BAL...   \n11546  [[JEPPESEN MAERSK, -, -, -, -, -], [MAERSK BAL...   \n11547  [[JENS MAERSK, -, -, -, -, -], [MAERSK BATUR, ...   \n11548  [[JENS MAERSK, -, -, -, -, -], [MAERSK BATUR, ...   \n11549  [[FORT DESAIX, -, -, -, -, -], [MAERSK BULAN, ...   \n11550  [[FORT DESAIX, -, -, -, -, -], [MAERSK BULAN, ...   \n\n                                                   Dates  \\\n11541  [2023-02-15 15:06:00, 2023-02-28 12:30:00, 202...   \n11542         [2023-02-15 15:06:00, 2023-03-22 09:59:00]   \n11543  [2023-02-22 15:30:00, 2023-03-07 12:30:00, 202...   \n11544         [2023-02-22 15:30:00, 2023-03-29 10:00:00]   \n11545  [2023-03-01 15:30:00, 2023-03-14 12:30:00, 202...   \n11546         [2023-03-01 15:30:00, 2023-04-07 10:00:00]   \n11547  [2023-03-08 15:30:00, 2023-03-21 12:30:00, 202...   \n11548         [2023-03-08 15:30:00, 2023-04-12 10:00:00]   \n11549  [2023-03-15 15:30:00, 2023-03-28 12:30:00, 202...   \n11550         [2023-03-15 15:30:00, 2023-04-21 09:59:00]   \n\n      LocalEstimatedDepartureTime LocalEstimatedArrivalTime  \n11541         2023-02-15 15:06:00       2023-03-22 09:59:00  \n11542         2023-02-15 15:06:00       2023-03-22 09:59:00  \n11543         2023-02-22 15:30:00       2023-03-29 10:00:00  \n11544         2023-02-22 15:30:00       2023-03-29 10:00:00  \n11545         2023-03-01 15:30:00       2023-04-07 10:00:00  \n11546         2023-03-01 15:30:00       2023-04-07 10:00:00  \n11547         2023-03-08 15:30:00       2023-04-12 10:00:00  \n11548         2023-03-08 15:30:00       2023-04-12 10:00:00  \n11549         2023-03-15 15:30:00       2023-04-21 09:59:00  \n11550         2023-03-15 15:30:00       2023-04-21 09:59:00  \n\n[10 rows x 40 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>ScrapingSite</th>\n      <th>EstimatedArrivalDatePolFormatted</th>\n      <th>Destination</th>\n      <th>ScrapingDate</th>\n      <th>MaritimeServiceName</th>\n      <th>OriginName</th>\n      <th>DestinationName</th>\n      <th>RouteHeaderId</th>\n      <th>PortOfLoadId</th>\n      <th>PortOfDischargeId</th>\n      <th>...</th>\n      <th>totalResults</th>\n      <th>TotalDistanceMeters</th>\n      <th>totalCo2EmissionsInKg</th>\n      <th>Departure time</th>\n      <th>Arrival time</th>\n      <th>Ports</th>\n      <th>Vessels</th>\n      <th>Dates</th>\n      <th>LocalEstimatedDepartureTime</th>\n      <th>LocalEstimatedArrivalTime</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>11541</th>\n      <td>Maersk</td>\n      <td>&lt;NA&gt;</td>\n      <td>Amsterdam Container Terminal Vrede Amsterdam</td>\n      <td>2023-02-16</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>&lt;NA&gt;</td>\n      <td>&lt;NA&gt;</td>\n      <td>&lt;NA&gt;</td>\n      <td>...</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>[Arica Terminal Puerto Arica (TPA), Balboa Bal...</td>\n      <td>[[JENS MAERSK, -, -, -, -, -], [MAERSK BUTON, ...</td>\n      <td>[2023-02-15 15:06:00, 2023-02-28 12:30:00, 202...</td>\n      <td>2023-02-15 15:06:00</td>\n      <td>2023-03-22 09:59:00</td>\n    </tr>\n    <tr>\n      <th>11542</th>\n      <td>Maersk</td>\n      <td>&lt;NA&gt;</td>\n      <td>Amsterdam Container Terminal Vrede Amsterdam</td>\n      <td>2023-02-16</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>&lt;NA&gt;</td>\n      <td>&lt;NA&gt;</td>\n      <td>&lt;NA&gt;</td>\n      <td>...</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>[Arica Terminal Puerto Arica (TPA), Amsterdam ...</td>\n      <td>[[JENS MAERSK, -, -, -, -, -], [MAERSK BUTON, ...</td>\n      <td>[2023-02-15 15:06:00, 2023-03-22 09:59:00]</td>\n      <td>2023-02-15 15:06:00</td>\n      <td>2023-03-22 09:59:00</td>\n    </tr>\n    <tr>\n      <th>11543</th>\n      <td>Maersk</td>\n      <td>&lt;NA&gt;</td>\n      <td>Amsterdam Container Terminal Vrede Amsterdam</td>\n      <td>2023-02-16</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>&lt;NA&gt;</td>\n      <td>&lt;NA&gt;</td>\n      <td>&lt;NA&gt;</td>\n      <td>...</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>[Arica Terminal Puerto Arica (TPA), Balboa Bal...</td>\n      <td>[[FORT DESAIX, -, -, -, -, -], [MAERSK BRATAN,...</td>\n      <td>[2023-02-22 15:30:00, 2023-03-07 12:30:00, 202...</td>\n      <td>2023-02-22 15:30:00</td>\n      <td>2023-03-29 10:00:00</td>\n    </tr>\n    <tr>\n      <th>11544</th>\n      <td>Maersk</td>\n      <td>&lt;NA&gt;</td>\n      <td>Amsterdam Container Terminal Vrede Amsterdam</td>\n      <td>2023-02-16</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>&lt;NA&gt;</td>\n      <td>&lt;NA&gt;</td>\n      <td>&lt;NA&gt;</td>\n      <td>...</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>[Arica Terminal Puerto Arica (TPA), Amsterdam ...</td>\n      <td>[[FORT DESAIX, -, -, -, -, -], [MAERSK BRATAN,...</td>\n      <td>[2023-02-22 15:30:00, 2023-03-29 10:00:00]</td>\n      <td>2023-02-22 15:30:00</td>\n      <td>2023-03-29 10:00:00</td>\n    </tr>\n    <tr>\n      <th>11545</th>\n      <td>Maersk</td>\n      <td>&lt;NA&gt;</td>\n      <td>Amsterdam Container Terminal Vrede Amsterdam</td>\n      <td>2023-02-16</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>&lt;NA&gt;</td>\n      <td>&lt;NA&gt;</td>\n      <td>&lt;NA&gt;</td>\n      <td>...</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>[Arica Terminal Puerto Arica (TPA), Balboa Bal...</td>\n      <td>[[JEPPESEN MAERSK, -, -, -, -, -], [MAERSK BAL...</td>\n      <td>[2023-03-01 15:30:00, 2023-03-14 12:30:00, 202...</td>\n      <td>2023-03-01 15:30:00</td>\n      <td>2023-04-07 10:00:00</td>\n    </tr>\n    <tr>\n      <th>11546</th>\n      <td>Maersk</td>\n      <td>&lt;NA&gt;</td>\n      <td>Amsterdam Container Terminal Vrede Amsterdam</td>\n      <td>2023-02-16</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>&lt;NA&gt;</td>\n      <td>&lt;NA&gt;</td>\n      <td>&lt;NA&gt;</td>\n      <td>...</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>[Arica Terminal Puerto Arica (TPA), Amsterdam ...</td>\n      <td>[[JEPPESEN MAERSK, -, -, -, -, -], [MAERSK BAL...</td>\n      <td>[2023-03-01 15:30:00, 2023-04-07 10:00:00]</td>\n      <td>2023-03-01 15:30:00</td>\n      <td>2023-04-07 10:00:00</td>\n    </tr>\n    <tr>\n      <th>11547</th>\n      <td>Maersk</td>\n      <td>&lt;NA&gt;</td>\n      <td>Amsterdam Container Terminal Vrede Amsterdam</td>\n      <td>2023-02-16</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>&lt;NA&gt;</td>\n      <td>&lt;NA&gt;</td>\n      <td>&lt;NA&gt;</td>\n      <td>...</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>[Arica Terminal Puerto Arica (TPA), Balboa Bal...</td>\n      <td>[[JENS MAERSK, -, -, -, -, -], [MAERSK BATUR, ...</td>\n      <td>[2023-03-08 15:30:00, 2023-03-21 12:30:00, 202...</td>\n      <td>2023-03-08 15:30:00</td>\n      <td>2023-04-12 10:00:00</td>\n    </tr>\n    <tr>\n      <th>11548</th>\n      <td>Maersk</td>\n      <td>&lt;NA&gt;</td>\n      <td>Amsterdam Container Terminal Vrede Amsterdam</td>\n      <td>2023-02-16</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>&lt;NA&gt;</td>\n      <td>&lt;NA&gt;</td>\n      <td>&lt;NA&gt;</td>\n      <td>...</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>[Arica Terminal Puerto Arica (TPA), Amsterdam ...</td>\n      <td>[[JENS MAERSK, -, -, -, -, -], [MAERSK BATUR, ...</td>\n      <td>[2023-03-08 15:30:00, 2023-04-12 10:00:00]</td>\n      <td>2023-03-08 15:30:00</td>\n      <td>2023-04-12 10:00:00</td>\n    </tr>\n    <tr>\n      <th>11549</th>\n      <td>Maersk</td>\n      <td>&lt;NA&gt;</td>\n      <td>Amsterdam Container Terminal Vrede Amsterdam</td>\n      <td>2023-02-16</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>&lt;NA&gt;</td>\n      <td>&lt;NA&gt;</td>\n      <td>&lt;NA&gt;</td>\n      <td>...</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>[Arica Terminal Puerto Arica (TPA), Balboa Bal...</td>\n      <td>[[FORT DESAIX, -, -, -, -, -], [MAERSK BULAN, ...</td>\n      <td>[2023-03-15 15:30:00, 2023-03-28 12:30:00, 202...</td>\n      <td>2023-03-15 15:30:00</td>\n      <td>2023-04-21 09:59:00</td>\n    </tr>\n    <tr>\n      <th>11550</th>\n      <td>Maersk</td>\n      <td>&lt;NA&gt;</td>\n      <td>Amsterdam Container Terminal Vrede Amsterdam</td>\n      <td>2023-02-16</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>&lt;NA&gt;</td>\n      <td>&lt;NA&gt;</td>\n      <td>&lt;NA&gt;</td>\n      <td>...</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>[Arica Terminal Puerto Arica (TPA), Amsterdam ...</td>\n      <td>[[FORT DESAIX, -, -, -, -, -], [MAERSK BULAN, ...</td>\n      <td>[2023-03-15 15:30:00, 2023-04-21 09:59:00]</td>\n      <td>2023-03-15 15:30:00</td>\n      <td>2023-04-21 09:59:00</td>\n    </tr>\n  </tbody>\n</table>\n<p>10 rows × 40 columns</p>\n</div>"
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Combine the dataframes. The column names are the same, some may have additional columns. Each row is a connection that was found by either MSC or Routescanner and should be in the combined dataframe.\n",
    "df = pd.concat([msc, routescanner, maersk], axis=\"index\", join=\"outer\")\n",
    "df.sort_values(by=['ScrapingDate', 'Origin', 'Destination'], inplace=True)\n",
    "df.head(10)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-11-09T11:45:43.358943200Z",
     "start_time": "2023-11-09T11:45:39.415110900Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "data": {
      "text/plain": "ScrapingSite\nMaersk          158323\nMSC             127273\nRoutescanner     15311\nName: count, dtype: int64"
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Count unique ScrapingSites\n",
    "df['ScrapingSite'].value_counts()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-11-09T11:45:43.371455Z",
     "start_time": "2023-11-09T11:45:43.365942800Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [],
   "source": [
    "# Save the combined dataframe to a pickle file.\n",
    "filename = \"all_connections_combined\"\n",
    "df.to_pickle(f\"../pickles/{filename}.pickle\")\n",
    "df.to_csv(f\"../data/{filename}.csv\")"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-11-09T11:46:02.677374900Z",
     "start_time": "2023-11-09T11:45:43.369455100Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "data": {
      "text/plain": "      ScrapingSite EstimatedArrivalDatePolFormatted  \\\n11541       Maersk                             <NA>   \n11542       Maersk                             <NA>   \n11543       Maersk                             <NA>   \n\n                                        Destination ScrapingDate  \\\n11541  Amsterdam Container Terminal Vrede Amsterdam   2023-02-16   \n11542  Amsterdam Container Terminal Vrede Amsterdam   2023-02-16   \n11543  Amsterdam Container Terminal Vrede Amsterdam   2023-02-16   \n\n      MaritimeServiceName OriginName DestinationName  RouteHeaderId  \\\n11541                <NA>        NaN             NaN           <NA>   \n11542                <NA>        NaN             NaN           <NA>   \n11543                <NA>        NaN             NaN           <NA>   \n\n       PortOfLoadId  PortOfDischargeId  ... totalResults  TotalDistanceMeters  \\\n11541          <NA>               <NA>  ...         <NA>                  NaN   \n11542          <NA>               <NA>  ...         <NA>                  NaN   \n11543          <NA>               <NA>  ...         <NA>                  NaN   \n\n      totalCo2EmissionsInKg Departure time  Arrival time  \\\n11541                  <NA>            NaN           NaN   \n11542                  <NA>            NaN           NaN   \n11543                  <NA>            NaN           NaN   \n\n                                                   Ports  \\\n11541  [Arica Terminal Puerto Arica (TPA), Balboa Bal...   \n11542  [Arica Terminal Puerto Arica (TPA), Amsterdam ...   \n11543  [Arica Terminal Puerto Arica (TPA), Balboa Bal...   \n\n                                                 Vessels  \\\n11541  [[JENS MAERSK, -, -, -, -, -], [MAERSK BUTON, ...   \n11542  [[JENS MAERSK, -, -, -, -, -], [MAERSK BUTON, ...   \n11543  [[FORT DESAIX, -, -, -, -, -], [MAERSK BRATAN,...   \n\n                                                   Dates  \\\n11541  [2023-02-15 15:06:00, 2023-02-28 12:30:00, 202...   \n11542         [2023-02-15 15:06:00, 2023-03-22 09:59:00]   \n11543  [2023-02-22 15:30:00, 2023-03-07 12:30:00, 202...   \n\n      LocalEstimatedDepartureTime LocalEstimatedArrivalTime  \n11541         2023-02-15 15:06:00       2023-03-22 09:59:00  \n11542         2023-02-15 15:06:00       2023-03-22 09:59:00  \n11543         2023-02-22 15:30:00       2023-03-29 10:00:00  \n\n[3 rows x 40 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>ScrapingSite</th>\n      <th>EstimatedArrivalDatePolFormatted</th>\n      <th>Destination</th>\n      <th>ScrapingDate</th>\n      <th>MaritimeServiceName</th>\n      <th>OriginName</th>\n      <th>DestinationName</th>\n      <th>RouteHeaderId</th>\n      <th>PortOfLoadId</th>\n      <th>PortOfDischargeId</th>\n      <th>...</th>\n      <th>totalResults</th>\n      <th>TotalDistanceMeters</th>\n      <th>totalCo2EmissionsInKg</th>\n      <th>Departure time</th>\n      <th>Arrival time</th>\n      <th>Ports</th>\n      <th>Vessels</th>\n      <th>Dates</th>\n      <th>LocalEstimatedDepartureTime</th>\n      <th>LocalEstimatedArrivalTime</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>11541</th>\n      <td>Maersk</td>\n      <td>&lt;NA&gt;</td>\n      <td>Amsterdam Container Terminal Vrede Amsterdam</td>\n      <td>2023-02-16</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>&lt;NA&gt;</td>\n      <td>&lt;NA&gt;</td>\n      <td>&lt;NA&gt;</td>\n      <td>...</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>[Arica Terminal Puerto Arica (TPA), Balboa Bal...</td>\n      <td>[[JENS MAERSK, -, -, -, -, -], [MAERSK BUTON, ...</td>\n      <td>[2023-02-15 15:06:00, 2023-02-28 12:30:00, 202...</td>\n      <td>2023-02-15 15:06:00</td>\n      <td>2023-03-22 09:59:00</td>\n    </tr>\n    <tr>\n      <th>11542</th>\n      <td>Maersk</td>\n      <td>&lt;NA&gt;</td>\n      <td>Amsterdam Container Terminal Vrede Amsterdam</td>\n      <td>2023-02-16</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>&lt;NA&gt;</td>\n      <td>&lt;NA&gt;</td>\n      <td>&lt;NA&gt;</td>\n      <td>...</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>[Arica Terminal Puerto Arica (TPA), Amsterdam ...</td>\n      <td>[[JENS MAERSK, -, -, -, -, -], [MAERSK BUTON, ...</td>\n      <td>[2023-02-15 15:06:00, 2023-03-22 09:59:00]</td>\n      <td>2023-02-15 15:06:00</td>\n      <td>2023-03-22 09:59:00</td>\n    </tr>\n    <tr>\n      <th>11543</th>\n      <td>Maersk</td>\n      <td>&lt;NA&gt;</td>\n      <td>Amsterdam Container Terminal Vrede Amsterdam</td>\n      <td>2023-02-16</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>&lt;NA&gt;</td>\n      <td>&lt;NA&gt;</td>\n      <td>&lt;NA&gt;</td>\n      <td>...</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>&lt;NA&gt;</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>[Arica Terminal Puerto Arica (TPA), Balboa Bal...</td>\n      <td>[[FORT DESAIX, -, -, -, -, -], [MAERSK BRATAN,...</td>\n      <td>[2023-02-22 15:30:00, 2023-03-07 12:30:00, 202...</td>\n      <td>2023-02-22 15:30:00</td>\n      <td>2023-03-29 10:00:00</td>\n    </tr>\n  </tbody>\n</table>\n<p>3 rows × 40 columns</p>\n</div>"
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_pickle('../pickles/all_connections_combined.pickle').head(3)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-11-09T11:46:06.551303700Z",
     "start_time": "2023-11-09T11:46:02.678379500Z"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}