Merge pull request #282 from adrienmellot/feature-transport-timeseries

Implement realistic timeseries for transport demand

CHANGELOG.md

@@ -4,7 +4,7 @@
 
 ### Added (models)
 
-* **ADD** fully-electrified road transportation (#270).
+* **ADD** fully-electrified road transportation (#270), (#271).
 
 * **ADD** nuclear power plant technology with capacity limits. Capacity limits can be equal to today or be bound by a minimum and maximum capacity to represent an available range in future. In either case, capacities are allocated at a subnational resolution based on linear scaling from current capacity geolocations, using the JRC power plant database (#78).
 

config/default.yaml

@@ -17,6 +17,7 @@ data-sources:
     eurostat-energy-balance: https://raw.githubusercontent.com/calliope-project/euro-calliope-datasets/feature-sector-coupling/eurostat/nrg_bal_c.tsv.gz # FIXME do not use cached data
     swiss-energy-balance: https://www.bfe.admin.ch/bfe/en/home/versorgung/statistik-und-geodaten/energiestatistiken/gesamtenergiestatistik.exturl.html/aHR0cHM6Ly9wdWJkYi5iZmUuYWRtaW4uY2gvZGUvcHVibGljYX/Rpb24vZG93bmxvYWQvNzUxOQ==.html
     swiss-industry-energy-balance: https://www.bfe.admin.ch/bfe/en/home/versorgung/statistik-und-geodaten/energiestatistiken/teilstatistiken.exturl.html/aHR0cHM6Ly9wdWJkYi5iZmUuYWRtaW4uY2gvZGUvcHVibGljYX/Rpb24vZG93bmxvYWQvODc4OA==.html
+    ev-data: https://zenodo.org/record/6579421/files/ramp-ev-consumption-profiles.csv.gz?download=1
 root-directory: .
 cluster-sync:
     url: euler.ethz.ch
@@ -134,19 +135,29 @@ parameters:
             coaches-and-buses: 3248
             passenger-cars: 324
             motorcycles: 200  # based on passenger car electrical efficiency scaled by relative diesel efficiency
-        names:
+        vehicle-type-names:
             light-duty-vehicles: Light duty vehicles
             heavy-duty-vehicles: Heavy duty vehicles
             coaches-and-buses: Motor coaches, buses and trolley buses
             passenger-cars: Passenger cars
             motorcycles: Powered 2-wheelers
         fill-missing-values:
-            ALB: ['BGR', 'HRV', 'HUN', 'ROU', 'GRC']
-            BIH: ['BGR', 'HRV', 'HUN', 'ROU', 'GRC']
-            MNE: ['BGR', 'HRV', 'HUN', 'ROU', 'GRC']
-            SRB: ['BGR', 'HRV', 'HUN', 'ROU', 'GRC']
-            NOR: ['SWE', 'DNK']
-            CHE: ['DEU', 'AUT', 'FRA', 'ITA']
+            annual-data:
+                ALB: ['BGR', 'HRV', 'HUN', 'ROU', 'GRC']
+                BIH: ['BGR', 'HRV', 'HUN', 'ROU', 'GRC']
+                MNE: ['BGR', 'HRV', 'HUN', 'ROU', 'GRC']
+                MKD: ['BGR', 'HRV', 'HUN', 'ROU', 'GRC']
+                SRB: ['BGR', 'HRV', 'HUN', 'ROU', 'GRC']
+                NOR: ['SWE', 'DNK']
+                CHE: ['DEU', 'AUT', 'FRA', 'ITA']
+            timeseries-data:
+                ALB: ['HRV']
+                MKD: ['HRV']
+                GRC: ['ROU']
+                BGR: ['ROU']
+                BIH: ['HRV', 'HUN']
+                MNE: ['HRV']
+                SRB: ['HUN']
     entsoe-tyndp:
         scenario: National Trends
         grid: Reference

config/schema.yaml

@@ -95,6 +95,10 @@ properties:
                 type: string
                 pattern: ^(https?|http?):\/\/.+
                 description: Web address of Swiss industry energy balance data.
+            ev-data:
+                type: string
+                pattern: ^(https?|http?):\/\/.+
+                description: Web address of electric vehicle data.
     root-directory:
         type: string
         description: Path to the root directory of euro-calliope containing scripts and template folders.
@@ -282,7 +286,7 @@ properties:
                             motorcycles:
                                 type: number
                                 description: Motorcylces.
-                    names:
+                    vehicle-type-names:
                         type: object
                         description: Names of vehicle types in JRC-IDEES.
                         additionalProperties: false
@@ -303,17 +307,34 @@ properties:
                                 type: string
                                 description: JRC-IDEES name of motorcycles.
                     fill-missing-values:
-                        description: Fill missing values in annual transport demand.
                         type: object
+                        description: Dictionaries to fill missing values
                         additionalProperties: false
-                        patternProperties:
-                            "^[A-Z][A-Z][A-Z]$":
-                                type: array
-                                description: Country to fill missing values for.
-                                items:
-                                    type: string
-                                    pattern: ^[A-Z][A-Z][A-Z]$
-                                    description: Country to fill missing values from.
+                        properties:
+                            annual-data:
+                                description: Fill missing values in annual transport demand.
+                                type: object
+                                additionalProperties: false
+                                patternProperties:
+                                    "^[A-Z][A-Z][A-Z]$":
+                                        type: array
+                                        description: Country to fill missing values for.
+                                        items:
+                                            type: string
+                                            pattern: ^[A-Z][A-Z][A-Z]$
+                                            description: Country to fill missing values from.
+                            timeseries-data:
+                                description: Fill missing countries in timeseries profile.
+                                type: object
+                                additionalProperties: false
+                                patternProperties:
+                                    "^[A-Z][A-Z][A-Z]$":
+                                        type: array
+                                        description: Country to fill missing values for.
+                                        items:
+                                            type: string
+                                            pattern: ^[A-Z][A-Z][A-Z]$
+                                            description: Country to fill missing values from.
             entsoe-tyndp:
                 type: object
                 description: Parameters to define scenario choice for data accessed from the ENTSO-E ten-year network development plan 2020. For more information, see https://2020.entsos-tyndp-scenarios.eu/

rules/data.smk

@@ -8,6 +8,7 @@ rule download_ch_energy_balances:
     params:
         url = lambda wildcards: config["data-sources"][f"swiss-{wildcards.dataset}"]
     output: protected("data/automatic/ch-{dataset}.xlsx")
+    conda: "../envs/shell.yaml"
     wildcard_constraints:
         dataset = "((energy-balance)|(industry-energy-balance))"
     localrule: True
@@ -21,6 +22,7 @@ rule download_eurostat_annual_energy_balances:
     message: "Download Eurostat Annual Energy Balances from euro-calliope datasets"
     params:
         url = config["data-sources"]["eurostat-energy-balance"]
+    conda: "../envs/shell.yaml"
     output: protected("data/automatic/eurostat-energy-balance.tsv.gz")
     localrule: True
     shell: "curl -sLo {output} {params.url}"
@@ -81,7 +83,9 @@ rule jrc_idees_transport_processed:
             country_code=EU28
         )
     output: "build/data/jrc-idees/transport/processed-{dataset}.csv"
+    params:
+        vehicle_type_names = config["parameters"]["transport"]["vehicle-type-names"],
     wildcard_constraints:
-        dataset = "((road-energy)|(road-distance)|(road-vehicles))"
+        dataset = "road-energy|road-distance|road-vehicles"
     conda: "../envs/default.yaml"
     script: "../scripts/transport/jrc_idees.py"

rules/transport.smk

@@ -1,5 +1,13 @@
 """Rules to process transport sector data."""
 
+rule download_transport_timeseries:
+    message: "Get EV data from RAMP"
+    params:
+        url = config["data-sources"]["ev-data"]
+    conda: "../envs/shell.yaml"
+    output: protected("data/automatic/ramp-ev-consumption-profiles.csv.gz")
+    localrule: True
+    shell: "curl -sLo {output} {params.url}"
 
 rule annual_transport_demand:
     message: "Calculate future transport energy demand based on JRC IDEES"
@@ -8,7 +16,7 @@ rule annual_transport_demand:
         jrc_road_energy = "build/data/jrc-idees/transport/processed-road-energy.csv",
         jrc_road_distance = "build/data/jrc-idees/transport/processed-road-distance.csv",
     params:
-        fill_missing_values = config["parameters"]["transport"]["fill-missing-values"],
+        fill_missing_values = config["parameters"]["transport"]["fill-missing-values"]["annual-data"],
         efficiency_quantile = config["parameters"]["transport"]["future-vehicle-efficiency-percentile"]
     conda: "../envs/default.yaml"
     output:
@@ -20,35 +28,39 @@ rule annual_transport_demand:
 rule create_road_transport_timeseries:
     message: "Create timeseries for road transport demand"
     input:
-        data = "build/data/transport/annual-road-transport-distance-demand.csv",
+        annual_data = "build/data/transport/annual-road-transport-distance-demand.csv",
+        timeseries = "data/automatic/ramp-ev-consumption-profiles.csv.gz"
     params:
         first_year = config["scope"]["temporal"]["first-year"],
         final_year = config["scope"]["temporal"]["final-year"],
         power_scaling_factor = config["scaling-factors"]["power"],
-        conversion_factor = lambda wildcards: config["parameters"]["transport"]["road-transport-conversion-factors"][wildcards.type],
-        type_name = lambda wildcards: config["parameters"]["transport"]["names"][wildcards.type],
-        historic = False
+        conversion_factor = lambda wildcards: config["parameters"]["transport"]["road-transport-conversion-factors"][wildcards.vehicle_type],
+        historic = False,
+        countries = config["scope"]["spatial"]["countries"],
+        country_neighbour_dict= config["parameters"]["transport"]["fill-missing-values"]["timeseries-data"],
     conda: "../envs/default.yaml"
     wildcard_constraints:
-        type = "light-duty-vehicles|heavy-duty-vehicles|coaches-and-buses|passenger-cars|motorcycles"
+        vehicle_type = "light-duty-vehicles|heavy-duty-vehicles|coaches-and-buses|passenger-cars|motorcycles"
     output:
-        main = "build/data/transport/timeseries/timeseries-{type}.csv",
+        main = "build/data/transport/timeseries/timeseries-{vehicle_type}.csv",
     script: "../scripts/transport/road_transport_timeseries.py"
 
 
 use rule create_road_transport_timeseries as create_road_transport_timeseries_historic_electrification with:
     message: "Create timeseries for historic electrified road transport demand"
     input:
-        data = "build/data/transport/annual-road-transport-historic-electrification.csv"
+        annual_data = "build/data/transport/annual-road-transport-historic-electrification.csv",
+        timeseries = "data/automatic/ramp-ev-consumption-profiles.csv.gz",
     params:
         first_year = config["scope"]["temporal"]["first-year"],
         final_year = config["scope"]["temporal"]["final-year"],
         power_scaling_factor = config["scaling-factors"]["power"],
-        conversion_factor = lambda wildcards: config["parameters"]["transport"]["road-transport-conversion-factors"][wildcards.type],
-        type_name = lambda wildcards: config["parameters"]["transport"]["names"][wildcards.type],
-        historic = True
+        conversion_factor = lambda wildcards: config["parameters"]["transport"]["road-transport-conversion-factors"][wildcards.vehicle_type],
+        historic = True,
+        countries = config["scope"]["spatial"]["countries"],
+        country_neighbour_dict= config["parameters"]["transport"]["fill-missing-values"]["timeseries-data"],
     output:
-        "build/data/transport/timeseries/timeseries-{type}-historic-electrification.csv"
+        "build/data/transport/timeseries/timeseries-{vehicle_type}-historic-electrification.csv"
 
 
 rule aggregate_timeseries: # TODO consider merge with other rules, as this is tiny atm
@@ -62,8 +74,6 @@ rule aggregate_timeseries: # TODO consider merge with other rules, as this is ti
             "build/data/transport/timeseries/timeseries-motorcycles.csv"),
         locations = "build/data/regional/units.csv",
         populations = "build/data/regional/population.csv"
-    params:
-        countries = config["scope"]["spatial"]["countries"]
     conda: "../envs/default.yaml"
     output:
         "build/models/{resolution}/timeseries/demand/electrified-road-transport.csv",

scripts/transport/aggregate_timeseries.py

@@ -2,22 +2,21 @@
 import pycountry
 
 
-def create_continental_timeseries(paths_to_input: list[str], country_codes: list[str]) -> pd.DataFrame:
-    ts = create_national_timeseries(paths_to_input, country_codes)
+def create_continental_timeseries(paths_to_input: list[str]) -> pd.DataFrame:
+    ts = create_national_timeseries(paths_to_input)
     return ts.sum(axis=1).rename("EUR")
 
 
-def create_national_timeseries(paths_to_input: list[str], country_codes: list[str]) -> pd.DataFrame:
+def create_national_timeseries(paths_to_input: list[str]) -> pd.DataFrame:
     all_ts = [
         pd.read_csv(path, index_col='utc-timestamp', parse_dates=True)
         for path in paths_to_input
     ]
-    return sum(all_ts).loc[:, country_codes]
+    return sum(all_ts)
 
 
 def create_regional_timeseries(
     paths_to_input: list[str],
-    country_codes: list[str],
     region_country_mapping: str,
     population: str,
 ) -> pd.DataFrame:
@@ -30,7 +29,7 @@ def create_regional_timeseries(
     ASSUME all road transport is subnationally distributed in proportion to population.
     """
 
-    df_national = create_national_timeseries(paths_to_input, country_codes)
+    df_national = create_national_timeseries(paths_to_input)
 
     region_country_mapping = (
         pd.read_csv(region_country_mapping, index_col=0)
@@ -66,17 +65,16 @@ def create_regional_timeseries(
 if __name__ == "__main__":
     resolution=snakemake.wildcards.resolution
     paths_to_input=snakemake.input.time_series
-    country_codes=[pycountry.countries.lookup(c).alpha_3 for c in snakemake.params.countries]
     path_to_output=snakemake.output[0]
     path_to_locations=snakemake.input.locations
     path_to_populations=snakemake.input.populations
 
     if resolution == "continental":
-        ts = create_continental_timeseries(paths_to_input, country_codes)
+        ts = create_continental_timeseries(paths_to_input)
     elif resolution == "national":
-        ts = create_national_timeseries(paths_to_input, country_codes)
+        ts = create_national_timeseries(paths_to_input)
     elif resolution == "regional":
-        ts = create_regional_timeseries(paths_to_input, country_codes, path_to_locations, path_to_populations)
+        ts = create_regional_timeseries(paths_to_input, path_to_locations, path_to_populations)
     else:
         raise ValueError("Input resolution not recognised.")
 

scripts/transport/jrc_idees.py

@@ -39,7 +39,7 @@
 }
 
 
-def process_jrc_transport_data(paths_to_data: list[str], dataset: object, out_path: str):
+def process_jrc_transport_data(paths_to_data: list[str], dataset: object, out_path: str, vehicle_type_names: dict[str, str]) -> None:
     paths_to_data = [Path(p) for p in paths_to_data]
     processed_data = pd.concat([
         read_transport_excel(path, **DATASET_PARAMS[dataset])
@@ -54,10 +54,10 @@ def process_jrc_transport_data(paths_to_data: list[str], dataset: object, out_pa
         .set_index("country_code", append=True)
         .stack('year')
         .rename('value')
+        .rename(index = vehicle_type_names, level = 'vehicle_type')
         .to_csv(out_path)
     )
 
-    processed_data
 
 
 def read_transport_excel(path: Path,
@@ -176,5 +176,6 @@ def remove_of_which(df: pd.DataFrame, main_carrier: str, of_which_carrier: str)
     process_jrc_transport_data(
         paths_to_data=snakemake.input.data,
         dataset=snakemake.wildcards.dataset,
-        out_path=snakemake.output[0]
+        out_path=snakemake.output[0],
+        vehicle_type_names={v:k for k,v in snakemake.params.vehicle_type_names.items()},
     )