Merge branch 'main' into sid/no-copy-inplace

xl2times/__main__.py

@@ -129,14 +129,14 @@ def convert_xl_to_times(
         transforms.process_transform_availability,
         transforms.process_flexible_import_tables,  # slow
         transforms.process_user_constraint_tables,
-        transforms.process_tradelinks,
+        transforms.harmonise_tradelinks,
         transforms.include_tables_source,
         transforms.process_processes,
         transforms.create_model_topology,
         transforms.fill_in_column_defaults,
         transforms.generate_uc_properties,
         transforms.expand_rows_parallel,  # slow
-        transforms.process_trade_links,
+        transforms.process_tradelinks,
         transforms.merge_tables,
         transforms.remove_invalid_values,
         transforms.include_cgs_in_topology,
@@ -148,7 +148,7 @@ def convert_xl_to_times(
         transforms.enforce_availability,
         transforms.complete_model_trade,
         transforms.create_model_cgs,
-        transforms.apply_fixups,
+        transforms.prepare_for_querying,
         transforms.apply_transform_tables,
         transforms.generate_implied_topology,
         transforms.verify_uc_topology,

xl2times/transforms.py

@@ -1202,12 +1202,18 @@ def _populate_calculated_defaults(df: DataFrame, model: TimesModel):
         )
 
 
-def apply_fixups(
+def prepare_for_querying(
     config: Config,
     tables: dict[str, DataFrame],
     model: TimesModel,
 ) -> dict[str, DataFrame]:
-    """Apply fixups to the tables."""
+    """Prepare tables for querying by harmonising them and filling in missing values.
+    This includes
+    - Adding expected columns with NA if they are missing.
+    - Expanding year columns if they contain ranges.
+    - Applying attribute-specific default values (incl. if specified under a different column).
+    - Filling in missing values in year and currency columns with defaults.
+    """
     exclude_tags = {Tag.tfm_mig, Tag.tfm_upd}
     attribute_tags = [Tag.fi_t, Tag.uc_t]
     currency = tables[Tag.currencies]["currency"][0]
@@ -1443,12 +1449,12 @@ def create_model_cgs(
     return tables
 
 
-def process_trade_links(
+def process_tradelinks(
     config: Config,
     tables: list[EmbeddedXlTable],
     model: TimesModel,
 ) -> list[EmbeddedXlTable]:
-    """Process trade links and create model trade (between internal regions)."""
+    """Process tradelinks and create model trade (between internal regions)."""
     cols_list = ["origin", "in", "destination", "out", "process"]
     cols_list.extend(dm_cols)
     result = []
@@ -1811,7 +1817,7 @@ def generate_dummy_processes(
     return tables
 
 
-def process_tradelinks(
+def harmonise_tradelinks(
     config: Config,
     tables: list[EmbeddedXlTable],
     model: TimesModel,
@@ -3183,7 +3189,7 @@ def apply_final_fixup(
     tables: dict[str, DataFrame],
     model: TimesModel,
 ) -> dict[str, DataFrame]:
-
+    """Clean up remaining issues. Includes handling of aliases that could not be generalised."""
     veda_ire_sets = model.custom_sets
     reg_com_flows = model.topology[["region", "process", "commodity"]].drop_duplicates(
         ignore_index=True