🐛 fix catalogPath for cases with multiple tables

etl/steps/__init__.py

@@ -780,16 +780,6 @@ def run(self) -> None:
 
                 table = gh._adapt_table_for_grapher(table)
 
-                # generate table with entity_id, year and value for every column
-                upsert = lambda t, catalog_path, verbose: gi.upsert_table(  # noqa: E731
-                    engine,
-                    t,
-                    dataset_upsert_results,
-                    catalog_path=catalog_path,
-                    dimensions=(t.iloc[:, 0].metadata.additional_info or {}).get("dimensions"),
-                    verbose=verbose,
-                )
-
                 for t in gh._yield_wide_table(table, na_action="drop"):
                     i += 1
                     assert len(t.columns) == 1
@@ -802,7 +792,18 @@ def run(self) -> None:
                             lambda: (time.sleep(10), log.info("upsert_dataset.continue_without_logging"))
                         )
 
-                    futures.append(thread_pool.submit(upsert, t, catalog_path=catalog_path, verbose=verbose))
+                    # generate table with entity_id, year and value for every column
+                    futures.append(
+                        thread_pool.submit(
+                            gi.upsert_table,
+                            engine,
+                            t,
+                            dataset_upsert_results,
+                            catalog_path=catalog_path,
+                            dimensions=(t.iloc[:, 0].metadata.additional_info or {}).get("dimensions"),
+                            verbose=verbose,
+                        )
+                    )
 
             variable_upsert_results = [future.result() for future in as_completed(futures)]
 

etl/steps/data/grapher/biodiversity/2023-01-11/cherry_blossom.py

@@ -1,8 +1,6 @@
 """Load a garden dataset and create a grapher dataset."""
 
-from typing import cast
 
-from owid.catalog import Dataset
 
 from etl.helpers import PathFinder, create_dataset, grapher_checks
 
@@ -15,7 +13,7 @@ def run(dest_dir: str) -> None:
     # Load inputs.
     #
     # Load garden dataset.
-    ds_garden = cast(Dataset, paths.load_dependency("cherry_blossom"))
+    ds_garden = paths.load_dataset("cherry_blossom")
 
     # Read table from garden dataset.
     tb = ds_garden["cherry_blossom"]

etl/steps/data/grapher/ihme_gbd/2019/gbd_cause.py

@@ -1,5 +1,3 @@
-from owid import catalog
-
 from etl.helpers import PathFinder
 
 from .shared import run_wrapper
@@ -8,6 +6,5 @@
 
 
 def run(dest_dir: str) -> None:
-    garden_dataset = paths.garden_dataset
-    dataset = catalog.Dataset.create_empty(dest_dir, garden_dataset.metadata)
-    dataset = run_wrapper(garden_dataset=garden_dataset, dataset=dataset)
+    ds_garden = paths.load_dataset("gbd_cause")
+    run_wrapper(dest_dir, garden_dataset=ds_garden)

etl/steps/data/grapher/ihme_gbd/2019/gbd_child_mortality.py

@@ -1,5 +1,3 @@
-from owid import catalog
-
 from etl.helpers import PathFinder
 
 from .shared import run_wrapper
@@ -8,8 +6,5 @@
 
 
 def run(dest_dir: str) -> None:
-    garden_dataset = paths.garden_dataset
-    dataset = catalog.Dataset.create_empty(dest_dir, garden_dataset.metadata)
-    # dataset.save()
-
-    run_wrapper(garden_dataset=garden_dataset, dataset=dataset)
+    ds_garden = paths.load_dataset("gbd_child_mortality")
+    run_wrapper(dest_dir, garden_dataset=ds_garden)

etl/steps/data/grapher/ihme_gbd/2019/gbd_mental_health.py

@@ -1,5 +1,3 @@
-from owid import catalog
-
 from etl.helpers import PathFinder
 
 from .shared import run_wrapper
@@ -8,6 +6,5 @@
 
 
 def run(dest_dir: str) -> None:
-    garden_dataset = paths.garden_dataset
-    dataset = catalog.Dataset.create_empty(dest_dir, garden_dataset.metadata)
-    dataset = run_wrapper(garden_dataset=garden_dataset, dataset=dataset)
+    ds_garden = paths.load_dataset("gbd_mental_health")
+    run_wrapper(dest_dir, garden_dataset=ds_garden)

etl/steps/data/grapher/ihme_gbd/2019/gbd_prevalence.py

@@ -1,5 +1,3 @@
-from owid import catalog
-
 from etl.helpers import PathFinder
 
 from .shared import run_wrapper
@@ -8,6 +6,5 @@
 
 
 def run(dest_dir: str) -> None:
-    garden_dataset = paths.garden_dataset
-    dataset = catalog.Dataset.create_empty(dest_dir, garden_dataset.metadata)
-    run_wrapper(garden_dataset=garden_dataset, dataset=dataset)
+    ds_garden = paths.load_dataset("gbd_prevalence")
+    run_wrapper(dest_dir, garden_dataset=ds_garden)

etl/steps/data/grapher/ihme_gbd/2019/gbd_risk.py

@@ -1,5 +1,3 @@
-from owid import catalog
-
 from etl.helpers import PathFinder
 
 from .shared import run_wrapper
@@ -8,6 +6,5 @@
 
 
 def run(dest_dir: str) -> None:
-    garden_dataset = paths.garden_dataset
-    dataset = catalog.Dataset.create_empty(dest_dir, garden_dataset.metadata)
-    run_wrapper(garden_dataset=garden_dataset, dataset=dataset)
+    ds_garden = paths.load_dataset("gbd_risk")
+    run_wrapper(dest_dir, garden_dataset=ds_garden)

etl/steps/data/grapher/ihme_gbd/2019/shared.py

@@ -1,13 +1,25 @@
 from owid.catalog import Dataset
-from structlog import get_logger
 
-log = get_logger()
+from etl.helpers import create_dataset, grapher_checks
 
 
-def run_wrapper(garden_dataset: Dataset, dataset: Dataset) -> Dataset:
-    # add tables to dataset
-    tables = garden_dataset.table_names
-    for table in tables:
-        tab = garden_dataset[table]
-        dataset.add(tab)
-    dataset.save()
+def run_wrapper(dest_dir: str, garden_dataset: Dataset) -> None:
+    # Read tables from garden dataset.
+    tables = []
+    for table_name in garden_dataset.table_names[:10] + ["diarrheal_diseases__both_sexes__age_standardized"]:
+        tb = garden_dataset[table_name]
+        tables.append(tb)
+
+    #
+    # Save outputs.
+    #
+    # Create a new grapher dataset with the same metadata as the garden dataset.
+    ds_grapher = create_dataset(dest_dir, tables=tables, default_metadata=garden_dataset.metadata)
+
+    #
+    # Checks.
+    #
+    grapher_checks(ds_grapher)
+
+    # Save changes in the new grapher dataset.
+    ds_grapher.save()