Refactor archive clean datasets (#3933)

* 🔨 archive and clean datasets

* add technology dag

* archive metaculus

* archive datasets

dag/archive/fasttrack.yml

@@ -23,3 +23,21 @@ steps:
     - snapshot://fasttrack/2024-06-17/guinea_worm.csv
   data://grapher/fasttrack/2023-06-28/guinea_worm:
     - snapshot://fasttrack/2023-06-28/guinea_worm.csv
+
+  # Others
+  data-private://grapher/fasttrack/latest/metaculus__complementary_data:
+    - snapshot-private://fasttrack/latest/metaculus__complementary_data.csv
+  data://grapher/fasttrack/2023-05-03/qubits:
+    - snapshot://fasttrack/2023-05-03/qubits.csv
+  data://grapher/fasttrack/latest/air_pollution_emissions_by_sector__ceds__2024:
+    - snapshot://fasttrack/latest/air_pollution_emissions_by_sector__ceds__2024.csv
+  data-private://grapher/fasttrack/latest/literate_population:
+    - snapshot-private://fasttrack/latest/literate_population.csv
+  data-private://grapher/fasttrack/latest/germany_hypothetical_constant_nuclear:
+    - snapshot-private://fasttrack/latest/germany_hypothetical_constant_nuclear.csv
+  data-private://grapher/fasttrack/latest/battery_costs_per_kwh__ziegler_et_al__and_bnef:
+    - snapshot-private://fasttrack/latest/battery_costs_per_kwh__ziegler_et_al__and_bnef.csv
+  data-private://grapher/fasttrack/latest/fiscal_top1_shares_country_standardized:
+    - snapshot-private://fasttrack/latest/fiscal_top1_shares_country_standardized.csv
+  data://grapher/fasttrack/latest/table_5__threatened_species_in_each_major_group_by_country__show_all__1:
+    - snapshot://fasttrack/latest/table_5__threatened_species_in_each_major_group_by_country__show_all__1.csv

dag/archive/main.yml

@@ -377,6 +377,15 @@ steps:
   data://grapher/wpf/2024-10-03/famines_by_place:
     - data://garden/wpf/2024-10-03/famines_by_place
 
+  # Missing Data - Children out of school
+  data://garden/missing_data/2024-03-26/children_out_of_school:
+    - data://garden/wb/2024-06-10/gender_statistics
+    - data://garden/wb/2024-03-11/income_groups
+    - data://garden/regions/2023-01-01/regions
+    - data://garden/demography/2023-03-31/population
+  data://grapher/missing_data/2024-03-26/children_out_of_school:
+    - data://garden/missing_data/2024-03-26/children_out_of_school
+
 include:
   # Include all active steps plus all archive steps.
   - dag/main.yml

dag/fasttrack.yml

@@ -6,24 +6,14 @@ steps:
   # Long-term homicide rates in Europe - Eisner (2014)
   data-private://grapher/fasttrack/2022-11-01/lighting_efficiency_uk:
     - snapshot-private://fasttrack/2022-11-01/lighting_efficiency_uk.csv
-  data-private://grapher/fasttrack/latest/metaculus__complementary_data:
-    - snapshot-private://fasttrack/latest/metaculus__complementary_data.csv
   data-private://grapher/fasttrack/latest/global_extreme_poverty_future_scenario:
     - snapshot-private://fasttrack/latest/global_extreme_poverty_future_scenario.csv
   data-private://grapher/fasttrack/latest/child_mortality_future_projections:
     - snapshot-private://fasttrack/latest/child_mortality_future_projections.csv
-  data-private://grapher/fasttrack/latest/literate_population:
-    - snapshot-private://fasttrack/latest/literate_population.csv
-  data-private://grapher/fasttrack/latest/battery_costs_per_kwh__ziegler_et_al__and_bnef:
-    - snapshot-private://fasttrack/latest/battery_costs_per_kwh__ziegler_et_al__and_bnef.csv
   data-private://grapher/fasttrack/latest/biosafety_level_4_facilities:
     - snapshot-private://fasttrack/latest/biosafety_level_4_facilities.csv
-  data-private://grapher/fasttrack/latest/germany_hypothetical_constant_nuclear:
-    - snapshot-private://fasttrack/latest/germany_hypothetical_constant_nuclear.csv
   data-private://grapher/fasttrack/latest/oil_prices:
     - snapshot-private://fasttrack/latest/oil_prices.csv
-  data://grapher/fasttrack/latest/table_5__threatened_species_in_each_major_group_by_country__show_all__1:
-    - snapshot://fasttrack/latest/table_5__threatened_species_in_each_major_group_by_country__show_all__1.csv
   data-private://grapher/fasttrack/latest/evs_per_charger:
     - snapshot-private://fasttrack/latest/evs_per_charger.csv
   data-private://grapher/fasttrack/latest/who_standard_pop:
@@ -58,8 +48,6 @@ steps:
     - snapshot-private://fasttrack/latest/lead_paint_regulation_who.csv
   data://grapher/fasttrack/latest/whm_treatment_gap_anxiety_disorders:
     - snapshot://fasttrack/latest/whm_treatment_gap_anxiety_disorders.csv
-  data-private://grapher/fasttrack/latest/fiscal_top1_shares_country_standardized:
-    - snapshot-private://fasttrack/latest/fiscal_top1_shares_country_standardized.csv
   data-private://grapher/fasttrack/latest/pain_hours_hen_systems:
     - snapshot-private://fasttrack/latest/pain_hours_hen_systems.csv
   data-private://grapher/fasttrack/latest/antibiotic_usage_livestock:
@@ -108,8 +96,6 @@ steps:
     - snapshot://fasttrack/latest/public_support_climate_andre.csv
   data://grapher/fasttrack/latest/public_support_climate_vlasceanu:
     - snapshot://fasttrack/latest/public_support_climate_vlasceanu.csv
-  data://grapher/fasttrack/2023-05-03/qubits:
-    - snapshot://fasttrack/2023-05-03/qubits.csv
   data://grapher/fasttrack/latest/completeness_disaster_emdat:
     - snapshot://fasttrack/latest/completeness_disaster_emdat.csv
   data-private://grapher/fasttrack/latest/vdem_women_executives:
@@ -148,8 +134,6 @@ steps:
     - snapshot-private://fasttrack/2024-04-25/joe_phd_short_period.csv
   data://grapher/fasttrack/latest/air_pollution_ceds:
     - snapshot://fasttrack/latest/air_pollution_ceds.csv
-  data://grapher/fasttrack/latest/air_pollution_emissions_by_sector__ceds__2024:
-    - snapshot://fasttrack/latest/air_pollution_emissions_by_sector__ceds__2024.csv
   data://grapher/fasttrack/latest/air_pollution_ceds_by_sector:
     - snapshot://fasttrack/latest/air_pollution_ceds_by_sector.csv
   data://grapher/fasttrack/latest/lives_saved_vaccination_who:

dag/main.yml

@@ -90,7 +90,7 @@ steps:
   data://garden/ggdc/2022-12-23/maddison_database:
     - data://meadow/ggdc/2022-12-23/maddison_database
 
-  #Penn World Table
+  # Penn World Table
   data://meadow/ggdc/2022-11-28/penn_world_table:
     - walden://ggdc/2021-06-18/penn_world_table
   data://meadow/ggdc/2022-11-28/penn_world_table_national_accounts:
@@ -102,14 +102,6 @@ steps:
   data://grapher/ggdc/2022-11-28/penn_world_table:
     - data://garden/ggdc/2022-11-28/penn_world_table
 
-  # Global mobile money dataset (GSMA)
-  data://meadow/technology/2024-05-30/mobile_money:
-    - snapshot://technology/2024-05-30/mobile_money.xlsx
-  data://garden/technology/2024-05-30/mobile_money:
-    - data://meadow/technology/2024-05-30/mobile_money
-  data://grapher/technology/2024-05-30/mobile_money:
-    - data://garden/technology/2024-05-30/mobile_money
-
   # Democracy and Human rights - V-Dem index
   data://meadow/democracy/2023-03-02/vdem:
     - snapshot://democracy/2023-03-02/vdem.csv
@@ -428,15 +420,6 @@ steps:
   data://grapher/survey/2023-08-04/trust_surveys:
     - data://garden/survey/2023-08-04/trust_surveys
 
-  # Missing Data - Children out of school
-  data://garden/missing_data/2024-03-26/children_out_of_school:
-    - data://garden/wb/2024-06-10/gender_statistics
-    - data://garden/wb/2024-03-11/income_groups
-    - data://garden/regions/2023-01-01/regions
-    - data://garden/demography/2023-03-31/population
-  data://grapher/missing_data/2024-03-26/children_out_of_school:
-    - data://garden/missing_data/2024-03-26/children_out_of_school
-
   # Missing Data - Suicides
   data://garden/missing_data/2024-03-26/who_md_suicides:
     - data://garden/who/2024-03-24/self_inflicted_injuries
@@ -759,9 +742,6 @@ steps:
     - data://garden/demography/2023-03-31/population
     - data://garden/regions/2023-01-01/regions
 
-
-
-
 include:
   - dag/open_numbers.yml
   - dag/faostat.yml
@@ -800,3 +780,4 @@ include:
   - dag/migration.yml
   - dag/equality.yml
   - dag/families.yml
+  - dag/technology.yml

dag/technology.yml

@@ -0,0 +1,8 @@
+steps:
+  # Global mobile money dataset (GSMA)
+  data://meadow/technology/2024-05-30/mobile_money:
+    - snapshot://technology/2024-05-30/mobile_money.xlsx
+  data://garden/technology/2024-05-30/mobile_money:
+    - data://meadow/technology/2024-05-30/mobile_money
+  data://grapher/technology/2024-05-30/mobile_money:
+    - data://garden/technology/2024-05-30/mobile_money