From a534ef47cd7f997cfaeba4da9f092efc2aa855a0 Mon Sep 17 00:00:00 2001 From: Veronika Samborska <32176660+veronikasamborska1994@users.noreply.github.com> Date: Mon, 30 Dec 2024 10:47:44 +0000 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A8=20archive=20steps=20for=20datasets?= =?UTF-8?q?=20with=20no=20charts=20(#3764)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dag/archive/artificial_intelligence.yml | 91 +++++++++++++++++++++---- dag/archive/climate.yml | 17 +++++ dag/archive/health.yml | 9 +++ dag/archive/main.yml | 5 ++ dag/archive/plastic_use.yml | 16 +++++ dag/artificial_intelligence.yml | 79 --------------------- dag/climate.yml | 24 ++----- dag/health.yml | 7 -- dag/main.yml | 8 --- dag/plastic_use.yml | 16 ----- 10 files changed, 132 insertions(+), 140 deletions(-) create mode 100644 dag/archive/plastic_use.yml diff --git a/dag/archive/artificial_intelligence.yml b/dag/archive/artificial_intelligence.yml index a69ed71a8d4..0920dfe3482 100644 --- a/dag/archive/artificial_intelligence.yml +++ b/dag/archive/artificial_intelligence.yml @@ -10,12 +10,6 @@ steps: data://grapher/artificial_intelligence/latest/epoch: - data://garden/artificial_intelligence/latest/epoch - # Artificial Intelligence (EPOCH, number of systems by country) - data://garden/artificial_intelligence/latest/epoch_system_countries: - - data://meadow/artificial_intelligence/latest/epoch - data://grapher/artificial_intelligence/latest/epoch_system_countries: - - data://garden/artificial_intelligence/latest/epoch_system_countries - # Aggregates by domain data://garden/artificial_intelligence/latest/epoch_aggregates_domain: - data://garden/artificial_intelligence/latest/epoch @@ -59,12 +53,6 @@ steps: data://grapher/artificial_intelligence/2024-06-19/epoch_compute_intensive_organizations: - data://garden/artificial_intelligence/2024-06-19/epoch_compute_intensive_organizations - # EPOCH notable systems, aggregates by organization - data://garden/artificial_intelligence/2024-06-03/epoch_aggregates_organizations: - - data://meadow/artificial_intelligence/2024-06-03/epoch - data://grapher/artificial_intelligence/2024-06-03/epoch_aggregates_organizations: - - data://garden/artificial_intelligence/2024-06-03/epoch_aggregates_organizations - # EPOCH notable systes, aggregates by country data://garden/artificial_intelligence/2024-06-03/epoch_aggregates_countries: - data://meadow/artificial_intelligence/2024-06-03/epoch @@ -381,3 +369,82 @@ steps: - data://meadow/artificial_intelligence/2024-01-30/yougov_job_automation data://grapher/artificial_intelligence/2024-01-30/yougov_job_automation: - data://garden/artificial_intelligence/2024-01-30/yougov_job_automation + + # CSET data on articles, patents and investment + data://meadow/artificial_intelligence/2023-07-25/cset: + - snapshot://artificial_intelligence/2023-07-25/cset.csv + data://garden/artificial_intelligence/2023-07-25/cset: + - data://meadow/artificial_intelligence/2023-07-25/cset + - data://garden/demography/2023-03-31/population + - data://garden/regions/2023-01-01/regions + - snapshot://wb/2023-09-19/us_cpi.csv + data://grapher/artificial_intelligence/2023-07-25/cset: + - data://garden/artificial_intelligence/2023-07-25/cset + + # DeepFakes + data://garden/artificial_intelligence/2023-06-14/ai_deepfakes: + - snapshot://artificial_intelligence/2023-06-14/ai_deepfakes.csv + data://grapher/artificial_intelligence/2023-06-14/ai_deepfakes: + - data://garden/artificial_intelligence/2023-06-14/ai_deepfakes + + # AI bills passed into law + data://garden/artificial_intelligence/2023-06-14/ai_bills: + - snapshot://artificial_intelligence/2023-06-14/ai_bills.csv + data://grapher/artificial_intelligence/2023-06-14/ai_bills: + - data://garden/artificial_intelligence/2023-06-14/ai_bills + + # AI employment by sector + data://garden/artificial_intelligence/2023-06-14/ai_employers: + - snapshot://artificial_intelligence/2023-06-14/ai_employers.csv + data://grapher/artificial_intelligence/2023-06-14/ai_employers: + - data://garden/artificial_intelligence/2023-06-14/ai_employers + + # AI github productivity + data://garden/artificial_intelligence/2023-06-14/ai_github: + - snapshot://artificial_intelligence/2023-06-14/ai_github.csv + data://grapher/artificial_intelligence/2023-06-14/ai_github: + - data://garden/artificial_intelligence/2023-06-14/ai_github + + # AI gender representation + data://garden/artificial_intelligence/2023-06-14/ai_gender_representation: + - snapshot://artificial_intelligence/2023-06-14/ai_gender_representation.csv + data://grapher/artificial_intelligence/2023-06-14/ai_gender_representation: + - data://garden/artificial_intelligence/2023-06-14/ai_gender_representation + + # AI PhDs + data://garden/artificial_intelligence/2023-06-14/ai_phds: + - snapshot://artificial_intelligence/2023-06-14/ai_phds.csv + data://grapher/artificial_intelligence/2023-06-14/ai_phds: + - data://garden/artificial_intelligence/2023-06-14/ai_phds + + # AI national strategy + data://meadow/artificial_intelligence/2023-06-14/ai_national_strategy: + - snapshot://artificial_intelligence/2023-06-14/ai_national_strategy.csv + data://garden/artificial_intelligence/2023-06-14/ai_national_strategy: + - data://meadow/artificial_intelligence/2023-06-14/ai_national_strategy + - data://garden/regions/2023-01-01/regions + data://grapher/artificial_intelligence/2023-06-14/ai_national_strategy: + - data://garden/artificial_intelligence/2023-06-14/ai_national_strategy + + # Monmouth University Poll + data://meadow/artificial_intelligence/2023-06-07/monmouth_poll: + - snapshot://artificial_intelligence/2023-06-07/monmouth_poll.pdf + data://garden/artificial_intelligence/2023-06-07/monmouth_poll: + - data://meadow/artificial_intelligence/2023-06-07/monmouth_poll + data://grapher/artificial_intelligence/2023-06-07/monmouth_poll: + - data://garden/artificial_intelligence/2023-06-07/monmouth_poll + + # YouGov AI surveys + data://meadow/artificial_intelligence/2023-06-08/yougov_jobs: + - snapshot://artificial_intelligence/2023-06-08/yougov_jobs.pdf + data://garden/artificial_intelligence/2023-06-08/yougov_jobs: + - data://meadow/artificial_intelligence/2023-06-08/yougov_jobs + data://grapher/artificial_intelligence/2023-06-08/yougov_jobs: + - data://garden/artificial_intelligence/2023-06-08/yougov_jobs + + data://meadow/artificial_intelligence/2023-06-08/yougov_end_of_humanity: + - snapshot://artificial_intelligence/2023-06-08/yougov_end_of_humanity.pdf + data://garden/artificial_intelligence/2023-06-08/yougov_end_of_humanity: + - data://meadow/artificial_intelligence/2023-06-08/yougov_end_of_humanity + data://grapher/artificial_intelligence/2023-06-08/yougov_end_of_humanity: + - data://garden/artificial_intelligence/2023-06-08/yougov_end_of_humanity \ No newline at end of file diff --git a/dag/archive/climate.yml b/dag/archive/climate.yml index 08b7e2a683e..1d19d79a372 100644 --- a/dag/archive/climate.yml +++ b/dag/archive/climate.yml @@ -783,3 +783,20 @@ steps: # data://grapher/climate/2024-09-30/sea_ice_anomalies_by_month: - data://garden/climate/2024-09-30/sea_ice_index + + # + # Emissions from wildfires. + # + data://meadow/climate/2024-02-19/monthly_fire_emissions: + - snapshot://climate/2024-02-19/monthly_fire_emissions.zip + data://garden/climate/2024-02-19/monthly_fire_emissions: + - data://meadow/climate/2024-02-19/monthly_fire_emissions + - data://garden/wb/2023-04-30/income_groups + - data://garden/regions/2023-01-01/regions + data://grapher/climate/2024-02-19/monthly_fire_emissions: + - data://garden/climate/2024-02-19/monthly_fire_emissions + data://grapher/climate/2024-02-19/yearly_fire_emissions: + - data://garden/climate/2024-02-19/monthly_fire_emissions + + data://grapher/climate/2024-02-19/monthly_burned_area: + - data://garden/climate/2024-02-19/monthly_burned_area \ No newline at end of file diff --git a/dag/archive/health.yml b/dag/archive/health.yml index de10fe68b56..6072c3aef42 100644 --- a/dag/archive/health.yml +++ b/dag/archive/health.yml @@ -175,3 +175,12 @@ steps: - data://meadow/health/2024-06-11/isaps_plastic_surgery data://grapher/health/2024-06-11/isaps_plastic_surgery: - data://garden/health/2024-06-11/isaps_plastic_surgery + + # Missing Data - Neuropsychiatric Conditions + data://garden/missing_data/2024-03-26/who_neuropsychiatric_conditions: + - data://garden/who/2024-07-26/mortality_database + - data://garden/wb/2024-03-11/income_groups + - data://garden/regions/2023-01-01/regions + - data://garden/demography/2023-03-31/population + data://grapher/missing_data/2024-03-26/who_neuropsychiatric_conditions: + - data://garden/missing_data/2024-03-26/who_neuropsychiatric_conditions \ No newline at end of file diff --git a/dag/archive/main.yml b/dag/archive/main.yml index 3b5774421e6..62bdaa5042d 100644 --- a/dag/archive/main.yml +++ b/dag/archive/main.yml @@ -319,9 +319,14 @@ include: - dag/archive/faostat.yml - dag/archive/energy.yml - dag/archive/emissions.yml + - dag/archive/health.yml - dag/archive/papers.yml - dag/archive/demography.yml - dag/archive/fasttrack.yml - dag/archive/climate.yml - dag/archive/environment.yml - dag/archive/equality.yml + - dag/archive/plastic_use.yml + - dag/archive/artificial_intelligence.yml + - dag/archive/urbanization.yml + - dag/archive/education.yml diff --git a/dag/archive/plastic_use.yml b/dag/archive/plastic_use.yml new file mode 100644 index 00000000000..49fc6962fd1 --- /dev/null +++ b/dag/archive/plastic_use.yml @@ -0,0 +1,16 @@ +steps: + # OECD plastic production and plastic waste datasets + data://meadow/oecd/2023-09-21/plastic_environment_leakage_projections: + - snapshot://oecd/2023-09-21/plastic_environment_leakage_projections.csv + data://garden/oecd/2023-09-21/plastic_environment_leakage_projections: + - data://meadow/oecd/2023-09-21/plastic_environment_leakage_projections + data://grapher/oecd/2023-09-21/plastic_environment_leakage_projections: + - data://garden/oecd/2023-09-21/plastic_environment_leakage_projections + + + data://meadow/oecd/2023-09-21/plastic_aquatic_leakage_projections: + - snapshot://oecd/2023-09-21/plastic_aquatic_leakage_projections.csv + data://garden/oecd/2023-09-21/plastic_aquatic_leakage_projections: + - data://meadow/oecd/2023-09-21/plastic_aquatic_leakage_projections + data://grapher/oecd/2023-09-21/plastic_aquatic_leakage_projections: + - data://garden/oecd/2023-09-21/plastic_aquatic_leakage_projections diff --git a/dag/artificial_intelligence.yml b/dag/artificial_intelligence.yml index 868a40ad852..7f75fb6bfb5 100644 --- a/dag/artificial_intelligence.yml +++ b/dag/artificial_intelligence.yml @@ -87,12 +87,6 @@ steps: # AI Index report - # DeepFakes - data://garden/artificial_intelligence/2023-06-14/ai_deepfakes: - - snapshot://artificial_intelligence/2023-06-14/ai_deepfakes.csv - data://grapher/artificial_intelligence/2023-06-14/ai_deepfakes: - - data://garden/artificial_intelligence/2023-06-14/ai_deepfakes - # AI adoption by organisations - AI Index Report 2024 data://garden/artificial_intelligence/2024-06-28/ai_adoption: @@ -102,54 +96,7 @@ steps: - data://garden/artificial_intelligence/2024-06-28/ai_adoption - # AI bills passed into law - data://garden/artificial_intelligence/2023-06-14/ai_bills: - - snapshot://artificial_intelligence/2023-06-14/ai_bills.csv - data://grapher/artificial_intelligence/2023-06-14/ai_bills: - - data://garden/artificial_intelligence/2023-06-14/ai_bills - - # AI employment by sector - data://garden/artificial_intelligence/2023-06-14/ai_employers: - - snapshot://artificial_intelligence/2023-06-14/ai_employers.csv - data://grapher/artificial_intelligence/2023-06-14/ai_employers: - - data://garden/artificial_intelligence/2023-06-14/ai_employers - - # AI github productivity - data://garden/artificial_intelligence/2023-06-14/ai_github: - - snapshot://artificial_intelligence/2023-06-14/ai_github.csv - data://grapher/artificial_intelligence/2023-06-14/ai_github: - - data://garden/artificial_intelligence/2023-06-14/ai_github - - # AI gender representation - data://garden/artificial_intelligence/2023-06-14/ai_gender_representation: - - snapshot://artificial_intelligence/2023-06-14/ai_gender_representation.csv - data://grapher/artificial_intelligence/2023-06-14/ai_gender_representation: - - data://garden/artificial_intelligence/2023-06-14/ai_gender_representation - - # AI PhDs - data://garden/artificial_intelligence/2023-06-14/ai_phds: - - snapshot://artificial_intelligence/2023-06-14/ai_phds.csv - data://grapher/artificial_intelligence/2023-06-14/ai_phds: - - data://garden/artificial_intelligence/2023-06-14/ai_phds - - # AI national strategy - data://meadow/artificial_intelligence/2023-06-14/ai_national_strategy: - - snapshot://artificial_intelligence/2023-06-14/ai_national_strategy.csv - data://garden/artificial_intelligence/2023-06-14/ai_national_strategy: - - data://meadow/artificial_intelligence/2023-06-14/ai_national_strategy - - data://garden/regions/2023-01-01/regions - data://grapher/artificial_intelligence/2023-06-14/ai_national_strategy: - - data://garden/artificial_intelligence/2023-06-14/ai_national_strategy - - # AI polls - # Monmouth University Poll - data://meadow/artificial_intelligence/2023-06-07/monmouth_poll: - - snapshot://artificial_intelligence/2023-06-07/monmouth_poll.pdf - data://garden/artificial_intelligence/2023-06-07/monmouth_poll: - - data://meadow/artificial_intelligence/2023-06-07/monmouth_poll - data://grapher/artificial_intelligence/2023-06-07/monmouth_poll: - - data://garden/artificial_intelligence/2023-06-07/monmouth_poll # YouGov AI surveys data://meadow/artificial_intelligence/2024-07-23/yougov_robots: @@ -166,20 +113,6 @@ steps: data://grapher/artificial_intelligence/2024-07-23/yougov_job_automation: - data://garden/artificial_intelligence/2024-07-23/yougov_job_automation - data://meadow/artificial_intelligence/2023-06-08/yougov_jobs: - - snapshot://artificial_intelligence/2023-06-08/yougov_jobs.pdf - data://garden/artificial_intelligence/2023-06-08/yougov_jobs: - - data://meadow/artificial_intelligence/2023-06-08/yougov_jobs - data://grapher/artificial_intelligence/2023-06-08/yougov_jobs: - - data://garden/artificial_intelligence/2023-06-08/yougov_jobs - - data://meadow/artificial_intelligence/2023-06-08/yougov_end_of_humanity: - - snapshot://artificial_intelligence/2023-06-08/yougov_end_of_humanity.pdf - data://garden/artificial_intelligence/2023-06-08/yougov_end_of_humanity: - - data://meadow/artificial_intelligence/2023-06-08/yougov_end_of_humanity - data://grapher/artificial_intelligence/2023-06-08/yougov_end_of_humanity: - - data://garden/artificial_intelligence/2023-06-08/yougov_end_of_humanity - # CSET data on semiconductor supply chain data://meadow/artificial_intelligence/2023-07-07/semiconductors_cset: - snapshot://artificial_intelligence/2023-07-07/semiconductors_cset.csv @@ -188,18 +121,6 @@ steps: data://grapher/artificial_intelligence/2023-07-07/semiconductors_cset: - data://garden/artificial_intelligence/2023-07-07/semiconductors_cset - # CSET data on articles, patents and investment - data://meadow/artificial_intelligence/2023-07-25/cset: - - snapshot://artificial_intelligence/2023-07-25/cset.csv - data://garden/artificial_intelligence/2023-07-25/cset: - - data://meadow/artificial_intelligence/2023-07-25/cset - - data://garden/demography/2023-03-31/population - - data://garden/regions/2023-01-01/regions - - snapshot://wb/2023-09-19/us_cpi.csv - - data://grapher/artificial_intelligence/2023-07-25/cset: - - data://garden/artificial_intelligence/2023-07-25/cset - # Chess dataset (ELO ratings) data://garden/artificial_intelligence/2024-02-05/chess: - snapshot://artificial_intelligence/2024-02-05/chess.csv diff --git a/dag/climate.yml b/dag/climate.yml index 4436974e2b2..9c9de234e72 100644 --- a/dag/climate.yml +++ b/dag/climate.yml @@ -66,6 +66,11 @@ steps: # data://explorers/climate/latest/climate_change_impacts: - data://garden/climate/2024-11-18/climate_change_impacts + + # + # Weekly area burnt and emissions data on wildfires. + # + # # Global Wildfire Information System - Monthly burned area. # @@ -75,26 +80,9 @@ steps: - data://meadow/climate/2024-02-19/monthly_burned_area - data://garden/wb/2023-04-30/income_groups - data://garden/regions/2023-01-01/regions - data://grapher/climate/2024-02-19/monthly_burned_area: - - data://garden/climate/2024-02-19/monthly_burned_area data://grapher/climate/2024-02-19/yearly_burned_area: - data://garden/climate/2024-02-19/monthly_burned_area - # - # Emissions from wildfires. - # - data://meadow/climate/2024-02-19/monthly_fire_emissions: - - snapshot://climate/2024-02-19/monthly_fire_emissions.zip - data://garden/climate/2024-02-19/monthly_fire_emissions: - - data://meadow/climate/2024-02-19/monthly_fire_emissions - - data://garden/wb/2023-04-30/income_groups - - data://garden/regions/2023-01-01/regions - data://grapher/climate/2024-02-19/monthly_fire_emissions: - - data://garden/climate/2024-02-19/monthly_fire_emissions - data://grapher/climate/2024-02-19/yearly_fire_emissions: - - data://garden/climate/2024-02-19/monthly_fire_emissions - # - # Weekly area burnt and emissions data on wildfires. - # + data://meadow/climate/latest/weekly_wildfires: - snapshot://climate/2024-02-22/weekly_wildfires_2003_2023.csv - snapshot://climate/latest/weekly_wildfires.csv diff --git a/dag/health.yml b/dag/health.yml index 77c95c1613c..2c6c452c2d9 100644 --- a/dag/health.yml +++ b/dag/health.yml @@ -801,13 +801,6 @@ steps: data://grapher/antibiotics/2024-10-09/gram_children: - data://garden/antibiotics/2024-10-09/gram_children - # Cervical cancer incidence rates GCO - Cancer Over Time - data://meadow/cancer/2024-10-13/gco_cancer_over_time_cervical: - - snapshot://cancer/2024-10-13/gco_cancer_over_time_cervical.csv - data://garden/cancer/2024-10-13/gco_cancer_over_time_cervical: - - data://meadow/cancer/2024-10-13/gco_cancer_over_time_cervical - data://grapher/cancer/2024-10-13/gco_cancer_over_time_cervical: - - data://garden/cancer/2024-10-13/gco_cancer_over_time_cervical # Cervical cancer incidence rates GCO - Cancer Today (2022) data://meadow/cancer/2024-10-13/gco_cancer_today_cervical: diff --git a/dag/main.yml b/dag/main.yml index 7ca958d7499..d8be36af51d 100644 --- a/dag/main.yml +++ b/dag/main.yml @@ -453,14 +453,6 @@ steps: data://grapher/missing_data/2024-03-26/children_out_of_school: - data://garden/missing_data/2024-03-26/children_out_of_school - # Missing Data - Neuropsychiatric Conditions - data://garden/missing_data/2024-03-26/who_neuropsychiatric_conditions: - - data://garden/who/2024-07-26/mortality_database - - data://garden/wb/2024-03-11/income_groups - - data://garden/regions/2023-01-01/regions - - data://garden/demography/2023-03-31/population - data://grapher/missing_data/2024-03-26/who_neuropsychiatric_conditions: - - data://garden/missing_data/2024-03-26/who_neuropsychiatric_conditions # Missing Data - Suicides data://garden/missing_data/2024-03-26/who_md_suicides: diff --git a/dag/plastic_use.yml b/dag/plastic_use.yml index ca204909e6b..e27c6eb0f03 100644 --- a/dag/plastic_use.yml +++ b/dag/plastic_use.yml @@ -63,22 +63,6 @@ steps: data://grapher/oecd/2023-09-21/plastic_fate_regions_projections: - data://garden/oecd/2023-09-21/plastic_fate_regions_projections - - data://meadow/oecd/2023-09-21/plastic_environment_leakage_projections: - - snapshot://oecd/2023-09-21/plastic_environment_leakage_projections.csv - data://garden/oecd/2023-09-21/plastic_environment_leakage_projections: - - data://meadow/oecd/2023-09-21/plastic_environment_leakage_projections - data://grapher/oecd/2023-09-21/plastic_environment_leakage_projections: - - data://garden/oecd/2023-09-21/plastic_environment_leakage_projections - - - data://meadow/oecd/2023-09-21/plastic_aquatic_leakage_projections: - - snapshot://oecd/2023-09-21/plastic_aquatic_leakage_projections.csv - data://garden/oecd/2023-09-21/plastic_aquatic_leakage_projections: - - data://meadow/oecd/2023-09-21/plastic_aquatic_leakage_projections - data://grapher/oecd/2023-09-21/plastic_aquatic_leakage_projections: - - data://garden/oecd/2023-09-21/plastic_aquatic_leakage_projections - # Geyer et al. 2017 historical plastic production data://meadow/plastic_waste/2023-09-26/geyer_2017: - snapshot://plastic_waste/2023-09-26/geyer_2017.pdf