From 8e508896c1e91d5abcb1584f889c0813e700e66c Mon Sep 17 00:00:00 2001 From: Daniel Bachler Date: Thu, 14 Nov 2024 11:17:50 +0100 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=90=9B=20fix=20broken=20zip/readme=20?= =?UTF-8?q?generation=20with=20generated=20columns?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- functions/_common/readmeTools.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/functions/_common/readmeTools.ts b/functions/_common/readmeTools.ts index 08559be2876..dc8475310de 100644 --- a/functions/_common/readmeTools.ts +++ b/functions/_common/readmeTools.ts @@ -240,7 +240,12 @@ export function constructReadme( columns: CoreColumn[] ): string { const isSingleColumn = columns.length === 1 - const sources = columns.flatMap((col) => [...columnReadmeText(col)]) + // Some computed columns have neither a source nor origins - filter these away + const sources = columns + .filter( + (column) => !!column.source.name || !isEmpty(column.def.origins) + ) + .flatMap((col) => [...columnReadmeText(col)]) let readme: string const downloadDate = formatDate(new Date()) // formats the date as "October 10, 2024" if (isSingleColumn) From a28c032c60b8d4ba5175df318ef2e3eb218c40e2 Mon Sep 17 00:00:00 2001 From: Daniel Bachler Date: Thu, 14 Nov 2024 11:18:16 +0100 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=A7=B9=20print=20"how=20we=20process?= =?UTF-8?q?=20data"=20only=20once=20in=20readme?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- functions/_common/readmeTools.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/functions/_common/readmeTools.ts b/functions/_common/readmeTools.ts index dc8475310de..81ba60a047a 100644 --- a/functions/_common/readmeTools.ts +++ b/functions/_common/readmeTools.ts @@ -13,6 +13,7 @@ import { prepareSourcesForDisplay, uniqBy, formatDate, + isEmpty, } from "@ourworldindata/utils" import { CoreColumn } from "@ourworldindata/core-table" import { Grapher } from "@ourworldindata/grapher" @@ -60,12 +61,6 @@ export function* getCitationLines( export function* getDataProcessingLines( def: OwidColumnDef ): Generator { - yield "" - yield "### How we process data at Our World In Data" - yield "" - yield `All data and visualizations on Our World in Data rely on data sourced from one or several original data providers. Preparing this original data involves several processing steps. Depending on the data, this can include standardizing country names and world region definitions, converting units, calculating derived indicators such as per capita measures, as well as adding or adapting metadata such as the name or the description given to an indicator.` - yield `At the link below you can find a detailed description of the structure of our data pipeline, including links to all the code used to prepare data across Our World in Data.` - yield `[Read about our data pipeline](https://docs.owid.io/projects/etl/)` if (def.descriptionProcessing) { yield "" yield `#### Notes on our processing step for this indicator` @@ -179,7 +174,6 @@ export function getAttribution(def: OwidColumnDef): string { const attributionFragments = getAttributionFragmentsFromVariable(def) ?? producers const attribution = attributionFragments.join(", ") - if (attribution === "") { return def.sourceName } else return attribution @@ -299,6 +293,10 @@ The .metadata.json file contains metadata about the data package. The "charts" k Our World in Data is almost never the original producer of the data - almost all of the data we use has been compiled by others. If you want to re-use data, it is your responsibility to ensure that you adhere to the sources' license and to credit them correctly. Please note that a single time series may have more than one source - e.g. when we stich together data from different time periods by different producers or when we calculate per capita metrics using population data from a second source. +### How we process data at Our World In Data +All data and visualizations on Our World in Data rely on data sourced from one or several original data providers. Preparing this original data involves several processing steps. Depending on the data, this can include standardizing country names and world region definitions, converting units, calculating derived indicators such as per capita measures, as well as adding or adapting metadata such as the name or the description given to an indicator. +[Read about our data pipeline](https://docs.owid.io/projects/etl/) + ## Detailed information about each time series ${sources.join("\n")}