From 7398cd893b8270ca8e514e5f3c96fe5c3e0bb4da Mon Sep 17 00:00:00 2001
From: Daniel Bachler <daniel@danielbachler.de>
Date: Thu, 14 Nov 2024 11:41:29 +0100
Subject: [PATCH] Fix bug with zip file/readme creation (#4162)

Grapher sometimes creates computed columns (e.g. for relative line charts). These then have neither sources nor origins which crashed the readme generation.

This PR fixes this by filtering out these columns.

It also moves a repeated section about our data processing out of the per-column section into the top section.
---
 functions/_common/readmeTools.ts | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/functions/_common/readmeTools.ts b/functions/_common/readmeTools.ts
index 08559be2876..81ba60a047a 100644
--- a/functions/_common/readmeTools.ts
+++ b/functions/_common/readmeTools.ts
@@ -13,6 +13,7 @@ import {
     prepareSourcesForDisplay,
     uniqBy,
     formatDate,
+    isEmpty,
 } from "@ourworldindata/utils"
 import { CoreColumn } from "@ourworldindata/core-table"
 import { Grapher } from "@ourworldindata/grapher"
@@ -60,12 +61,6 @@ export function* getCitationLines(
 export function* getDataProcessingLines(
     def: OwidColumnDef
 ): Generator<string, void, unknown> {
-    yield ""
-    yield "### How we process data at Our World In Data"
-    yield ""
-    yield `All data and visualizations on Our World in Data rely on data sourced from one or several original data providers. Preparing this original data involves several processing steps. Depending on the data, this can include standardizing country names and world region definitions, converting units, calculating derived indicators such as per capita measures, as well as adding or adapting metadata such as the name or the description given to an indicator.`
-    yield `At the link below you can find a detailed description of the structure of our data pipeline, including links to all the code used to prepare data across Our World in Data.`
-    yield `[Read about our data pipeline](https://docs.owid.io/projects/etl/)`
     if (def.descriptionProcessing) {
         yield ""
         yield `#### Notes on our processing step for this indicator`
@@ -179,7 +174,6 @@ export function getAttribution(def: OwidColumnDef): string {
     const attributionFragments =
         getAttributionFragmentsFromVariable(def) ?? producers
     const attribution = attributionFragments.join(", ")
-
     if (attribution === "") {
         return def.sourceName
     } else return attribution
@@ -240,7 +234,12 @@ export function constructReadme(
     columns: CoreColumn[]
 ): string {
     const isSingleColumn = columns.length === 1
-    const sources = columns.flatMap((col) => [...columnReadmeText(col)])
+    // Some computed columns have neither a source nor origins - filter these away
+    const sources = columns
+        .filter(
+            (column) => !!column.source.name || !isEmpty(column.def.origins)
+        )
+        .flatMap((col) => [...columnReadmeText(col)])
     let readme: string
     const downloadDate = formatDate(new Date()) // formats the date as "October 10, 2024"
     if (isSingleColumn)
@@ -294,6 +293,10 @@ The .metadata.json file contains metadata about the data package. The "charts" k
 
 Our World in Data is almost never the original producer of the data - almost all of the data we use has been compiled by others. If you want to re-use data, it is your responsibility to ensure that you adhere to the sources' license and to credit them correctly. Please note that a single time series may have more than one source - e.g. when we stich together data from different time periods by different producers or when we calculate per capita metrics using population data from a second source.
 
+### How we process data at Our World In Data
+All data and visualizations on Our World in Data rely on data sourced from one or several original data providers. Preparing this original data involves several processing steps. Depending on the data, this can include standardizing country names and world region definitions, converting units, calculating derived indicators such as per capita measures, as well as adding or adapting metadata such as the name or the description given to an indicator.
+[Read about our data pipeline](https://docs.owid.io/projects/etl/)
+
 ## Detailed information about each time series
 
 ${sources.join("\n")}