diff --git a/adminSiteServer/multiDim.ts b/adminSiteServer/multiDim.ts index d018dcbaa5..ab6f49fe38 100644 --- a/adminSiteServer/multiDim.ts +++ b/adminSiteServer/multiDim.ts @@ -231,6 +231,8 @@ async function cleanUpOrphanedChartConfigs( } } +// TODO: Do we need to manually delete unpublished multi-dim pages? +// - from R2? export async function createMultiDimConfig( knex: db.KnexReadWriteTransaction, slug: string, diff --git a/baker/GrapherBaker.tsx b/baker/GrapherBaker.tsx index 74f3fa4eb8..86897b067f 100644 --- a/baker/GrapherBaker.tsx +++ b/baker/GrapherBaker.tsx @@ -58,6 +58,7 @@ import { logErrorAndMaybeCaptureInSentry } from "../serverUtils/errorLog.js" import { getTagToSlugMap } from "./GrapherBakingUtils.js" import { knexRaw } from "../db/db.js" import { getRelatedChartsForVariable } from "../db/model/Chart.js" +import { getAllMultiDimDataPageSlugs } from "../db/model/MultiDimDataPage.js" import pMap from "p-map" const renderDatapageIfApplicable = async ( @@ -308,7 +309,6 @@ const bakeGrapherPageAndVariablesPngAndSVGIfChanged = async ( imageMetadataDictionary ) ) - console.log(outPath) const variableIds = lodash.uniq( grapher.dimensions?.map((d) => d.variableId) @@ -411,7 +411,6 @@ export const bakeAllChangedGrapherPagesVariablesPngSvgAndDeleteRemovedGraphers = ` ) - const newSlugs = chartsToBake.map((row) => row.slug) await fs.mkdirp(bakedSiteDir + "/grapher") // Prefetch imageMetadata instead of each grapher page fetching @@ -432,7 +431,7 @@ export const bakeAllChangedGrapherPagesVariablesPngSvgAndDeleteRemovedGraphers = ) const progressBar = new ProgressBar( - "bake grapher page [:bar] :current/:total :elapseds :rate/s :etas :name\n", + "bake grapher page [:bar] :current/:total :elapseds :rate/s :name\n", { width: 20, total: chartsToBake.length + 1, @@ -451,11 +450,18 @@ export const bakeAllChangedGrapherPagesVariablesPngSvgAndDeleteRemovedGraphers = async (knex) => await bakeSingleGrapherChart(job, knex), db.TransactionCloseMode.KeepOpen ) - progressBar.tick({ name: `slug ${job.slug}` }) + progressBar.tick({ name: job.slug }) }, { concurrency: 10 } ) - await deleteOldGraphers(bakedSiteDir, excludeUndefined(newSlugs)) + // Multi-dim data pages are baked into the same directory as graphers + // and they are handled separately. + const multiDimSlugs = await getAllMultiDimDataPageSlugs(knex) + const newSlugs = excludeUndefined([ + ...chartsToBake.map((row) => row.slug), + ...multiDimSlugs, + ]) + await deleteOldGraphers(bakedSiteDir, newSlugs) progressBar.tick({ name: `✅ Deleted old graphers` }) } diff --git a/baker/GrapherImageBaker.tsx b/baker/GrapherImageBaker.tsx index 51362d1caf..59694a128c 100644 --- a/baker/GrapherImageBaker.tsx +++ b/baker/GrapherImageBaker.tsx @@ -41,9 +41,7 @@ export async function bakeGrapherToSvgAndPng( if (optimizeSvgs) svgCode = await optimizeSvg(svgCode) return Promise.all([ - fs - .writeFile(`${outPath}.svg`, svgCode) - .then(() => console.log(`${outPath}.svg`)), + fs.writeFile(`${outPath}.svg`, svgCode), sharp(Buffer.from(grapher.staticSVG), { density: 144 }) .png() .resize(grapher.defaultBounds.width, grapher.defaultBounds.height) diff --git a/baker/MultiDimBaker.tsx b/baker/MultiDimBaker.tsx index 1756c72505..383dfb486e 100644 --- a/baker/MultiDimBaker.tsx +++ b/baker/MultiDimBaker.tsx @@ -1,5 +1,7 @@ +import { glob } from "glob" import fs from "fs-extra" import path from "path" +import ProgressBar from "progress" import { ImageMetadata, MultiDimDataPageConfigPreProcessed, @@ -32,8 +34,10 @@ import { resolveFaqsForVariable, } from "./DatapageHelpers.js" import { logErrorAndMaybeCaptureInSentry } from "../serverUtils/errorLog.js" +import { getAllPublishedChartSlugs } from "../db/model/Chart.js" import { getAllMultiDimDataPages, + getAllPublishedMultiDimDataPageSlugs, getMultiDimDataPageBySlug, } from "../db/model/MultiDimDataPage.js" @@ -219,12 +223,42 @@ export const bakeMultiDimDataPage = async ( await fs.writeFile(outPath, renderedHtml) } +async function deleteOldMultiDimPages( + bakedSiteDir: string, + newSlugs: Set +) { + const oldSlugs = new Set( + glob + .sync(`${bakedSiteDir}/grapher/*.html`) + .map((slug) => + slug + .replace(`${bakedSiteDir}/grapher/`, "") + .replace(".html", "") + ) + ) + const toRemove = oldSlugs.difference(newSlugs) + for (const slug of toRemove) { + console.log(`DELETING ${slug}`) + const path = `${bakedSiteDir}/grapher/${slug}.html` + await fs.unlink(path) + console.log(path) + } +} + export const bakeAllMultiDimDataPages = async ( knex: db.KnexReadonlyTransaction, bakedSiteDir: string, imageMetadata: Record ) => { const multiDimsBySlug = await getAllMultiDimDataPages(knex) + const progressBar = new ProgressBar( + "bake multi-dim page [:bar] :current/:total :elapseds :rate/s :name\n", + { + width: 20, + total: multiDimsBySlug.size + 1, + renderThrottle: 0, + } + ) for (const [slug, row] of multiDimsBySlug.entries()) { await bakeMultiDimDataPage( knex, @@ -233,5 +267,11 @@ export const bakeAllMultiDimDataPages = async ( row.config, imageMetadata ) + progressBar.tick({ name: slug }) } + const publishedSlugs = await getAllPublishedMultiDimDataPageSlugs(knex) + const chartSlugs = await getAllPublishedChartSlugs(knex) + const newSlugs = new Set([...publishedSlugs, ...chartSlugs]) + await deleteOldMultiDimPages(bakedSiteDir, newSlugs) + progressBar.tick({ name: `✅ Deleted old multi-dim pages` }) } diff --git a/db/model/Chart.ts b/db/model/Chart.ts index 7a74b18698..dfe049a98e 100644 --- a/db/model/Chart.ts +++ b/db/model/Chart.ts @@ -673,3 +673,20 @@ export const getRedirectsByChartId = async ( ORDER BY id ASC`, [chartId] ) + +export async function getAllPublishedChartSlugs( + knex: db.KnexReadonlyTransaction +): Promise { + const rows = await db.knexRaw<{ + slug: string + }>( + knex, + `-- sql + SELECT cc.slug + FROM charts c + JOIN chart_configs cc ON c.configId = cc.id + WHERE cc.full->>"$.isPublished" = "true" + ` + ) + return rows.map((row) => row.slug) +} diff --git a/db/model/MultiDimDataPage.ts b/db/model/MultiDimDataPage.ts index 50fc52e0c2..a9e0d6908c 100644 --- a/db/model/MultiDimDataPage.ts +++ b/db/model/MultiDimDataPage.ts @@ -84,6 +84,24 @@ export async function getAllLinkedPublishedMultiDimDataPages( return rows.map(enrichRow) } +export async function getAllMultiDimDataPageSlugs( + knex: KnexReadonlyTransaction +): Promise { + const rows = await knex( + MultiDimDataPagesTableName + ).select("slug") + return rows.map((row) => row.slug) +} + +export async function getAllPublishedMultiDimDataPageSlugs( + knex: KnexReadonlyTransaction +): Promise { + const rows = await knex(MultiDimDataPagesTableName) + .select("slug") + .where({ published: true }) + return rows.map((row) => row.slug) +} + export const getMultiDimDataPageBySlug = async ( knex: KnexReadonlyTransaction, slug: string, diff --git a/devTools/tsconfigs/tsconfig.base.json b/devTools/tsconfigs/tsconfig.base.json index 3507ceb1f3..e6fcc81fa5 100644 --- a/devTools/tsconfigs/tsconfig.base.json +++ b/devTools/tsconfigs/tsconfig.base.json @@ -9,7 +9,23 @@ "declaration": true, "declarationMap": true, - "lib": ["dom", "dom.iterable", "es2020", "es2021", "es2022", "es2023"], + // To get newer APIs like Set.prototype.intersection(), we need to use + // ESNext lib on Node 22 and current version of TypeScript (5.7.2). + // However, using the ESNext option doesn't work with + // @types/node@20.8.3, which we are currently pinned to, to fix an + // unrelated bug with types in Cloudflare Functions, thus we only use + // ESNext.Collection. + // https://github.com/microsoft/TypeScript/issues/59919 + // https://developers.cloudflare.com/workers/languages/typescript/#transitive-loading-of-typesnode-overrides-cloudflareworkers-types + "lib": [ + "dom", + "dom.iterable", + "es2020", + "es2021", + "es2022", + "es2023", + "ESNext.Collection" + ], // Using es2022 as a `target` caused the following error in wrangler: // "Uncaught TypeError: PointVector is not a constructor". // This seems to be related to a change in how classes are compiled in