feat(assets): Use entity-tags to revalidate cached remote images

.changeset/red-poems-pay.md

@@ -0,0 +1,5 @@
+---
+'astro': minor
+---
+
+Improve asset caching allowing stale assets to be revalidated with entity tags

packages/astro/src/assets/build/generate.ts

@@ -16,18 +16,18 @@ import { getConfiguredImageService } from '../internal.js';
 import type { LocalImageService } from '../services/service.js';
 import type { AssetsGlobalStaticImagesList, ImageMetadata, ImageTransform } from '../types.js';
 import { isESMImportedImage } from '../utils/imageKind.js';
-import { type RemoteCacheEntry, loadRemoteImage } from './remote.js';
+import { type RemoteCacheEntry, loadRemoteImage, revalidateRemoteImage } from './remote.js';
 
 interface GenerationDataUncached {
-	cached: false;
+	cached: 'miss';
 	weight: {
 		before: number;
 		after: number;
 	};
 }
 
 interface GenerationDataCached {
-	cached: true;
+	cached: 'revalidated' | 'hit';
 }
 
 type GenerationData = GenerationDataUncached | GenerationDataCached;
@@ -44,7 +44,11 @@ type AssetEnv = {
 	assetsFolder: AstroConfig['build']['assets'];
 };
 
-type ImageData = { data: Uint8Array; expires: number };
+type ImageData = {
+	data: Uint8Array;
+	expires: number;
+	etag?: string | null;
+};
 
 export async function prepareAssetsGenerationEnv(
 	pipeline: BuildPipeline,
@@ -135,9 +139,12 @@ export async function generateImagesForPath(
 		const timeEnd = performance.now();
 		const timeChange = getTimeStat(timeStart, timeEnd);
 		const timeIncrease = `(+${timeChange})`;
-		const statsText = generationData.cached
-			? `(reused cache entry)`
-			: `(before: ${generationData.weight.before}kB, after: ${generationData.weight.after}kB)`;
+		const statsText =
+			generationData.cached !== 'miss'
+				? generationData.cached === 'hit'
+					? `(reused cache entry)`
+					: `(revalidated cache entry)`
+				: `(before: ${generationData.weight.before}kB, after: ${generationData.weight.after}kB)`;
 		const count = `(${env.count.current}/${env.count.total})`;
 		env.logger.info(
 			null,
@@ -156,7 +163,7 @@ export async function generateImagesForPath(
 		const finalFolderURL = new URL('./', finalFileURL);
 		await fs.promises.mkdir(finalFolderURL, { recursive: true });
 
-		// For remote images, instead of saving the image directly, we save a JSON file with the image data and expiration date from the server
+		// For remote images, instead of saving the image directly, we save a JSON file with the image data, expiration date and etag from the server
 		const cacheFile = basename(filepath) + (isLocalImage ? '' : '.json');
 		const cachedFileURL = new URL(cacheFile, env.assetsCacheDir);
 
@@ -166,7 +173,7 @@ export async function generateImagesForPath(
 				await fs.promises.copyFile(cachedFileURL, finalFileURL, fs.constants.COPYFILE_FICLONE);
 
 				return {
-					cached: true,
+					cached: 'hit',
 				};
 			} else {
 				const JSONData = JSON.parse(readFileSync(cachedFileURL, 'utf-8')) as RemoteCacheEntry;
@@ -184,11 +191,29 @@ export async function generateImagesForPath(
 					await fs.promises.writeFile(finalFileURL, Buffer.from(JSONData.data, 'base64'));
 
 					return {
-						cached: true,
+						cached: 'hit',
 					};
-				} else {
-					await fs.promises.unlink(cachedFileURL);
 				}
+
+				// Try to revalidate the cache
+				if (JSONData.etag) {
+					const revalidatedData = await revalidateRemoteImage(options.src as string, JSONData.etag);
+
+					if (revalidatedData.data.length) {
+						// Image cache was stale, update original image to avoid redownload
+						originalImage = revalidatedData;
+					} else {
+						revalidatedData.data = Buffer.from(JSONData.data, 'base64');
+
+						// Freshen cache on disk
+						await writeRemoteCacheFile(cachedFileURL, revalidatedData, env);
+
+						await fs.promises.writeFile(finalFileURL, revalidatedData.data);
+						return { cached: 'revalidated' };
+					}
+				}
+
+				await fs.promises.unlink(cachedFileURL);
 			}
 		} catch (e: any) {
 			if (e.code !== 'ENOENT') {
@@ -209,6 +234,7 @@ export async function generateImagesForPath(
 		let resultData: Partial<ImageData> = {
 			data: undefined,
 			expires: originalImage.expires,
+			etag: originalImage.etag,
 		};
 
 		const imageService = (await getConfiguredImageService()) as LocalImageService;
@@ -239,13 +265,7 @@ export async function generateImagesForPath(
 				if (isLocalImage) {
 					await fs.promises.writeFile(cachedFileURL, resultData.data);
 				} else {
-					await fs.promises.writeFile(
-						cachedFileURL,
-						JSON.stringify({
-							data: Buffer.from(resultData.data).toString('base64'),
-							expires: resultData.expires,
-						}),
-					);
+					await writeRemoteCacheFile(cachedFileURL, resultData as ImageData, env);
 				}
 			}
 		} catch (e) {
@@ -259,7 +279,7 @@ export async function generateImagesForPath(
 		}
 
 		return {
-			cached: false,
+			cached: 'miss',
 			weight: {
 				// Divide by 1024 to get size in kilobytes
 				before: Math.trunc(originalImage.data.byteLength / 1024),
@@ -269,6 +289,24 @@ export async function generateImagesForPath(
 	}
 }
 
+async function writeRemoteCacheFile(cachedFileURL: URL, resultData: ImageData, env: AssetEnv) {
+	try {
+		return await fs.promises.writeFile(
+			cachedFileURL,
+			JSON.stringify({
+				data: Buffer.from(resultData.data).toString('base64'),
+				expires: resultData.expires,
+				etag: resultData.etag,
+			}),
+		);
+	} catch (e) {
+		env.logger.warn(
+			null,
+			`An error was encountered while writing the cache file for a remote asset. Proceeding without caching this asset. Error: ${e}`,
+		);
+	}
+}
+
 export function getStaticImageList(): AssetsGlobalStaticImagesList {
 	if (!globalThis?.astroAsset?.staticImages) {
 		return new Map();
@@ -279,11 +317,7 @@ export function getStaticImageList(): AssetsGlobalStaticImagesList {
 
 async function loadImage(path: string, env: AssetEnv): Promise<ImageData> {
 	if (isRemotePath(path)) {
-		const remoteImage = await loadRemoteImage(path);
-		return {
-			data: remoteImage.data,
-			expires: remoteImage.expires,
-		};
+		return await loadRemoteImage(path);
 	}
 
 	return {

packages/astro/src/assets/build/remote.ts

@@ -1,6 +1,6 @@
 import CachePolicy from 'http-cache-semantics';
 
-export type RemoteCacheEntry = { data: string; expires: number };
+export type RemoteCacheEntry = { data: string; expires: number; etag?: string };
 
 export async function loadRemoteImage(src: string) {
 	const req = new Request(src);
@@ -19,6 +19,48 @@ export async function loadRemoteImage(src: string) {
 	return {
 		data: Buffer.from(await res.arrayBuffer()),
 		expires: Date.now() + expires,
+		etag: res.headers.get('Etag'),
+	};
+}
+
+/**
+ * Revalidate a cached remote asset using its entity-tag.
+ * Uses the If-None-Match header to check with the remote server if the cached version of a remote asset is still up to date.
+ * The remote server may respond that the cached asset is still up-to-date if the entity-tag matches (304 Not Modified), or respond with an updated asset (200 OK)
+ * @param src - url to remote asset
+ * @param etag - the stored Entity-Tag of the cached asset
+ * @returns An ImageData object containing the asset data, a new expiry time, and the asset's etag. The data buffer will be empty if the asset was not modified.
+ */
+export async function revalidateRemoteImage(src: string, etag: string) {
+	const req = new Request(src, { headers: { 'If-None-Match': etag } });
+	const res = await fetch(req);
+
+	if (!res.ok && res.status != 304) {
+		throw new Error(
+			`Failed to revalidate cached remote image ${src}. The request did not return a 200 OK / 304 NOT MODIFIED response. (received ${res.status}))`,
+		);
+	}
+
+	const data = Buffer.from(await res.arrayBuffer());
+
+	if (res.ok && !data.length) {
+		// Server did not include body but indicated cache was stale
+		return await loadRemoteImage(src);
+	}
+
+	// calculate an expiration date based on the response's TTL
+	const policy = new CachePolicy(
+		webToCachePolicyRequest(req),
+		webToCachePolicyResponse(
+			res.ok ? res : new Response(null, { status: 200, headers: res.headers }),
+		), // 304 responses themselves are not cachable, so just pretend to get the refreshed TTL
+	);
+	const expires = policy.storable() ? policy.timeToLive() : 0;
+
+	return {
+		data,
+		expires: Date.now() + expires,
+		etag: res.headers.get('Etag'),
 	};
 }