feat: image support for content layer (#11469)

* wip

* wip

* Add image to benchmark

* Stub assets if missing

* Resolve assets in data

* Ignore virtual module

* Format

* rm log

* Handle images when using cached data

* Fix CCC

* Add a comment

* Changes from review

* Format

* Use relative paths for asset files

* Pass all md props to getImage

* Ensure dotastro dir exists

* Fix tests

* Changes from review

* Don't use temp array in getcollection

* Add error handling

* Format

* Handle paths that are already relative

benchmark/make-project/markdown-cc1.js

@@ -8,6 +8,7 @@ export async function run(projectDir) {
 	await fs.rm(projectDir, { recursive: true, force: true });
 	await fs.mkdir(new URL('./src/pages/blog', projectDir), { recursive: true });
 	await fs.mkdir(new URL('./src/content/blog', projectDir), { recursive: true });
+	await fs.copyFile(new URL('./image.jpg', import.meta.url), new URL('./src/image.jpg', projectDir));
 
 	const promises = [];
 
@@ -17,6 +18,10 @@ export async function run(projectDir) {
 # Article ${i}
 
 ${loremIpsumMd}
+
+![image ${i}](../../image.jpg)
+
+
 `;
 		promises.push(
 			fs.writeFile(new URL(`./src/content/blog/article-${i}.md`, projectDir), content, 'utf-8')

benchmark/make-project/markdown-cc2.js

@@ -9,6 +9,7 @@ export async function run(projectDir) {
 	await fs.mkdir(new URL('./src/pages/blog', projectDir), { recursive: true });
 	await fs.mkdir(new URL('./data/blog', projectDir), { recursive: true });
 	await fs.mkdir(new URL('./src/content', projectDir), { recursive: true });
+	await fs.copyFile(new URL('./image.jpg', import.meta.url), new URL('./image.jpg', projectDir));
 
 	const promises = [];
 
@@ -17,6 +18,9 @@ export async function run(projectDir) {
 # Article ${i}
 
 ${loremIpsumMd}
+
+![image ${i}](../../image.jpg)
+
 `;
 		promises.push(
 			fs.writeFile(new URL(`./data/blog/article-${i}.md`, projectDir), content, 'utf-8')

benchmark/package.json

@@ -16,6 +16,7 @@
     "markdown-table": "^3.0.3",
     "mri": "^1.2.0",
     "port-authority": "^2.0.1",
-    "pretty-bytes": "^6.1.1"
+    "pretty-bytes": "^6.1.1",
+    "sharp": "^0.33.3"
   }
 }

packages/astro/package.json

@@ -170,6 +170,7 @@
     "magic-string": "^0.30.10",
     "micromatch": "^4.0.7",
     "mrmime": "^2.0.0",
+    "neotraverse": "^0.6.9",
     "ora": "^8.0.1",
     "p-limit": "^6.1.0",
     "p-queue": "^8.0.1",

packages/astro/src/assets/utils/resolveImports.ts

@@ -0,0 +1,40 @@
+import { isRemotePath, removeBase } from '@astrojs/internal-helpers/path';
+import { CONTENT_IMAGE_FLAG, IMAGE_IMPORT_PREFIX } from '../../content/consts.js';
+import { shorthash } from '../../runtime/server/shorthash.js';
+import { VALID_INPUT_FORMATS } from '../consts.js';
+
+/**
+ * Resolves an image src from a content file (such as markdown) to a module ID or import that can be resolved by Vite.
+ *
+ * @param imageSrc The src attribute of an image tag
+ * @param filePath The path to the file that contains the imagem relative to the site root
+ * @returns A module id of the image that can be rsolved by Vite, or undefined if it is not a local image
+ */
+export function imageSrcToImportId(imageSrc: string, filePath: string): string | undefined {
+	// If the import is coming from the data store it will have a special prefix to identify it
+	// as an image import. We remove this prefix so that we can resolve the image correctly.
+	imageSrc = removeBase(imageSrc, IMAGE_IMPORT_PREFIX);
+
+	// We only care about local imports
+	if (isRemotePath(imageSrc) || imageSrc.startsWith('/')) {
+		return;
+	}
+	// We only care about images
+	const ext = imageSrc.split('.').at(-1) as (typeof VALID_INPUT_FORMATS)[number] | undefined;
+	if (!ext || !VALID_INPUT_FORMATS.includes(ext)) {
+		return;
+	}
+
+	// The import paths are relative to the content (md) file, but when it's actually resolved it will
+	// be in a single assets file, so relative paths will no longer work. To deal with this we use
+	// a query parameter to store the original path to the file and append a query param flag.
+	// This allows our Vite plugin to intercept the import and resolve the path relative to the
+	// importer and get the correct full path for the imported image.
+
+	const params = new URLSearchParams(CONTENT_IMAGE_FLAG);
+	params.set('importer', filePath);
+	return `${imageSrc}?${params.toString()}`;
+}
+
+export const importIdToSymbolName = (importId: string) =>
+	`__ASTRO_IMAGE_IMPORT_${shorthash(importId)}`;

packages/astro/src/content/consts.ts

@@ -2,22 +2,28 @@ export const PROPAGATED_ASSET_FLAG = 'astroPropagatedAssets';
 export const CONTENT_RENDER_FLAG = 'astroRenderContent';
 export const CONTENT_FLAG = 'astroContentCollectionEntry';
 export const DATA_FLAG = 'astroDataCollectionEntry';
+export const CONTENT_IMAGE_FLAG = 'astroContentImageFlag';
 
 export const VIRTUAL_MODULE_ID = 'astro:content';
 export const RESOLVED_VIRTUAL_MODULE_ID = '\0' + VIRTUAL_MODULE_ID;
 export const DATA_STORE_VIRTUAL_ID = 'astro:data-layer-content';
 export const RESOLVED_DATA_STORE_VIRTUAL_ID = '\0' + DATA_STORE_VIRTUAL_ID;
+export const ASSET_IMPORTS_VIRTUAL_ID = 'astro:asset-imports';
+export const ASSET_IMPORTS_RESOLVED_STUB_ID = '\0' + ASSET_IMPORTS_VIRTUAL_ID;
 export const LINKS_PLACEHOLDER = '@@ASTRO-LINKS@@';
 export const STYLES_PLACEHOLDER = '@@ASTRO-STYLES@@';
 export const SCRIPTS_PLACEHOLDER = '@@ASTRO-SCRIPTS@@';
+export const IMAGE_IMPORT_PREFIX = '__ASTRO_IMAGE_';
 
 export const CONTENT_FLAGS = [
 	CONTENT_FLAG,
 	CONTENT_RENDER_FLAG,
 	DATA_FLAG,
 	PROPAGATED_ASSET_FLAG,
+	CONTENT_IMAGE_FLAG,
 ] as const;
 
 export const CONTENT_TYPES_FILE = 'types.d.ts';
 
 export const DATA_STORE_FILE = 'data-store.json';
+export const ASSET_IMPORTS_FILE = 'assets.mjs';

packages/astro/src/content/data-store.ts

@@ -1,10 +1,15 @@
 import { promises as fs, type PathLike, existsSync } from 'fs';
+import { imageSrcToImportId, importIdToSymbolName } from '../assets/utils/resolveImports.js';
+import { AstroError, AstroErrorData } from '../core/errors/index.js';
 
 const SAVE_DEBOUNCE_MS = 500;
 
 export interface RenderedContent {
 	html: string;
-	metadata?: Record<string, unknown>;
+	metadata?: {
+		imagePaths: Array<string>;
+		[key: string]: unknown;
+	};
 }
 
 export interface DataEntry {
@@ -21,9 +26,15 @@ export class DataStore {
 
 	#file?: PathLike;
 
+	#assetsFile?: PathLike;
+
 	#saveTimeout: NodeJS.Timeout | undefined;
+	#assetsSaveTimeout: NodeJS.Timeout | undefined;
 
 	#dirty = false;
+	#assetsDirty = false;
+
+	#assetImports = new Set<string>();
 
 	constructor() {
 		this.#collections = new Map();
@@ -77,7 +88,77 @@ export class DataStore {
 		return this.#collections;
 	}
 
-	#saveToDiskDebounced = () => {
+	addAssetImport(assetImport: string, filePath: string) {
+		const id = imageSrcToImportId(assetImport, filePath);
+		if (id) {
+			this.#assetImports.add(id);
+			// We debounce the writes to disk because addAssetImport is called for every image in every file,
+			// and can be called many times in quick succession by a filesystem watcher. We only want to write
+			// the file once, after all the imports have been added.
+			this.#writeAssetsImportsDebounced();
+		}
+	}
+
+	addAssetImports(assets: Array<string>, filePath: string) {
+		assets.forEach((asset) => this.addAssetImport(asset, filePath));
+	}
+
+	async writeAssetImports(filePath: PathLike) {
+		this.#assetsFile = filePath;
+
+		if (this.#assetImports.size === 0) {
+			try {
+				await fs.writeFile(filePath, 'export default new Map();');
+			} catch (err) {
+				throw new AstroError({
+					...(err as Error),
+					...AstroErrorData.ContentLayerWriteError,
+				});
+			}
+		}
+
+		if (!this.#assetsDirty && existsSync(filePath)) {
+			return;
+		}
+		// Import the assets, with a symbol name that is unique to the import id. The import
+		// for each asset is an object with path, format and dimensions.
+		// We then export them all, mapped by the import id, so we can find them again in the build.
+		const imports: Array<string> = [];
+		const exports: Array<string> = [];
+		this.#assetImports.forEach((id) => {
+			const symbol = importIdToSymbolName(id);
+			imports.push(`import ${symbol} from '${id}';`);
+			exports.push(`[${JSON.stringify(id)}, ${symbol}]`);
+		});
+		const code = /* js */ `
+${imports.join('\n')}
+export default new Map([${exports.join(', ')}]);
+		`;
+		try {
+			await fs.writeFile(filePath, code);
+		} catch (err) {
+			throw new AstroError({
+				...(err as Error),
+				...AstroErrorData.ContentLayerWriteError,
+			});
+		}
+		this.#assetsDirty = false;
+	}
+
+	#writeAssetsImportsDebounced() {
+		this.#assetsDirty = true;
+		if (this.#assetsFile) {
+			if (this.#assetsSaveTimeout) {
+				clearTimeout(this.#assetsSaveTimeout);
+			}
+			this.#assetsSaveTimeout = setTimeout(() => {
+				this.#assetsSaveTimeout = undefined;
+				this.writeAssetImports(this.#assetsFile!);
+			}, SAVE_DEBOUNCE_MS);
+		}
+	}
+
+	#saveToDiskDebounced() {
 		this.#dirty = true;
 		// Only save to disk if it has already been saved once
 		if (this.#file) {
@@ -89,7 +170,7 @@ export class DataStore {
 				this.writeToDisk(this.#file!);
 			}, SAVE_DEBOUNCE_MS);
 		}
-	};
+	}
 
 	scopedStore(collectionName: string): ScopedDataStore {
 		return {
@@ -118,6 +199,9 @@ export class DataStore {
 					entry.body = body;
 				}
 				if (filePath) {
+					if (filePath.startsWith('/')) {
+						throw new Error(`File path must be relative to the site root. Got: ${filePath}`);
+					}
 					entry.filePath = filePath;
 				}
 				if (digest) {
@@ -133,6 +217,10 @@ export class DataStore {
 			delete: (key: string) => this.delete(collectionName, key),
 			clear: () => this.clear(collectionName),
 			has: (key: string) => this.has(collectionName, key),
+			addAssetImport: (assetImport: string, fileName: string) =>
+				this.addAssetImport(assetImport, fileName),
+			addAssetImports: (assets: Array<string>, fileName: string) =>
+				this.addAssetImports(assets, fileName),
 		};
 	}
 
@@ -162,8 +250,11 @@ export class DataStore {
 			await fs.writeFile(filePath, this.toString());
 			this.#file = filePath;
 			this.#dirty = false;
-		} catch {
-			throw new Error(`Failed to save data store to disk`);
+		} catch (err) {
+			throw new AstroError({
+				...(err as Error),
+				...AstroErrorData.ContentLayerWriteError,
+			});
 		}
 	}
 
@@ -203,6 +294,18 @@ export class DataStore {
 export interface ScopedDataStore {
 	get: (key: string) => DataEntry | undefined;
 	entries: () => Array<[id: string, DataEntry]>;
+	/**
+	 * Adds a new entry to the store. If an entry with the same ID already exists,
+	 * it will be replaced.
+	 * @param opts
+	 * @param opts.id The ID of the entry. Must be unique per collection.
+	 * @param opts.data The data to store.
+	 * @param opts.body The raw body of the content, if applicable.
+	 * @param opts.filePath The file path of the content, if applicable. Relative to the site root.
+	 * @param opts.digest A content digest, to check if the content has changed.
+	 * @param opts.rendered The rendered content, if applicable.
+	 * @returns
+	 */
 	set: (opts: {
 		id: string;
 		data: Record<string, unknown>;
@@ -216,6 +319,21 @@ export interface ScopedDataStore {
 	delete: (key: string) => void;
 	clear: () => void;
 	has: (key: string) => boolean;
+	/**
+	 * Adds image etc assets to the store. These assets will be transformed
+	 * by Vite, and the URLs will be available in the final build.
+	 * @param assets An array of asset src values, relative to the importing file.
+	 * @param fileName The full path of the file that is importing the assets.
+	 */
+	addAssetImports: (assets: Array<string>, fileName: string) => void;
+	/**
+	 * Adds a single asset to the store. This asset will be transformed
+	 * by Vite, and the URL will be available in the final build.
+	 * @param assetImport
+	 * @param fileName
+	 * @returns
+	 */
+	addAssetImport: (assetImport: string, fileName: string) => void;
 }
 
 /**

packages/astro/src/content/loaders/file.ts

@@ -1,5 +1,6 @@
-import { promises as fs, existsSync } from 'fs';
-import { fileURLToPath } from 'url';
+import { promises as fs, existsSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { posixRelative } from '../utils.js';
 import type { Loader, LoaderContext } from './types.js';
 
 /**
@@ -13,7 +14,7 @@ export function file(fileName: string): Loader {
 		throw new Error('Glob patterns are not supported in `file` loader. Use `glob` loader instead.');
 	}
 
-	async function syncData(filePath: string, { logger, parseData, store }: LoaderContext) {
+	async function syncData(filePath: string, { logger, parseData, store, settings }: LoaderContext) {
 		let json: Array<Record<string, unknown>>;
 
 		try {
@@ -38,7 +39,11 @@ export function file(fileName: string): Loader {
 					continue;
 				}
 				const data = await parseData({ id, data: rawItem, filePath });
-				store.set({ id, data, filePath });
+				store.set({
+					id,
+					data,
+					filePath: posixRelative(fileURLToPath(settings.config.root), filePath),
+				});
 			}
 		} else if (typeof json === 'object') {
 			const entries = Object.entries<Record<string, unknown>>(json);