feat: allow integrations to refresh content layer data

.changeset/curvy-walls-kneel.md

@@ -0,0 +1,47 @@
+---
+'astro': patch
+---
+
+Adds support for allowing integrations to refresh the content layer
+
+This adds a new object to the `astro:server:setup` hook that allows integrations to refresh the content layer. This can be used for example to register a webhook endpoint during dev, or to open a socket to a CMS to listen for changes.
+
+The hook is passed a function called `refreshContent` that can be called to refresh the content layer. It can optionally be passed a `loaders` property, which is an array of loader names. If provided, only collections that use those loaders will be refreshed. If not provided, all loaders will be refreshed. A CMS integration could use this to only refresh its own collections.
+
+It can also pass a `context` object, which will be passed to the loaders. This can be used to pass arbitrary data, such as the webhook body or an event from the websocket.
+
+```ts
+ {
+    name: 'my-integration',
+    hooks: {
+        'astro:server:setup': async ({ server, refreshContent }) => {
+            server.middlewares.use('/_refresh', async (req, res) => {
+                if(req.method !== 'POST') {
+                  res.statusCode = 405
+                  res.end('Method Not Allowed');
+                  return
+                }
+                let body = '';
+                req.on('data', chunk => {
+                    body += chunk.toString();
+                });
+                req.on('end', async () => {
+                    try {
+                        const webhookBody = JSON.parse(body);
+                        await refreshContent({
+                          context: { webhookBody },
+                          loaders: ['my-loader']
+                        });
+                        res.writeHead(200, { 'Content-Type': 'application/json' });
+                        res.end(JSON.stringify({ message: 'Content refreshed successfully' }));
+                    } catch (error) {
+                        res.writeHead(500, { 'Content-Type': 'application/json' });
+                        res.end(JSON.stringify({ error: 'Failed to refresh content: ' + error.message }));
+                    }
+                });
+            });
+        }
+    }
+}
+```
+

packages/astro/src/content/content-layer.ts

@@ -76,6 +76,11 @@ export class ContentLayer {
 		this.#unsubscribe?.();
 	}
 
+	dispose() {
+		this.#queue.kill();
+		this.#unsubscribe?.();
+	}
+
 	async #getGenerateDigest() {
 		if (this.#generateDigest) {
 			return this.#generateDigest;
@@ -301,13 +306,13 @@ function contentLayerSingleton() {
 	let instance: ContentLayer | null = null;
 	return {
 		init: (options: ContentLayerOptions) => {
-			instance?.unwatchContentConfig();
+			instance?.dispose();
 			instance = new ContentLayer(options);
 			return instance;
 		},
 		get: () => instance,
 		dispose: () => {
-			instance?.unwatchContentConfig();
+			instance?.dispose();
 			instance = null;
 		},
 	};

packages/astro/src/content/loaders/types.ts

@@ -33,6 +33,7 @@ export interface LoaderContext {
 	/** When running in dev, this is a filesystem watcher that can be used to trigger updates */
 	watcher?: FSWatcher;
 
+	/** If the loader has been triggered by an integration, this may optionally contain extra data set by that integration */
 	refreshContextData?: Record<string, unknown>;
 	entryTypes: Map<string, ContentEntryType>;
 }

packages/astro/src/integrations/hooks.ts

@@ -5,6 +5,9 @@ import { bold } from 'kleur/colors';
 import type { InlineConfig, ViteDevServer } from 'vite';
 import astroIntegrationActionsRouteHandler from '../actions/integration.js';
 import { isActionsFilePresent } from '../actions/utils.js';
+import { CONTENT_LAYER_TYPE } from '../content/consts.js';
+import { globalContentLayer } from '../content/content-layer.js';
+import { globalContentConfigObserver } from '../content/utils.js';
 import type { SerializedSSRManifest } from '../core/app/types.js';
 import type { PageBuildData } from '../core/build/types.js';
 import { buildClientDirectiveEntrypoint } from '../core/client-directive/index.js';
@@ -13,7 +16,11 @@ import type { AstroIntegrationLogger, Logger } from '../core/logger/core.js';
 import { isServerLikeOutput } from '../core/util.js';
 import type { AstroSettings } from '../types/astro.js';
 import type { AstroConfig } from '../types/public/config.js';
-import type { ContentEntryType, DataEntryType } from '../types/public/content.js';
+import type {
+	ContentEntryType,
+	DataEntryType,
+	RefreshContentOptions,
+} from '../types/public/content.js';
 import type {
 	AstroIntegration,
 	AstroRenderer,
@@ -367,6 +374,24 @@ export async function runHookServerSetup({
 	server: ViteDevServer;
 	logger: Logger;
 }) {
+	let refreshContent: undefined | ((options: RefreshContentOptions) => Promise<void>);
+	if (config.experimental?.contentLayer) {
+		refreshContent = async (options: RefreshContentOptions) => {
+			const contentConfig = globalContentConfigObserver.get();
+			if (
+				contentConfig.status !== 'loaded' ||
+				!Object.values(contentConfig.config.collections).some(
+					(collection) => collection.type === CONTENT_LAYER_TYPE,
+				)
+			) {
+				return;
+			}
+
+			const contentLayer = await globalContentLayer.get();
+			await contentLayer?.sync(options);
+		};
+	}
+
 	for (const integration of config.integrations) {
 		if (integration?.hooks?.['astro:server:setup']) {
 			await withTakingALongTimeMsg({
@@ -376,6 +401,7 @@ export async function runHookServerSetup({
 					server,
 					logger: getLogger(integration, logger),
 					toolbar: getToolbarServerCommunicationHelpers(server),
+					refreshContent,
 				}),
 				logger,
 			});

packages/astro/src/types/public/integrations.ts

@@ -6,6 +6,7 @@ import type { AstroIntegrationLogger } from '../../core/logger/core.js';
 import type { getToolbarServerCommunicationHelpers } from '../../integrations/hooks.js';
 import type { DeepPartial } from '../../type-utils.js';
 import type { AstroConfig } from './config.js';
+import type { RefreshContentOptions } from './content.js';
 import type { RouteData } from './internal.js';
 import type { DevToolbarAppEntry } from './toolbar.js';
 
@@ -187,6 +188,7 @@ export interface BaseIntegrationHooks {
 		server: ViteDevServer;
 		logger: AstroIntegrationLogger;
 		toolbar: ReturnType<typeof getToolbarServerCommunicationHelpers>;
+		refreshContent?: (options: RefreshContentOptions) => Promise<void>;
 	}) => void | Promise<void>;
 	'astro:server:start': (options: {
 		address: AddressInfo;

packages/astro/test/content-layer.test.js

@@ -290,5 +290,21 @@ describe('Content Layer', () => {
 			assert.ok(updated.fileLoader[0].data.temperament.includes('Bouncy'));
 			await fixture.resetAllFiles();
 		});
+
+		it('reloads data when an integration triggers a content refresh', async () => {
+			const rawJsonResponse = await fixture.fetch('/collections.json');
+			const initialJson = devalue.parse(await rawJsonResponse.text());
+			assert.equal(initialJson.increment.data.lastValue, 1);
+
+			const refreshResponse = await fixture.fetch('/_refresh', {
+				method: 'POST',
+				body: JSON.stringify({}),
+			});
+			const refreshData = await refreshResponse.json();
+			assert.equal(refreshData.message, 'Content refreshed successfully');
+			const updatedJsonResponse = await fixture.fetch('/collections.json');
+			const updated = devalue.parse(await updatedJsonResponse.text());
+			assert.equal(updated.increment.data.lastValue, 2);
+		});
 	});
 });

packages/astro/test/fixtures/content-layer/astro.config.mjs

@@ -3,7 +3,38 @@ import { defineConfig } from 'astro/config';
 import { fileURLToPath } from 'node:url';
 
 export default defineConfig({
-  integrations: [mdx()],
+  integrations: [mdx(), {
+		name: '@astrojs/my-integration',
+		hooks: {
+				'astro:server:setup': async ({ server, refreshContent }) => {
+						server.middlewares.use('/_refresh', async (req, res) => {
+								if(req.method !== 'POST') {
+									res.statusCode = 405
+									res.end('Method Not Allowed');
+									return
+								}
+								let body = '';
+								req.on('data', chunk => {
+										body += chunk.toString();
+								});
+								req.on('end', async () => {
+										try {
+												const webhookBody = JSON.parse(body);
+												await refreshContent({
+													context: { webhookBody },
+													loaders: ['increment-loader']
+												});
+												res.writeHead(200, { 'Content-Type': 'application/json' });
+												res.end(JSON.stringify({ message: 'Content refreshed successfully' }));
+										} catch (error) {
+												res.writeHead(500, { 'Content-Type': 'application/json' });
+												res.end(JSON.stringify({ error: 'Failed to refresh content: ' + error.message }));
+										}
+								});
+						});
+				}
+		}
+}],
 	vite: {
 		resolve: {
 			alias: {