feat: adds simple loader (#11386)

* wip

* Add simple loader

* Fix type guard

* Tighten loader schema

* Add loader function to type

packages/astro/src/content/loaders.ts

@@ -16,6 +16,11 @@ export interface ParseDataOptions {
 	filePath?: string;
 }
 
+export type DataWithId = {
+	id: string;
+	[key: string]: unknown;
+};
+
 export interface LoaderContext {
 	/** The unique name of the collection */
 	collection: string;
@@ -28,9 +33,7 @@ export interface LoaderContext {
 	settings: AstroSettings;
 
 	/** Validates and parses the data according to the collection schema */
-	parseData<T extends Record<string, unknown> = Record<string, unknown>>(
-		props: ParseDataOptions
-	): T;
+	parseData(props: ParseDataOptions): Promise<DataWithId>;
 
 	/** When running in dev, this is a filesystem watcher that can be used to trigger updates */
 	watcher?: FSWatcher;
@@ -83,7 +86,7 @@ export async function syncContentLayer({
 
 			let { schema } = collection;
 
-			if (!schema) {
+			if (!schema && typeof collection.loader === 'object') {
 				schema = collection.loader.schema;
 			}
 
@@ -97,12 +100,8 @@ export async function syncContentLayer({
 
 			const collectionWithResolvedSchema = { ...collection, schema };
 
-			function parseData<T extends Record<string, unknown> = Record<string, unknown>>({
-				id,
-				data,
-				filePath = '',
-			}: { id: string; data: T; filePath?: string }): T {
-				return getEntryData(
+			const parseData: LoaderContext['parseData'] = ({ id, data, filePath = '' }) =>
+				getEntryData(
 					{
 						id,
 						collection: name,
@@ -114,18 +113,27 @@ export async function syncContentLayer({
 					},
 					collectionWithResolvedSchema,
 					false
-				) as unknown as T;
-			}
+				) as Promise<DataWithId>;
 
-			return collection.loader.load({
+			const payload: LoaderContext = {
 				collection: name,
 				store: store.scopedStore(name),
 				meta: store.metaStore(name),
 				logger: globalLogger.forkIntegrationLogger(collection.loader.name ?? 'content'),
 				settings,
 				parseData,
 				watcher,
-			});
+			};
+
+			if (typeof collection.loader === 'function') {
+				return simpleLoader(collection.loader, payload);
+			}
+
+			if (!collection.loader.load) {
+				throw new Error(`Collection loader for ${name} does not have a load method`);
+			}
+
+			return collection.loader.load(payload);
 		})
 	);
 	const cacheFile = new URL(DATA_STORE_FILE, settings.config.cacheDir);
@@ -135,3 +143,15 @@ export async function syncContentLayer({
 	await store.writeToDisk(cacheFile);
 	logger.info('Synced content');
 }
+
+export async function simpleLoader(
+	handler: () => Array<DataWithId> | Promise<Array<DataWithId>>,
+	context: LoaderContext
+) {
+	const data = await handler();
+	context.store.clear();
+	for (const raw of data) {
+		const item = await context.parseData({ id: raw.id, data: raw });
+		context.store.set(raw.id, item);
+	}
+}

packages/astro/src/content/types-generator.ts

@@ -369,7 +369,11 @@ async function typeForCollection<T extends keyof ContentConfig['collections']>(
 		return `InferEntrySchema<${collectionKey}>`;
 	}
 
-	if (collection?.type === 'experimental_data' && collection.loader.schema) {
+	if (
+		collection?.type === 'experimental_data' &&
+		typeof collection.loader === 'object' &&
+		collection.loader.schema
+	) {
 		let schema = collection.loader.schema;
 		if (typeof schema === 'function') {
 			schema = await schema();

packages/astro/src/content/utils.ts

@@ -38,27 +38,49 @@ export const collectionConfigParser = z.union([
 	z.object({
 		type: z.literal('experimental_data'),
 		schema: z.any().optional(),
-		loader: z.object({
-			name: z.string(),
-			load: z.function(
-				z.tuple(
-					[
-						z.object({
-							collection: z.string(),
-							store: z.any(),
-							meta: z.any(),
-							logger: z.any(),
-							settings: z.any(),
-							parseData: z.any(),
-							watcher: z.any().optional(),
-						}),
-					],
-					z.unknown()
-				)
+		loader: z.union([
+			z.function().returns(
+				z.union([
+					z.array(
+						z
+							.object({
+								id: z.string(),
+							})
+							.catchall(z.unknown())
+					),
+					z.promise(
+						z.array(
+							z
+								.object({
+									id: z.string(),
+								})
+								.catchall(z.unknown())
+						)
+					),
+				])
 			),
-			schema: z.any().optional(),
-			render: z.function(z.tuple([z.any()], z.unknown())).optional(),
-		}),
+			z.object({
+				name: z.string(),
+				load: z.function(
+					z.tuple(
+						[
+							z.object({
+								collection: z.string(),
+								store: z.any(),
+								meta: z.any(),
+								logger: z.any(),
+								settings: z.any(),
+								parseData: z.any(),
+								watcher: z.any().optional(),
+							}),
+						],
+						z.unknown()
+					)
+				),
+				schema: z.any().optional(),
+				render: z.function(z.tuple([z.any()], z.unknown())).optional(),
+			}),
+		]),
 	}),
 ]);
 

packages/astro/test/content-layer.test.js

@@ -86,6 +86,27 @@ describe('Content Layer', () => {
 				},
 			});
 		});
+
+		it('returns collection from a simple loader', async () => {
+			assert.ok(json.hasOwnProperty('simpleLoader'));
+			assert.ok(Array.isArray(json.simpleLoader));
+
+			const item = json.simpleLoader[0];
+			assert.equal(json.simpleLoader.length, 4);
+			assert.deepEqual(item, {
+				id: 'siamese',
+				collection: 'cats',
+				data: {
+					breed: 'Siamese',
+					id: 'siamese',
+					size: 'Medium',
+					origin: 'Thailand',
+					lifespan: '15 years',
+					temperament: ['Active', 'Affectionate', 'Social', 'Playful'],
+				},
+				type: 'experimental_data',
+			});
+		});
 	});
 
 	describe('Dev', () => {

packages/astro/test/fixtures/content-layer/src/content/config.ts

@@ -18,4 +18,52 @@ const dogs = defineCollection({
 		temperament: z.array(z.string())
 	}),
 })
-export const collections = { blog, dogs };
+
+const cats = defineCollection({
+	type: "experimental_data",
+	loader: async function() {
+		return  [{
+				"breed": "Siamese",
+				"id": "siamese",
+				"size": "Medium",
+				"origin": "Thailand",
+				"lifespan": "15 years",
+				"temperament": ["Active", "Affectionate", "Social", "Playful"]
+			},
+			 {
+				"breed": "Persian",
+				"id": "persian",
+				"size": "Medium",
+				"origin": "Iran",
+				"lifespan": "15 years",
+				"temperament": ["Calm", "Affectionate", "Social"]
+			},
+			 {
+				"breed": "Tabby",
+				"id": "tabby",
+				"size": "Medium",
+				"origin": "Egypt",
+				"lifespan": "15 years",
+				"temperament": ["Curious", "Playful", "Independent"]
+			},
+			{
+				"breed": "Ragdoll",
+				"id": "ragdoll",
+				"size": "Medium",
+				"origin": "United States",
+				"lifespan": "15 years",
+				"temperament": ["Calm", "Affectionate", "Social"]
+			}
+		];
+	},
+	schema: z.object({
+		breed: z.string(),
+		id: z.string(),
+		size: z.string(),
+		origin: z.string(),
+		lifespan: z.string(),
+		temperament: z.array(z.string())
+	}),
+})
+
+export const collections = { blog, dogs, cats };

packages/astro/test/fixtures/content-layer/src/pages/collections.json.js

@@ -1,12 +1,14 @@
 import { getCollection, getDataEntryById } from 'astro:content';
 
 export async function GET() {
-	const customLoader = await getCollection('blog');
+	const customLoader = (await getCollection('blog')).slice(0, 10);
 	const fileLoader = await getCollection('dogs');
 
 	const dataEntryById = await getDataEntryById('dogs', 'beagle');
 
+	const simpleLoader = await getCollection('cats');
+
 	return new Response(
-		JSON.stringify({ customLoader, fileLoader, dataEntryById }),
+		JSON.stringify({ customLoader, fileLoader, dataEntryById, simpleLoader }),
 	);
 }

packages/astro/types/content.d.ts

@@ -81,10 +81,15 @@ declare module 'astro:content' {
 
 	export type SchemaContext = { image: ImageFunction };
 
+	export interface DataWithId {
+		id: string;
+		[key: string]: unknown;
+	}
+
 	type ContentCollectionV2Config<S extends BaseSchema> = {
 		type: 'experimental_data';
 		schema?: S | ((context: SchemaContext) => S);
-		loader: Loader;
+		loader: Loader | (() => Array<DataWithId> | Promise<Array<DataWithId>>);
 	};
 
 	type DataCollectionConfig<S extends BaseSchema> = {