Add integrations API for db config/seed files (#10321)

* Add integrations API for adding db config/seed files

* Fix seeding when user seed file is present

* Add basic test and fixture for integrations API

* Freeze that lockfile

* Test to see if this is a Windows fix

* Don’t import.meta.glob integration seed files

* Make integration seed files export a default function

* style: rejiggle

* Fix temporary file conflicts

* Remove changes to Astro’s core types, type utility method instead

* Use `astro:db` instead of `@astrojs/db`

* Revert unnecessarily cautious temporary path name

This reverts commit ef2156e.

* Add changeset

* Fix entrypoints and `asDrizzleTable` usage in changeset

* Getting Nate in on the co-author action

Co-authored-by: Nate Moore <natemoo-re@users.noreply.github.com>

* Fix user seed file in integrations fixture

* Update `seedLocal()` after merge

* Provide empty `seedFiles` array in `db execute`

---------

Co-authored-by: Nate Moore <natemoo-re@users.noreply.github.com>

.changeset/purple-poets-sin.md

@@ -0,0 +1,58 @@
+---
+"@astrojs/db": minor
+---
+
+Adds support for integrations providing `astro:db` configuration and seed files, using the new `astro:db:setup` hook.
+
+To get TypeScript support for the `astro:db:setup` hook, wrap your integration object in the `defineDbIntegration()` utility:
+
+```js
+import { defineDbIntegration } from '@astrojs/db/utils';
+
+export default function MyDbIntegration() {
+  return defineDbIntegration({
+    name: 'my-astro-db-powered-integration',
+    hooks: {
+      'astro:db:setup': ({ extendDb }) => {
+        extendDb({
+          configEntrypoint: '@astronaut/my-package/config',
+          seedEntrypoint: '@astronaut/my-package/seed',
+        });
+      },
+    },
+  });
+}
+```
+
+Use the `extendDb` method to register additional `astro:db` config and seed files.
+
+Integration config and seed files follow the same format as their user-defined equivalents. However, often while working on integrations, you may not be able to benefit from Astro’s generated table types exported from `astro:db`. For full type safety and autocompletion support, use the `asDrizzleTable()` utility to wrap your table definitions in the seed file.
+
+```js
+// config.ts
+import { defineTable, column } from 'astro:db';
+
+export const Pets = defineTable({
+  columns: {
+    name: column.text(),
+    age: column.number(),
+  },
+});
+```
+
+```js
+// seed.ts
+import { asDrizzleTable } from '@astrojs/db/utils';
+import { db } from 'astro:db';
+import { Pets } from './config';
+
+export default async function() {
+  // Convert the Pets table into a format ready for querying.
+  const typeSafePets = asDrizzleTable('Pets', Pets);
+
+  await db.insert(typeSafePets).values([
+    { name: 'Palomita', age: 7 },
+    { name: 'Pan', age: 3.5 },
+  ]);
+}
+```

packages/db/src/core/cli/commands/execute/index.ts

@@ -43,6 +43,7 @@ export async function cmd({
 			tables: dbConfig.tables ?? {},
 			root: astroConfig.root,
 			shouldSeed: false,
+			seedFiles: [],
 		});
 	}
 	const { code } = await bundleFile({ virtualModContents, root: astroConfig.root, fileUrl });

packages/db/src/core/cli/index.ts

@@ -1,7 +1,6 @@
 import type { AstroConfig } from 'astro';
 import type { Arguments } from 'yargs-parser';
-import { loadDbConfigFile } from '../load-file.js';
-import { dbConfigSchema } from '../types.js';
+import { resolveDbConfig } from '../load-file.js';
 
 export async function cli({
 	flags,
@@ -14,9 +13,7 @@ export async function cli({
 	// Most commands are `astro db foo`, but for now login/logout
 	// are also handled by this package, so first check if this is a db command.
 	const command = args[2] === 'db' ? args[3] : args[2];
-	const { mod } = await loadDbConfigFile(astroConfig.root);
-	// TODO: parseConfigOrExit()
-	const dbConfig = dbConfigSchema.parse(mod?.default ?? {});
+	const { dbConfig } = await resolveDbConfig(astroConfig);
 
 	switch (command) {
 		case 'shell': {

packages/db/src/core/errors.ts

@@ -71,3 +71,13 @@ export const FOREIGN_KEY_REFERENCES_EMPTY_ERROR = (tableName: string) => {
 		tableName
 	)} is misconfigured. \`references\` array cannot be empty.`;
 };
+
+export const INTEGRATION_TABLE_CONFLICT_ERROR = (
+	integrationName: string,
+	tableName: string,
+	isUserConflict: boolean
+) => {
+	return red('▶ Conflicting table name in integration ' + bold(integrationName)) + isUserConflict
+		? `\n  A user-defined table named ${bold(tableName)} already exists`
+		: `\n  Another integration already added a table named ${bold(tableName)}`;
+};

packages/db/src/core/integration/index.ts

@@ -6,22 +6,19 @@ import { mkdir, rm, writeFile } from 'fs/promises';
 import { blue, yellow } from 'kleur/colors';
 import parseArgs from 'yargs-parser';
 import { CONFIG_FILE_NAMES, DB_PATH } from '../consts.js';
-import { loadDbConfigFile } from '../load-file.js';
+import { resolveDbConfig } from '../load-file.js';
 import { type ManagedAppToken, getManagedAppTokenOrExit } from '../tokens.js';
-import { type DBConfig, dbConfigSchema } from '../types.js';
 import { type VitePlugin, getDbDirectoryUrl } from '../utils.js';
-import { errorMap } from './error-map.js';
 import { fileURLIntegration } from './file-url.js';
 import { typegen } from './typegen.js';
-import { type LateTables, vitePluginDb } from './vite-plugin-db.js';
+import { type LateTables, vitePluginDb, type LateSeedFiles } from './vite-plugin-db.js';
 import { vitePluginInjectEnvTs } from './vite-plugin-inject-env-ts.js';
 
 function astroDBIntegration(): AstroIntegration {
 	let connectToStudio = false;
 	let configFileDependencies: string[] = [];
 	let root: URL;
 	let appToken: ManagedAppToken | undefined;
-	let dbConfig: DBConfig;
 
 	// Make table loading "late" to pass to plugins from `config:setup`,
 	// but load during `config:done` to wait for integrations to settle.
@@ -30,6 +27,11 @@ function astroDBIntegration(): AstroIntegration {
 			throw new Error('[astro:db] INTERNAL Tables not loaded yet');
 		},
 	};
+	let seedFiles: LateSeedFiles = {
+		get() {
+			throw new Error('[astro:db] INTERNAL Seed files not loaded yet');
+		},
+	};
 	let command: 'dev' | 'build' | 'preview';
 	return {
 		name: 'astro:db',
@@ -57,6 +59,7 @@ function astroDBIntegration(): AstroIntegration {
 					dbPlugin = vitePluginDb({
 						connectToStudio: false,
 						tables,
+						seedFiles,
 						root: config.root,
 						srcDir: config.srcDir,
 					});
@@ -74,13 +77,10 @@ function astroDBIntegration(): AstroIntegration {
 
 				// TODO: refine where we load tables
 				// @matthewp: may want to load tables by path at runtime
-				const { mod, dependencies } = await loadDbConfigFile(config.root);
+				const { dbConfig, dependencies, integrationSeedPaths } = await resolveDbConfig(config);
+				tables.get = () => dbConfig.tables;
+				seedFiles.get = () => integrationSeedPaths;
 				configFileDependencies = dependencies;
-				dbConfig = dbConfigSchema.parse(mod?.default ?? {}, {
-					errorMap,
-				});
-				// TODO: resolve integrations here?
-				tables.get = () => dbConfig.tables ?? {};
 
 				if (!connectToStudio) {
 					const dbUrl = new URL(DB_PATH, config.root);

packages/db/src/core/integration/vite-plugin-db.ts

@@ -1,3 +1,4 @@
+import { resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { type SQL, sql } from 'drizzle-orm';
 import { SQLiteAsyncDialect } from 'drizzle-orm/sqlite-core';
@@ -23,11 +24,15 @@ const resolved = {
 export type LateTables = {
 	get: () => DBTables;
 };
+export type LateSeedFiles = {
+	get: () => Array<string | URL>;
+};
 
 type VitePluginDBParams =
 	| {
 			connectToStudio: false;
 			tables: LateTables;
+			seedFiles: LateSeedFiles;
 			srcDir: URL;
 			root: URL;
 	  }
@@ -81,6 +86,7 @@ export function vitePluginDb(params: VitePluginDBParams): VitePlugin {
 			return getLocalVirtualModContents({
 				root: params.root,
 				tables: params.tables.get(),
+				seedFiles: params.seedFiles.get(),
 				shouldSeed: id === resolved.seedVirtual,
 			});
 		},
@@ -94,17 +100,26 @@ export function getConfigVirtualModContents() {
 export function getLocalVirtualModContents({
 	tables,
 	root,
+	seedFiles,
 	shouldSeed,
 }: {
 	tables: DBTables;
+	seedFiles: Array<string | URL>;
 	root: URL;
 	shouldSeed: boolean;
 }) {
-	const seedFilePaths = SEED_DEV_FILE_NAME.map(
+	const userSeedFilePaths = SEED_DEV_FILE_NAME.map(
 		// Format as /db/[name].ts
 		// for Vite import.meta.glob
 		(name) => new URL(name, getDbDirectoryUrl('file:///')).pathname
 	);
+	const resolveId = (id: string) => (id.startsWith('.') ? resolve(fileURLToPath(root), id) : id);
+	const integrationSeedFilePaths = seedFiles.map((pathOrUrl) =>
+		typeof pathOrUrl === 'string' ? resolveId(pathOrUrl) : pathOrUrl.pathname
+	);
+	const integrationSeedImports = integrationSeedFilePaths.map(
+		(filePath) => `() => import(${JSON.stringify(filePath)})`
+	);
 
 	const dbUrl = new URL(DB_PATH, root);
 	return `
@@ -117,7 +132,8 @@ export const db = createLocalDatabaseClient({ dbUrl });
 ${
 	shouldSeed
 		? `await seedLocal({
-	fileGlob: import.meta.glob(${JSON.stringify(seedFilePaths)}, { eager: true }),
+	userSeedGlob: import.meta.glob(${JSON.stringify(userSeedFilePaths)}, { eager: true }),
+	integrationSeedImports: [${integrationSeedImports.join(',')}],
 });`
 		: ''
 }

packages/db/src/core/load-file.ts

@@ -1,12 +1,74 @@
+import type { AstroConfig, AstroIntegration } from 'astro';
+import { build as esbuild } from 'esbuild';
 import { existsSync } from 'node:fs';
 import { unlink, writeFile } from 'node:fs/promises';
-import { fileURLToPath } from 'node:url';
-import { build as esbuild } from 'esbuild';
+import { createRequire } from 'node:module';
+import { fileURLToPath, pathToFileURL } from 'node:url';
 import { CONFIG_FILE_NAMES, VIRTUAL_MODULE_ID } from './consts.js';
+import { INTEGRATION_TABLE_CONFLICT_ERROR } from './errors.js';
+import { errorMap } from './integration/error-map.js';
 import { getConfigVirtualModContents } from './integration/vite-plugin-db.js';
+import { dbConfigSchema, type AstroDbIntegration } from './types.js';
 import { getDbDirectoryUrl } from './utils.js';
 
-export async function loadDbConfigFile(
+const isDbIntegration = (integration: AstroIntegration): integration is AstroDbIntegration =>
+	'astro:db:setup' in integration.hooks;
+
+/**
+ * Load a user’s `astro:db` configuration file and additional configuration files provided by integrations.
+ */
+export async function resolveDbConfig({ root, integrations }: AstroConfig) {
+	const { mod, dependencies } = await loadUserConfigFile(root);
+	const userDbConfig = dbConfigSchema.parse(mod?.default ?? {}, { errorMap });
+	/** Resolved `astro:db` config including tables provided by integrations. */
+	const dbConfig = { tables: userDbConfig.tables ?? {} };
+
+	// Collect additional config and seed files from integrations.
+	const integrationDbConfigPaths: Array<{ name: string; configEntrypoint: string | URL }> = [];
+	const integrationSeedPaths: Array<string | URL> = [];
+	for (const integration of integrations) {
+		if (!isDbIntegration(integration)) continue;
+		const { name, hooks } = integration;
+		if (hooks['astro:db:setup']) {
+			hooks['astro:db:setup']({
+				extendDb({ configEntrypoint, seedEntrypoint }) {
+					if (configEntrypoint) {
+						integrationDbConfigPaths.push({ name, configEntrypoint });
+					}
+					if (seedEntrypoint) {
+						integrationSeedPaths.push(seedEntrypoint);
+					}
+				},
+			});
+		}
+	}
+	for (const { name, configEntrypoint } of integrationDbConfigPaths) {
+		// TODO: config file dependencies are not tracked for integrations for now.
+		const loadedConfig = await loadIntegrationConfigFile(root, configEntrypoint);
+		const integrationDbConfig = dbConfigSchema.parse(loadedConfig.mod?.default ?? {}, {
+			errorMap,
+		});
+		for (const key in integrationDbConfig.tables) {
+			if (key in dbConfig.tables) {
+				const isUserConflict = key in (userDbConfig.tables ?? {});
+				throw new Error(INTEGRATION_TABLE_CONFLICT_ERROR(name, key, isUserConflict));
+			} else {
+				dbConfig.tables[key] = integrationDbConfig.tables[key];
+			}
+		}
+	}
+
+	return {
+		/** Resolved `astro:db` config, including tables added by integrations. */
+		dbConfig,
+		/** Dependencies imported into the user config file. */
+		dependencies,
+		/** Additional `astro:db` seed file paths provided by integrations. */
+		integrationSeedPaths,
+	};
+}
+
+async function loadUserConfigFile(
 	root: URL
 ): Promise<{ mod: { default?: unknown } | undefined; dependencies: string[] }> {
 	let configFileUrl: URL | undefined;
@@ -16,13 +78,35 @@ export async function loadDbConfigFile(
 			configFileUrl = fileUrl;
 		}
 	}
-	if (!configFileUrl) {
+	return await loadAndBundleDbConfigFile({ root, fileUrl: configFileUrl });
+}
+
+async function loadIntegrationConfigFile(root: URL, filePathOrUrl: string | URL) {
+	let fileUrl: URL;
+	if (typeof filePathOrUrl === 'string') {
+		const { resolve } = createRequire(root);
+		const resolvedFilePath = resolve(filePathOrUrl);
+		fileUrl = pathToFileURL(resolvedFilePath);
+	} else {
+		fileUrl = filePathOrUrl;
+	}
+	return await loadAndBundleDbConfigFile({ root, fileUrl });
+}
+
+async function loadAndBundleDbConfigFile({
+	root,
+	fileUrl,
+}: {
+	root: URL;
+	fileUrl: URL | undefined;
+}): Promise<{ mod: { default?: unknown } | undefined; dependencies: string[] }> {
+	if (!fileUrl) {
 		return { mod: undefined, dependencies: [] };
 	}
 	const { code, dependencies } = await bundleFile({
 		virtualModContents: getConfigVirtualModContents(),
 		root,
-		fileUrl: configFileUrl,
+		fileUrl,
 	});
 	return {
 		mod: await importBundledFile({ code, root }),

packages/db/src/core/types.ts

@@ -3,6 +3,7 @@ import { SQLiteAsyncDialect } from 'drizzle-orm/sqlite-core';
 import { type ZodTypeDef, z } from 'zod';
 import { SERIALIZED_SQL_KEY, type SerializedSQL } from '../runtime/types.js';
 import { errorMap } from './integration/error-map.js';
+import type { AstroIntegration } from 'astro';
 
 export type MaybePromise<T> = T | Promise<T>;
 export type MaybeArray<T> = T | T[];
@@ -271,3 +272,14 @@ export type ResolvedCollectionConfig<TColumns extends ColumnsConfig = ColumnsCon
 // since Omit collapses our union type on primary key.
 export type NumberColumnOpts = z.input<typeof numberColumnOptsSchema>;
 export type TextColumnOpts = z.input<typeof textColumnOptsSchema>;
+
+export type AstroDbIntegration = AstroIntegration & {
+	hooks: {
+		'astro:db:setup'?: (options: {
+			extendDb: (options: {
+				configEntrypoint?: URL | string;
+				seedEntrypoint?: URL | string;
+			}) => void;
+		}) => void | Promise<void>;
+	};
+};

packages/db/src/core/utils.ts

@@ -1,5 +1,6 @@
-import type { AstroConfig } from 'astro';
+import type { AstroConfig, AstroIntegration } from 'astro';
 import { loadEnv } from 'vite';
+import type { AstroDbIntegration } from './types.js';
 
 export type VitePlugin = Required<AstroConfig['vite']>['plugins'][number];
 
@@ -21,3 +22,7 @@ export function getAstroStudioUrl(): string {
 export function getDbDirectoryUrl(root: URL | string) {
 	return new URL('db/', root);
 }
+
+export function defineDbIntegration(integration: AstroDbIntegration): AstroIntegration {
+	return integration;
+}