Inject latest Durable Object migration state into Miniflare (#6647)

* Update workerd.capnp and generated code to make use of enableSql

* Inject latest Durable Object migration state into Miniflare

* Add warnings in remote mode

---------

Co-authored-by: Samuel Macleod <smacleod@cloudflare.com>

.changeset/lovely-experts-double.md

@@ -0,0 +1,6 @@
+---
+"miniflare": minor
+"wrangler": minor
+---
+
+feat: Configure SQLite backed Durable Objects in local dev

packages/miniflare/src/index.ts

@@ -312,6 +312,7 @@ function getDurableObjectClassNames(
 				className,
 				// Fallback to current worker service if name not defined
 				serviceName = workerServiceName,
+				enableSql,
 				unsafeUniqueKey,
 				unsafePreventEviction,
 			} = normaliseDurableObject(designator);
@@ -325,6 +326,14 @@ function getDurableObjectClassNames(
 				// If we've already seen this class in this service, make sure the
 				// unsafe unique keys and unsafe prevent eviction values match
 				const existingInfo = classNames.get(className);
+				if (existingInfo?.enableSql !== enableSql) {
+					throw new MiniflareCoreError(
+						"ERR_DIFFERENT_STORAGE_BACKEND",
+						`Different storage backends defined for Durable Object "${className}" in "${serviceName}": ${JSON.stringify(
+							enableSql
+						)} and ${JSON.stringify(existingInfo?.enableSql)}`
+					);
+				}
 				if (existingInfo?.unsafeUniqueKey !== unsafeUniqueKey) {
 					throw new MiniflareCoreError(
 						"ERR_DIFFERENT_UNIQUE_KEYS",
@@ -343,7 +352,11 @@ function getDurableObjectClassNames(
 				}
 			} else {
 				// Otherwise, just add it
-				classNames.set(className, { unsafeUniqueKey, unsafePreventEviction });
+				classNames.set(className, {
+					enableSql,
+					unsafeUniqueKey,
+					unsafePreventEviction,
+				});
 			}
 		}
 	}

packages/miniflare/src/plugins/core/index.ts

@@ -612,16 +612,21 @@ export const CORE_PLUGIN: Plugin<
 					bindings: workerBindings,
 					durableObjectNamespaces:
 						classNamesEntries.map<Worker_DurableObjectNamespace>(
-							([className, { unsafeUniqueKey, unsafePreventEviction }]) => {
+							([
+								className,
+								{ enableSql, unsafeUniqueKey, unsafePreventEviction },
+							]) => {
 								if (unsafeUniqueKey === kUnsafeEphemeralUniqueKey) {
 									return {
 										className,
+										enableSql,
 										ephemeralLocal: kVoid,
 										preventEviction: unsafePreventEviction,
 									};
 								} else {
 									return {
 										className,
+										enableSql,
 										// This `uniqueKey` will (among other things) be used as part of the
 										// path when persisting to the file-system. `-` is invalid in
 										// JavaScript class names, but safe on filesystems (incl. Windows).

packages/miniflare/src/plugins/do/index.ts

@@ -19,6 +19,7 @@ export const DurableObjectsOptionsSchema = z.object({
 				z.object({
 					className: z.string(),
 					scriptName: z.string().optional(),
+					useSQLite: z.boolean().optional(),
 					// Allow `uniqueKey` to be customised. We use in Wrangler when setting
 					// up stub Durable Objects that proxy requests to Durable Objects in
 					// another `workerd` process, to ensure the IDs created by the stub
@@ -44,6 +45,7 @@ export function normaliseDurableObject(
 ): {
 	className: string;
 	serviceName?: string;
+	enableSql?: boolean;
 	unsafeUniqueKey?: UnsafeUniqueKey;
 	unsafePreventEviction?: boolean;
 } {
@@ -53,11 +55,18 @@ export function normaliseDurableObject(
 		isObject && designator.scriptName !== undefined
 			? getUserServiceName(designator.scriptName)
 			: undefined;
+	const enableSql = isObject ? designator.useSQLite : undefined;
 	const unsafeUniqueKey = isObject ? designator.unsafeUniqueKey : undefined;
 	const unsafePreventEviction = isObject
 		? designator.unsafePreventEviction
 		: undefined;
-	return { className, serviceName, unsafeUniqueKey, unsafePreventEviction };
+	return {
+		className,
+		serviceName,
+		enableSql,
+		unsafeUniqueKey,
+		unsafePreventEviction,
+	};
 }
 
 export const DURABLE_OBJECTS_PLUGIN_NAME = "do";

packages/miniflare/src/plugins/shared/index.ts

@@ -44,6 +44,7 @@ export type DurableObjectClassNames = Map<
 	Map<
 		/* className */ string,
 		{
+			enableSql?: boolean;
 			unsafeUniqueKey?: UnsafeUniqueKey;
 			unsafePreventEviction?: boolean;
 		}

packages/miniflare/src/runtime/config/workerd.capnp

@@ -25,7 +25,7 @@
 # to see and restrict what each Worker can access. Instead, the default is that a Worker has
 # access to no privileged resources at all, and you must explicitly declare "bindings" to give
 # it access to specific resources. A binding gives the Worker a JavaScript API object that points
-# to a specific resource. This means that by changing config alone, you can fully controll which
+# to a specific resource. This means that by changing config alone, you can fully control which
 # resources an Worker connects to. (You can even disallow access to the public internet, although
 # public internet access is granted by default.)
 #
@@ -39,6 +39,7 @@
 # 2. added to `tryImportBulitin` in workerd.c++ (grep for '"/workerd/workerd.capnp"').
 using Cxx = import "/capnp/c++.capnp";
 $Cxx.namespace("workerd::server::config");
+$Cxx.allowCancellation;
 
 struct Config {
   # Top-level configuration for a workerd instance.
@@ -206,7 +207,7 @@ struct Worker {
     # event handlers.
     #
     # The value of this field is the raw source code. When using Cap'n Proto text format, use the
-    # `embed` directive to read the code from an exnternal file:
+    # `embed` directive to read the code from an external file:
     #
     #     serviceWorkerScript = embed "worker.js"
 
@@ -271,6 +272,10 @@ struct Worker {
       # Pyodide (https://pyodide.org/en/stable/usage/packages-in-pyodide.html). All packages listed
       # will be installed prior to the execution of the worker.
     }
+
+    namedExports @10 :List(Text);
+    # For commonJsModule and nodeJsCompatModule, this is a list of named exports that the
+    # module expects to be exported once the evaluation is complete.
   }
 
   compatibilityDate @3 :Text;
@@ -345,7 +350,7 @@ struct Worker {
 
       kvNamespace @11 :ServiceDesignator;
       # A KV namespace, implemented by the named service. The Worker sees a KvNamespace-typed
-      # binding. Requests to the namespace will be converted into HTTP requests targetting the
+      # binding. Requests to the namespace will be converted into HTTP requests targeting the
       # given service name.
 
       r2Bucket @12 :ServiceDesignator;
@@ -358,7 +363,7 @@ struct Worker {
 
       queue @15 :ServiceDesignator;
       # A Queue binding, implemented by the named service. Requests to the
-      # namespace will be converted into HTTP requests targetting the given
+      # namespace will be converted into HTTP requests targeting the given
       # service name.
 
       fromEnvironment @16 :Text;
@@ -520,7 +525,7 @@ struct Worker {
     }
   }
 
-  globalOutbound @6 :ServiceDesignator = ( name = "internet" );
+  globalOutbound @6 :ServiceDesignator = "internet";
   # Where should the global "fetch" go to? The default is the service called "internet", which
   # should usually be configured to talk to the public internet.
 
@@ -555,10 +560,10 @@ struct Worker {
       # Instances of this class are ephemeral -- they have no durable storage at all. The
       # `state.storage` API will not be present. Additionally, this namespace will allow arbitrary
       # strings as IDs. There are no `idFromName()` nor `newUniqueId()` methods; `get()` takes any
-      # string as a paremeter.
+      # string as a parameter.
       #
       # Ephemeral objects are NOT globally unique, only "locally" unique, for some definition of
-      # "local". For exmaple, on Cloudflare's network, these objects are unique per-colo.
+      # "local". For example, on Cloudflare's network, these objects are unique per-colo.
       #
       # WARNING: Cloudflare Workers currently limits this feature to Cloudflare-internal users
       #   only, because using them correctly requires deep understanding of Cloudflare network
@@ -574,6 +579,14 @@ struct Worker {
     # pinned to memory forever, so we provide this flag to change the default behavior.
     #
     # Note that this is only supported in Workerd; production Durable Objects cannot toggle eviction.
+
+    enableSql @4 :Bool;
+    # Whether or not Durable Objects in this namespace can use the `storage.sql` API to execute SQL
+    # queries.
+    #
+    # workerd uses SQLite to back all Durable Objects, but the SQL API is hidden by default to
+    # emulate behavior of traditional DO namespaces on Cloudflare that aren't SQLite-backed. This
+    # flag should be enabled when testing code that will run on a SQLite-backed namespace.
   }
 
   durableObjectUniqueKeyModifier @8 :Text;
@@ -730,7 +743,7 @@ struct DiskDirectory {
   # particular, no attempt is made to guess the `Content-Type` header. You normally would wrap
   # this in a Worker that fills in the metadata in the way you want.
   #
-  # A GET request targetting a directory (rather than a file) will return a basic JSAN directory
+  # A GET request targeting a directory (rather than a file) will return a basic JSAN directory
   # listing like:
   #
   #     [{"name":"foo","type":"file"},{"name":"bar","type":"directory"}]
@@ -921,4 +934,4 @@ struct Extension {
     esModule @2 :Text;
     # Raw source code of ES module.
   }
-}
+}

packages/miniflare/src/runtime/config/workerd.capnp.d.ts

@@ -229,6 +229,12 @@ export declare class Worker_Module extends __S {
     getPythonRequirement(): string;
     isPythonRequirement(): boolean;
     setPythonRequirement(value: string): void;
+    adoptNamedExports(value: capnp.Orphan<capnp.List<string>>): void;
+    disownNamedExports(): capnp.Orphan<capnp.List<string>>;
+    getNamedExports(): capnp.List<string>;
+    hasNamedExports(): boolean;
+    initNamedExports(length: number): capnp.List<string>;
+    setNamedExports(value: capnp.List<string>): void;
     toString(): string;
     which(): Worker_Module_Which;
 }
@@ -671,6 +677,8 @@ export declare class Worker_DurableObjectNamespace extends __S {
     setEphemeralLocal(): void;
     getPreventEviction(): boolean;
     setPreventEviction(value: boolean): void;
+    getEnableSql(): boolean;
+    setEnableSql(value: boolean): void;
     toString(): string;
     which(): Worker_DurableObjectNamespace_Which;
 }

packages/miniflare/src/runtime/config/workerd.capnp.js

@@ -336,6 +336,12 @@ class Worker_Module extends capnp_ts_1.Struct {
         capnp_ts_1.Struct.setUint16(0, 8, this);
         capnp_ts_1.Struct.setText(1, value, this);
     }
+    adoptNamedExports(value) { capnp_ts_1.Struct.adopt(value, capnp_ts_1.Struct.getPointer(2, this)); }
+    disownNamedExports() { return capnp_ts_1.Struct.disown(this.getNamedExports()); }
+    getNamedExports() { return capnp_ts_1.Struct.getList(2, capnp.TextList, this); }
+    hasNamedExports() { return !capnp_ts_1.Struct.isNull(capnp_ts_1.Struct.getPointer(2, this)); }
+    initNamedExports(length) { return capnp_ts_1.Struct.initList(2, capnp.TextList, length, this); }
+    setNamedExports(value) { capnp_ts_1.Struct.copyFrom(value, capnp_ts_1.Struct.getPointer(2, this)); }
     toString() { return "Worker_Module_" + super.toString(); }
     which() { return capnp_ts_1.Struct.getUint16(0, this); }
 }
@@ -349,7 +355,7 @@ Worker_Module.JSON = Worker_Module_Which.JSON;
 Worker_Module.NODE_JS_COMPAT_MODULE = Worker_Module_Which.NODE_JS_COMPAT_MODULE;
 Worker_Module.PYTHON_MODULE = Worker_Module_Which.PYTHON_MODULE;
 Worker_Module.PYTHON_REQUIREMENT = Worker_Module_Which.PYTHON_REQUIREMENT;
-Worker_Module._capnp = { displayName: "Module", id: "d9d87a63770a12f3", size: new capnp_ts_1.ObjectSize(8, 2) };
+Worker_Module._capnp = { displayName: "Module", id: "d9d87a63770a12f3", size: new capnp_ts_1.ObjectSize(8, 3) };
 var Worker_Binding_Type_Which;
 (function (Worker_Binding_Type_Which) {
     Worker_Binding_Type_Which[Worker_Binding_Type_Which["UNSPECIFIED"] = 0] = "UNSPECIFIED";
@@ -995,6 +1001,8 @@ class Worker_DurableObjectNamespace extends capnp_ts_1.Struct {
     setEphemeralLocal() { capnp_ts_1.Struct.setUint16(0, 1, this); }
     getPreventEviction() { return capnp_ts_1.Struct.getBit(16, this); }
     setPreventEviction(value) { capnp_ts_1.Struct.setBit(16, value, this); }
+    getEnableSql() { return capnp_ts_1.Struct.getBit(17, this); }
+    setEnableSql(value) { capnp_ts_1.Struct.setBit(17, value, this); }
     toString() { return "Worker_DurableObjectNamespace_" + super.toString(); }
     which() { return capnp_ts_1.Struct.getUint16(0, this); }
 }

packages/miniflare/src/runtime/config/workerd.ts

@@ -174,6 +174,7 @@ export interface Worker_Binding_MemoryCacheLimits {
 export type Worker_DurableObjectNamespace = {
 	className?: string;
 	preventEviction?: boolean;
+	enableSql?: boolean;
 } & ({ uniqueKey?: string } | { ephemeralLocal?: Void });
 
 export type ExternalServer = { address?: string } & (

packages/miniflare/src/shared/error.ts

@@ -28,6 +28,7 @@ export type MiniflareCoreErrorCode =
 	| "ERR_NO_WORKERS" // No workers defined
 	| "ERR_VALIDATION" // Options failed to parse
 	| "ERR_DUPLICATE_NAME" // Multiple workers defined with same name
+	| "ERR_DIFFERENT_STORAGE_BACKEND" // Multiple Durable Object bindings declared for same class with different storage backends
 	| "ERR_DIFFERENT_UNIQUE_KEYS" // Multiple Durable Object bindings declared for same class with different unsafe unique keys
 	| "ERR_DIFFERENT_PREVENT_EVICTION" // Multiple Durable Object bindings declared for same class with different unsafe prevent eviction values
 	| "ERR_MULTIPLE_OUTBOUNDS" // Both `outboundService` and `fetchMock` specified

packages/miniflare/test/plugins/do/index.spec.ts

@@ -383,6 +383,75 @@ test("prevent Durable Object eviction", async (t) => {
 	t.is(await res.text(), original);
 });
 
+const MINIFLARE_WITH_SQLITE = (useSQLite: boolean) =>
+	new Miniflare({
+		verbose: true,
+		modules: true,
+		script: `export class SQLiteDurableObject {
+			constructor(ctx) { this.ctx = ctx; }
+			fetch() {
+				try {
+					return new Response(this.ctx.storage.sql.databaseSize);
+				} catch (error) {
+					if (error instanceof Error) {
+						return new Response(error.message);
+					}
+					throw error;
+				}
+			}
+		}
+		export default {
+			fetch(req, env, ctx) {
+				const id = env.SQLITE_DURABLE_OBJECT.idFromName("foo");
+				console.log({id})
+				const stub = env.SQLITE_DURABLE_OBJECT.get(id);
+				return stub.fetch(req);
+			}
+		}`,
+		durableObjects: {
+			SQLITE_DURABLE_OBJECT: {
+				className: "SQLiteDurableObject",
+				useSQLite,
+			},
+		},
+	});
+
+test("SQLite is available in SQLite backed Durable Objects", async (t) => {
+	const mf = MINIFLARE_WITH_SQLITE(true);
+	t.teardown(() => mf.dispose());
+
+	let res = await mf.dispatchFetch("http://localhost");
+	t.is(await res.text(), "4096");
+
+	const ns = await mf.getDurableObjectNamespace("SQLITE_DURABLE_OBJECT");
+	const id = ns.newUniqueId();
+	const stub = ns.get(id);
+	res = await stub.fetch("http://localhost");
+	t.is(await res.text(), "4096");
+});
+
+test("SQLite is not available in default Durable Objects", async (t) => {
+	const mf = MINIFLARE_WITH_SQLITE(false);
+	t.teardown(() => mf.dispose());
+
+	let res = await mf.dispatchFetch("http://localhost");
+	t.assert(
+		(await res.text()).startsWith(
+			"SQL is not enabled for this Durable Object class."
+		)
+	);
+
+	const ns = await mf.getDurableObjectNamespace("SQLITE_DURABLE_OBJECT");
+	const id = ns.newUniqueId();
+	const stub = ns.get(id);
+	res = await stub.fetch("http://localhost");
+	t.assert(
+		(await res.text()).startsWith(
+			"SQL is not enabled for this Durable Object class."
+		)
+	);
+});
+
 test("colo-local actors", async (t) => {
 	const mf = new Miniflare({
 		modules: true,

packages/wrangler/src/api/integrations/platform/index.ts

@@ -159,6 +159,7 @@ async function getMiniflareOptionsFromConfig(
 		queueConsumers: undefined,
 		services: rawConfig.services,
 		serviceBindings: {},
+		migrations: rawConfig.migrations,
 	});
 
 	const persistOptions = getMiniflarePersistOptions(options.persist);
@@ -264,6 +265,7 @@ export function unstable_getMiniflareWorkerOptions(
 		queueConsumers: config.queues.consumers,
 		services: [],
 		serviceBindings: {},
+		migrations: config.migrations,
 	});
 
 	// This function is currently only exported for the Workers Vitest pool.
@@ -285,7 +287,11 @@ export function unstable_getMiniflareWorkerOptions(
 		bindingOptions.durableObjects = Object.fromEntries(
 			bindings.durable_objects.bindings.map((binding) => [
 				binding.name,
-				{ className: binding.class_name, scriptName: binding.script_name },
+				{
+					className: binding.class_name,
+					scriptName: binding.script_name,
+					binding,
+				},
 			])
 		);
 	}