From d68e8c996ba40eaaf4a3b237f89880bdaafd0113 Mon Sep 17 00:00:00 2001 From: Josh Howard Date: Thu, 12 Sep 2024 16:45:34 -0500 Subject: [PATCH] Inject latest Durable Object migration state into Miniflare (#6647) * Update workerd.capnp and generated code to make use of enableSql * Inject latest Durable Object migration state into Miniflare * Add warnings in remote mode --------- Co-authored-by: Samuel Macleod --- .changeset/lovely-experts-double.md | 6 ++ packages/miniflare/src/index.ts | 15 +++- packages/miniflare/src/plugins/core/index.ts | 7 +- packages/miniflare/src/plugins/do/index.ts | 11 ++- .../miniflare/src/plugins/shared/index.ts | 1 + .../src/runtime/config/workerd.capnp | 31 +++++--- .../src/runtime/config/workerd.capnp.d.ts | 8 ++ .../src/runtime/config/workerd.capnp.js | 10 ++- .../miniflare/src/runtime/config/workerd.ts | 1 + packages/miniflare/src/shared/error.ts | 1 + .../miniflare/test/plugins/do/index.spec.ts | 69 +++++++++++++++++ .../src/api/integrations/platform/index.ts | 8 +- .../api/startDevWorker/ConfigController.ts | 13 ++++ .../startDevWorker/LocalRuntimeController.ts | 2 +- .../wrangler/src/api/startDevWorker/types.ts | 2 + packages/wrangler/src/config/environment.ts | 35 +++++---- packages/wrangler/src/dev.tsx | 14 ++++ packages/wrangler/src/dev/dev.tsx | 5 +- packages/wrangler/src/dev/local.tsx | 2 + packages/wrangler/src/dev/miniflare.ts | 77 +++++++++++++++++-- packages/wrangler/src/dev/start-server.ts | 3 +- 21 files changed, 284 insertions(+), 37 deletions(-) create mode 100644 .changeset/lovely-experts-double.md diff --git a/.changeset/lovely-experts-double.md b/.changeset/lovely-experts-double.md new file mode 100644 index 000000000000..35fbfdf36244 --- /dev/null +++ b/.changeset/lovely-experts-double.md @@ -0,0 +1,6 @@ +--- +"miniflare": minor +"wrangler": minor +--- + +feat: Configure SQLite backed Durable Objects in local dev diff --git a/packages/miniflare/src/index.ts b/packages/miniflare/src/index.ts index ce3f0f51a1d3..2251c9883810 100644 --- a/packages/miniflare/src/index.ts +++ b/packages/miniflare/src/index.ts @@ -312,6 +312,7 @@ function getDurableObjectClassNames( className, // Fallback to current worker service if name not defined serviceName = workerServiceName, + enableSql, unsafeUniqueKey, unsafePreventEviction, } = normaliseDurableObject(designator); @@ -325,6 +326,14 @@ function getDurableObjectClassNames( // If we've already seen this class in this service, make sure the // unsafe unique keys and unsafe prevent eviction values match const existingInfo = classNames.get(className); + if (existingInfo?.enableSql !== enableSql) { + throw new MiniflareCoreError( + "ERR_DIFFERENT_STORAGE_BACKEND", + `Different storage backends defined for Durable Object "${className}" in "${serviceName}": ${JSON.stringify( + enableSql + )} and ${JSON.stringify(existingInfo?.enableSql)}` + ); + } if (existingInfo?.unsafeUniqueKey !== unsafeUniqueKey) { throw new MiniflareCoreError( "ERR_DIFFERENT_UNIQUE_KEYS", @@ -343,7 +352,11 @@ function getDurableObjectClassNames( } } else { // Otherwise, just add it - classNames.set(className, { unsafeUniqueKey, unsafePreventEviction }); + classNames.set(className, { + enableSql, + unsafeUniqueKey, + unsafePreventEviction, + }); } } } diff --git a/packages/miniflare/src/plugins/core/index.ts b/packages/miniflare/src/plugins/core/index.ts index 22b878dcd8b6..c03752d7688f 100644 --- a/packages/miniflare/src/plugins/core/index.ts +++ b/packages/miniflare/src/plugins/core/index.ts @@ -612,16 +612,21 @@ export const CORE_PLUGIN: Plugin< bindings: workerBindings, durableObjectNamespaces: classNamesEntries.map( - ([className, { unsafeUniqueKey, unsafePreventEviction }]) => { + ([ + className, + { enableSql, unsafeUniqueKey, unsafePreventEviction }, + ]) => { if (unsafeUniqueKey === kUnsafeEphemeralUniqueKey) { return { className, + enableSql, ephemeralLocal: kVoid, preventEviction: unsafePreventEviction, }; } else { return { className, + enableSql, // This `uniqueKey` will (among other things) be used as part of the // path when persisting to the file-system. `-` is invalid in // JavaScript class names, but safe on filesystems (incl. Windows). diff --git a/packages/miniflare/src/plugins/do/index.ts b/packages/miniflare/src/plugins/do/index.ts index e28af3135160..7b75d8abf110 100644 --- a/packages/miniflare/src/plugins/do/index.ts +++ b/packages/miniflare/src/plugins/do/index.ts @@ -19,6 +19,7 @@ export const DurableObjectsOptionsSchema = z.object({ z.object({ className: z.string(), scriptName: z.string().optional(), + useSQLite: z.boolean().optional(), // Allow `uniqueKey` to be customised. We use in Wrangler when setting // up stub Durable Objects that proxy requests to Durable Objects in // another `workerd` process, to ensure the IDs created by the stub @@ -44,6 +45,7 @@ export function normaliseDurableObject( ): { className: string; serviceName?: string; + enableSql?: boolean; unsafeUniqueKey?: UnsafeUniqueKey; unsafePreventEviction?: boolean; } { @@ -53,11 +55,18 @@ export function normaliseDurableObject( isObject && designator.scriptName !== undefined ? getUserServiceName(designator.scriptName) : undefined; + const enableSql = isObject ? designator.useSQLite : undefined; const unsafeUniqueKey = isObject ? designator.unsafeUniqueKey : undefined; const unsafePreventEviction = isObject ? designator.unsafePreventEviction : undefined; - return { className, serviceName, unsafeUniqueKey, unsafePreventEviction }; + return { + className, + serviceName, + enableSql, + unsafeUniqueKey, + unsafePreventEviction, + }; } export const DURABLE_OBJECTS_PLUGIN_NAME = "do"; diff --git a/packages/miniflare/src/plugins/shared/index.ts b/packages/miniflare/src/plugins/shared/index.ts index 49ec49049779..e8623e7de702 100644 --- a/packages/miniflare/src/plugins/shared/index.ts +++ b/packages/miniflare/src/plugins/shared/index.ts @@ -44,6 +44,7 @@ export type DurableObjectClassNames = Map< Map< /* className */ string, { + enableSql?: boolean; unsafeUniqueKey?: UnsafeUniqueKey; unsafePreventEviction?: boolean; } diff --git a/packages/miniflare/src/runtime/config/workerd.capnp b/packages/miniflare/src/runtime/config/workerd.capnp index 295a91de7efe..4125421e68b5 100644 --- a/packages/miniflare/src/runtime/config/workerd.capnp +++ b/packages/miniflare/src/runtime/config/workerd.capnp @@ -25,7 +25,7 @@ # to see and restrict what each Worker can access. Instead, the default is that a Worker has # access to no privileged resources at all, and you must explicitly declare "bindings" to give # it access to specific resources. A binding gives the Worker a JavaScript API object that points -# to a specific resource. This means that by changing config alone, you can fully controll which +# to a specific resource. This means that by changing config alone, you can fully control which # resources an Worker connects to. (You can even disallow access to the public internet, although # public internet access is granted by default.) # @@ -39,6 +39,7 @@ # 2. added to `tryImportBulitin` in workerd.c++ (grep for '"/workerd/workerd.capnp"'). using Cxx = import "/capnp/c++.capnp"; $Cxx.namespace("workerd::server::config"); +$Cxx.allowCancellation; struct Config { # Top-level configuration for a workerd instance. @@ -206,7 +207,7 @@ struct Worker { # event handlers. # # The value of this field is the raw source code. When using Cap'n Proto text format, use the - # `embed` directive to read the code from an exnternal file: + # `embed` directive to read the code from an external file: # # serviceWorkerScript = embed "worker.js" @@ -271,6 +272,10 @@ struct Worker { # Pyodide (https://pyodide.org/en/stable/usage/packages-in-pyodide.html). All packages listed # will be installed prior to the execution of the worker. } + + namedExports @10 :List(Text); + # For commonJsModule and nodeJsCompatModule, this is a list of named exports that the + # module expects to be exported once the evaluation is complete. } compatibilityDate @3 :Text; @@ -345,7 +350,7 @@ struct Worker { kvNamespace @11 :ServiceDesignator; # A KV namespace, implemented by the named service. The Worker sees a KvNamespace-typed - # binding. Requests to the namespace will be converted into HTTP requests targetting the + # binding. Requests to the namespace will be converted into HTTP requests targeting the # given service name. r2Bucket @12 :ServiceDesignator; @@ -358,7 +363,7 @@ struct Worker { queue @15 :ServiceDesignator; # A Queue binding, implemented by the named service. Requests to the - # namespace will be converted into HTTP requests targetting the given + # namespace will be converted into HTTP requests targeting the given # service name. fromEnvironment @16 :Text; @@ -520,7 +525,7 @@ struct Worker { } } - globalOutbound @6 :ServiceDesignator = ( name = "internet" ); + globalOutbound @6 :ServiceDesignator = "internet"; # Where should the global "fetch" go to? The default is the service called "internet", which # should usually be configured to talk to the public internet. @@ -555,10 +560,10 @@ struct Worker { # Instances of this class are ephemeral -- they have no durable storage at all. The # `state.storage` API will not be present. Additionally, this namespace will allow arbitrary # strings as IDs. There are no `idFromName()` nor `newUniqueId()` methods; `get()` takes any - # string as a paremeter. + # string as a parameter. # # Ephemeral objects are NOT globally unique, only "locally" unique, for some definition of - # "local". For exmaple, on Cloudflare's network, these objects are unique per-colo. + # "local". For example, on Cloudflare's network, these objects are unique per-colo. # # WARNING: Cloudflare Workers currently limits this feature to Cloudflare-internal users # only, because using them correctly requires deep understanding of Cloudflare network @@ -574,6 +579,14 @@ struct Worker { # pinned to memory forever, so we provide this flag to change the default behavior. # # Note that this is only supported in Workerd; production Durable Objects cannot toggle eviction. + + enableSql @4 :Bool; + # Whether or not Durable Objects in this namespace can use the `storage.sql` API to execute SQL + # queries. + # + # workerd uses SQLite to back all Durable Objects, but the SQL API is hidden by default to + # emulate behavior of traditional DO namespaces on Cloudflare that aren't SQLite-backed. This + # flag should be enabled when testing code that will run on a SQLite-backed namespace. } durableObjectUniqueKeyModifier @8 :Text; @@ -730,7 +743,7 @@ struct DiskDirectory { # particular, no attempt is made to guess the `Content-Type` header. You normally would wrap # this in a Worker that fills in the metadata in the way you want. # - # A GET request targetting a directory (rather than a file) will return a basic JSAN directory + # A GET request targeting a directory (rather than a file) will return a basic JSAN directory # listing like: # # [{"name":"foo","type":"file"},{"name":"bar","type":"directory"}] @@ -921,4 +934,4 @@ struct Extension { esModule @2 :Text; # Raw source code of ES module. } -} +} \ No newline at end of file diff --git a/packages/miniflare/src/runtime/config/workerd.capnp.d.ts b/packages/miniflare/src/runtime/config/workerd.capnp.d.ts index f0eec5e4d647..1294b3815e7b 100644 --- a/packages/miniflare/src/runtime/config/workerd.capnp.d.ts +++ b/packages/miniflare/src/runtime/config/workerd.capnp.d.ts @@ -229,6 +229,12 @@ export declare class Worker_Module extends __S { getPythonRequirement(): string; isPythonRequirement(): boolean; setPythonRequirement(value: string): void; + adoptNamedExports(value: capnp.Orphan>): void; + disownNamedExports(): capnp.Orphan>; + getNamedExports(): capnp.List; + hasNamedExports(): boolean; + initNamedExports(length: number): capnp.List; + setNamedExports(value: capnp.List): void; toString(): string; which(): Worker_Module_Which; } @@ -671,6 +677,8 @@ export declare class Worker_DurableObjectNamespace extends __S { setEphemeralLocal(): void; getPreventEviction(): boolean; setPreventEviction(value: boolean): void; + getEnableSql(): boolean; + setEnableSql(value: boolean): void; toString(): string; which(): Worker_DurableObjectNamespace_Which; } diff --git a/packages/miniflare/src/runtime/config/workerd.capnp.js b/packages/miniflare/src/runtime/config/workerd.capnp.js index bb8be0dc6c1d..0c8d94c5a792 100644 --- a/packages/miniflare/src/runtime/config/workerd.capnp.js +++ b/packages/miniflare/src/runtime/config/workerd.capnp.js @@ -336,6 +336,12 @@ class Worker_Module extends capnp_ts_1.Struct { capnp_ts_1.Struct.setUint16(0, 8, this); capnp_ts_1.Struct.setText(1, value, this); } + adoptNamedExports(value) { capnp_ts_1.Struct.adopt(value, capnp_ts_1.Struct.getPointer(2, this)); } + disownNamedExports() { return capnp_ts_1.Struct.disown(this.getNamedExports()); } + getNamedExports() { return capnp_ts_1.Struct.getList(2, capnp.TextList, this); } + hasNamedExports() { return !capnp_ts_1.Struct.isNull(capnp_ts_1.Struct.getPointer(2, this)); } + initNamedExports(length) { return capnp_ts_1.Struct.initList(2, capnp.TextList, length, this); } + setNamedExports(value) { capnp_ts_1.Struct.copyFrom(value, capnp_ts_1.Struct.getPointer(2, this)); } toString() { return "Worker_Module_" + super.toString(); } which() { return capnp_ts_1.Struct.getUint16(0, this); } } @@ -349,7 +355,7 @@ Worker_Module.JSON = Worker_Module_Which.JSON; Worker_Module.NODE_JS_COMPAT_MODULE = Worker_Module_Which.NODE_JS_COMPAT_MODULE; Worker_Module.PYTHON_MODULE = Worker_Module_Which.PYTHON_MODULE; Worker_Module.PYTHON_REQUIREMENT = Worker_Module_Which.PYTHON_REQUIREMENT; -Worker_Module._capnp = { displayName: "Module", id: "d9d87a63770a12f3", size: new capnp_ts_1.ObjectSize(8, 2) }; +Worker_Module._capnp = { displayName: "Module", id: "d9d87a63770a12f3", size: new capnp_ts_1.ObjectSize(8, 3) }; var Worker_Binding_Type_Which; (function (Worker_Binding_Type_Which) { Worker_Binding_Type_Which[Worker_Binding_Type_Which["UNSPECIFIED"] = 0] = "UNSPECIFIED"; @@ -995,6 +1001,8 @@ class Worker_DurableObjectNamespace extends capnp_ts_1.Struct { setEphemeralLocal() { capnp_ts_1.Struct.setUint16(0, 1, this); } getPreventEviction() { return capnp_ts_1.Struct.getBit(16, this); } setPreventEviction(value) { capnp_ts_1.Struct.setBit(16, value, this); } + getEnableSql() { return capnp_ts_1.Struct.getBit(17, this); } + setEnableSql(value) { capnp_ts_1.Struct.setBit(17, value, this); } toString() { return "Worker_DurableObjectNamespace_" + super.toString(); } which() { return capnp_ts_1.Struct.getUint16(0, this); } } diff --git a/packages/miniflare/src/runtime/config/workerd.ts b/packages/miniflare/src/runtime/config/workerd.ts index a7bcc1aa6d8c..49a80b0c8a10 100644 --- a/packages/miniflare/src/runtime/config/workerd.ts +++ b/packages/miniflare/src/runtime/config/workerd.ts @@ -174,6 +174,7 @@ export interface Worker_Binding_MemoryCacheLimits { export type Worker_DurableObjectNamespace = { className?: string; preventEviction?: boolean; + enableSql?: boolean; } & ({ uniqueKey?: string } | { ephemeralLocal?: Void }); export type ExternalServer = { address?: string } & ( diff --git a/packages/miniflare/src/shared/error.ts b/packages/miniflare/src/shared/error.ts index 53d57891cc2e..baaaea62efcd 100644 --- a/packages/miniflare/src/shared/error.ts +++ b/packages/miniflare/src/shared/error.ts @@ -28,6 +28,7 @@ export type MiniflareCoreErrorCode = | "ERR_NO_WORKERS" // No workers defined | "ERR_VALIDATION" // Options failed to parse | "ERR_DUPLICATE_NAME" // Multiple workers defined with same name + | "ERR_DIFFERENT_STORAGE_BACKEND" // Multiple Durable Object bindings declared for same class with different storage backends | "ERR_DIFFERENT_UNIQUE_KEYS" // Multiple Durable Object bindings declared for same class with different unsafe unique keys | "ERR_DIFFERENT_PREVENT_EVICTION" // Multiple Durable Object bindings declared for same class with different unsafe prevent eviction values | "ERR_MULTIPLE_OUTBOUNDS" // Both `outboundService` and `fetchMock` specified diff --git a/packages/miniflare/test/plugins/do/index.spec.ts b/packages/miniflare/test/plugins/do/index.spec.ts index f0ca9c1e6f8b..da382b59eeeb 100644 --- a/packages/miniflare/test/plugins/do/index.spec.ts +++ b/packages/miniflare/test/plugins/do/index.spec.ts @@ -383,6 +383,75 @@ test("prevent Durable Object eviction", async (t) => { t.is(await res.text(), original); }); +const MINIFLARE_WITH_SQLITE = (useSQLite: boolean) => + new Miniflare({ + verbose: true, + modules: true, + script: `export class SQLiteDurableObject { + constructor(ctx) { this.ctx = ctx; } + fetch() { + try { + return new Response(this.ctx.storage.sql.databaseSize); + } catch (error) { + if (error instanceof Error) { + return new Response(error.message); + } + throw error; + } + } + } + export default { + fetch(req, env, ctx) { + const id = env.SQLITE_DURABLE_OBJECT.idFromName("foo"); + console.log({id}) + const stub = env.SQLITE_DURABLE_OBJECT.get(id); + return stub.fetch(req); + } + }`, + durableObjects: { + SQLITE_DURABLE_OBJECT: { + className: "SQLiteDurableObject", + useSQLite, + }, + }, + }); + +test("SQLite is available in SQLite backed Durable Objects", async (t) => { + const mf = MINIFLARE_WITH_SQLITE(true); + t.teardown(() => mf.dispose()); + + let res = await mf.dispatchFetch("http://localhost"); + t.is(await res.text(), "4096"); + + const ns = await mf.getDurableObjectNamespace("SQLITE_DURABLE_OBJECT"); + const id = ns.newUniqueId(); + const stub = ns.get(id); + res = await stub.fetch("http://localhost"); + t.is(await res.text(), "4096"); +}); + +test("SQLite is not available in default Durable Objects", async (t) => { + const mf = MINIFLARE_WITH_SQLITE(false); + t.teardown(() => mf.dispose()); + + let res = await mf.dispatchFetch("http://localhost"); + t.assert( + (await res.text()).startsWith( + "SQL is not enabled for this Durable Object class." + ) + ); + + const ns = await mf.getDurableObjectNamespace("SQLITE_DURABLE_OBJECT"); + const id = ns.newUniqueId(); + const stub = ns.get(id); + res = await stub.fetch("http://localhost"); + t.assert( + (await res.text()).startsWith( + "SQL is not enabled for this Durable Object class." + ) + ); +}); + test("colo-local actors", async (t) => { const mf = new Miniflare({ modules: true, diff --git a/packages/wrangler/src/api/integrations/platform/index.ts b/packages/wrangler/src/api/integrations/platform/index.ts index 5073d538bb6a..a9e38e79b0aa 100644 --- a/packages/wrangler/src/api/integrations/platform/index.ts +++ b/packages/wrangler/src/api/integrations/platform/index.ts @@ -159,6 +159,7 @@ async function getMiniflareOptionsFromConfig( queueConsumers: undefined, services: rawConfig.services, serviceBindings: {}, + migrations: rawConfig.migrations, }); const persistOptions = getMiniflarePersistOptions(options.persist); @@ -264,6 +265,7 @@ export function unstable_getMiniflareWorkerOptions( queueConsumers: config.queues.consumers, services: [], serviceBindings: {}, + migrations: config.migrations, }); // This function is currently only exported for the Workers Vitest pool. @@ -285,7 +287,11 @@ export function unstable_getMiniflareWorkerOptions( bindingOptions.durableObjects = Object.fromEntries( bindings.durable_objects.bindings.map((binding) => [ binding.name, - { className: binding.class_name, scriptName: binding.script_name }, + { + className: binding.class_name, + scriptName: binding.script_name, + binding, + }, ]) ); } diff --git a/packages/wrangler/src/api/startDevWorker/ConfigController.ts b/packages/wrangler/src/api/startDevWorker/ConfigController.ts index 1e9649feb9c7..437bebb48930 100644 --- a/packages/wrangler/src/api/startDevWorker/ConfigController.ts +++ b/packages/wrangler/src/api/startDevWorker/ConfigController.ts @@ -236,6 +236,7 @@ async function resolveConfig( entrypoint: entry.file, directory: entry.directory, bindings, + migrations: input.migrations ?? config.migrations, sendMetrics: input.sendMetrics ?? config.send_metrics, triggers: await resolveTriggers(config, input), env: input.env, @@ -317,6 +318,18 @@ async function resolveConfig( "Queues are currently in Beta and are not supported in wrangler dev remote mode." ); } + + if ( + resolved.migrations?.some( + (m) => + Array.isArray(m.new_sqlite_classes) && m.new_sqlite_classes.length > 0 + ) && + resolved.dev?.remote + ) { + throw new UserError( + "SQLite in Durable Objects is only supported in local mode." + ); + } return resolved; } export class ConfigController extends Controller { diff --git a/packages/wrangler/src/api/startDevWorker/LocalRuntimeController.ts b/packages/wrangler/src/api/startDevWorker/LocalRuntimeController.ts index 2785fd05b58d..10dcd4e30d6e 100644 --- a/packages/wrangler/src/api/startDevWorker/LocalRuntimeController.ts +++ b/packages/wrangler/src/api/startDevWorker/LocalRuntimeController.ts @@ -91,6 +91,7 @@ async function convertToConfigBundle( compatibilityDate: event.config.compatibilityDate, compatibilityFlags: event.config.compatibilityFlags, bindings, + migrations: event.config.migrations, workerDefinitions: event.config.dev?.registry, legacyAssetPaths: event.config.legacy?.site?.bucket ? { @@ -154,7 +155,6 @@ export class LocalRuntimeController extends RuntimeController { options.liveReload = false; // TODO: set in buildMiniflareOptions once old code path is removed if (this.#mf === undefined) { logger.log(chalk.dim("⎔ Starting local server...")); - this.#mf = new Miniflare(options); } else { logger.log(chalk.dim("⎔ Reloading local server...")); diff --git a/packages/wrangler/src/api/startDevWorker/types.ts b/packages/wrangler/src/api/startDevWorker/types.ts index f5c7a3178415..e2a02e40bb0f 100644 --- a/packages/wrangler/src/api/startDevWorker/types.ts +++ b/packages/wrangler/src/api/startDevWorker/types.ts @@ -1,6 +1,7 @@ import type { Config } from "../../config"; import type { CustomDomainRoute, + DurableObjectMigration, Rule, ZoneIdRoute, ZoneNameRoute, @@ -74,6 +75,7 @@ export interface StartDevWorkerInput { /** The bindings available to the worker. The specified bindind type will be exposed to the worker on the `env` object under the same key. */ bindings?: Record; // Type level constraint for bindings not sharing names + migrations?: DurableObjectMigration[]; /** The triggers which will cause the worker's exported default handlers to be called. */ triggers?: Trigger[]; diff --git a/packages/wrangler/src/config/environment.ts b/packages/wrangler/src/config/environment.ts index e37ca59ba172..d23dd0aee148 100644 --- a/packages/wrangler/src/config/environment.ts +++ b/packages/wrangler/src/config/environment.ts @@ -39,6 +39,25 @@ export type CloudchamberConfig = { ipv4?: boolean; }; +/** + * Configuration in wrangler for Durable Object Migrations + */ +export type DurableObjectMigration = { + /** A unique identifier for this migration. */ + tag: string; + /** The new Durable Objects being defined. */ + new_classes?: string[]; + /** The new SQLite Durable Objects being defined. */ + new_sqlite_classes?: string[]; + /** The Durable Objects being renamed. */ + renamed_classes?: { + from: string; + to: string; + }[]; + /** The Durable Objects being removed. */ + deleted_classes?: string[]; +}; + /** * The `EnvironmentInheritable` interface declares all the configuration fields for an environment * that can be inherited (and overridden) from the top-level environment. @@ -183,21 +202,7 @@ interface EnvironmentInheritable { * @default [] * @inheritable */ - migrations: { - /** A unique identifier for this migration. */ - tag: string; - /** The new Durable Objects being defined. */ - new_classes?: string[]; - /** The new SQLite Durable Objects being defined. */ - new_sqlite_classes?: string[]; - /** The Durable Objects being renamed. */ - renamed_classes?: { - from: string; - to: string; - }[]; - /** The Durable Objects being removed. */ - deleted_classes?: string[]; - }[]; + migrations: DurableObjectMigration[]; /** * "Cron" definitions to trigger a Worker's "scheduled" function. diff --git a/packages/wrangler/src/dev.tsx b/packages/wrangler/src/dev.tsx index 8bb0b0a2ce9c..714ce3ece361 100644 --- a/packages/wrangler/src/dev.tsx +++ b/packages/wrangler/src/dev.tsx @@ -618,6 +618,18 @@ export async function startDev(args: StartDevOptions) { ); } + if ( + args.remote && + config.migrations?.some( + (m) => + Array.isArray(m.new_sqlite_classes) && m.new_sqlite_classes.length > 0 + ) + ) { + throw new UserError( + "SQLite in Durable Objects is only supported in local mode." + ); + } + const projectRoot = configPath && path.dirname(configPath); const devEnv = new DevEnv(); @@ -973,6 +985,7 @@ export async function startDev(args: StartDevOptions) { } usageModel={configParam.usage_model} bindings={bindings} + migrations={configParam.migrations} crons={configParam.triggers.crons} queueConsumers={configParam.queues.consumers} onReady={args.onReady} @@ -1155,6 +1168,7 @@ export async function startApiDev(args: StartDevOptions) { args.compatibilityFlags ?? configParam.compatibility_flags, usageModel: configParam.usage_model, bindings: bindings, + migrations: configParam.migrations, crons: configParam.triggers.crons, queueConsumers: configParam.queues.consumers, onReady: args.onReady, diff --git a/packages/wrangler/src/dev/dev.tsx b/packages/wrangler/src/dev/dev.tsx index 49d82560bdc0..21825ccf04a4 100644 --- a/packages/wrangler/src/dev/dev.tsx +++ b/packages/wrangler/src/dev/dev.tsx @@ -227,6 +227,7 @@ export type DevProps = { localPersistencePath: string | null; liveReload: boolean; bindings: CfWorkerInit["bindings"]; + migrations: Config["migrations"] | undefined; define: Config["define"]; alias: Config["alias"]; crons: Config["triggers"]["crons"]; @@ -413,7 +414,7 @@ function DevSession(props: DevSessionProps) { entrypoint: props.entry.file, directory: props.entry.directory, bindings: convertCfWorkerInitBindingstoBindings(props.bindings), - + migrations: props.migrations, triggers: [...routes, ...queueConsumers, ...crons], env: props.env, build: { @@ -493,6 +494,7 @@ function DevSession(props: DevSessionProps) { props.compatibilityDate, props.compatibilityFlags, props.bindings, + props.migrations, props.entry, props.legacyAssetPaths, props.isWorkersSite, @@ -689,6 +691,7 @@ function DevSession(props: DevSessionProps) { compatibilityFlags={props.compatibilityFlags} usageModel={props.usageModel} bindings={props.bindings} + migrations={props.migrations} workerDefinitions={workerDefinitions} legacyAssetPaths={props.legacyAssetPaths} experimentalAssets={props.experimentalAssets} diff --git a/packages/wrangler/src/dev/local.tsx b/packages/wrangler/src/dev/local.tsx index c340a6f9b76b..c98e2d2cd14d 100644 --- a/packages/wrangler/src/dev/local.tsx +++ b/packages/wrangler/src/dev/local.tsx @@ -30,6 +30,7 @@ export interface LocalProps { compatibilityFlags: string[] | undefined; usageModel: "bundled" | "unbound" | undefined; bindings: CfWorkerInit["bindings"]; + migrations: Config["migrations"] | undefined; workerDefinitions: WorkerRegistry | undefined; legacyAssetPaths: LegacyAssetPaths | undefined; experimentalAssets: ExperimentalAssetsOptions | undefined; @@ -90,6 +91,7 @@ export async function localPropsToConfigBundle( compatibilityFlags: props.compatibilityFlags, inspectorPort: props.runtimeInspectorPort, bindings: props.bindings, + migrations: props.migrations, workerDefinitions: props.workerDefinitions, legacyAssetPaths: props.legacyAssetPaths, experimentalAssets: props.experimentalAssets, diff --git a/packages/wrangler/src/dev/miniflare.ts b/packages/wrangler/src/dev/miniflare.ts index 1e4feb173b36..972b2a985a91 100644 --- a/packages/wrangler/src/dev/miniflare.ts +++ b/packages/wrangler/src/dev/miniflare.ts @@ -171,6 +171,7 @@ export interface ConfigBundle { compatibilityDate: string | undefined; compatibilityFlags: string[] | undefined; bindings: CfWorkerInit["bindings"]; + migrations: Config["migrations"] | undefined; workerDefinitions: WorkerRegistry | undefined; legacyAssetPaths: LegacyAssetPaths | undefined; experimentalAssets: ExperimentalAssetsOptions | undefined; @@ -370,6 +371,7 @@ type WorkerOptionsBindings = Pick< type MiniflareBindingsConfig = Pick< ConfigBundle, | "bindings" + | "migrations" | "workerDefinitions" | "queueConsumers" | "name" @@ -494,6 +496,8 @@ export function buildMiniflareBindingOptions(config: MiniflareBindingsConfig): { } } + const classNameToUseSQLite = getClassNamesWhichUseSQLite(config.migrations); + // Partition Durable Objects based on whether they're internal (defined by // this session's worker), or external (defined by another session's worker // registered in the dev registry) @@ -511,10 +515,13 @@ export function buildMiniflareBindingOptions(config: MiniflareBindingsConfig): { // Bind all internal objects, so they're accessible by all other sessions // that proxy requests for our objects to this worker durableObjects: Object.fromEntries( - internalObjects.map(({ class_name }) => [ - class_name, - { className: class_name, scriptName: getName(config) }, - ]) + internalObjects.map(({ class_name }) => { + const useSQLite = classNameToUseSQLite.get(class_name); + return [ + class_name, + { className: class_name, scriptName: getName(config), useSQLite }, + ]; + }) ), // Use this worker instead of the user worker if the pathname is // `/${EXTERNAL_SERVICE_WORKER_NAME}` @@ -622,14 +629,25 @@ export function buildMiniflareBindingOptions(config: MiniflareBindingsConfig): { ), durableObjects: Object.fromEntries([ - ...internalObjects.map(({ name, class_name }) => [name, class_name]), + ...internalObjects.map(({ name, class_name }) => { + const useSQLite = classNameToUseSQLite.get(class_name); + return [ + name, + { + className: class_name, + useSQLite, + }, + ]; + }), ...externalObjects.map(({ name, class_name, script_name }) => { const identifier = getIdentifier(`do_${script_name}_${class_name}`); + const useSQLite = classNameToUseSQLite.get(class_name); return [ name, { className: identifier, scriptName: EXTERNAL_SERVICE_WORKER_NAME, + useSQLite, // Matches the unique key Miniflare will generate for this object in // the target session. We need to do this so workerd generates the // same IDs it would if this were part of the same process. workerd @@ -658,6 +676,55 @@ export function buildMiniflareBindingOptions(config: MiniflareBindingsConfig): { }; } +function getClassNamesWhichUseSQLite( + migrations: Config["migrations"] | undefined +) { + const classNameToUseSQLite = new Map(); + (migrations || []).forEach((migration) => { + migration.deleted_classes?.forEach((deleted_class) => { + if (!classNameToUseSQLite.delete(deleted_class)) { + throw new UserError( + `Cannot apply deleted_classes migration to non-existent class ${deleted_class}` + ); + } + }); + + migration.renamed_classes?.forEach(({ from, to }) => { + const useSQLite = classNameToUseSQLite.get(from); + if (useSQLite === undefined) { + throw new UserError( + `Cannot apply renamed_classes migration to non-existent class ${from}` + ); + } else { + classNameToUseSQLite.delete(from); + classNameToUseSQLite.set(to, useSQLite); + } + }); + + migration.new_classes?.forEach((new_class) => { + if (classNameToUseSQLite.has(new_class)) { + throw new UserError( + `Cannot apply new_classes migration to existing class ${new_class}` + ); + } else { + classNameToUseSQLite.set(new_class, false); + } + }); + + migration.new_sqlite_classes?.forEach((new_class) => { + if (classNameToUseSQLite.has(new_class)) { + throw new UserError( + `Cannot apply new_sqlite_classes migration to existing class ${new_class}` + ); + } else { + classNameToUseSQLite.set(new_class, true); + } + }); + }); + + return classNameToUseSQLite; +} + type PickTemplate = { [P in keyof T & K]: T[P]; }; diff --git a/packages/wrangler/src/dev/start-server.ts b/packages/wrangler/src/dev/start-server.ts index 815b0da1f326..5058b78a521a 100644 --- a/packages/wrangler/src/dev/start-server.ts +++ b/packages/wrangler/src/dev/start-server.ts @@ -107,6 +107,7 @@ export async function startDevServer( ...(typeof r === "string" ? { pattern: r } : r), })), bindings: convertCfWorkerInitBindingstoBindings(props.bindings), + migrations: props.migrations, dev: { server: { hostname: props.initialIp, @@ -258,7 +259,6 @@ export async function startDevServer( config: fakeResolvedInput(startDevWorkerOptions), bundle, }); - const { stop } = await startLocalServer({ name: props.name, bundle: bundle, @@ -266,6 +266,7 @@ export async function startDevServer( compatibilityDate: props.compatibilityDate, compatibilityFlags: props.compatibilityFlags, bindings: props.bindings, + migrations: props.migrations, legacyAssetPaths: props.legacyAssetPaths, experimentalAssets: props.experimentalAssets, initialPort: undefined, // hard-code for userworker, DevEnv-ProxyWorker now uses this prop value