From 05c56073b4e8c71ab6e0b287adddddc00d763170 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Somhairle=20MacLe=C3=B2id?= Date: Wed, 19 Jun 2024 17:44:37 +0100 Subject: [PATCH] File based dev registry (#5214) --- .changeset/yellow-coins-serve.md | 5 + fixtures/ai-app/tests/index.test.ts | 3 +- .../a/wrangler.toml | 5 + .../b/wrangler.toml | 1 + .../c/wrangler.toml | 1 + .../tests/index.test.ts | 30 +++-- .../tests/index.test.ts | 37 ++++++- .../tests/get-bindings-proxy.bindings.test.ts | 45 ++++---- .../service-bindings-app/tests/index.test.ts | 19 ++-- packages/wrangler/package.json | 1 + packages/wrangler/src/api/dev.ts | 12 +- .../src/api/integrations/platform/index.ts | 17 ++- packages/wrangler/src/dev-registry.ts | 103 ++++++++++++++++++ packages/wrangler/src/dev.tsx | 24 +++- packages/wrangler/src/dev/miniflare.ts | 10 +- packages/wrangler/src/experimental-flags.ts | 27 +++++ packages/wrangler/src/pages/dev.ts | 8 ++ pnpm-lock.yaml | 26 +++-- 18 files changed, 315 insertions(+), 59 deletions(-) create mode 100644 .changeset/yellow-coins-serve.md create mode 100644 packages/wrangler/src/experimental-flags.ts diff --git a/.changeset/yellow-coins-serve.md b/.changeset/yellow-coins-serve.md new file mode 100644 index 000000000000..d3d825e5e90d --- /dev/null +++ b/.changeset/yellow-coins-serve.md @@ -0,0 +1,5 @@ +--- +"wrangler": patch +--- + +feat: Experimental file based service discovery when running multiple Wrangler instances locally. To try it out, make sure all your local Wrangler instances are running with the `--x-registry` flag. diff --git a/fixtures/ai-app/tests/index.test.ts b/fixtures/ai-app/tests/index.test.ts index 838ee7bb0007..94d1eaae7848 100644 --- a/fixtures/ai-app/tests/index.test.ts +++ b/fixtures/ai-app/tests/index.test.ts @@ -27,8 +27,7 @@ describe("'wrangler dev' correctly renders pages", () => { expect((content as Record).run).toEqual("function"); }); - // TODO: unskip when https://github.com/cloudflare/workerd/pull/2095 is merged and released - it.skip("ai binding properties", async ({ expect }) => { + it("ai binding properties", async ({ expect }) => { const response = await fetch(`http://${ip}:${port}/`); const content = await response.json(); expect((content as Record).binding).toEqual({ diff --git a/fixtures/external-durable-objects-app/a/wrangler.toml b/fixtures/external-durable-objects-app/a/wrangler.toml index 359e75cd58fc..9e5383b33b0f 100644 --- a/fixtures/external-durable-objects-app/a/wrangler.toml +++ b/fixtures/external-durable-objects-app/a/wrangler.toml @@ -1,6 +1,11 @@ name = "a" +compatibility_date = "2024-06-10" [durable_objects] bindings = [ { name = "MY_DO", class_name = "MyDurableObject" } ] + +[[migrations]] +tag = "v1" +new_classes = ["MyDurableObject"] \ No newline at end of file diff --git a/fixtures/external-durable-objects-app/b/wrangler.toml b/fixtures/external-durable-objects-app/b/wrangler.toml index a6685761c5af..42a678344f62 100644 --- a/fixtures/external-durable-objects-app/b/wrangler.toml +++ b/fixtures/external-durable-objects-app/b/wrangler.toml @@ -1,4 +1,5 @@ name = "b" +compatibility_date = "2024-06-10" [durable_objects] bindings = [ diff --git a/fixtures/external-durable-objects-app/c/wrangler.toml b/fixtures/external-durable-objects-app/c/wrangler.toml index 2b23b054b2fc..4bb01a0fd9fa 100644 --- a/fixtures/external-durable-objects-app/c/wrangler.toml +++ b/fixtures/external-durable-objects-app/c/wrangler.toml @@ -1,4 +1,5 @@ name = "c" +compatibility_date = "2024-06-10" [durable_objects] bindings = [ diff --git a/fixtures/external-durable-objects-app/tests/index.test.ts b/fixtures/external-durable-objects-app/tests/index.test.ts index 7cab3713d9a3..c32cbe169ac8 100644 --- a/fixtures/external-durable-objects-app/tests/index.test.ts +++ b/fixtures/external-durable-objects-app/tests/index.test.ts @@ -1,5 +1,6 @@ import { fork } from "child_process"; import * as path from "path"; +import { setTimeout } from "timers/promises"; import { fetch } from "undici"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { unstable_dev } from "wrangler"; @@ -8,7 +9,7 @@ import type { UnstableDevWorker } from "wrangler"; // TODO: reenable when https://github.com/cloudflare/workers-sdk/pull/4241 lands // and improves reliability of this test. -describe.skip( +describe( "Pages Functions", () => { let a: UnstableDevWorker; @@ -26,15 +27,28 @@ describe.skip( beforeAll(async () => { a = await unstable_dev(path.join(__dirname, "../a/index.ts"), { config: path.join(__dirname, "../a/wrangler.toml"), + experimental: { + fileBasedRegistry: true, + disableExperimentalWarning: true, + }, }); + await setTimeout(1000); b = await unstable_dev(path.join(__dirname, "../b/index.ts"), { config: path.join(__dirname, "../b/wrangler.toml"), + experimental: { + fileBasedRegistry: true, + disableExperimentalWarning: true, + }, }); - + await setTimeout(1000); c = await unstable_dev(path.join(__dirname, "../c/index.ts"), { config: path.join(__dirname, "../c/wrangler.toml"), + experimental: { + fileBasedRegistry: true, + disableExperimentalWarning: true, + }, }); - + await setTimeout(1000); dWranglerProcess = fork( path.join( "..", @@ -48,10 +62,10 @@ describe.skip( [ "pages", "dev", + "--x-registry", "public", + "--compatibility-date=2024-03-04", "--do=PAGES_REFERENCED_DO=MyDurableObject@a", - "--port=0", - "--inspector-port=0", ], { stdio: ["ignore", "ignore", "ignore", "ipc"], @@ -63,6 +77,7 @@ describe.skip( dPort = parsedMessage.port; dResolveReadyPromise(undefined); }); + await setTimeout(1000); }); afterAll(async () => { @@ -85,10 +100,7 @@ describe.skip( it("connects up Durable Objects and keeps state across wrangler instances", async () => { await dReadyPromise; - - // Service registry is polled every 300ms, - // so let's give all the Workers a little time to find each other - await new Promise((resolve) => setTimeout(resolve, 700)); + await setTimeout(1000); const responseA = await a.fetch(`/`, { headers: { diff --git a/fixtures/external-service-bindings-app/tests/index.test.ts b/fixtures/external-service-bindings-app/tests/index.test.ts index 2542f163e647..44a9d7a23af4 100644 --- a/fixtures/external-service-bindings-app/tests/index.test.ts +++ b/fixtures/external-service-bindings-app/tests/index.test.ts @@ -29,7 +29,7 @@ type WranglerInstance = { port: string; }; -describe.skip("Pages Functions", () => { +describe("Pages Functions", () => { let wranglerInstances: (WranglerInstance | UnstableDevWorker)[] = []; let pagesAppPort: string; @@ -38,34 +38,64 @@ describe.skip("Pages Functions", () => { path.join(__dirname, "../module-worker-a/index.ts"), { config: path.join(__dirname, "../module-worker-a/wrangler.toml"), + experimental: { + fileBasedRegistry: true, + disableExperimentalWarning: true, + }, } ); + await setTimeout(1000); + wranglerInstances[1] = await unstable_dev( path.join(__dirname, "../module-worker-b/index.ts"), { config: path.join(__dirname, "../module-worker-b/wrangler.toml"), + experimental: { + fileBasedRegistry: true, + disableExperimentalWarning: true, + }, } ); + await setTimeout(1000); + wranglerInstances[2] = await unstable_dev( path.join(__dirname, "../service-worker-a/index.ts"), { config: path.join(__dirname, "../service-worker-a/wrangler.toml"), + experimental: { + fileBasedRegistry: true, + disableExperimentalWarning: true, + }, } ); + await setTimeout(1000); + wranglerInstances[3] = await unstable_dev( path.join(__dirname, "../module-worker-c/index.ts"), { config: path.join(__dirname, "../module-worker-c/wrangler.toml"), env: "staging", + experimental: { + fileBasedRegistry: true, + disableExperimentalWarning: true, + }, } ); + await setTimeout(1000); + wranglerInstances[4] = await unstable_dev( path.join(__dirname, "../module-worker-d/index.ts"), { config: path.join(__dirname, "../module-worker-d/wrangler.toml"), env: "production", + experimental: { + fileBasedRegistry: true, + disableExperimentalWarning: true, + }, } ); + await setTimeout(1000); + wranglerInstances[5] = await getWranglerInstance({ pages: true, dirName: "pages-functions-app", @@ -79,6 +109,7 @@ describe.skip("Pages Functions", () => { }); pagesAppPort = wranglerInstances[5].port; + await setTimeout(1000); }); afterAll(async () => { @@ -111,7 +142,7 @@ describe.skip("Pages Functions", () => { expect(json).toMatchInlineSnapshot(` { "moduleWorkerCResponse": "Hello from module worker c (staging)", - "moduleWorkerDResponse": "You should start up wrangler dev --local on the STAGING_MODULE_D_SERVICE worker", + "moduleWorkerDResponse": "[wrangler] Couldn't find \`wrangler dev\` session for service "module-worker-d-staging" to proxy to", } `); }); @@ -132,9 +163,9 @@ async function getWranglerInstance({ [ ...(pages ? ["pages"] : []), "dev", + "--x-registry", ...(pages ? ["public"] : ["index.ts"]), "--local", - `--port=0`, ...extraArgs, ], { diff --git a/fixtures/get-bindings-proxy/tests/get-bindings-proxy.bindings.test.ts b/fixtures/get-bindings-proxy/tests/get-bindings-proxy.bindings.test.ts index 334b2a45c39e..02819121ddf4 100644 --- a/fixtures/get-bindings-proxy/tests/get-bindings-proxy.bindings.test.ts +++ b/fixtures/get-bindings-proxy/tests/get-bindings-proxy.bindings.test.ts @@ -1,12 +1,21 @@ import { readdir } from "fs/promises"; import path from "path"; +import { setTimeout } from "timers/promises"; import { D1Database, DurableObjectNamespace, Fetcher, R2Bucket, } from "@cloudflare/workers-types"; -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; import { unstable_dev } from "wrangler"; import { getBindingsProxy } from "./shared"; import type { KVNamespace } from "@cloudflare/workers-types"; @@ -36,18 +45,13 @@ describe("getBindingsProxy - bindings", () => { vi.spyOn(console, "log").mockImplementation(() => {}); }); - // Note: we're skipping the service workers and durable object tests - // so there's no need to start separate workers right now, the - // following beforeAll and afterAll should be un-commented when - // we reenable the tests - - // beforeAll(async () => { - // devWorkers = await startWorkers(); - // }); + beforeAll(async () => { + devWorkers = await startWorkers(); + }); - // afterAll(async () => { - // await Promise.allSettled(devWorkers.map((i) => i.stop())); - // }); + afterAll(async () => { + await Promise.allSettled(devWorkers.map((i) => i.stop())); + }); describe("var bindings", () => { it("correctly obtains var bindings from both wrangler.toml and .dev.vars", async () => { @@ -134,11 +138,10 @@ describe("getBindingsProxy - bindings", () => { } }); - // Note: the following test is skipped due to flakiness caused by the local registry not working reliably - // when we run all our fixtures together (possibly because of race condition issues) - it.skip("provides service bindings to external local workers", async () => { + it("provides service bindings to external local workers", async () => { const { bindings, dispose } = await getBindingsProxy({ configPath: wranglerTomlFilePath, + experimentalRegistry: true, }); try { const { MY_SERVICE_A, MY_SERVICE_B } = bindings; @@ -164,11 +167,10 @@ describe("getBindingsProxy - bindings", () => { await dispose(); }); - // Note: the following test is skipped due to flakiness caused by the local registry not working reliably - // when we run all our fixtures together (possibly because of race condition issues) - it.skip("correctly obtains functioning DO bindings (provided by external local workers)", async () => { + it("correctly obtains functioning DO bindings (provided by external local workers)", async () => { const { bindings, dispose } = await getBindingsProxy({ configPath: wranglerTomlFilePath, + experimentalRegistry: true, }); try { const { MY_DO_A, MY_DO_B } = bindings; @@ -235,11 +237,16 @@ async function startWorkers(): Promise { const workersDirPath = path.join(__dirname, "..", "workers"); const workers = await readdir(workersDirPath); return await Promise.all( - workers.map((workerName) => { + workers.map(async (workerName) => { const workerPath = path.join(workersDirPath, workerName); + await setTimeout(2000); return unstable_dev(path.join(workerPath, "index.ts"), { config: path.join(workerPath, "wrangler.toml"), ip: "127.0.0.1", + experimental: { + fileBasedRegistry: true, + disableExperimentalWarning: true, + }, }); }) ); diff --git a/fixtures/service-bindings-app/tests/index.test.ts b/fixtures/service-bindings-app/tests/index.test.ts index 0004c7ef2b63..f7ebfc233fe5 100644 --- a/fixtures/service-bindings-app/tests/index.test.ts +++ b/fixtures/service-bindings-app/tests/index.test.ts @@ -1,10 +1,9 @@ import path from "node:path"; +import { setTimeout } from "node:timers/promises"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { unstable_dev, UnstableDevWorker } from "wrangler"; -// TODO: reenable when https://github.com/cloudflare/workers-sdk/pull/4241 lands -// and improves reliability of this test. -describe.skip("Service Bindings", () => { +describe("Service Bindings", () => { let aWorker: UnstableDevWorker; let bWorker: UnstableDevWorker; @@ -12,14 +11,20 @@ describe.skip("Service Bindings", () => { beforeAll(async () => { bWorker = await unstable_dev(path.join(__dirname, "../b/index.ts"), { config: path.join(__dirname, "../b/wrangler.toml"), + experimental: { + fileBasedRegistry: true, + disableExperimentalWarning: true, + }, }); + await setTimeout(1000); aWorker = await unstable_dev(path.join(__dirname, "../a/index.ts"), { config: path.join(__dirname, "../a/wrangler.toml"), + experimental: { + fileBasedRegistry: true, + disableExperimentalWarning: true, + }, }); - // Service registry is polled every 300ms, - // so let's give worker A some time to find B - - await new Promise((resolve) => setTimeout(resolve, 700)); + await setTimeout(1000); }); it("connects up Durable Objects and keeps state across wrangler instances", async () => { diff --git a/packages/wrangler/package.json b/packages/wrangler/package.json index 08e37f3644b0..d1dcb073684e 100644 --- a/packages/wrangler/package.json +++ b/packages/wrangler/package.json @@ -74,6 +74,7 @@ "@esbuild-plugins/node-modules-polyfill": "^0.2.2", "blake3-wasm": "^2.1.5", "chokidar": "^3.5.3", + "date-fns": "^3.6.0", "esbuild": "0.17.19", "miniflare": "workspace:*", "nanoid": "^3.3.3", diff --git a/packages/wrangler/src/api/dev.ts b/packages/wrangler/src/api/dev.ts index e71de6ac9fe4..2151634c6446 100644 --- a/packages/wrangler/src/api/dev.ts +++ b/packages/wrangler/src/api/dev.ts @@ -1,5 +1,6 @@ import { fetch, Request } from "undici"; import { startApiDev, startDev } from "../dev"; +import { run } from "../experimental-flags"; import { logger } from "../logger"; import type { Environment } from "../config"; import type { Rule } from "../config/environment"; @@ -78,6 +79,7 @@ export interface UnstableDevOptions { testScheduled?: boolean; // Test scheduled events by visiting /__scheduled in browser watch?: boolean; // unstable_dev doesn't support watch-mode yet in testMode devEnv?: boolean; + fileBasedRegistry?: boolean; }; } @@ -122,6 +124,7 @@ export async function unstable_dev( testMode, testScheduled, devEnv = false, + fileBasedRegistry = false, // 2. options for alpha/beta products/libs d1Databases, enablePagesAssetsServiceBinding, @@ -214,6 +217,7 @@ export async function unstable_dev( port: options?.port ?? 0, experimentalVersions: undefined, experimentalDevEnv: devEnv, + experimentalRegistry: fileBasedRegistry, }; //due to Pages adoption of unstable_dev, we can't *just* disable rebuilds and watching. instead, we'll have two versions of startDev, which will converge. @@ -245,7 +249,13 @@ export async function unstable_dev( }; } else { //outside of test mode, rebuilds work fine, but only one instance of wrangler will work at a time - const devServer = await startDev(devOptions); + const devServer = await run( + { + DEV_ENV: devEnv, + FILE_BASED_REGISTRY: fileBasedRegistry, + }, + () => startDev(devOptions) + ); const { port, address, proxyData } = await readyPromise; return { port, diff --git a/packages/wrangler/src/api/integrations/platform/index.ts b/packages/wrangler/src/api/integrations/platform/index.ts index d25ab15ac08b..fb1ba52cf2f5 100644 --- a/packages/wrangler/src/api/integrations/platform/index.ts +++ b/packages/wrangler/src/api/integrations/platform/index.ts @@ -8,6 +8,7 @@ import { buildMiniflareBindingOptions, buildSitesOptions, } from "../../../dev/miniflare"; +import { run } from "../../../experimental-flags"; import { getAssetPaths, getSiteAssetPaths } from "../../../sites"; import { CacheStorage } from "./caches"; import { ExecutionContext } from "./executionContext"; @@ -41,6 +42,12 @@ export type GetPlatformProxyOptions = { * If `false` is specified no data is persisted on the filesystem. */ persist?: boolean | { path: string }; + /** + * Use the experimental file-based dev registry for service discovery + * + * Note: this feature is experimental + */ + experimentalRegistry?: boolean; }; /** @@ -93,10 +100,12 @@ export async function getPlatformProxy< env, }); - const miniflareOptions = await getMiniflareOptionsFromConfig( - rawConfig, - env, - options + const miniflareOptions = await run( + { + FILE_BASED_REGISTRY: Boolean(options.experimentalRegistry), + DEV_ENV: false, + }, + () => getMiniflareOptionsFromConfig(rawConfig, env, options) ); const mf = new Miniflare({ diff --git a/packages/wrangler/src/dev-registry.ts b/packages/wrangler/src/dev-registry.ts index 1b478b2099cc..0b53aa346711 100644 --- a/packages/wrangler/src/dev-registry.ts +++ b/packages/wrangler/src/dev-registry.ts @@ -1,10 +1,25 @@ import events from "node:events"; +import { utimesSync } from "node:fs"; +import { + mkdir, + readdir, + readFile, + stat, + unlink, + writeFile, +} from "node:fs/promises"; import { createServer } from "node:http"; import net from "node:net"; +import path from "node:path"; import bodyParser from "body-parser"; +import { watch } from "chokidar"; +import { subMinutes } from "date-fns"; import express from "express"; import { createHttpTerminator } from "http-terminator"; import { fetch } from "undici"; +import { version as wranglerVersion } from "../package.json"; +import { getFlag } from "./experimental-flags"; +import { getGlobalWranglerConfigPath } from "./global-wrangler-config-path"; import { logger } from "./logger"; import type { Config } from "./config"; import type { HttpTerminator } from "http-terminator"; @@ -18,9 +33,16 @@ if (Number.isNaN(DEV_REGISTRY_PORT)) { } const DEV_REGISTRY_HOST = `http://127.0.0.1:${DEV_REGISTRY_PORT}`; +const DEV_REGISTRY_PATH = path.join(getGlobalWranglerConfigPath(), "registry"); + let globalServer: Server | null; let globalTerminator: HttpTerminator; +let globalWatcher: ReturnType | undefined; +let globalWorkers: WorkerRegistry | undefined; + +const heartbeats = new Map>(); + export type WorkerRegistry = Record; export type WorkerEntrypointsDefinition = Record< @@ -40,6 +62,34 @@ export type WorkerDefinition = { durableObjectsPort?: number; }; +async function loadWorkerDefinitions(): Promise { + await mkdir(DEV_REGISTRY_PATH, { recursive: true }); + globalWorkers ??= {}; + const newWorkers = new Set(); + const workerDefinitions = await readdir(DEV_REGISTRY_PATH); + for (const workerName of workerDefinitions) { + const file = await readFile( + path.join(DEV_REGISTRY_PATH, workerName), + "utf8" + ); + const stats = await stat(path.join(DEV_REGISTRY_PATH, workerName)); + // Cleanup old workers + if (stats.mtime < subMinutes(new Date(), 10)) { + await unregisterWorker(workerName); + } else { + globalWorkers[workerName] = JSON.parse(file); + newWorkers.add(workerName); + } + } + + for (const worker of Object.keys(globalWorkers)) { + if (!newWorkers.has(worker)) { + delete globalWorkers[worker]; + } + } + return globalWorkers; +} + /** * A helper function to check whether our service registry is already running */ @@ -102,6 +152,14 @@ export async function startWorkerRegistryServer(port: number) { * services, as well as getting the state of the registry. */ export async function startWorkerRegistry() { + if (getFlag("FILE_BASED_REGISTRY")) { + globalWatcher ??= watch(DEV_REGISTRY_PATH, { + persistent: true, + }).on("all", async () => { + await loadWorkerDefinitions(); + }); + return; + } if ((await isPortAvailable()) && !globalServer) { const result = await startWorkerRegistryServer(DEV_REGISTRY_PORT); globalServer = result.server; @@ -131,6 +189,13 @@ export async function startWorkerRegistry() { * Stop the service registry. */ export async function stopWorkerRegistry() { + if (getFlag("FILE_BASED_REGISTRY") || globalWatcher) { + await globalWatcher?.close(); + for (const heartbeat of heartbeats) { + clearInterval(heartbeat[1]); + } + return; + } await globalTerminator?.terminate(); globalServer = null; } @@ -142,6 +207,27 @@ export async function registerWorker( name: string, definition: WorkerDefinition ) { + if (getFlag("FILE_BASED_REGISTRY")) { + const existingHeartbeat = heartbeats.get(name); + if (existingHeartbeat) { + clearInterval(existingHeartbeat); + } + await mkdir(DEV_REGISTRY_PATH, { recursive: true }); + await writeFile( + path.join(DEV_REGISTRY_PATH, name), + // We don't currently do anything with the stored Wrangler version, + // but if we need to make breaking changes to this format in the future + // we can use this field to present useful messaging + JSON.stringify({ ...definition, wranglerVersion }, null, 2) + ); + heartbeats.set( + name, + setInterval(() => { + utimesSync(path.join(DEV_REGISTRY_PATH, name), new Date(), new Date()); + }, 30_000) + ); + return; + } /** * Prevent the dev registry be closed. */ @@ -171,6 +257,18 @@ export async function registerWorker( * Unregister a worker from the registry. */ export async function unregisterWorker(name: string) { + if (getFlag("FILE_BASED_REGISTRY")) { + try { + await unlink(path.join(DEV_REGISTRY_PATH, name)); + const existingHeartbeat = heartbeats.get(name); + if (existingHeartbeat) { + clearInterval(existingHeartbeat); + } + } catch (e) { + logger.debug("failed to unregister worker", e); + } + return; + } try { await fetch(`${DEV_REGISTRY_HOST}/workers/${name}`, { method: "DELETE", @@ -193,6 +291,11 @@ export async function unregisterWorker(name: string) { export async function getRegisteredWorkers(): Promise< WorkerRegistry | undefined > { + if (getFlag("FILE_BASED_REGISTRY")) { + globalWorkers = await loadWorkerDefinitions(); + return globalWorkers; + } + try { const response = await fetch(`${DEV_REGISTRY_HOST}/workers`); return (await response.json()) as WorkerRegistry; diff --git a/packages/wrangler/src/dev.tsx b/packages/wrangler/src/dev.tsx index 6309c299dd4c..e6dd26a20df6 100644 --- a/packages/wrangler/src/dev.tsx +++ b/packages/wrangler/src/dev.tsx @@ -11,6 +11,7 @@ import { getVarsForDev } from "./dev/dev-vars"; import { getLocalPersistencePath } from "./dev/get-local-persistence-path"; import { startDevServer } from "./dev/start-server"; import { UserError } from "./errors"; +import { run } from "./experimental-flags"; import { logger } from "./logger"; import * as metrics from "./metrics"; import { getAssetPaths, getSiteAssetPaths } from "./sites"; @@ -295,6 +296,13 @@ export function devOptions(yargs: CommonYargsArgv) { "Use the experimental DevEnv instantiation (unified across wrangler dev and unstable_dev)", default: false, }) + .option("experimental-registry", { + alias: ["x-registry"], + type: "boolean", + describe: + "Use the experimental file based dev registry for multi-worker development", + default: false, + }) ); } @@ -323,7 +331,13 @@ This is currently not supported 😭, but we think that we'll get it to work soo let watcher; try { - const devInstance = await startDev(args); + const devInstance = await run( + { + DEV_ENV: args.experimentalDevEnv, + FILE_BASED_REGISTRY: args.experimentalRegistry, + }, + () => startDev(args) + ); watcher = devInstance.watcher; const { waitUntilExit } = devInstance.devReactElement; await waitUntilExit(); @@ -659,7 +673,13 @@ export async function startApiDev(args: StartDevOptions) { }); } - const devServer = await getDevServer(config); + const devServer = await run( + { + DEV_ENV: args.experimentalDevEnv, + FILE_BASED_REGISTRY: args.experimentalRegistry, + }, + () => getDevServer(config) + ); if (!devServer) { const error = new Error("Failed to start dev server."); logger.error(error.message); diff --git a/packages/wrangler/src/dev/miniflare.ts b/packages/wrangler/src/dev/miniflare.ts index 513c8dbafd42..b869b7fdf92f 100644 --- a/packages/wrangler/src/dev/miniflare.ts +++ b/packages/wrangler/src/dev/miniflare.ts @@ -108,7 +108,7 @@ function createDurableObjectClass({ className, proxyUrl }) { // Forward regular HTTP requests to the other "wrangler dev" session klass.prototype.fetch = function(request) { if (proxyUrl === undefined) { - return new Response(\`[wrangler] Couldn't find \\\`wrangler dev\\\` session for class "\${className}" to proxy to\`, { status: 503 }); + return new Response(\`\${className} \${proxyUrl}[wrangler] Couldn't find \\\`wrangler dev\\\` session for class "\${className}" to proxy to\`, { status: 503 }); } const proxyRequest = new Request(proxyUrl, request); proxyRequest.headers.set(HEADER_URL, request.url); @@ -434,6 +434,7 @@ export function buildMiniflareBindingOptions(config: MiniflareBindingsConfig): { // services support JSRPC over HTTP CONNECT using a special hostname. // Refer to https://github.com/cloudflare/workerd/pull/1757 for details. let address: `${string}:${number}`; + let style = HttpOptions_Style.PROXY; if (service.entrypoint !== undefined) { // If the user has requested a named entrypoint... if (target.entrypointAddresses === undefined) { @@ -459,7 +460,7 @@ export function buildMiniflareBindingOptions(config: MiniflareBindingsConfig): { target.entrypointAddresses?.["default"]; if (defaultEntrypointAddress === undefined) { // If the "server" `wrangler` is too old to provide direct entrypoint - // addresses, fallback to sending requests directly to the target... + // addresses (or uses service-worker syntax), fallback to sending requests directly to the target... if (target.protocol === "https") { // ...unless the target is listening on HTTPS, in which case throw. // We can't support this as `workerd` requires us to explicitly @@ -471,6 +472,8 @@ export function buildMiniflareBindingOptions(config: MiniflareBindingsConfig): { ); } address = `${target.host}:${target.port}`; + // Removing this line causes `Internal Service Error` responses from service-worker syntax workers, since they don't seem to support the PROXY protocol + style = HttpOptions_Style.HOST; } else { address = `${defaultEntrypointAddress.host}:${defaultEntrypointAddress.port}`; } @@ -480,7 +483,7 @@ export function buildMiniflareBindingOptions(config: MiniflareBindingsConfig): { external: { address, http: { - style: HttpOptions_Style.PROXY, + style, cfBlobHeader: CoreHeaders.CF_BLOB, }, }, @@ -532,6 +535,7 @@ export function buildMiniflareBindingOptions(config: MiniflareBindingsConfig): { const identifier = getIdentifier(`do_${script_name}_${class_name}`); const classNameJson = JSON.stringify(class_name); + if ( target?.host === undefined || target.port === undefined || diff --git a/packages/wrangler/src/experimental-flags.ts b/packages/wrangler/src/experimental-flags.ts new file mode 100644 index 000000000000..319f288d385f --- /dev/null +++ b/packages/wrangler/src/experimental-flags.ts @@ -0,0 +1,27 @@ +import { AsyncLocalStorage } from "async_hooks"; +import { logger } from "./logger"; + +type ExperimentalFlags = { + // TODO: use this + DEV_ENV: boolean; + FILE_BASED_REGISTRY: boolean; +}; + +const flags = new AsyncLocalStorage(); + +export const run = (flagValues: ExperimentalFlags, cb: () => V) => + flags.run(flagValues, cb); + +export const getFlag = (flag: F) => { + const store = flags.getStore(); + if (store === undefined) { + logger.debug("No experimental flag store instantiated"); + } + const value = flags.getStore()?.[flag]; + if (value === undefined) { + logger.debug( + `Attempted to use flag "${flag}" which has not been instantiated` + ); + } + return value; +}; diff --git a/packages/wrangler/src/pages/dev.ts b/packages/wrangler/src/pages/dev.ts index 6ca094f04b72..05816e9218fa 100644 --- a/packages/wrangler/src/pages/dev.ts +++ b/packages/wrangler/src/pages/dev.ts @@ -242,6 +242,13 @@ export function Options(yargs: CommonYargsArgv) { "Show interactive dev session (defaults to true if the terminal supports interactivity)", type: "boolean", }, + "experimental-registry": { + alias: ["x-registry"], + type: "boolean", + describe: + "Use the experimental file based dev registry for multi-worker development", + default: false, + }, }); } @@ -680,6 +687,7 @@ export const Handler = async (args: PagesDevArguments) => { showInteractiveDevSession: args.showInteractiveDevSession, testMode: false, watch: true, + fileBasedRegistry: args.experimentalRegistry, }, }); await metrics.sendMetricsEvent("run pages dev"); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 130147a3b7b9..ddaa8490d8f5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1573,6 +1573,9 @@ importers: chokidar: specifier: ^3.5.3 version: 3.5.3 + date-fns: + specifier: ^3.6.0 + version: 3.6.0 esbuild: specifier: 0.17.19 version: 0.17.19 @@ -5704,6 +5707,9 @@ packages: resolution: {integrity: sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==} engines: {node: '>=0.11'} + date-fns@3.6.0: + resolution: {integrity: sha512-fRHTG8g/Gif+kSh50gaGEdToemgfj74aRX3swtiouboip5JDLAyDE9F11nHMIcvOaXeOC6D7SpNhi7uFyB7Uww==} + date-time@3.1.0: resolution: {integrity: sha512-uqCUKXE5q1PNBXjPqvwhwJf9SwMoAHBgWJ6DcrnS5o+W2JOiIILl0JEdVD8SGujrNS02GGxgwAg2PN2zONgtjg==} engines: {node: '>=6'} @@ -13236,8 +13242,8 @@ snapshots: '@typescript-eslint/parser': 5.62.0(eslint@8.57.0)(typescript@5.4.5) eslint: 8.57.0 eslint-import-resolver-node: 0.3.7 - eslint-import-resolver-typescript: 3.5.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.7)(eslint-plugin-import@2.27.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0))(eslint@8.57.0) - eslint-plugin-import: 2.27.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.5.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.7)(eslint-plugin-import@2.27.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0) + eslint-import-resolver-typescript: 3.5.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.7)(eslint-plugin-import@2.27.5)(eslint@8.57.0) + eslint-plugin-import: 2.27.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.5.5)(eslint@8.57.0) eslint-plugin-jest: 26.9.0(@typescript-eslint/eslint-plugin@5.62.0(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5) eslint-plugin-jest-dom: 4.0.3(eslint@8.57.0) eslint-plugin-jsx-a11y: 6.7.1(eslint@8.57.0) @@ -15283,6 +15289,8 @@ snapshots: dependencies: '@babel/runtime': 7.22.5 + date-fns@3.6.0: {} + date-time@3.1.0: dependencies: time-zone: 1.0.0 @@ -15819,13 +15827,13 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-import-resolver-typescript@3.5.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.7)(eslint-plugin-import@2.27.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0))(eslint@8.57.0): + eslint-import-resolver-typescript@3.5.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.7)(eslint-plugin-import@2.27.5)(eslint@8.57.0): dependencies: debug: 4.3.4(supports-color@9.2.2) enhanced-resolve: 5.14.1 eslint: 8.57.0 - eslint-module-utils: 2.8.0(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.7)(eslint-import-resolver-typescript@3.5.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.7)(eslint-plugin-import@2.27.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0) - eslint-plugin-import: 2.27.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.5.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.7)(eslint-plugin-import@2.27.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0) + eslint-module-utils: 2.8.0(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.7)(eslint-import-resolver-typescript@3.5.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.7)(eslint-plugin-import@2.27.5)(eslint@8.57.0))(eslint@8.57.0) + eslint-plugin-import: 2.27.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.5.5)(eslint@8.57.0) get-tsconfig: 4.6.0 globby: 13.1.4 is-core-module: 2.13.0 @@ -15837,14 +15845,14 @@ snapshots: - eslint-import-resolver-webpack - supports-color - eslint-module-utils@2.8.0(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.7)(eslint-import-resolver-typescript@3.5.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.7)(eslint-plugin-import@2.27.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0): + eslint-module-utils@2.8.0(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.7)(eslint-import-resolver-typescript@3.5.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.7)(eslint-plugin-import@2.27.5)(eslint@8.57.0))(eslint@8.57.0): dependencies: debug: 3.2.7 optionalDependencies: '@typescript-eslint/parser': 5.62.0(eslint@8.57.0)(typescript@5.4.5) eslint: 8.57.0 eslint-import-resolver-node: 0.3.7 - eslint-import-resolver-typescript: 3.5.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.7)(eslint-plugin-import@2.27.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0))(eslint@8.57.0) + eslint-import-resolver-typescript: 3.5.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.7)(eslint-plugin-import@2.27.5)(eslint@8.57.0) transitivePeerDependencies: - supports-color @@ -15926,7 +15934,7 @@ snapshots: - eslint-import-resolver-webpack - supports-color - eslint-plugin-import@2.27.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.5.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.7)(eslint-plugin-import@2.27.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0): + eslint-plugin-import@2.27.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.5.5)(eslint@8.57.0): dependencies: array-includes: 3.1.6 array.prototype.flat: 1.3.1 @@ -15935,7 +15943,7 @@ snapshots: doctrine: 2.1.0 eslint: 8.57.0 eslint-import-resolver-node: 0.3.7 - eslint-module-utils: 2.8.0(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.7)(eslint-import-resolver-typescript@3.5.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.7)(eslint-plugin-import@2.27.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0) + eslint-module-utils: 2.8.0(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.7)(eslint-import-resolver-typescript@3.5.5(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.7)(eslint-plugin-import@2.27.5)(eslint@8.57.0))(eslint@8.57.0) has: 1.0.3 is-core-module: 2.13.0 is-glob: 4.0.3