From 4107f573b85eb86cc163c4acadf2b85138f76d97 Mon Sep 17 00:00:00 2001 From: Sunil Pai Date: Tue, 10 Sep 2024 22:27:19 +0100 Subject: [PATCH] feat: support analytics engine in local/remote dev (#6666) This adds "support" for analytics engine datasets for `wrangler dev`. Specifically, it simply mocks the AE bindings so that they exist while developing (and don't throw when accessed). This does NOT add support in Pages, though we very well could do so in a similar way in a followup. --- .changeset/long-rules-wait.md | 9 ++ packages/wrangler/e2e/dev.test.ts | 98 +++++++++++++++++++ .../__tests__/navigator-user-agent.test.ts | 2 + .../api/startDevWorker/BundlerController.ts | 3 + packages/wrangler/src/deploy/deploy.ts | 2 + .../wrangler/src/deployment-bundle/bundle.ts | 17 ++++ packages/wrangler/src/dev/dev.tsx | 1 + packages/wrangler/src/dev/start-server.ts | 4 + packages/wrangler/src/dev/use-esbuild.ts | 7 ++ .../src/pages/functions/buildPlugin.ts | 2 + .../src/pages/functions/buildWorker.ts | 4 + packages/wrangler/src/versions/upload.ts | 1 + .../middleware-mock-analytics-engine.d.ts | 3 + .../middleware-mock-analytics-engine.ts | 30 ++++++ .../middleware-serve-static-assets.ts | 4 +- 15 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 .changeset/long-rules-wait.md create mode 100644 packages/wrangler/templates/middleware/middleware-mock-analytics-engine.d.ts create mode 100644 packages/wrangler/templates/middleware/middleware-mock-analytics-engine.ts diff --git a/.changeset/long-rules-wait.md b/.changeset/long-rules-wait.md new file mode 100644 index 000000000000..93673d00bc25 --- /dev/null +++ b/.changeset/long-rules-wait.md @@ -0,0 +1,9 @@ +--- +"wrangler": minor +--- + +feat: support analytics engine in local/remote dev + +This adds "support" for analytics engine datasets for `wrangler dev`. Specifically, it simply mocks the AE bindings so that they exist while developing (and don't throw when accessed). + +This does NOT add support in Pages, though we very well could do so in a similar way in a followup. diff --git a/packages/wrangler/e2e/dev.test.ts b/packages/wrangler/e2e/dev.test.ts index 2aa436cd2b85..bcc0c5fc29cd 100644 --- a/packages/wrangler/e2e/dev.test.ts +++ b/packages/wrangler/e2e/dev.test.ts @@ -602,6 +602,104 @@ describe("writes debug logs to hidden file", () => { }); }); +describe("analytics engine", () => { + describe.each([ + { cmd: "wrangler dev" }, + { cmd: "wrangler dev --x-dev-env" }, + { cmd: "wrangler dev --remote" }, + { cmd: "wrangler dev --remote --x-dev-env" }, + ])("mock analytics engine datasets: $cmd", ({ cmd }) => { + describe("module worker", () => { + it("analytics engine datasets are mocked in dev", async () => { + const helper = new WranglerE2ETestHelper(); + await helper.seed({ + "wrangler.toml": dedent` + name = "${workerName}" + main = "src/index.ts" + compatibility_date = "2024-08-08" + + [[analytics_engine_datasets]] + binding = "ANALYTICS_BINDING" + dataset = "ANALYTICS_DATASET" + `, + "src/index.ts": dedent` + export default { + fetch(request, env) { + // let's make an analytics call + env.ANALYTICS_BINDING.writeDataPoint({ + 'blobs': ["Seattle", "USA", "pro_sensor_9000"], // City, State + 'doubles': [25, 0.5], + 'indexes': ["a3cd45"] + }); + // and return a response + return new Response("successfully wrote datapoint from module worker"); + } + }`, + "package.json": dedent` + { + "name": "worker", + "version": "0.0.0", + "private": true + } + `, + }); + const worker = helper.runLongLived(cmd); + + const { url } = await worker.waitForReady(); + + const text = await fetchText(url); + expect(text).toContain( + `successfully wrote datapoint from module worker` + ); + }); + }); + + describe("service worker", async () => { + it("analytics engine datasets are mocked in dev", async () => { + const helper = new WranglerE2ETestHelper(); + await helper.seed({ + "wrangler.toml": dedent` + name = "${workerName}" + main = "src/index.ts" + compatibility_date = "2024-08-08" + + [[analytics_engine_datasets]] + binding = "ANALYTICS_BINDING" + dataset = "ANALYTICS_DATASET" + `, + "src/index.ts": dedent` + addEventListener("fetch", (event) => { + // let's make an analytics call + ANALYTICS_BINDING.writeDataPoint({ + blobs: ["Seattle", "USA", "pro_sensor_9000"], // City, State + doubles: [25, 0.5], + indexes: ["a3cd45"], + }); + // and return a response + event.respondWith(new Response("successfully wrote datapoint from service worker")); + }); + `, + "package.json": dedent` + { + "name": "worker", + "version": "0.0.0", + "private": true + } + `, + }); + const worker = helper.runLongLived(cmd); + + const { url } = await worker.waitForReady(); + + const text = await fetchText(url); + expect(text).toContain( + `successfully wrote datapoint from service worker` + ); + }); + }); + }); +}); + describe("zone selection", () => { it("defaults to a workers.dev preview", async () => { const helper = new WranglerE2ETestHelper(); diff --git a/packages/wrangler/src/__tests__/navigator-user-agent.test.ts b/packages/wrangler/src/__tests__/navigator-user-agent.test.ts index 44b3b71eb766..cda656d441c1 100644 --- a/packages/wrangler/src/__tests__/navigator-user-agent.test.ts +++ b/packages/wrangler/src/__tests__/navigator-user-agent.test.ts @@ -114,6 +114,7 @@ describe("defineNavigatorUserAgent is respected", () => { additionalModules: [], moduleCollector: noopModuleCollector, serveLegacyAssetsFromWorker: false, + mockAnalyticsEngineDatasets: [], doBindings: [], define: {}, alias: {}, @@ -176,6 +177,7 @@ describe("defineNavigatorUserAgent is respected", () => { doBindings: [], define: {}, alias: {}, + mockAnalyticsEngineDatasets: [], checkFetch: false, targetConsumer: "deploy", local: true, diff --git a/packages/wrangler/src/api/startDevWorker/BundlerController.ts b/packages/wrangler/src/api/startDevWorker/BundlerController.ts index 614f38e11359..dfcdb738116a 100644 --- a/packages/wrangler/src/api/startDevWorker/BundlerController.ts +++ b/packages/wrangler/src/api/startDevWorker/BundlerController.ts @@ -115,6 +115,8 @@ export class BundlerController extends Controller { nodejsCompatMode: config.build.nodejsCompatMode, define: config.build.define, checkFetch: true, + mockAnalyticsEngineDatasets: + bindings.analytics_engine_datasets ?? [], alias: config.build.alias, legacyAssets: config.legacy?.legacyAssets, // enable the cache when publishing @@ -227,6 +229,7 @@ export class BundlerController extends Controller { noBundle: !config.build?.bundle, findAdditionalModules: config.build?.findAdditionalModules, durableObjects: bindings?.durable_objects ?? { bindings: [] }, + mockAnalyticsEngineDatasets: bindings.analytics_engine_datasets ?? [], local: !config.dev?.remote, // startDevWorker only applies to "dev" targetConsumer: "dev", diff --git a/packages/wrangler/src/deploy/deploy.ts b/packages/wrangler/src/deploy/deploy.ts index 04c81a5c4547..165c4f965c24 100644 --- a/packages/wrangler/src/deploy/deploy.ts +++ b/packages/wrangler/src/deploy/deploy.ts @@ -551,6 +551,8 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m checkFetch: false, alias: config.alias, legacyAssets: config.legacy_assets, + // We do not mock AE datasets when deploying + mockAnalyticsEngineDatasets: [], // enable the cache when publishing bypassAssetCache: false, // We want to know if the build is for development or publishing diff --git a/packages/wrangler/src/deployment-bundle/bundle.ts b/packages/wrangler/src/deployment-bundle/bundle.ts index dc5826804765..523dceebf826 100644 --- a/packages/wrangler/src/deployment-bundle/bundle.ts +++ b/packages/wrangler/src/deployment-bundle/bundle.ts @@ -86,6 +86,7 @@ export type BundleOptions = { define: Config["define"]; alias: Config["alias"]; checkFetch: boolean; + mockAnalyticsEngineDatasets: Config["analytics_engine_datasets"]; targetConsumer: "dev" | "deploy"; testScheduled?: boolean; inject?: string[]; @@ -122,6 +123,7 @@ export async function bundleWorker( alias, define, checkFetch, + mockAnalyticsEngineDatasets, legacyAssets, bypassAssetCache, targetConsumer, @@ -148,6 +150,21 @@ export async function bundleWorker( // At this point, we take the opportunity to "wrap" the worker with middleware. const middlewareToLoad: MiddlewareLoader[] = []; + if ( + targetConsumer === "dev" && + mockAnalyticsEngineDatasets && + mockAnalyticsEngineDatasets.length > 0 + ) { + middlewareToLoad.push({ + name: "mock-analytics-engine", + path: "templates/middleware/middleware-mock-analytics-engine.ts", + config: { + bindings: mockAnalyticsEngineDatasets.map(({ binding }) => binding), + }, + supports: ["modules", "service-worker"], + }); + } + if ( targetConsumer === "dev" && !process.env.WRANGLER_DISABLE_REQUEST_BODY_DRAINING diff --git a/packages/wrangler/src/dev/dev.tsx b/packages/wrangler/src/dev/dev.tsx index 170681596698..49d82560bdc0 100644 --- a/packages/wrangler/src/dev/dev.tsx +++ b/packages/wrangler/src/dev/dev.tsx @@ -606,6 +606,7 @@ function DevSession(props: DevSessionProps) { alias: props.alias, noBundle: props.noBundle, findAdditionalModules: props.findAdditionalModules, + mockAnalyticsEngineDatasets: props.bindings.analytics_engine_datasets ?? [], legacyAssets: props.legacyAssetsConfig, durableObjects: props.bindings.durable_objects || { bindings: [] }, local: props.local, diff --git a/packages/wrangler/src/dev/start-server.ts b/packages/wrangler/src/dev/start-server.ts index 92ca4e693b58..815b0da1f326 100644 --- a/packages/wrangler/src/dev/start-server.ts +++ b/packages/wrangler/src/dev/start-server.ts @@ -243,6 +243,7 @@ export async function startDevServer( testScheduled: props.testScheduled, local: props.local, doBindings: props.bindings.durable_objects?.bindings ?? [], + mockAnalyticsEngineDatasets: props.bindings.analytics_engine_datasets ?? [], projectRoot: props.projectRoot, defineNavigatorUserAgent: isNavigatorDefined( props.compatibilityDate, @@ -416,6 +417,7 @@ async function runEsbuild({ testScheduled, local, doBindings, + mockAnalyticsEngineDatasets, projectRoot, defineNavigatorUserAgent, }: { @@ -438,6 +440,7 @@ async function runEsbuild({ testScheduled?: boolean; local: boolean; doBindings: DurableObjectBindings; + mockAnalyticsEngineDatasets: Config["analytics_engine_datasets"]; projectRoot: string | undefined; defineNavigatorUserAgent: boolean; }): Promise { @@ -475,6 +478,7 @@ async function runEsbuild({ nodejsCompatMode, define, checkFetch: true, + mockAnalyticsEngineDatasets, alias, legacyAssets, // disable the cache in dev diff --git a/packages/wrangler/src/dev/use-esbuild.ts b/packages/wrangler/src/dev/use-esbuild.ts index 5c9ec363683e..2ba0016c9358 100644 --- a/packages/wrangler/src/dev/use-esbuild.ts +++ b/packages/wrangler/src/dev/use-esbuild.ts @@ -51,6 +51,7 @@ export type EsbuildBundleProps = { noBundle: boolean; findAdditionalModules: boolean | undefined; durableObjects: Config["durable_objects"]; + mockAnalyticsEngineDatasets: Config["analytics_engine_datasets"]; local: boolean; targetConsumer: "dev" | "deploy"; testScheduled: boolean; @@ -78,6 +79,7 @@ export function runBuild( alias, noBundle, findAdditionalModules, + mockAnalyticsEngineDatasets, durableObjects, local, targetConsumer, @@ -103,6 +105,7 @@ export function runBuild( noBundle: boolean; findAdditionalModules: boolean | undefined; durableObjects: Config["durable_objects"]; + mockAnalyticsEngineDatasets: Config["analytics_engine_datasets"]; local: boolean; targetConsumer: "dev" | "deploy"; testScheduled: boolean; @@ -183,6 +186,7 @@ export function runBuild( alias, define, checkFetch: true, + mockAnalyticsEngineDatasets, legacyAssets, // disable the cache in dev bypassAssetCache: true, @@ -264,6 +268,7 @@ export function useEsbuild({ define, noBundle, findAdditionalModules, + mockAnalyticsEngineDatasets, durableObjects, local, targetConsumer, @@ -295,6 +300,7 @@ export function useEsbuild({ noBundle, findAdditionalModules, durableObjects, + mockAnalyticsEngineDatasets, local, targetConsumer, testScheduled, @@ -327,6 +333,7 @@ export function useEsbuild({ alias, define, legacyAssets, + mockAnalyticsEngineDatasets, durableObjects, local, targetConsumer, diff --git a/packages/wrangler/src/pages/functions/buildPlugin.ts b/packages/wrangler/src/pages/functions/buildPlugin.ts index 45dc029d75ef..4f0d8346e075 100644 --- a/packages/wrangler/src/pages/functions/buildPlugin.ts +++ b/packages/wrangler/src/pages/functions/buildPlugin.ts @@ -106,6 +106,8 @@ export function buildPluginFromFunctions({ ], serveLegacyAssetsFromWorker: false, checkFetch: local, + // TODO: mock AE datasets in Pages functions for dev + mockAnalyticsEngineDatasets: [], targetConsumer: local ? "dev" : "deploy", forPages: true, local, diff --git a/packages/wrangler/src/pages/functions/buildWorker.ts b/packages/wrangler/src/pages/functions/buildWorker.ts index 014e548e065b..9792b4314b39 100644 --- a/packages/wrangler/src/pages/functions/buildWorker.ts +++ b/packages/wrangler/src/pages/functions/buildWorker.ts @@ -72,6 +72,8 @@ export function buildWorkerFromFunctions({ sourcemap, watch, nodejsCompatMode, + // TODO: mock AE datasets in Pages functions for dev + mockAnalyticsEngineDatasets: [], define: { __FALLBACK_SERVICE__: JSON.stringify(fallbackService), }, @@ -153,6 +155,8 @@ export function buildRawWorker({ sourcemap, watch, nodejsCompatMode, + // TODO: mock AE datasets in Pages functions for dev + mockAnalyticsEngineDatasets: [], define: {}, alias: {}, doBindings: [], // Pages functions don't support internal Durable Objects diff --git a/packages/wrangler/src/versions/upload.ts b/packages/wrangler/src/versions/upload.ts index c638adf522ed..9493713a4b9b 100644 --- a/packages/wrangler/src/versions/upload.ts +++ b/packages/wrangler/src/versions/upload.ts @@ -311,6 +311,7 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m alias: { ...config.alias, ...props.alias }, checkFetch: false, legacyAssets: config.legacy_assets, + mockAnalyticsEngineDatasets: [], // enable the cache when publishing bypassAssetCache: false, // We want to know if the build is for development or publishing diff --git a/packages/wrangler/templates/middleware/middleware-mock-analytics-engine.d.ts b/packages/wrangler/templates/middleware/middleware-mock-analytics-engine.d.ts new file mode 100644 index 000000000000..16a46469b16c --- /dev/null +++ b/packages/wrangler/templates/middleware/middleware-mock-analytics-engine.d.ts @@ -0,0 +1,3 @@ +declare module "config:middleware/mock-analytics-engine" { + export const bindings: string[]; +} diff --git a/packages/wrangler/templates/middleware/middleware-mock-analytics-engine.ts b/packages/wrangler/templates/middleware/middleware-mock-analytics-engine.ts new file mode 100644 index 000000000000..77f0d315f08a --- /dev/null +++ b/packages/wrangler/templates/middleware/middleware-mock-analytics-engine.ts @@ -0,0 +1,30 @@ +/// + +import { bindings } from "config:middleware/mock-analytics-engine"; +import type { Middleware } from "./common"; + +const bindingsEnv = Object.fromEntries( + bindings.map((binding) => [ + binding, + { + writeDataPoint() { + // no op in dev + }, + }, + ]) +) satisfies Record; + +const analyticsEngine: Middleware = async ( + request, + env, + _ctx, + middlewareCtx +) => { + // we're going to directly modify env so it maintains referential equality + for (const binding of bindings) { + env[binding] ??= bindingsEnv[binding]; + } + return await middlewareCtx.next(request, env); +}; + +export default analyticsEngine; diff --git a/packages/wrangler/templates/middleware/middleware-serve-static-assets.ts b/packages/wrangler/templates/middleware/middleware-serve-static-assets.ts index d97698a2b7bd..76964b14f7a5 100644 --- a/packages/wrangler/templates/middleware/middleware-serve-static-assets.ts +++ b/packages/wrangler/templates/middleware/middleware-serve-static-assets.ts @@ -14,7 +14,7 @@ import type * as kvAssetHandler from "@cloudflare/kv-asset-handler"; const ASSET_MANIFEST = JSON.parse(manifest); -const staticAssets: Middleware = async (request, env, _ctx, middlewareCtx) => { +const staticAssets: Middleware = async (request, env, ctx, middlewareCtx) => { let options: Partial = { ASSET_MANIFEST, ASSET_NAMESPACE: env.__STATIC_CONTENT, @@ -27,7 +27,7 @@ const staticAssets: Middleware = async (request, env, _ctx, middlewareCtx) => { { request, waitUntil(promise) { - return _ctx.waitUntil(promise); + return ctx.waitUntil(promise); }, }, options