diff --git a/.changeset/pink-camels-hug.md b/.changeset/pink-camels-hug.md new file mode 100644 index 000000000000..29ac107b1602 --- /dev/null +++ b/.changeset/pink-camels-hug.md @@ -0,0 +1,9 @@ +--- +"miniflare": minor +--- + +feature: respect incoming `Accept-Encoding` header and ensure `Accept-Encoding`/`request.cf.clientAcceptEncoding` set correctly + +Previously, Miniflare would pass through the incoming `Accept-Encoding` header to your Worker code. This change ensures this header is always set to `Accept-Encoding: br, gzip` for incoming requests to your Worker. The original value of `Accept-Encoding` will be stored in `request.cf.clientAcceptEncoding`. This matches [deployed behaviour](https://developers.cloudflare.com/fundamentals/reference/http-request-headers/#accept-encoding). + +Fixes #5246 diff --git a/fixtures/worker-app/src/index.js b/fixtures/worker-app/src/index.js index e02a2b6d2b24..8117d0848001 100644 --- a/fixtures/worker-app/src/index.js +++ b/fixtures/worker-app/src/index.js @@ -40,6 +40,18 @@ export default { ], }); + if (pathname === "/content-encoding") { + return Response.json({ + AcceptEncoding: request.headers.get("Accept-Encoding"), + clientAcceptEncoding: request.cf.clientAcceptEncoding, + }); + } + if (pathname === "/content-encoding/gzip") { + return new Response("x".repeat(100), { + headers: { "Content-Encoding": "gzip" }, + }); + } + if (request.headers.get("X-Test-URL") !== null) { return new Response(request.url); } diff --git a/fixtures/worker-app/tests/index.test.ts b/fixtures/worker-app/tests/index.test.ts index 1356f90e89a8..d88588abfe5d 100644 --- a/fixtures/worker-app/tests/index.test.ts +++ b/fixtures/worker-app/tests/index.test.ts @@ -157,4 +157,22 @@ describe("'wrangler dev' correctly renders pages", () => { tag: expect.any(String), }); }); + + it("passes through client content encoding", async ({ expect }) => { + // https://github.com/cloudflare/workers-sdk/issues/5246 + const response = await fetch(`http://${ip}:${port}/content-encoding`, { + headers: { "Accept-Encoding": "hello" }, + }); + expect(await response.json()).toStrictEqual({ + AcceptEncoding: "br, gzip", + clientAcceptEncoding: "hello", + }); + }); + + it("supports encoded responses", async ({ expect }) => { + const response = await fetch(`http://${ip}:${port}/content-encoding/gzip`, { + headers: { "Accept-Encoding": "gzip" }, + }); + expect(await response.text()).toEqual("x".repeat(100)); + }); }); diff --git a/packages/miniflare/src/plugins/core/index.ts b/packages/miniflare/src/plugins/core/index.ts index bea399ac892a..26871e530e4c 100644 --- a/packages/miniflare/src/plugins/core/index.ts +++ b/packages/miniflare/src/plugins/core/index.ts @@ -738,7 +738,11 @@ export function getGlobalServices({ worker: { modules: [{ name: "entry.worker.js", esModule: SCRIPT_ENTRY() }], compatibilityDate: "2023-04-04", - compatibilityFlags: ["nodejs_compat", "service_binding_extra_handlers"], + compatibilityFlags: [ + "nodejs_compat", + "service_binding_extra_handlers", + "brotli_content_encoding", + ], bindings: serviceEntryBindings, durableObjectNamespaces: [ { diff --git a/packages/miniflare/src/workers/core/entry.worker.ts b/packages/miniflare/src/workers/core/entry.worker.ts index f10c94f348ba..947bd5c26545 100644 --- a/packages/miniflare/src/workers/core/entry.worker.ts +++ b/packages/miniflare/src/workers/core/entry.worker.ts @@ -89,9 +89,19 @@ function getUserRequest( // See https://github.com/cloudflare/workerd/issues/1122 for more details. request = new Request(url, request); if (request.cf === undefined) { - request = new Request(request, { cf: env[CoreBindings.JSON_CF_BLOB] }); + const cf: IncomingRequestCfProperties = { + ...env[CoreBindings.JSON_CF_BLOB], + // Defaulting to empty string to preserve undefined `Accept-Encoding` + // through Wrangler's proxy worker. + clientAcceptEncoding: request.headers.get("Accept-Encoding") ?? "", + }; + request = new Request(request, { cf }); } + // `Accept-Encoding` is always set to "br, gzip" in Workers: + // https://developers.cloudflare.com/fundamentals/reference/http-request-headers/#accept-encoding + request.headers.set("Accept-Encoding", "br, gzip"); + if (rewriteHeadersFromOriginalUrl) { request.headers.set("Host", url.host); } @@ -175,6 +185,92 @@ function maybeInjectLiveReload( }); } +const acceptEncodingElement = + /^(?[a-z]+|\*)(?:\s*;\s*q=(?\d+(?:.\d+)?))?$/; +interface AcceptedEncoding { + coding: string; + weight: number; +} +function maybeParseAcceptEncodingElement( + element: string +): AcceptedEncoding | undefined { + const match = acceptEncodingElement.exec(element); + if (match?.groups == null) return; + return { + coding: match.groups.coding, + weight: + match.groups.weight === undefined ? 1 : parseFloat(match.groups.weight), + }; +} +function parseAcceptEncoding(header: string): AcceptedEncoding[] { + const encodings: AcceptedEncoding[] = []; + for (const element of header.split(",")) { + const maybeEncoding = maybeParseAcceptEncodingElement(element.trim()); + if (maybeEncoding !== undefined) encodings.push(maybeEncoding); + } + // `Array#sort()` is stable, so original ordering preserved for same weights + return encodings.sort((a, b) => b.weight - a.weight); +} +function ensureAcceptableEncoding( + clientAcceptEncoding: string | null, + response: Response +): Response { + // https://www.rfc-editor.org/rfc/rfc9110#section-12.5.3 + + // If the client hasn't specified any acceptable encodings, assume anything is + if (clientAcceptEncoding === null) return response; + const encodings = parseAcceptEncoding(clientAcceptEncoding); + if (encodings.length === 0) return response; + + const contentEncoding = response.headers.get("Content-Encoding"); + + // If `Content-Encoding` is defined, but unknown, return the response as is + if ( + contentEncoding !== null && + contentEncoding !== "gzip" && + contentEncoding !== "br" + ) { + return response; + } + + let desiredEncoding: "gzip" | "br" | undefined; + let identityDisallowed = false; + + for (const encoding of encodings) { + if (encoding.weight === 0) { + // If we have an `identity;q=0` or `*;q=0` entry, disallow no encoding + if (encoding.coding === "identity" || encoding.coding === "*") { + identityDisallowed = true; + } + } else if (encoding.coding === "gzip" || encoding.coding === "br") { + // If the client accepts one of our supported encodings, use that + desiredEncoding = encoding.coding; + break; + } else if (encoding.coding === "identity") { + // If the client accepts no encoding, use that + break; + } + } + + if (desiredEncoding === undefined) { + if (identityDisallowed) { + return new Response("Unsupported Media Type", { + status: 415 /* Unsupported Media Type */, + headers: { "Accept-Encoding": "br, gzip" }, + }); + } + if (contentEncoding === null) return response; + response = new Response(response.body, response); // Ensure mutable headers + response.headers.delete("Content-Encoding"); // Use identity + return response; + } else { + if (contentEncoding === desiredEncoding) return response; + response = new Response(response.body, response); // Ensure mutable headers + response.headers.set("Content-Encoding", desiredEncoding); // Use desired + return response; + } +} + function colourFromHTTPStatus(status: number): Colorize { if (200 <= status && status < 300) return green; if (400 <= status && status < 500) return yellow; @@ -250,6 +346,8 @@ export default >{ const disablePrettyErrorPage = request.headers.get(CoreHeaders.DISABLE_PRETTY_ERROR) !== null; + const clientAcceptEncoding = request.headers.get("Accept-Encoding"); + try { request = getUserRequest(request, env); } catch (e) { @@ -274,6 +372,7 @@ export default >{ response = await maybePrettifyError(request, response, env); } response = maybeInjectLiveReload(response, env, ctx); + response = ensureAcceptableEncoding(clientAcceptEncoding, response); maybeLogRequest(request, response, env, ctx, startTime); return response; } catch (e: any) { diff --git a/packages/miniflare/test/index.spec.ts b/packages/miniflare/test/index.spec.ts index 67f466eeaca9..6bf3b3202c98 100644 --- a/packages/miniflare/test/index.spec.ts +++ b/packages/miniflare/test/index.spec.ts @@ -289,6 +289,7 @@ test("Miniflare: custom service using Content-Encoding header", async (t) => { initialStream.end(); }); const mf = new Miniflare({ + compatibilityFlags: ["brotli_content_encoding"], script: `addEventListener("fetch", (event) => { event.respondWith(CUSTOM.fetch(event.request)); })`, @@ -304,7 +305,6 @@ test("Miniflare: custom service using Content-Encoding header", async (t) => { const res = await mf.dispatchFetch("http://localhost", { headers: { "X-Test-Encoding": encoding }, }); - t.is(res.headers.get("Content-Encoding"), encoding); t.is(await res.text(), testBody, encoding); }; @@ -318,6 +318,135 @@ test("Miniflare: custom service using Content-Encoding header", async (t) => { // await test("deflate, gzip"); }); +test("Miniflare: negotiates acceptable encoding", async (t) => { + const testBody = "x".repeat(100); + const mf = new Miniflare({ + bindings: { TEST_BODY: testBody }, + compatibilityFlags: ["brotli_content_encoding"], + modules: true, + script: ` + export default { + async fetch(request, env, ctx) { + const { pathname } = new URL(request.url); + if (pathname === "/") { + return Response.json({ + AcceptEncoding: request.headers.get("Accept-Encoding"), + clientAcceptEncoding: request.cf.clientAcceptEncoding, + }); + } else if (pathname === "/gzip") { + return new Response(env.TEST_BODY, { headers: { "Content-Encoding": "gzip" } }); + } else if (pathname === "/br") { + return new Response(env.TEST_BODY, { headers: { "Content-Encoding": "br" } }); + } else if (pathname === "/deflate") { + // workerd doesn't automatically encode "deflate" + const response = new Response(env.TEST_BODY); + const compressionStream = new CompressionStream("deflate"); + const compressedBody = response.body.pipeThrough(compressionStream); + return new Response(compressedBody, { + headers: { "Content-Encoding": "deflate" }, + encodeBody: "manual", + }); + } else { + return new Response(null, { status: 404 }); + } + }, + }; + `, + }); + t.teardown(() => mf.dispose()); + + // Using `fetch()` directly to simulate eyeball + const url = await mf.ready; + const gzipUrl = new URL("/gzip", url); + const brUrl = new URL("/br", url); + const deflateUrl = new URL("/deflate", url); + + // https://github.com/cloudflare/workers-sdk/issues/5246 + let res = await fetch(url, { + headers: { "Accept-Encoding": "hello" }, + }); + t.deepEqual(await res.json(), { + AcceptEncoding: "br, gzip", + clientAcceptEncoding: "hello", + }); + + // Check all encodings supported + res = await fetch(gzipUrl); + t.is(await res.text(), testBody); + res = await fetch(brUrl); + t.is(await res.text(), testBody); + res = await fetch(deflateUrl); + t.is(await res.text(), testBody); + + // Check with `Accept-Encoding: gzip` + res = await fetch(gzipUrl, { headers: { "Accept-Encoding": "gzip" } }); + t.is(res.headers.get("Content-Encoding"), "gzip"); + t.is(await res.text(), testBody); + res = await fetch(brUrl, { headers: { "Accept-Encoding": "gzip" } }); + t.is(res.headers.get("Content-Encoding"), "gzip"); + t.is(await res.text(), testBody); + // "deflate" isn't an accepted encoding inside Workers, so returned as is + res = await fetch(deflateUrl, { headers: { "Accept-Encoding": "gzip" } }); + t.is(res.headers.get("Content-Encoding"), "deflate"); + t.is(await res.text(), testBody); + + // Check with `Accept-Encoding: br` + res = await fetch(gzipUrl, { headers: { "Accept-Encoding": "br" } }); + t.is(res.headers.get("Content-Encoding"), "br"); + t.is(await res.text(), testBody); + res = await fetch(brUrl, { headers: { "Accept-Encoding": "br" } }); + t.is(res.headers.get("Content-Encoding"), "br"); + t.is(await res.text(), testBody); + // "deflate" isn't an accepted encoding inside Workers, so returned as is + res = await fetch(deflateUrl, { headers: { "Accept-Encoding": "br" } }); + t.is(res.headers.get("Content-Encoding"), "deflate"); + t.is(await res.text(), testBody); + + // Check with mixed `Accept-Encoding` + res = await fetch(gzipUrl, { headers: { "Accept-Encoding": "gzip, br" } }); + t.is(res.headers.get("Content-Encoding"), "gzip"); + t.is(await res.text(), testBody); + res = await fetch(gzipUrl, { headers: { "Accept-Encoding": "br, gzip" } }); + t.is(res.headers.get("Content-Encoding"), "br"); + t.is(await res.text(), testBody); + res = await fetch(gzipUrl, { + headers: { "Accept-Encoding": "br;q=0.5, gzip" }, + }); + t.is(res.headers.get("Content-Encoding"), "gzip"); + t.is(await res.text(), testBody); + + // Check empty `Accept-Encoding` + res = await fetch(gzipUrl, { headers: { "Accept-Encoding": "" } }); + t.is(res.headers.get("Content-Encoding"), "gzip"); + t.is(await res.text(), testBody); + + // Check identity encoding + res = await fetch(gzipUrl, { headers: { "Accept-Encoding": "identity" } }); + t.is(res.headers.get("Content-Encoding"), null); + t.is(await res.text(), testBody); + res = await fetch(gzipUrl, { headers: { "Accept-Encoding": "*" } }); + t.is(res.headers.get("Content-Encoding"), null); + t.is(await res.text(), testBody); + res = await fetch(gzipUrl, { headers: { "Accept-Encoding": "zstd, *" } }); + t.is(res.headers.get("Content-Encoding"), null); + t.is(await res.text(), testBody); + res = await fetch(gzipUrl, { + headers: { "Accept-Encoding": "zstd, identity;q=0" }, + }); + t.is(res.status, 415); + t.is(res.headers.get("Accept-Encoding"), "br, gzip"); + t.is(await res.text(), "Unsupported Media Type"); + res = await fetch(gzipUrl, { headers: { "Accept-Encoding": "zstd, *;q=0" } }); + t.is(res.status, 415); + t.is(res.headers.get("Accept-Encoding"), "br, gzip"); + t.is(await res.text(), "Unsupported Media Type"); + + // Check malformed `Accept-Encoding` + res = await fetch(gzipUrl, { headers: { "Accept-Encoding": ",(,br,,,q=," } }); + t.is(res.headers.get("Content-Encoding"), "br"); + t.is(await res.text(), testBody); +}); + test("Miniflare: custom service using Set-Cookie header", async (t) => { const testCookies = [ "key1=value1; Max-Age=3600", @@ -690,14 +819,14 @@ test("Miniflare: can send GET request with body", async (t) => { let res = await get(); t.deepEqual(await json(res), { - cf: { key: "value" }, + cf: { key: "value", clientAcceptEncoding: "" }, contentLength: null, hasBody: false, }); res = await get({ headers: { "content-length": "0" } }); t.deepEqual(await json(res), { - cf: { key: "value" }, + cf: { key: "value", clientAcceptEncoding: "" }, contentLength: "0", hasBody: true, }); diff --git a/packages/wrangler/templates/startDevWorker/ProxyWorker.ts b/packages/wrangler/templates/startDevWorker/ProxyWorker.ts index aa59846e3c46..d171e1910b39 100644 --- a/packages/wrangler/templates/startDevWorker/ProxyWorker.ts +++ b/packages/wrangler/templates/startDevWorker/ProxyWorker.ts @@ -126,6 +126,11 @@ export class ProxyWorker implements DurableObject { headers.set("MF-Original-URL", innerUrl.href); headers.set("MF-Disable-Pretty-Error", "true"); // disables the UserWorker miniflare instance from rendering the pretty error -- instead the ProxyWorker miniflare instance will intercept the json error response and render the pretty error page + // Preserve client `Accept-Encoding`, rather than using Worker's default + // of `Accept-Encoding: br, gzip` + const encoding = request.cf?.clientAcceptEncoding; + if (encoding !== undefined) headers.set("Accept-Encoding", encoding); + rewriteUrlRelatedHeaders(headers, outerUrl, innerUrl); // merge proxyData headers with the request headers