feature: respect incoming Accept-Encoding header (#5409)

Fixes #5246

.changeset/pink-camels-hug.md

@@ -0,0 +1,9 @@
+---
+"miniflare": minor
+---
+
+feature: respect incoming `Accept-Encoding` header and ensure `Accept-Encoding`/`request.cf.clientAcceptEncoding` set correctly
+
+Previously, Miniflare would pass through the incoming `Accept-Encoding` header to your Worker code. This change ensures this header is always set to `Accept-Encoding: br, gzip` for incoming requests to your Worker. The original value of `Accept-Encoding` will be stored in `request.cf.clientAcceptEncoding`. This matches [deployed behaviour](https://developers.cloudflare.com/fundamentals/reference/http-request-headers/#accept-encoding).
+
+Fixes #5246

fixtures/worker-app/src/index.js

@@ -40,6 +40,18 @@ export default {
 				],
 			});
 
+		if (pathname === "/content-encoding") {
+			return Response.json({
+				AcceptEncoding: request.headers.get("Accept-Encoding"),
+				clientAcceptEncoding: request.cf.clientAcceptEncoding,
+			});
+		}
+		if (pathname === "/content-encoding/gzip") {
+			return new Response("x".repeat(100), {
+				headers: { "Content-Encoding": "gzip" },
+			});
+		}
+
 		if (request.headers.get("X-Test-URL") !== null) {
 			return new Response(request.url);
 		}

fixtures/worker-app/tests/index.test.ts

@@ -157,4 +157,22 @@ describe("'wrangler dev' correctly renders pages", () => {
 			tag: expect.any(String),
 		});
 	});
+
+	it("passes through client content encoding", async ({ expect }) => {
+		// https://github.com/cloudflare/workers-sdk/issues/5246
+		const response = await fetch(`http://${ip}:${port}/content-encoding`, {
+			headers: { "Accept-Encoding": "hello" },
+		});
+		expect(await response.json()).toStrictEqual({
+			AcceptEncoding: "br, gzip",
+			clientAcceptEncoding: "hello",
+		});
+	});
+
+	it("supports encoded responses", async ({ expect }) => {
+		const response = await fetch(`http://${ip}:${port}/content-encoding/gzip`, {
+			headers: { "Accept-Encoding": "gzip" },
+		});
+		expect(await response.text()).toEqual("x".repeat(100));
+	});
 });

packages/miniflare/src/plugins/core/index.ts

@@ -738,7 +738,11 @@ export function getGlobalServices({
 			worker: {
 				modules: [{ name: "entry.worker.js", esModule: SCRIPT_ENTRY() }],
 				compatibilityDate: "2023-04-04",
-				compatibilityFlags: ["nodejs_compat", "service_binding_extra_handlers"],
+				compatibilityFlags: [
+					"nodejs_compat",
+					"service_binding_extra_handlers",
+					"brotli_content_encoding",
+				],
 				bindings: serviceEntryBindings,
 				durableObjectNamespaces: [
 					{

packages/miniflare/src/workers/core/entry.worker.ts

@@ -89,9 +89,19 @@ function getUserRequest(
 	// See https://github.com/cloudflare/workerd/issues/1122 for more details.
 	request = new Request(url, request);
 	if (request.cf === undefined) {
-		request = new Request(request, { cf: env[CoreBindings.JSON_CF_BLOB] });
+		const cf: IncomingRequestCfProperties = {
+			...env[CoreBindings.JSON_CF_BLOB],
+			// Defaulting to empty string to preserve undefined `Accept-Encoding`
+			// through Wrangler's proxy worker.
+			clientAcceptEncoding: request.headers.get("Accept-Encoding") ?? "",
+		};
+		request = new Request(request, { cf });
 	}
 
+	// `Accept-Encoding` is always set to "br, gzip" in Workers:
+	// https://developers.cloudflare.com/fundamentals/reference/http-request-headers/#accept-encoding
+	request.headers.set("Accept-Encoding", "br, gzip");
+
 	if (rewriteHeadersFromOriginalUrl) {
 		request.headers.set("Host", url.host);
 	}
@@ -175,6 +185,92 @@ function maybeInjectLiveReload(
 	});
 }
 
+const acceptEncodingElement =
+	/^(?<coding>[a-z]+|\*)(?:\s*;\s*q=(?<weight>\d+(?:.\d+)?))?$/;
+interface AcceptedEncoding {
+	coding: string;
+	weight: number;
+}
+function maybeParseAcceptEncodingElement(
+	element: string
+): AcceptedEncoding | undefined {
+	const match = acceptEncodingElement.exec(element);
+	if (match?.groups == null) return;
+	return {
+		coding: match.groups.coding,
+		weight:
+			match.groups.weight === undefined ? 1 : parseFloat(match.groups.weight),
+	};
+}
+function parseAcceptEncoding(header: string): AcceptedEncoding[] {
+	const encodings: AcceptedEncoding[] = [];
+	for (const element of header.split(",")) {
+		const maybeEncoding = maybeParseAcceptEncodingElement(element.trim());
+		if (maybeEncoding !== undefined) encodings.push(maybeEncoding);
+	}
+	// `Array#sort()` is stable, so original ordering preserved for same weights
+	return encodings.sort((a, b) => b.weight - a.weight);
+}
+function ensureAcceptableEncoding(
+	clientAcceptEncoding: string | null,
+	response: Response
+): Response {
+	// https://www.rfc-editor.org/rfc/rfc9110#section-12.5.3
+
+	// If the client hasn't specified any acceptable encodings, assume anything is
+	if (clientAcceptEncoding === null) return response;
+	const encodings = parseAcceptEncoding(clientAcceptEncoding);
+	if (encodings.length === 0) return response;
+
+	const contentEncoding = response.headers.get("Content-Encoding");
+
+	// If `Content-Encoding` is defined, but unknown, return the response as is
+	if (
+		contentEncoding !== null &&
+		contentEncoding !== "gzip" &&
+		contentEncoding !== "br"
+	) {
+		return response;
+	}
+
+	let desiredEncoding: "gzip" | "br" | undefined;
+	let identityDisallowed = false;
+
+	for (const encoding of encodings) {
+		if (encoding.weight === 0) {
+			// If we have an `identity;q=0` or `*;q=0` entry, disallow no encoding
+			if (encoding.coding === "identity" || encoding.coding === "*") {
+				identityDisallowed = true;
+			}
+		} else if (encoding.coding === "gzip" || encoding.coding === "br") {
+			// If the client accepts one of our supported encodings, use that
+			desiredEncoding = encoding.coding;
+			break;
+		} else if (encoding.coding === "identity") {
+			// If the client accepts no encoding, use that
+			break;
+		}
+	}
+
+	if (desiredEncoding === undefined) {
+		if (identityDisallowed) {
+			return new Response("Unsupported Media Type", {
+				status: 415 /* Unsupported Media Type */,
+				headers: { "Accept-Encoding": "br, gzip" },
+			});
+		}
+		if (contentEncoding === null) return response;
+		response = new Response(response.body, response); // Ensure mutable headers
+		response.headers.delete("Content-Encoding"); // Use identity
+		return response;
+	} else {
+		if (contentEncoding === desiredEncoding) return response;
+		response = new Response(response.body, response); // Ensure mutable headers
+		response.headers.set("Content-Encoding", desiredEncoding); // Use desired
+		return response;
+	}
+}
+
 function colourFromHTTPStatus(status: number): Colorize {
 	if (200 <= status && status < 300) return green;
 	if (400 <= status && status < 500) return yellow;
@@ -250,6 +346,8 @@ export default <ExportedHandler<Env>>{
 		const disablePrettyErrorPage =
 			request.headers.get(CoreHeaders.DISABLE_PRETTY_ERROR) !== null;
 
+		const clientAcceptEncoding = request.headers.get("Accept-Encoding");
+
 		try {
 			request = getUserRequest(request, env);
 		} catch (e) {
@@ -274,6 +372,7 @@ export default <ExportedHandler<Env>>{
 				response = await maybePrettifyError(request, response, env);
 			}
 			response = maybeInjectLiveReload(response, env, ctx);
+			response = ensureAcceptableEncoding(clientAcceptEncoding, response);
 			maybeLogRequest(request, response, env, ctx, startTime);
 			return response;
 		} catch (e: any) {

packages/miniflare/test/index.spec.ts

@@ -289,6 +289,7 @@ test("Miniflare: custom service using Content-Encoding header", async (t) => {
 		initialStream.end();
 	});
 	const mf = new Miniflare({
+		compatibilityFlags: ["brotli_content_encoding"],
 		script: `addEventListener("fetch", (event) => {
       event.respondWith(CUSTOM.fetch(event.request));
     })`,
@@ -304,7 +305,6 @@ test("Miniflare: custom service using Content-Encoding header", async (t) => {
 		const res = await mf.dispatchFetch("http://localhost", {
 			headers: { "X-Test-Encoding": encoding },
 		});
-		t.is(res.headers.get("Content-Encoding"), encoding);
 		t.is(await res.text(), testBody, encoding);
 	};
 
@@ -318,6 +318,135 @@ test("Miniflare: custom service using Content-Encoding header", async (t) => {
 	// await test("deflate, gzip");
 });
 
+test("Miniflare: negotiates acceptable encoding", async (t) => {
+	const testBody = "x".repeat(100);
+	const mf = new Miniflare({
+		bindings: { TEST_BODY: testBody },
+		compatibilityFlags: ["brotli_content_encoding"],
+		modules: true,
+		script: `
+		export default {
+			async fetch(request, env, ctx) {
+				const { pathname } = new URL(request.url);
+				if (pathname === "/") {
+					return Response.json({
+						AcceptEncoding: request.headers.get("Accept-Encoding"),
+						clientAcceptEncoding: request.cf.clientAcceptEncoding,
+					});
+				} else if (pathname === "/gzip") {
+					return new Response(env.TEST_BODY, { headers: { "Content-Encoding": "gzip" } });
+				} else if (pathname === "/br") {
+					return new Response(env.TEST_BODY, { headers: { "Content-Encoding": "br" } });
+				} else if (pathname === "/deflate") {
+					// workerd doesn't automatically encode "deflate"
+					const response = new Response(env.TEST_BODY);
+					const compressionStream = new CompressionStream("deflate");
+					const compressedBody = response.body.pipeThrough(compressionStream);
+					return new Response(compressedBody, {
+						headers: { "Content-Encoding": "deflate" },
+						encodeBody: "manual",
+					});
+				} else {
+					return new Response(null, { status: 404 });
+				}
+			},
+		};
+		`,
+	});
+	t.teardown(() => mf.dispose());
+
+	// Using `fetch()` directly to simulate eyeball
+	const url = await mf.ready;
+	const gzipUrl = new URL("/gzip", url);
+	const brUrl = new URL("/br", url);
+	const deflateUrl = new URL("/deflate", url);
+
+	// https://github.com/cloudflare/workers-sdk/issues/5246
+	let res = await fetch(url, {
+		headers: { "Accept-Encoding": "hello" },
+	});
+	t.deepEqual(await res.json(), {
+		AcceptEncoding: "br, gzip",
+		clientAcceptEncoding: "hello",
+	});
+
+	// Check all encodings supported
+	res = await fetch(gzipUrl);
+	t.is(await res.text(), testBody);
+	res = await fetch(brUrl);
+	t.is(await res.text(), testBody);
+	res = await fetch(deflateUrl);
+	t.is(await res.text(), testBody);
+
+	// Check with `Accept-Encoding: gzip`
+	res = await fetch(gzipUrl, { headers: { "Accept-Encoding": "gzip" } });
+	t.is(res.headers.get("Content-Encoding"), "gzip");
+	t.is(await res.text(), testBody);
+	res = await fetch(brUrl, { headers: { "Accept-Encoding": "gzip" } });
+	t.is(res.headers.get("Content-Encoding"), "gzip");
+	t.is(await res.text(), testBody);
+	// "deflate" isn't an accepted encoding inside Workers, so returned as is
+	res = await fetch(deflateUrl, { headers: { "Accept-Encoding": "gzip" } });
+	t.is(res.headers.get("Content-Encoding"), "deflate");
+	t.is(await res.text(), testBody);
+
+	// Check with `Accept-Encoding: br`
+	res = await fetch(gzipUrl, { headers: { "Accept-Encoding": "br" } });
+	t.is(res.headers.get("Content-Encoding"), "br");
+	t.is(await res.text(), testBody);
+	res = await fetch(brUrl, { headers: { "Accept-Encoding": "br" } });
+	t.is(res.headers.get("Content-Encoding"), "br");
+	t.is(await res.text(), testBody);
+	// "deflate" isn't an accepted encoding inside Workers, so returned as is
+	res = await fetch(deflateUrl, { headers: { "Accept-Encoding": "br" } });
+	t.is(res.headers.get("Content-Encoding"), "deflate");
+	t.is(await res.text(), testBody);
+
+	// Check with mixed `Accept-Encoding`
+	res = await fetch(gzipUrl, { headers: { "Accept-Encoding": "gzip, br" } });
+	t.is(res.headers.get("Content-Encoding"), "gzip");
+	t.is(await res.text(), testBody);
+	res = await fetch(gzipUrl, { headers: { "Accept-Encoding": "br, gzip" } });
+	t.is(res.headers.get("Content-Encoding"), "br");
+	t.is(await res.text(), testBody);
+	res = await fetch(gzipUrl, {
+		headers: { "Accept-Encoding": "br;q=0.5, gzip" },
+	});
+	t.is(res.headers.get("Content-Encoding"), "gzip");
+	t.is(await res.text(), testBody);
+
+	// Check empty `Accept-Encoding`
+	res = await fetch(gzipUrl, { headers: { "Accept-Encoding": "" } });
+	t.is(res.headers.get("Content-Encoding"), "gzip");
+	t.is(await res.text(), testBody);
+
+	// Check identity encoding
+	res = await fetch(gzipUrl, { headers: { "Accept-Encoding": "identity" } });
+	t.is(res.headers.get("Content-Encoding"), null);
+	t.is(await res.text(), testBody);
+	res = await fetch(gzipUrl, { headers: { "Accept-Encoding": "*" } });
+	t.is(res.headers.get("Content-Encoding"), null);
+	t.is(await res.text(), testBody);
+	res = await fetch(gzipUrl, { headers: { "Accept-Encoding": "zstd, *" } });
+	t.is(res.headers.get("Content-Encoding"), null);
+	t.is(await res.text(), testBody);
+	res = await fetch(gzipUrl, {
+		headers: { "Accept-Encoding": "zstd, identity;q=0" },
+	});
+	t.is(res.status, 415);
+	t.is(res.headers.get("Accept-Encoding"), "br, gzip");
+	t.is(await res.text(), "Unsupported Media Type");
+	res = await fetch(gzipUrl, { headers: { "Accept-Encoding": "zstd, *;q=0" } });
+	t.is(res.status, 415);
+	t.is(res.headers.get("Accept-Encoding"), "br, gzip");
+	t.is(await res.text(), "Unsupported Media Type");
+
+	// Check malformed `Accept-Encoding`
+	res = await fetch(gzipUrl, { headers: { "Accept-Encoding": ",(,br,,,q=," } });
+	t.is(res.headers.get("Content-Encoding"), "br");
+	t.is(await res.text(), testBody);
+});
+
 test("Miniflare: custom service using Set-Cookie header", async (t) => {
 	const testCookies = [
 		"key1=value1; Max-Age=3600",
@@ -690,14 +819,14 @@ test("Miniflare: can send GET request with body", async (t) => {
 
 	let res = await get();
 	t.deepEqual(await json(res), {
-		cf: { key: "value" },
+		cf: { key: "value", clientAcceptEncoding: "" },
 		contentLength: null,
 		hasBody: false,
 	});
 
 	res = await get({ headers: { "content-length": "0" } });
 	t.deepEqual(await json(res), {
-		cf: { key: "value" },
+		cf: { key: "value", clientAcceptEncoding: "" },
 		contentLength: "0",
 		hasBody: true,
 	});

packages/wrangler/templates/startDevWorker/ProxyWorker.ts

@@ -126,6 +126,11 @@ export class ProxyWorker implements DurableObject {
 			headers.set("MF-Original-URL", innerUrl.href);
 			headers.set("MF-Disable-Pretty-Error", "true"); // disables the UserWorker miniflare instance from rendering the pretty error -- instead the ProxyWorker miniflare instance will intercept the json error response and render the pretty error page
 
+			// Preserve client `Accept-Encoding`, rather than using Worker's default
+			// of `Accept-Encoding: br, gzip`
+			const encoding = request.cf?.clientAcceptEncoding;
+			if (encoding !== undefined) headers.set("Accept-Encoding", encoding);
+
 			rewriteUrlRelatedHeaders(headers, outerUrl, innerUrl);
 
 			// merge proxyData headers with the request headers