From 6009bb44185e6a8a464528c945ce5a47eb992837 Mon Sep 17 00:00:00 2001 From: Rahul Sethi <5822355+RamIdeas@users.noreply.github.com> Date: Tue, 1 Oct 2024 11:40:42 +0100 Subject: [PATCH] Retry deployments (#6801) * retry wrangler deploy + versions upload --- .changeset/chilled-trainers-switch.md | 5 ++ .../versions/versions.upload.test.ts | 10 ++- packages/wrangler/src/deploy/deploy.ts | 61 ++++++++++--------- packages/wrangler/src/utils/retry.ts | 18 ++++++ packages/wrangler/src/versions/upload.ts | 25 ++++---- 5 files changed, 77 insertions(+), 42 deletions(-) create mode 100644 .changeset/chilled-trainers-switch.md create mode 100644 packages/wrangler/src/utils/retry.ts diff --git a/.changeset/chilled-trainers-switch.md b/.changeset/chilled-trainers-switch.md new file mode 100644 index 000000000000..0450e7bedd27 --- /dev/null +++ b/.changeset/chilled-trainers-switch.md @@ -0,0 +1,5 @@ +--- +"wrangler": minor +--- + +feat: implement retries within `wrangler deploy` and `wrangler versions upload` to workaround spotty network connections and service flakes diff --git a/packages/wrangler/src/__tests__/versions/versions.upload.test.ts b/packages/wrangler/src/__tests__/versions/versions.upload.test.ts index 00d9bdbb5b18..d2632e1bd2ff 100644 --- a/packages/wrangler/src/__tests__/versions/versions.upload.test.ts +++ b/packages/wrangler/src/__tests__/versions/versions.upload.test.ts @@ -37,11 +37,16 @@ describe("versions upload", () => { ) ); } - function mockUploadVersion(has_preview: boolean) { + function mockUploadVersion(has_preview: boolean, flakeCount = 1) { msw.use( http.post( `*/accounts/:accountId/workers/scripts/:scriptName/versions`, ({ params }) => { + if (flakeCount > 0) { + flakeCount--; + return HttpResponse.error(); + } + expect(params.scriptName).toEqual("test-worker"); return HttpResponse.json( @@ -53,8 +58,7 @@ describe("versions upload", () => { }, }) ); - }, - { once: true } + } ) ); } diff --git a/packages/wrangler/src/deploy/deploy.ts b/packages/wrangler/src/deploy/deploy.ts index 4f355826ac7d..1f2addecf685 100644 --- a/packages/wrangler/src/deploy/deploy.ts +++ b/packages/wrangler/src/deploy/deploy.ts @@ -46,6 +46,7 @@ import { maybeRetrieveFileSourceMap, } from "../sourcemap"; import triggersDeploy from "../triggers/deploy"; +import { retryOnError } from "../utils/retry"; import { createDeployment, patchNonVersionedScriptSettings, @@ -814,13 +815,15 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m // If we're using the new APIs, first upload the version if (canUseNewVersionsDeploymentsApi) { // Upload new version - const versionResult = await fetchResult( - `/accounts/${accountId}/workers/scripts/${scriptName}/versions`, - { - method: "POST", - body: createWorkerUploadForm(worker), - headers: await getMetricsUsageHeaders(config.send_metrics), - } + const versionResult = await retryOnError(async () => + fetchResult( + `/accounts/${accountId}/workers/scripts/${scriptName}/versions`, + { + method: "POST", + body: createWorkerUploadForm(worker), + headers: await getMetricsUsageHeaders(config.send_metrics), + } + ) ); // Deploy new version to 100% @@ -852,27 +855,29 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m startup_time_ms: versionResult.startup_time_ms, }; } else { - result = await fetchResult<{ - available_on_subdomain: boolean; - id: string | null; - etag: string | null; - pipeline_hash: string | null; - mutable_pipeline_id: string | null; - deployment_id: string | null; - startup_time_ms: number; - }>( - workerUrl, - { - method: "PUT", - body: createWorkerUploadForm(worker), - headers: await getMetricsUsageHeaders(config.send_metrics), - }, - new URLSearchParams({ - include_subdomain_availability: "true", - // pass excludeScript so the whole body of the - // script doesn't get included in the response - excludeScript: "true", - }) + result = await retryOnError(async () => + fetchResult<{ + available_on_subdomain: boolean; + id: string | null; + etag: string | null; + pipeline_hash: string | null; + mutable_pipeline_id: string | null; + deployment_id: string | null; + startup_time_ms: number; + }>( + workerUrl, + { + method: "PUT", + body: createWorkerUploadForm(worker), + headers: await getMetricsUsageHeaders(config.send_metrics), + }, + new URLSearchParams({ + include_subdomain_availability: "true", + // pass excludeScript so the whole body of the + // script doesn't get included in the response + excludeScript: "true", + }) + ) ); } diff --git a/packages/wrangler/src/utils/retry.ts b/packages/wrangler/src/utils/retry.ts new file mode 100644 index 000000000000..ac35efef4ab2 --- /dev/null +++ b/packages/wrangler/src/utils/retry.ts @@ -0,0 +1,18 @@ +import { setTimeout } from "node:timers/promises"; + +export async function retryOnError( + action: () => T | Promise, + backoff = 2_000, + attempts = 3 +): Promise { + try { + return await action(); + } catch (err) { + if (attempts <= 1) { + throw err; + } + + await setTimeout(backoff); + return retryOnError(action, backoff, attempts - 1); + } +} diff --git a/packages/wrangler/src/versions/upload.ts b/packages/wrangler/src/versions/upload.ts index 4f1dc4971d5c..944fc9f3f377 100644 --- a/packages/wrangler/src/versions/upload.ts +++ b/packages/wrangler/src/versions/upload.ts @@ -35,6 +35,7 @@ import { getSourceMappedString, maybeRetrieveFileSourceMap, } from "../sourcemap"; +import { retryOnError } from "../utils/retry"; import type { AssetsOptions } from "../assets"; import type { Config } from "../config"; import type { Rule } from "../config/environment"; @@ -466,17 +467,19 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m try { const body = createWorkerUploadForm(worker); - const result = await fetchResult<{ - id: string; - startup_time_ms: number; - metadata: { - has_preview: boolean; - }; - }>(`${workerUrl}/versions`, { - method: "POST", - body, - headers: await getMetricsUsageHeaders(config.send_metrics), - }); + const result = await retryOnError(async () => + fetchResult<{ + id: string; + startup_time_ms: number; + metadata: { + has_preview: boolean; + }; + }>(`${workerUrl}/versions`, { + method: "POST", + body, + headers: await getMetricsUsageHeaders(config.send_metrics), + }) + ); logger.log("Worker Startup Time:", result.startup_time_ms, "ms"); bindingsPrinted = true;