diff --git a/packages/vite/src/node/plugins/worker.ts b/packages/vite/src/node/plugins/worker.ts
index c4ff6dff78bb2e..411386f25e8a3e 100644
--- a/packages/vite/src/node/plugins/worker.ts
+++ b/packages/vite/src/node/plugins/worker.ts
@@ -44,6 +44,7 @@ function saveEmitWorkerAsset(
 // Ensure that only one rollup build is called at the same time to avoid
 // leaking state in plugins between worker builds.
 // TODO: Review if we can parallelize the bundling of workers.
+const workerKey = { key: 'vite:worker' }
 const workerConfigSemaphore = new WeakMap<
   ResolvedConfig,
   Promise<OutputChunk>
@@ -53,14 +54,14 @@ export async function bundleWorkerEntry(
   id: string,
   query: Record<string, string> | null,
 ): Promise<OutputChunk> {
-  const processing = workerConfigSemaphore.get(config)
+  const processing = workerConfigSemaphore.get(workerKey)
   if (processing) {
     await processing
     return bundleWorkerEntry(config, id, query)
   }
   const promise = serialBundleWorkerEntry(config, id, query)
-  workerConfigSemaphore.set(config, promise)
-  promise.then(() => workerConfigSemaphore.delete(config))
+  workerConfigSemaphore.set(workerKey, promise)
+  promise.then(() => workerConfigSemaphore.delete(workerKey))
   return promise
 }