diff --git a/packages/next/src/build/swc/options.ts b/packages/next/src/build/swc/options.ts
index 0336df67ace12..546f2ebfc743f 100644
--- a/packages/next/src/build/swc/options.ts
+++ b/packages/next/src/build/swc/options.ts
@@ -1,3 +1,4 @@
+import { WEBPACK_LAYERS, type WebpackLayerName } from '../../lib/constants'
 import type {
   NextConfig,
   ExperimentalConfig,
@@ -9,6 +10,8 @@ import type { ResolvedBaseUrl } from '../load-jsconfig'
 const nextDistPath =
   /(next[\\/]dist[\\/]shared[\\/]lib)|(next[\\/]dist[\\/]client)|(next[\\/]dist[\\/]pages)/
 
+const nodeModulesPath = /[\\/]node_modules[\\/]/
+
 const regeneratorRuntimePath = require.resolve(
   'next/dist/compiled/regenerator-runtime'
 )
@@ -44,7 +47,7 @@ function getBaseSWCOptions({
   jsConfig,
   swcCacheDir,
   serverComponents,
-  isReactServerLayer,
+  bundleLayer,
 }: {
   filename: string
   jest?: boolean
@@ -59,8 +62,10 @@ function getBaseSWCOptions({
   jsConfig: any
   swcCacheDir?: string
   serverComponents?: boolean
-  isReactServerLayer?: boolean
+  bundleLayer?: WebpackLayerName
 }) {
+  const isReactServerLayer =
+    bundleLayer === WEBPACK_LAYERS.reactServerComponents
   const parserConfig = getParserOptions({ filename, jsConfig })
   const paths = jsConfig?.compilerOptions?.paths
   const enableDecorators = Boolean(
@@ -177,7 +182,7 @@ function getBaseSWCOptions({
     serverComponents:
       serverComponents && !jest
         ? {
-            isReactServerLayer: !!isReactServerLayer,
+            isReactServerLayer,
           }
         : undefined,
     serverActions:
@@ -186,7 +191,7 @@ function getBaseSWCOptions({
             // always enable server actions
             // TODO: remove this option
             enabled: true,
-            isReactServerLayer: !!isReactServerLayer,
+            isReactServerLayer,
           }
         : undefined,
     // For app router we prefer to bundle ESM,
@@ -295,8 +300,8 @@ export function getJestSWCOptions({
     resolvedBaseUrl,
     esm,
     // Don't apply server layer transformations for Jest
-    isReactServerLayer: false,
     // Disable server / client graph assertions for Jest
+    bundleLayer: undefined,
     serverComponents: false,
   })
 
@@ -339,7 +344,7 @@ export function getLoaderSWCOptions({
   swcCacheDir,
   relativeFilePathFromRoot,
   serverComponents,
-  isReactServerLayer,
+  bundleLayer,
   esm,
 }: {
   filename: string
@@ -362,7 +367,7 @@ export function getLoaderSWCOptions({
   relativeFilePathFromRoot: string
   esm?: boolean
   serverComponents?: boolean
-  isReactServerLayer?: boolean
+  bundleLayer?: WebpackLayerName
 }) {
   let baseOptions: any = getBaseSWCOptions({
     filename,
@@ -375,7 +380,7 @@ export function getLoaderSWCOptions({
     jsConfig,
     // resolvedBaseUrl,
     swcCacheDir,
-    isReactServerLayer,
+    bundleLayer,
     serverComponents,
     esm: !!esm,
   })
@@ -418,9 +423,12 @@ export function getLoaderSWCOptions({
   }
 
   const isNextDist = nextDistPath.test(filename)
+  const isNodeModules = nodeModulesPath.test(filename)
+  const isAppBrowserLayer = bundleLayer === WEBPACK_LAYERS.appPagesBrowser
 
+  let options: any
   if (isServer) {
-    return {
+    options = {
       ...baseOptions,
       // Disables getStaticProps/getServerSideProps tree shaking on the server compilation for pages
       disableNextSsg: true,
@@ -440,7 +448,7 @@ export function getLoaderSWCOptions({
       ...getModuleOptions(esm),
     }
   } else {
-    const options = {
+    options = {
       ...baseOptions,
       // Ensure Next.js internals are output as commonjs modules
       ...(isNextDist
@@ -468,6 +476,17 @@ export function getLoaderSWCOptions({
       // Matches default @babel/preset-env behavior
       options.jsc.target = 'es5'
     }
-    return options
   }
+
+  // For node_modules in app browser layer, we don't need to do any server side transformation.
+  // Only keep server actions transform to discover server actions from client components.
+  if (isAppBrowserLayer && isNodeModules) {
+    options.disableNextSsg = true
+    options.disablePageConfig = true
+    options.isPageFile = false
+    options.optimizeServerReact = undefined
+    options.cjsRequireOptimizer = undefined
+  }
+
+  return options
 }
diff --git a/packages/next/src/build/webpack-config.ts b/packages/next/src/build/webpack-config.ts
index df034b4e1bd08..6f24d83cd3aa6 100644
--- a/packages/next/src/build/webpack-config.ts
+++ b/packages/next/src/build/webpack-config.ts
@@ -445,18 +445,22 @@ export default async function getBaseWebpackConfig(
   // RSC loaders, prefer ESM, set `esm` to true
   const swcServerLayerLoader = getSwcLoader({
     serverComponents: true,
-    isReactServerLayer: true,
+    bundleLayer: WEBPACK_LAYERS.reactServerComponents,
     esm: true,
   })
-  const swcClientLayerLoader = getSwcLoader({
+  const swcSSRLayerLoader = getSwcLoader({
     serverComponents: true,
-    isReactServerLayer: false,
+    bundleLayer: WEBPACK_LAYERS.serverSideRendering,
+    esm: true,
+  })
+  const swcBrowserLayerLoader = getSwcLoader({
+    serverComponents: true,
+    bundleLayer: WEBPACK_LAYERS.appPagesBrowser,
     esm: true,
   })
   // Default swc loaders for pages doesn't prefer ESM.
   const swcDefaultLoader = getSwcLoader({
     serverComponents: true,
-    isReactServerLayer: false,
     esm: false,
   })
 
@@ -475,27 +479,30 @@ export default async function getBaseWebpackConfig(
       ].filter(Boolean)
     : []
 
-  const swcLoaderForMiddlewareLayer = useSWCLoader
-    ? getSwcLoader({
-        serverComponents: false,
-        isReactServerLayer: false,
-      })
-    : // When using Babel, we will have to use SWC to do the optimization
-      // for middleware to tree shake the unused default optimized imports like "next/server".
-      // This will cause some performance overhead but
-      // acceptable as Babel will not be recommended.
-      [
-        getSwcLoader({
-          serverComponents: false,
-          isReactServerLayer: false,
-        }),
-      ]
+  const swcLoaderForMiddlewareLayer = [
+    // When using Babel, we will have to use SWC to do the optimization
+    // for middleware to tree shake the unused default optimized imports like "next/server".
+    // This will cause some performance overhead but
+    // acceptable as Babel will not be recommended.
+    getSwcLoader({
+      serverComponents: false,
+      bundleLayer: WEBPACK_LAYERS.middleware,
+    }),
+    babelLoader,
+  ].filter(Boolean)
 
   const reactRefreshLoaders =
     dev && isClient ? [require.resolve(reactRefreshLoaderName)] : []
 
-  // client components layers: SSR + browser
-  const swcLoaderForClientLayer = [
+  // client components layers: SSR or browser
+  const createSwcLoaderForClientLayer = (isBrowserLayer: boolean) => [
+    ...(dev && isClient
+      ? [
+          require.resolve(
+            'next/dist/compiled/@next/react-refresh-utils/dist/loader'
+          ),
+        ]
+      : []),
     {
       // This loader handles actions and client entries
       // in the client layer.
@@ -507,12 +514,15 @@ export default async function getBaseWebpackConfig(
           // as an additional pass to handle RSC correctly.
           // This will cause some performance overhead but
           // acceptable as Babel will not be recommended.
-          swcClientLayerLoader,
+          isBrowserLayer ? swcBrowserLayerLoader : swcSSRLayerLoader,
           babelLoader,
         ].filter(Boolean)
       : []),
   ]
 
+  const swcLoaderForBrowserLayer = createSwcLoaderForClientLayer(true)
+  const swcLoaderForSSRLayer = createSwcLoaderForClientLayer(false)
+
   // Loader for API routes needs to be differently configured as it shouldn't
   // have RSC transpiler enabled, so syntax checks such as invalid imports won't
   // be performed.
@@ -520,7 +530,7 @@ export default async function getBaseWebpackConfig(
     hasAppDir && useSWCLoader
       ? getSwcLoader({
           serverComponents: false,
-          isReactServerLayer: false,
+          bundleLayer: WEBPACK_LAYERS.api,
         })
       : defaultLoaders.babel
 
@@ -1405,7 +1415,7 @@ export default async function getBaseWebpackConfig(
                   {
                     test: codeCondition.test,
                     issuerLayer: WEBPACK_LAYERS.appPagesBrowser,
-                    use: swcLoaderForClientLayer,
+                    use: swcLoaderForBrowserLayer,
                     resolve: {
                       mainFields: getMainField(compilerType, true),
                     },
@@ -1413,7 +1423,7 @@ export default async function getBaseWebpackConfig(
                   {
                     test: codeCondition.test,
                     issuerLayer: WEBPACK_LAYERS.serverSideRendering,
-                    use: swcLoaderForClientLayer,
+                    use: swcLoaderForSSRLayer,
                     resolve: {
                       mainFields: getMainField(compilerType, true),
                     },
diff --git a/packages/next/src/build/webpack/loaders/next-swc-loader.ts b/packages/next/src/build/webpack/loaders/next-swc-loader.ts
index aff07d2700b61..b1650fa4b8334 100644
--- a/packages/next/src/build/webpack/loaders/next-swc-loader.ts
+++ b/packages/next/src/build/webpack/loaders/next-swc-loader.ts
@@ -27,6 +27,7 @@ DEALINGS IN THE SOFTWARE.
 */
 
 import type { NextConfig } from '../../../../types'
+import type { WebpackLayerName } from '../../../lib/constants'
 import { isWasm, transform } from '../../swc'
 import { getLoaderSWCOptions } from '../../swc/options'
 import path, { isAbsolute } from 'path'
@@ -43,7 +44,7 @@ export interface SWCLoaderOptions {
   supportedBrowsers: string[] | undefined
   swcCacheDir: string
   serverComponents?: boolean
-  isReactServerLayer?: boolean
+  bundleLayer?: WebpackLayerName
   esm?: boolean
 }
 
@@ -69,7 +70,7 @@ async function loaderTransform(
     supportedBrowsers,
     swcCacheDir,
     serverComponents,
-    isReactServerLayer,
+    bundleLayer,
     esm,
   } = loaderOptions
   const isPageFile = filename.startsWith(pagesDir)
@@ -93,7 +94,7 @@ async function loaderTransform(
     swcCacheDir,
     relativeFilePathFromRoot,
     serverComponents,
-    isReactServerLayer,
+    bundleLayer,
     esm,
   })