diff --git a/packages/runtime-core/__tests__/components/Teleport.spec.ts b/packages/runtime-core/__tests__/components/Teleport.spec.ts
index f6babd9263b..8b234e2b1e6 100644
--- a/packages/runtime-core/__tests__/components/Teleport.spec.ts
+++ b/packages/runtime-core/__tests__/components/Teleport.spec.ts
@@ -8,7 +8,9 @@ import {
   ref,
   nextTick,
   markRaw,
-  defineComponent
+  defineComponent,
+  withDirectives,
+  createApp
 } from '@vue/runtime-test'
 import { createVNode, Fragment } from '../../src/vnode'
 import { compile, render as domRender } from 'vue'
@@ -432,4 +434,42 @@ describe('renderer: teleport', () => {
       `"<div>teleported</div><span>false</span><!--v-if-->"`
     )
   })
+
+  // #3497
+  test(`the dir hooks of the Teleport's children should be called correctly`, async () => {
+    const target = nodeOps.createElement('div')
+    const root = nodeOps.createElement('div')
+    const toggle = ref(true)
+    const dir = {
+      mounted: jest.fn(),
+      unmounted: jest.fn()
+    }
+
+    const app = createApp({
+      setup() {
+        return () => {
+          return toggle.value
+            ? h(Teleport, { to: target }, [
+                withDirectives(h('div', ['foo']), [[dir]])
+              ])
+            : null
+        }
+      }
+    })
+    app.mount(root)
+
+    expect(serializeInner(root)).toMatchInlineSnapshot(
+      `"<!--teleport start--><!--teleport end-->"`
+    )
+    expect(serializeInner(target)).toMatchInlineSnapshot(`"<div>foo</div>"`)
+    expect(dir.mounted).toHaveBeenCalledTimes(1)
+    expect(dir.unmounted).toHaveBeenCalledTimes(0)
+
+    toggle.value = false
+    await nextTick()
+    expect(serializeInner(root)).toMatchInlineSnapshot(`"<!---->"`)
+    expect(serializeInner(target)).toMatchInlineSnapshot(`""`)
+    expect(dir.mounted).toHaveBeenCalledTimes(1)
+    expect(dir.unmounted).toHaveBeenCalledTimes(1)
+  })
 })
diff --git a/packages/runtime-core/src/renderer.ts b/packages/runtime-core/src/renderer.ts
index 4375dcdbf10..5122e7a9131 100644
--- a/packages/runtime-core/src/renderer.ts
+++ b/packages/runtime-core/src/renderer.ts
@@ -2096,7 +2096,16 @@ function baseCreateRenderer(
         invokeDirectiveHook(vnode, null, parentComponent, 'beforeUnmount')
       }
 
-      if (
+      if (shapeFlag & ShapeFlags.TELEPORT) {
+        ;(vnode.type as typeof TeleportImpl).remove(
+          vnode,
+          parentComponent,
+          parentSuspense,
+          optimized,
+          internals,
+          doRemove
+        )
+      } else if (
         dynamicChildren &&
         // #1153: fast path should not be taken for non-stable (v-for) fragments
         (type !== Fragment ||
@@ -2119,17 +2128,6 @@ function baseCreateRenderer(
         unmountChildren(children as VNode[], parentComponent, parentSuspense)
       }
 
-      if (shapeFlag & ShapeFlags.TELEPORT) {
-        ;(vnode.type as typeof TeleportImpl).remove(
-          vnode,
-          parentComponent,
-          parentSuspense,
-          optimized,
-          internals,
-          doRemove
-        )
-      }
-
       if (doRemove) {
         remove(vnode)
       }