diff --git a/lib/perf_hooks.js b/lib/perf_hooks.js
index 14b1aa44dddc26..00f188bfc7db4d 100644
--- a/lib/perf_hooks.js
+++ b/lib/perf_hooks.js
@@ -15,7 +15,8 @@ const {
   timeOriginTimestamp,
   timerify,
   constants,
-  setupGarbageCollectionTracking
+  installGarbageCollectionTracking,
+  removeGarbageCollectionTracking
 } = internalBinding('performance');
 
 const {
@@ -280,8 +281,6 @@ class PerformanceObserverEntryList {
   }
 }
 
-let gcTrackingIsEnabled = false;
-
 class PerformanceObserver extends AsyncResource {
   constructor(callback) {
     if (typeof callback !== 'function') {
@@ -318,6 +317,7 @@ class PerformanceObserver extends AsyncResource {
   }
 
   disconnect() {
+    const observerCountsGC = observerCounts[NODE_PERFORMANCE_ENTRY_TYPE_GC];
     const types = this[kTypes];
     const keys = Object.keys(types);
     for (var n = 0; n < keys.length; n++) {
@@ -328,6 +328,10 @@ class PerformanceObserver extends AsyncResource {
       }
     }
     this[kTypes] = {};
+    if (observerCountsGC === 1 &&
+      observerCounts[NODE_PERFORMANCE_ENTRY_TYPE_GC] === 0) {
+      removeGarbageCollectionTracking();
+    }
   }
 
   observe(options) {
@@ -341,12 +345,8 @@ class PerformanceObserver extends AsyncResource {
     if (entryTypes.length === 0) {
       throw new ERR_VALID_PERFORMANCE_ENTRY_TYPE();
     }
-    if (entryTypes.includes(NODE_PERFORMANCE_ENTRY_TYPE_GC) &&
-      !gcTrackingIsEnabled) {
-      setupGarbageCollectionTracking();
-      gcTrackingIsEnabled = true;
-    }
     this.disconnect();
+    const observerCountsGC = observerCounts[NODE_PERFORMANCE_ENTRY_TYPE_GC];
     this[kBuffer][kEntries] = [];
     L.init(this[kBuffer][kEntries]);
     this[kBuffering] = Boolean(options.buffered);
@@ -358,6 +358,10 @@ class PerformanceObserver extends AsyncResource {
       L.append(list, item);
       observerCounts[entryType]++;
     }
+    if (observerCountsGC === 0 &&
+      observerCounts[NODE_PERFORMANCE_ENTRY_TYPE_GC] === 1) {
+      installGarbageCollectionTracking();
+    }
   }
 }
 
diff --git a/src/node_perf.cc b/src/node_perf.cc
index 3efaca60658310..da711fee846bb4 100644
--- a/src/node_perf.cc
+++ b/src/node_perf.cc
@@ -277,7 +277,13 @@ void MarkGarbageCollectionEnd(Isolate* isolate,
   });
 }
 
-static void SetupGarbageCollectionTracking(
+void GarbageCollectionCleanupHook(void* data) {
+  Environment* env = static_cast<Environment*>(data);
+  env->isolate()->RemoveGCPrologueCallback(MarkGarbageCollectionStart, data);
+  env->isolate()->RemoveGCEpilogueCallback(MarkGarbageCollectionEnd, data);
+}
+
+static void InstallGarbageCollectionTracking(
     const FunctionCallbackInfo<Value>& args) {
   Environment* env = Environment::GetCurrent(args);
 
@@ -285,11 +291,15 @@ static void SetupGarbageCollectionTracking(
                                         static_cast<void*>(env));
   env->isolate()->AddGCEpilogueCallback(MarkGarbageCollectionEnd,
                                         static_cast<void*>(env));
-  env->AddCleanupHook([](void* data) {
-    Environment* env = static_cast<Environment*>(data);
-    env->isolate()->RemoveGCPrologueCallback(MarkGarbageCollectionStart, data);
-    env->isolate()->RemoveGCEpilogueCallback(MarkGarbageCollectionEnd, data);
-  }, env);
+  env->AddCleanupHook(GarbageCollectionCleanupHook, env);
+}
+
+static void RemoveGarbageCollectionTracking(
+  const FunctionCallbackInfo<Value> &args) {
+  Environment* env = Environment::GetCurrent(args);
+
+  env->RemoveCleanupHook(GarbageCollectionCleanupHook, env);
+  GarbageCollectionCleanupHook(env);
 }
 
 // Gets the name of a function
@@ -575,8 +585,12 @@ void Initialize(Local<Object> target,
   env->SetMethod(target, "markMilestone", MarkMilestone);
   env->SetMethod(target, "setupObservers", SetupPerformanceObservers);
   env->SetMethod(target, "timerify", Timerify);
-  env->SetMethod(
-      target, "setupGarbageCollectionTracking", SetupGarbageCollectionTracking);
+  env->SetMethod(target,
+                 "installGarbageCollectionTracking",
+                 InstallGarbageCollectionTracking);
+  env->SetMethod(target,
+                 "removeGarbageCollectionTracking",
+                 RemoveGarbageCollectionTracking);
   env->SetMethod(target, "notify", Notify);
 
   Local<Object> constants = Object::New(isolate);