diff --git a/src/cdk/overlay/keyboard/overlay-keyboard-dispatcher.ts b/src/cdk/overlay/keyboard/overlay-keyboard-dispatcher.ts
index 55c1f2c87d91..7b2d0e8f7c2d 100644
--- a/src/cdk/overlay/keyboard/overlay-keyboard-dispatcher.ts
+++ b/src/cdk/overlay/keyboard/overlay-keyboard-dispatcher.ts
@@ -26,10 +26,7 @@ export class OverlayKeyboardDispatcher implements OnDestroy {
   private _keydownEventSubscription: Subscription | null;
 
   ngOnDestroy() {
-    if (this._keydownEventSubscription) {
-      this._keydownEventSubscription.unsubscribe();
-      this._keydownEventSubscription = null;
-    }
+    this._unsubscribeFromKeydownEvents();
   }
 
   /** Add a new overlay to the list of attached overlay refs. */
@@ -45,9 +42,15 @@ export class OverlayKeyboardDispatcher implements OnDestroy {
   /** Remove an overlay from the list of attached overlay refs. */
   remove(overlayRef: OverlayRef): void {
     const index = this._attachedOverlays.indexOf(overlayRef);
+
     if (index > -1) {
       this._attachedOverlays.splice(index, 1);
     }
+
+    // Remove the global listener once there are no more overlays.
+    if (this._attachedOverlays.length === 0) {
+      this._unsubscribeFromKeydownEvents();
+    }
   }
 
   /**
@@ -65,6 +68,14 @@ export class OverlayKeyboardDispatcher implements OnDestroy {
     });
   }
 
+  /** Removes the global keydown subscription. */
+  private _unsubscribeFromKeydownEvents(): void {
+    if (this._keydownEventSubscription) {
+      this._keydownEventSubscription.unsubscribe();
+      this._keydownEventSubscription = null;
+    }
+  }
+
   /** Select the appropriate overlay from a keydown event. */
   private _selectOverlayFromEvent(event: KeyboardEvent): OverlayRef {
     // Check if any overlays contain the event