diff --git a/src/connection-manager/index.ts b/src/connection-manager/index.ts
index f4bc74e159..aa4a14a2ba 100644
--- a/src/connection-manager/index.ts
+++ b/src/connection-manager/index.ts
@@ -153,7 +153,7 @@ export class DefaultConnectionManager extends EventEmitter<ConnectionManagerEven
   private readonly opts: ConnectionManagerInit
   private readonly connections: Map<string, Connection[]>
   private started: boolean
-  private readonly latencyMonitor: LatencyMonitor
+  private readonly latencyMonitor?: LatencyMonitor
   private readonly startupReconnectTimeout: number
   private connectOnStartupController?: TimeoutController
   private readonly dialTimeout: number
@@ -182,10 +182,12 @@ export class DefaultConnectionManager extends EventEmitter<ConnectionManagerEven
 
     this.started = false
 
-    this.latencyMonitor = new LatencyMonitor({
-      latencyCheckIntervalMs: init.pollInterval,
-      dataEmitIntervalMs: init.pollInterval
-    })
+    if (init.maxEventLoopDelay != null && init.maxEventLoopDelay > 0 && init.maxEventLoopDelay !== Infinity) {
+      this.latencyMonitor = new LatencyMonitor({
+        latencyCheckIntervalMs: init.pollInterval,
+        dataEmitIntervalMs: init.pollInterval
+      })
+    }
 
     try {
       // This emitter gets listened to a lot
@@ -297,9 +299,9 @@ export class DefaultConnectionManager extends EventEmitter<ConnectionManagerEven
     })
 
     // latency monitor
-    this.latencyMonitor.start()
+    this.latencyMonitor?.start()
     this._onLatencyMeasure = this._onLatencyMeasure.bind(this)
-    this.latencyMonitor.addEventListener('data', this._onLatencyMeasure)
+    this.latencyMonitor?.addEventListener('data', this._onLatencyMeasure)
 
     this.started = true
     log('started')
@@ -361,8 +363,8 @@ export class DefaultConnectionManager extends EventEmitter<ConnectionManagerEven
    * Stops the Connection Manager
    */
   async stop () {
-    this.latencyMonitor.removeEventListener('data', this._onLatencyMeasure)
-    this.latencyMonitor.stop()
+    this.latencyMonitor?.removeEventListener('data', this._onLatencyMeasure)
+    this.latencyMonitor?.stop()
 
     this.started = false
     await this._close()
diff --git a/test/connection-manager/index.node.ts b/test/connection-manager/index.node.ts
index 77e4ce9dcf..8de955eeb1 100644
--- a/test/connection-manager/index.node.ts
+++ b/test/connection-manager/index.node.ts
@@ -570,4 +570,43 @@ describe('libp2p.connections', () => {
       expect(denyOutboundUpgradedConnection.getCall(0)).to.have.nested.property('args[0].multihash.digest').that.equalBytes(remoteLibp2p.peerId.multihash.digest)
     })
   })
+
+  describe('latency monitor', () => {
+    let libp2p: Libp2pNode
+
+    afterEach(async () => {
+      if (libp2p != null) {
+        await libp2p.stop()
+      }
+    })
+
+    it('should only start latency monitor if a limit is configured', async () => {
+      libp2p = await createNode({
+        config: createBaseOptions({
+          peerId: peerIds[0],
+          addresses: {
+            listen: ['/ip4/127.0.0.1/tcp/0/ws']
+          }
+        })
+      })
+
+      expect(libp2p).to.not.have.nested.property('connectionManager.latencyMonitor')
+
+      await libp2p.stop()
+
+      libp2p = await createNode({
+        config: createBaseOptions({
+          peerId: peerIds[0],
+          addresses: {
+            listen: ['/ip4/127.0.0.1/tcp/0/ws']
+          },
+          connectionManager: {
+            maxEventLoopDelay: 10000
+          }
+        })
+      })
+
+      expect(libp2p).to.have.nested.property('connectionManager.latencyMonitor')
+    })
+  })
 })