diff --git a/lib/matplotlib/contour.py b/lib/matplotlib/contour.py
index adb72ef46017..ec343763cce0 100644
--- a/lib/matplotlib/contour.py
+++ b/lib/matplotlib/contour.py
@@ -350,6 +350,11 @@ def _split_path_and_get_label_rotation(self, path, idx, screen_pos, lw, spacing=
         taken into account when breaking the path, but not when computing the angle.
         """
         if hasattr(self, "_old_style_split_collections"):
+            vis = False
+            for coll in self._old_style_split_collections:
+                vis |= coll.get_visible()
+                coll.remove()
+            self.set_visible(vis)
             del self._old_style_split_collections  # Invalidate them.
 
         xys = path.vertices
diff --git a/lib/matplotlib/tests/baseline_images/test_contour/contour_disconnected_segments.png b/lib/matplotlib/tests/baseline_images/test_contour/contour_disconnected_segments.png
new file mode 100644
index 000000000000..ceb700e09de2
Binary files /dev/null and b/lib/matplotlib/tests/baseline_images/test_contour/contour_disconnected_segments.png differ
diff --git a/lib/matplotlib/tests/test_contour.py b/lib/matplotlib/tests/test_contour.py
index b3b7f575b07a..74165faad9fc 100644
--- a/lib/matplotlib/tests/test_contour.py
+++ b/lib/matplotlib/tests/test_contour.py
@@ -116,11 +116,31 @@ def test_contour_manual_labels(split_collections):
 
     plt.figure(figsize=(6, 2), dpi=200)
     cs = plt.contour(x, y, z)
+
+    _maybe_split_collections(split_collections)
+
     pts = np.array([(1.0, 3.0), (1.0, 4.4), (1.0, 6.0)])
     plt.clabel(cs, manual=pts)
     pts = np.array([(2.0, 3.0), (2.0, 4.4), (2.0, 6.0)])
     plt.clabel(cs, manual=pts, fontsize='small', colors=('r', 'g'))
 
+
+@pytest.mark.parametrize("split_collections", [False, True])
+@image_comparison(['contour_disconnected_segments'],
+                  remove_text=True, style='mpl20', extensions=['png'])
+def test_contour_label_with_disconnected_segments(split_collections):
+    x, y = np.mgrid[-1:1:21j, -1:1:21j]
+    z = 1 / np.sqrt(0.01 + (x + 0.3) ** 2 + y ** 2)
+    z += 1 / np.sqrt(0.01 + (x - 0.3) ** 2 + y ** 2)
+
+    plt.figure()
+    cs = plt.contour(x, y, z, levels=[7])
+
+    # Adding labels should invalidate the old style
+    _maybe_split_collections(split_collections)
+
+    cs.clabel(manual=[(0.2, 0.1)])
+
     _maybe_split_collections(split_collections)
 
 
@@ -232,6 +252,9 @@ def test_labels(split_collections):
     disp_units = [(216, 177), (359, 290), (521, 406)]
     data_units = [(-2, .5), (0, -1.5), (2.8, 1)]
 
+    # Adding labels should invalidate the old style
+    _maybe_split_collections(split_collections)
+
     CS.clabel()
 
     for x, y in data_units: