diff --git a/packages/format-library/src/link/inline.js b/packages/format-library/src/link/inline.js
index 7d0594981c290c..b01f37e9657bbb 100644
--- a/packages/format-library/src/link/inline.js
+++ b/packages/format-library/src/link/inline.js
@@ -21,7 +21,6 @@ import {
 import {
 	__experimentalLinkControl as LinkControl,
 	store as blockEditorStore,
-	useCachedTruthy,
 } from '@wordpress/block-editor';
 import { useSelect } from '@wordpress/data';
 
@@ -195,28 +194,9 @@ function InlineLinkUI( {
 
 	const popoverAnchor = useAnchor( {
 		editableContentElement: contentRef.current,
-		settings,
+		settings: { ...settings, isActive },
 	} );
 
-	//  As you change the link by interacting with the Link UI
-	//  the return value of document.getSelection jumps to the field you're editing,
-	//  not the highlighted text. Given that useAnchor uses document.getSelection,
-	//  it will return null, since it can't find the <mark> element within the Link UI.
-	//  This caches the last truthy value of the selection anchor reference.
-	// This ensures the Popover is positioned correctly on initial submission of the link.
-	const cachedRect = useCachedTruthy( popoverAnchor.getBoundingClientRect() );
-
-	// If the link is not active (i.e. it is a new link) then we need to
-	// override the getBoundingClientRect method on the anchor element
-	// to return the cached value of the selection represented by the text
-	// that the user selected to be linked.
-	// If the link is active (i.e. it is an existing link) then we allow
-	// the default behaviour of the popover anchor to be used. This will get
-	// the anchor based on the `<a>` element in the rich text.
-	if ( ! isActive ) {
-		popoverAnchor.getBoundingClientRect = () => cachedRect;
-	}
-
 	async function handleCreate( pageTitle ) {
 		const page = await createPageEntity( {
 			title: pageTitle,
diff --git a/packages/format-library/src/text-color/index.js b/packages/format-library/src/text-color/index.js
index 8a28cb69549cdc..464c2eec7c065c 100644
--- a/packages/format-library/src/text-color/index.js
+++ b/packages/format-library/src/text-color/index.js
@@ -120,6 +120,7 @@ function TextColorEdit( {
 					value={ value }
 					onChange={ onChange }
 					contentRef={ contentRef }
+					isActive={ isActive }
 				/>
 			) }
 		</>
diff --git a/packages/format-library/src/text-color/inline.js b/packages/format-library/src/text-color/inline.js
index 98ced2cca01e29..8bd4ae33c78443 100644
--- a/packages/format-library/src/text-color/inline.js
+++ b/packages/format-library/src/text-color/inline.js
@@ -15,7 +15,6 @@ import {
 	getColorObjectByColorValue,
 	getColorObjectByAttributeValues,
 	store as blockEditorStore,
-	useCachedTruthy,
 } from '@wordpress/block-editor';
 import {
 	Popover,
@@ -147,22 +146,13 @@ export default function InlineColorUI( {
 	onChange,
 	onClose,
 	contentRef,
+	isActive,
 } ) {
 	const popoverAnchor = useAnchor( {
 		editableContentElement: contentRef.current,
-		settings,
+		settings: { ...settings, isActive },
 	} );
 
-	/*
-	 As you change the text color by typing a HEX value into a field,
-	 the return value of document.getSelection jumps to the field you're editing,
-	 not the highlighted text. Given that useAnchor uses document.getSelection,
-	 it will return null, since it can't find the <mark> element within the HEX input.
-	 This caches the last truthy value of the selection anchor reference.
-	 */
-	const cachedRect = useCachedTruthy( popoverAnchor.getBoundingClientRect() );
-	popoverAnchor.getBoundingClientRect = () => cachedRect;
-
 	return (
 		<Popover
 			onClose={ onClose }
diff --git a/packages/rich-text/src/component/use-anchor.js b/packages/rich-text/src/component/use-anchor.js
index 7ea3281cb60d3a..b24aa9c8f7653c 100644
--- a/packages/rich-text/src/component/use-anchor.js
+++ b/packages/rich-text/src/component/use-anchor.js
@@ -1,6 +1,7 @@
 /**
  * WordPress dependencies
  */
+import { usePrevious } from '@wordpress/compose';
 import { useState, useLayoutEffect } from '@wordpress/element';
 
 /** @typedef {import('../register-format-type').WPFormat} WPFormat */
@@ -137,44 +138,31 @@ function getAnchor( editableContentElement, tagName, className ) {
  * @return {Element|VirtualAnchorElement|undefined|null} The active element or selection range.
  */
 export function useAnchor( { editableContentElement, settings = {} } ) {
-	const { tagName, className } = settings;
+	const { tagName, className, isActive } = settings;
 	const [ anchor, setAnchor ] = useState( () =>
 		getAnchor( editableContentElement, tagName, className )
 	);
+	const wasActive = usePrevious( isActive );
 
 	useLayoutEffect( () => {
 		if ( ! editableContentElement ) return;
 
 		const { ownerDocument } = editableContentElement;
 
-		function callback() {
+		if (
+			editableContentElement === ownerDocument.activeElement ||
+			// When a link is created, we need to attach the popover to the newly created anchor.
+			( ! wasActive && isActive ) ||
+			// Sometimes we're _removing_ an active anchor, such as the inline color popover.
+			// When we add the color, it switches from a virtual anchor to a `<mark>` element.
+			// When we _remove_ the color, it switches from a `<mark>` element to a virtual anchor.
+			( wasActive && ! isActive )
+		) {
 			setAnchor(
 				getAnchor( editableContentElement, tagName, className )
 			);
 		}
-
-		function attach() {
-			ownerDocument.addEventListener( 'selectionchange', callback );
-		}
-
-		function detach() {
-			ownerDocument.removeEventListener( 'selectionchange', callback );
-		}
-
-		if ( editableContentElement === ownerDocument.activeElement ) {
-			attach();
-		}
-
-		editableContentElement.addEventListener( 'focusin', attach );
-		editableContentElement.addEventListener( 'focusout', detach );
-
-		return () => {
-			detach();
-
-			editableContentElement.removeEventListener( 'focusin', attach );
-			editableContentElement.removeEventListener( 'focusout', detach );
-		};
-	}, [ editableContentElement, tagName, className ] );
+	}, [ editableContentElement, tagName, className, isActive, wasActive ] );
 
 	return anchor;
 }