diff --git a/packages/rich-text/src/component/index.js b/packages/rich-text/src/component/index.js
index 8e8d1fa8f1ffe..926dcc7057fca 100644
--- a/packages/rich-text/src/component/index.js
+++ b/packages/rich-text/src/component/index.js
@@ -238,7 +238,6 @@ export function useRichText( {
 			applyRecord,
 			createRecord,
 			handleChange,
-			isSelected,
 			onSelectionChange,
 		} ),
 		useSelectionChangeCompat(),
diff --git a/packages/rich-text/src/component/use-input-and-selection.js b/packages/rich-text/src/component/use-input-and-selection.js
index 2e7ebfc6c4097..870bfe170635c 100644
--- a/packages/rich-text/src/component/use-input-and-selection.js
+++ b/packages/rich-text/src/component/use-input-and-selection.js
@@ -113,18 +113,11 @@ export function useInputAndSelection( props ) {
 
 		/**
 		 * Syncs the selection to local state. A callback for the
-		 * `selectionchange` native events.
-		 *
-		 * @param {Event} event
+		 * `selectionchange` event.
 		 */
-		function handleSelectionChange( event ) {
-			const {
-				record,
-				applyRecord,
-				createRecord,
-				isSelected,
-				onSelectionChange,
-			} = propsRef.current;
+		function handleSelectionChange() {
+			const { record, applyRecord, createRecord, onSelectionChange } =
+				propsRef.current;
 
 			// Check if the implementor disabled editing. `contentEditable`
 			// does disable input, but not text selection, so we must ignore
@@ -178,10 +171,6 @@ export function useInputAndSelection( props ) {
 				return;
 			}
 
-			if ( event.type !== 'selectionchange' && ! isSelected ) {
-				return;
-			}
-
 			// In case of a keyboard event, ignore selection changes during
 			// composition.
 			if ( isComposing ) {
diff --git a/packages/rich-text/src/component/use-selection-change-compat.js b/packages/rich-text/src/component/use-selection-change-compat.js
index e2efd4f93cebb..41783c9e034d1 100644
--- a/packages/rich-text/src/component/use-selection-change-compat.js
+++ b/packages/rich-text/src/component/use-selection-change-compat.js
@@ -15,6 +15,14 @@ import { useRefEffect } from '@wordpress/compose';
 export function useSelectionChangeCompat() {
 	return useRefEffect( ( element ) => {
 		const { ownerDocument } = element;
+		const { defaultView } = ownerDocument;
+		const selection = defaultView.getSelection();
+
+		let range;
+
+		function getRange() {
+			return selection.rangeCount ? selection.getRangeAt( 0 ) : null;
+		}
 
 		function onDown( event ) {
 			const type = event.type === 'keydown' ? 'keyup' : 'pointerup';
@@ -30,12 +38,15 @@ export function useSelectionChangeCompat() {
 
 			function onUp() {
 				onCancel();
+				if ( range === getRange() ) return;
 				ownerDocument.dispatchEvent( new Event( 'selectionchange' ) );
 			}
 
 			ownerDocument.addEventListener( type, onUp );
 			ownerDocument.addEventListener( 'selectionchange', onCancel );
 			ownerDocument.addEventListener( 'input', onCancel );
+
+			range = getRange();
 		}
 
 		element.addEventListener( 'pointerdown', onDown );