diff --git a/js/src/components/contact-information/phone-number-card/verification-code-control.js b/js/src/components/contact-information/phone-number-card/verification-code-control.js
index 7beb35f480..c281f33628 100644
--- a/js/src/components/contact-information/phone-number-card/verification-code-control.js
+++ b/js/src/components/contact-information/phone-number-card/verification-code-control.js
@@ -56,13 +56,13 @@ export default function VerificationCodeControl( {
 	};
 
 	const handleKeyDown = ( e ) => {
-		const { dataset, selectionStart, value } = e.target;
+		const { dataset, selectionStart, selectionEnd, value } = e.target;
 		const idx = Number( dataset.idx );
 
 		switch ( e.keyCode ) {
 			case KEY_CODE_LEFT:
 			case KEY_CODE_BACKSPACE:
-				if ( selectionStart === 0 ) {
+				if ( selectionStart === 0 && selectionEnd === 0 ) {
 					maybeMoveFocus( idx - 1 );
 				}
 				break;