diff --git a/packages/edit-site/src/components/global-styles/font-library-modal/context.js b/packages/edit-site/src/components/global-styles/font-library-modal/context.js
index 52a40d7dc9070..e02dcb2e8a0af 100644
--- a/packages/edit-site/src/components/global-styles/font-library-modal/context.js
+++ b/packages/edit-site/src/components/global-styles/font-library-modal/context.js
@@ -451,13 +451,13 @@ function FontLibraryProvider( { children } ) {
 		);
 
 		if ( isFaceActivated ) {
+			unloadFontFaceInBrowser( face, 'all' );
+		} else {
 			loadFontFaceInBrowser(
 				face,
 				getDisplaySrcFromFontFace( face?.src ),
 				'all'
 			);
-		} else {
-			unloadFontFaceInBrowser( face, 'all' );
 		}
 	};