diff --git a/x-pack/test/usage_collection/test_suites/application_usage/index.ts b/x-pack/test/usage_collection/test_suites/application_usage/index.ts
index 43d7714dfea1d..fc53c8ddf5ed3 100644
--- a/x-pack/test/usage_collection/test_suites/application_usage/index.ts
+++ b/x-pack/test/usage_collection/test_suites/application_usage/index.ts
@@ -16,8 +16,14 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
     const browser = getService('browser');
 
     it('keys in the schema match the registered application IDs', async () => {
-      await common.navigateToApp('home'); // Navigate to Home to make sure all the appIds are loaded
+      await common.navigateToApp('home'); // Navigate to Home
+      await common.isChromeVisible(); // Make sure the page is fully loaded
       const appIds = await browser.execute(() => window.__applicationIds__);
+      if (!appIds || !Array.isArray(appIds)) {
+        throw new Error(
+          'Failed to retrieve all the existing applications in Kibana. Did it fail to boot or to navigate to home?'
+        );
+      }
       try {
         expect(Object.keys(applicationUsageSchema).sort()).to.eql(appIds.sort());
       } catch (err) {