diff --git a/code/lib/manager-api/src/lib/stories.ts b/code/lib/manager-api/src/lib/stories.ts
index b48cbd51b877..c3c40ce513ed 100644
--- a/code/lib/manager-api/src/lib/stories.ts
+++ b/code/lib/manager-api/src/lib/stories.ts
@@ -49,7 +49,7 @@ export const transformSetStoriesStoryDataToStoriesHash = (
 ) =>
   transformStoryIndexToStoriesHash(transformSetStoriesStoryDataToPreparedStoryIndex(data), options);
 
-const transformSetStoriesStoryDataToPreparedStoryIndex = (
+export const transformSetStoriesStoryDataToPreparedStoryIndex = (
   stories: SetStoriesStoryData
 ): API_PreparedStoryIndex => {
   const entries: API_PreparedStoryIndex['entries'] = Object.entries(stories).reduce(
diff --git a/code/lib/manager-api/src/modules/refs.ts b/code/lib/manager-api/src/modules/refs.ts
index 02884665cf5a..20c3f6cc887e 100644
--- a/code/lib/manager-api/src/modules/refs.ts
+++ b/code/lib/manager-api/src/modules/refs.ts
@@ -8,10 +8,11 @@ import type {
   SetStoriesStoryData,
   API_IndexHash,
   API_StoryMapper,
+  StoryIndex,
 } from '@storybook/types';
 // eslint-disable-next-line import/no-cycle
 import {
-  transformSetStoriesStoryDataToStoriesHash,
+  transformSetStoriesStoryDataToPreparedStoryIndex,
   transformStoryIndexToStoriesHash,
 } from '../lib/stories';
 
@@ -104,8 +105,12 @@ async function handleRequest(
 
   try {
     const response = await request;
-    if (response === false || response === true) throw new Error('Unexpected boolean response');
-    if (!response.ok) throw new Error(`Unexpected response not OK: ${response.statusText}`);
+    if (response === false || response === true) {
+      throw new Error('Unexpected boolean response');
+    }
+    if (!response.ok) {
+      throw new Error(`Unexpected response not OK: ${response.statusText}`);
+    }
 
     const json = await response.json();
 
@@ -276,26 +281,33 @@ export const init: ModuleFn<SubAPI, SubState> = (
       if (singleStory) {
         return;
       }
+      // eslint-disable-next-line @typescript-eslint/naming-convention
+      let internal_index: StoryIndex | undefined;
+      let index: API_IndexHash | undefined;
+      const { filters } = store.getState();
       const { storyMapper = defaultStoryMapper } = provider.getConfig();
       const ref = api.getRefs()[id];
 
-      let index: API_IndexHash;
-      if (setStoriesData) {
-        index = transformSetStoriesStoryDataToStoriesHash(
-          map(setStoriesData, ref, { storyMapper }),
-          { provider, docsOptions, filters: {}, status: {} }
-        );
-      } else if (storyIndex) {
+      if (storyIndex || setStoriesData) {
+        internal_index = setStoriesData
+          ? transformSetStoriesStoryDataToPreparedStoryIndex(
+              map(setStoriesData, ref, { storyMapper })
+            )
+          : storyIndex;
+
         index = transformStoryIndexToStoriesHash(storyIndex, {
           provider,
           docsOptions,
-          filters: {},
+          filters,
           status: {},
         });
       }
-      if (index) index = addRefIds(index, ref);
 
-      api.updateRef(id, { index, ...rest });
+      if (index) {
+        index = addRefIds(index, ref);
+      }
+
+      api.updateRef(id, { ...ref, ...rest, index, internal_index });
     },
 
     updateRef: (id, data) => {
diff --git a/code/lib/manager-api/src/modules/stories.ts b/code/lib/manager-api/src/modules/stories.ts
index 72c4198a3863..5f96f2ed7d48 100644
--- a/code/lib/manager-api/src/modules/stories.ts
+++ b/code/lib/manager-api/src/modules/stories.ts
@@ -615,14 +615,29 @@ export const init: ModuleFn<SubAPI, SubState> = ({
       });
 
       await store.setState({ status: newStatus }, { persistence: 'session' });
+
       if (index) {
+        // We need to re-prepare the index
         await api.setIndex(index);
+
+        const refs = await fullAPI.getRefs();
+        Object.entries(refs).forEach(([refId, { internal_index, ...ref }]) => {
+          fullAPI.setRef(refId, { ...ref, storyIndex: internal_index }, true);
+        });
       }
     },
     experimental_setFilter: async (id, filterFunction) => {
       const { internal_index: index } = store.getState();
       await store.setState({ filters: { ...store.getState().filters, [id]: filterFunction } });
-      await api.setIndex(index);
+
+      if (index) {
+        await api.setIndex(index);
+
+        const refs = await fullAPI.getRefs();
+        Object.entries(refs).forEach(([refId, { internal_index, ...ref }]) => {
+          fullAPI.setRef(refId, { ...ref, storyIndex: internal_index }, true);
+        });
+      }
     },
   };
 
diff --git a/code/lib/manager-api/src/tests/refs.test.ts b/code/lib/manager-api/src/tests/refs.test.ts
index f0556d560ba3..d07275337b04 100644
--- a/code/lib/manager-api/src/tests/refs.test.ts
+++ b/code/lib/manager-api/src/tests/refs.test.ts
@@ -1,5 +1,9 @@
 import { global } from '@storybook/global';
+import type { StoryIndex } from 'lib/types/src';
+import type { State } from '..';
+import { transformStoryIndexToStoriesHash } from '../lib/stories';
 import { getSourceType, init as initRefs } from '../modules/refs';
+import type Store from '../store';
 
 const { fetch } = global;
 
@@ -40,6 +44,7 @@ const provider = {
 
 const store = {
   getState: jest.fn().mockReturnValue({
+    filters: {},
     refs: {
       fake: {
         id: 'fake',
@@ -51,6 +56,17 @@ const store = {
   setState: jest.fn((a: any) => {}),
 };
 
+function createMockStore(initialState: Partial<State> = {}) {
+  let state = initialState;
+  return {
+    getState: jest.fn(() => state),
+    setState: jest.fn((s: typeof state) => {
+      state = { ...state, ...s };
+      return Promise.resolve(state);
+    }),
+  } as any as Store;
+}
+
 interface ResponseResult {
   ok?: boolean;
   err?: Error;
@@ -279,6 +295,7 @@ describe('Refs API', () => {
 
         Please check your dev-tools network tab.",
               },
+              "internal_index": undefined,
               "title": "Fake",
               "type": "auto-inject",
               "url": "https://example.com",
@@ -347,6 +364,7 @@ describe('Refs API', () => {
 
         Please check your dev-tools network tab.",
               },
+              "internal_index": undefined,
               "title": "Fake",
               "type": "auto-inject",
               "url": "https://example.com",
@@ -479,6 +497,10 @@ describe('Refs API', () => {
             "fake": Object {
               "id": "fake",
               "index": Object {},
+              "internal_index": Object {
+                "entries": Object {},
+                "v": 4,
+              },
               "title": "Fake",
               "type": "lazy",
               "url": "https://example.com",
@@ -493,6 +515,10 @@ describe('Refs API', () => {
             "fake": Object {
               "id": "fake",
               "index": Object {},
+              "internal_index": Object {
+                "entries": Object {},
+                "v": 4,
+              },
               "title": "Fake",
               "type": "lazy",
               "url": "https://example.com",
@@ -568,6 +594,10 @@ describe('Refs API', () => {
             "fake": Object {
               "id": "fake",
               "index": Object {},
+              "internal_index": Object {
+                "entries": Object {},
+                "v": 4,
+              },
               "title": "Fake",
               "type": "lazy",
               "url": "https://example.com",
@@ -645,6 +675,10 @@ describe('Refs API', () => {
             "fake": Object {
               "id": "fake",
               "index": Object {},
+              "internal_index": Object {
+                "entries": Object {},
+                "v": 4,
+              },
               "title": "Fake",
               "type": "lazy",
               "url": "https://example.com",
@@ -722,6 +756,7 @@ describe('Refs API', () => {
             "fake": Object {
               "id": "fake",
               "index": undefined,
+              "internal_index": undefined,
               "loginUrl": "https://example.com/login",
               "title": "Fake",
               "type": "auto-inject",
@@ -863,6 +898,7 @@ describe('Refs API', () => {
             "fake": Object {
               "id": "fake",
               "index": undefined,
+              "internal_index": undefined,
               "loginUrl": "https://example.com/login",
               "title": "Fake",
               "type": "auto-inject",
@@ -944,6 +980,10 @@ describe('Refs API', () => {
             "fake": Object {
               "id": "fake",
               "index": Object {},
+              "internal_index": Object {
+                "entries": Object {},
+                "v": 4,
+              },
               "title": "Fake",
               "type": "lazy",
               "url": "https://example.com",
@@ -1021,6 +1061,10 @@ describe('Refs API', () => {
             "fake": Object {
               "id": "fake",
               "index": Object {},
+              "internal_index": Object {
+                "entries": Object {},
+                "v": 4,
+              },
               "title": "Fake",
               "type": "lazy",
               "url": "https://example.com",
@@ -1152,6 +1196,71 @@ describe('Refs API', () => {
     });
   });
 
+  describe('setRef', () => {
+    it('can filter', async () => {
+      const index: StoryIndex = {
+        v: 4,
+        entries: {
+          'a--1': {
+            id: 'a--1',
+            title: 'A',
+            name: '1',
+            importPath: './path/to/a1.ts',
+            type: 'story',
+          },
+          'a--2': {
+            id: 'a--2',
+            title: 'A',
+            name: '2',
+            importPath: './path/to/a2.ts',
+            type: 'story',
+          },
+        },
+      };
+
+      const initialState: Partial<State> = {
+        refs: {
+          fake: {
+            id: 'fake',
+            url: 'https://example.com',
+            previewInitialized: true,
+            index: transformStoryIndexToStoriesHash(index, {
+              provider: provider as any,
+              docsOptions: {},
+              filters: {},
+              status: {},
+            }),
+            internal_index: index,
+          },
+        },
+      };
+      // eslint-disable-next-line @typescript-eslint/no-shadow
+      const store = createMockStore(initialState);
+      const { api } = initRefs({ provider, store } as any, { runCheck: false });
+
+      await expect(api.getRefs().fake.index).toEqual(
+        expect.objectContaining({ 'a--2': expect.anything() })
+      );
+
+      const stateWithFilters: Partial<State> = {
+        filters: {
+          fake: (a) => a.name.includes('1'),
+        },
+      };
+
+      await store.setState(stateWithFilters);
+
+      await api.setRef('fake', { storyIndex: index });
+
+      await expect(api.getRefs().fake.index).toEqual(
+        expect.objectContaining({ 'a--1': expect.anything() })
+      );
+      await expect(api.getRefs().fake.index).not.toEqual(
+        expect.objectContaining({ 'a--2': expect.anything() })
+      );
+    });
+  });
+
   it('errors on unknown version', async () => {
     // given
     const { api } = initRefs({ provider, store } as any, { runCheck: false });
diff --git a/code/lib/manager-api/src/tests/stories.test.ts b/code/lib/manager-api/src/tests/stories.test.ts
index 7733f569e5c5..a07d6d202171 100644
--- a/code/lib/manager-api/src/tests/stories.test.ts
+++ b/code/lib/manager-api/src/tests/stories.test.ts
@@ -72,7 +72,7 @@ function createMockModuleArgs({
   const store = createMockStore({ filters: {}, status: {}, ...initialState });
   const provider = createMockProvider();
 
-  return { navigate, store, provider, fullAPI };
+  return { navigate, store, provider, fullAPI: { ...fullAPI, getRefs: () => ({}) } };
 }
 
 describe('stories API', () => {
diff --git a/code/lib/types/src/modules/api.ts b/code/lib/types/src/modules/api.ts
index 434bd32b378f..be510b4bdfe4 100644
--- a/code/lib/types/src/modules/api.ts
+++ b/code/lib/types/src/modules/api.ts
@@ -175,6 +175,8 @@ export interface API_ComposedRef extends API_LoadedRefData {
   versions?: API_Versions;
   loginUrl?: string;
   version?: string;
+  /** DO NOT USE THIS */
+  internal_index?: StoryIndex;
 }
 
 export type API_ComposedRefUpdate = Partial<
@@ -189,6 +191,7 @@ export type API_ComposedRefUpdate = Partial<
     | 'version'
     | 'indexError'
     | 'previewInitialized'
+    | 'internal_index'
   >
 >;
 
diff --git a/code/ui/manager/src/components/sidebar/RefBlocks.tsx b/code/ui/manager/src/components/sidebar/RefBlocks.tsx
index 943e2ac9bfa6..efeb56c2e6d9 100644
--- a/code/ui/manager/src/components/sidebar/RefBlocks.tsx
+++ b/code/ui/manager/src/components/sidebar/RefBlocks.tsx
@@ -151,7 +151,10 @@ export const EmptyBlock: FC<any> = ({ isMain }) => (
               </ul>{' '}
             </>
           ) : (
-            <>Yikes! Something went wrong loading these stories.</>
+            <>
+              This composed storybook is empty, maybe you're using filter-functions, and all stories
+              are filtered away.
+            </>
           )}
         </Text>
       </WideSpaced>
diff --git a/code/ui/manager/src/components/sidebar/Sidebar.stories.tsx b/code/ui/manager/src/components/sidebar/Sidebar.stories.tsx
index 1d9c02d914a4..3e7b7e2b23f3 100644
--- a/code/ui/manager/src/components/sidebar/Sidebar.stories.tsx
+++ b/code/ui/manager/src/components/sidebar/Sidebar.stories.tsx
@@ -67,6 +67,14 @@ const refsError = {
   },
 };
 
+const refsEmpty = {
+  optimized: {
+    ...refs.optimized,
+    // type: 'auto-inject',
+    index: {} as IndexHash,
+  },
+};
+
 export const Simple: Story = {
   args: { previewInitialized: true },
   render: (args) => (
@@ -186,6 +194,24 @@ export const LoadingWithRefError: Story = {
   ),
 };
 
+export const WithRefEmpty: Story = {
+  args: {
+    previewInitialized: true,
+  },
+  render: (args) => (
+    <Sidebar
+      {...args}
+      menu={menu}
+      extra={[]}
+      index={{}}
+      storyId={storyId}
+      refId={DEFAULT_REF_ID}
+      refs={refsEmpty}
+      status={{}}
+    />
+  ),
+};
+
 export const StatusesCollapsed: Story = {
   args: {
     previewInitialized: true,