From 58ecbd531140939d9bb6e3a434209f4bd2bebdf7 Mon Sep 17 00:00:00 2001
From: Manav Rathi <manav@mrmr.io>
Date: Mon, 2 Dec 2024 13:48:53 +0530
Subject: [PATCH] Pagination

---
 web/apps/auth/src/services/remote.ts          | 70 ++++++++++++++-----
 .../new/photos/services/user-entity/remote.ts |  9 +++
 2 files changed, 60 insertions(+), 19 deletions(-)

diff --git a/web/apps/auth/src/services/remote.ts b/web/apps/auth/src/services/remote.ts
index e312b70986..0ae02c93d0 100644
--- a/web/apps/auth/src/services/remote.ts
+++ b/web/apps/auth/src/services/remote.ts
@@ -68,7 +68,17 @@ const RemoteAuthenticatorEntityChange = z.object({
      * Will be `null` when isDeleted is true.
      */
     header: z.string().nullable(),
+    /**
+     * `true` if the corresponding entity was deleted.
+     */
     isDeleted: z.boolean(),
+    /**
+     * Epoch milliseconds when this entity was last updated.
+     *
+     * This value is suitable for being passed as the `sinceTime` in the diff
+     * requests to implement pagination.
+     */
+    updatedAt: z.number(),
 });
 
 /**
@@ -86,26 +96,48 @@ export const authenticatorEntityDiff = async (
             authenticatorKey,
         );
 
-    // Always fetch all data from server for now.
-    const params = new URLSearchParams({
-        sinceTime: "0",
-        limit: "2500",
-    });
-    const url = await apiURL("/authenticator/entity/diff");
-    const res = await fetch(`${url}?${params.toString()}`, {
-        headers: await authenticatedRequestHeaders(),
-    });
-    ensureOk(res);
-    const diff = z
-        .object({ diff: z.array(RemoteAuthenticatorEntityChange) })
-        .parse(await res.json()).diff;
+    // Fetch all the entities, paginating the requests.
+    const entities = new Map<
+        string,
+        { id: string; encryptedData: string; header: string }
+    >();
+    let sinceTime = 0;
+    const batchSize = 2500;
+
+    while (true) {
+        const params = new URLSearchParams({
+            sinceTime: `${sinceTime}`,
+            limit: `${batchSize}`,
+        });
+        const url = await apiURL("/authenticator/entity/diff");
+        const res = await fetch(`${url}?${params.toString()}`, {
+            headers: await authenticatedRequestHeaders(),
+        });
+        ensureOk(res);
+        const diff = z
+            .object({ diff: z.array(RemoteAuthenticatorEntityChange) })
+            .parse(await res.json()).diff;
+        if (diff.length == 0) break;
+
+        for (const change of diff) {
+            sinceTime = Math.max(sinceTime, change.updatedAt);
+            if (change.isDeleted) {
+                entities.delete(change.id);
+            } else {
+                entities.set(change.id, {
+                    id: change.id,
+                    encryptedData: change.encryptedData!,
+                    header: change.header!,
+                });
+            }
+        }
+    }
+
     return Promise.all(
-        diff
-            .filter((entity) => !entity.isDeleted)
-            .map(async ({ id, encryptedData, header }) => ({
-                id,
-                data: await decrypt(encryptedData!, header!),
-            })),
+        entities.values().map(async ({ id, encryptedData, header }) => ({
+            id,
+            data: await decrypt(encryptedData, header),
+        })),
     );
 };
 
diff --git a/web/packages/new/photos/services/user-entity/remote.ts b/web/packages/new/photos/services/user-entity/remote.ts
index b97ea7b467..d56a7fbff1 100644
--- a/web/packages/new/photos/services/user-entity/remote.ts
+++ b/web/packages/new/photos/services/user-entity/remote.ts
@@ -78,7 +78,16 @@ const RemoteUserEntityChange = z.object({
      * Will be `null` when isDeleted is true.
      */
     header: z.string().nullable(),
+    /**
+     * `true` if the corresponding entity was deleted.
+     */
     isDeleted: z.boolean(),
+    /**
+     * Epoch milliseconds when this entity was last updated.
+     *
+     * This value is suitable for being passed as the `sinceTime` in the diff
+     * requests to implement pagination.
+     */
     updatedAt: z.number(),
 });