feat: homepage pagination and local db seeding (#221)

* run simple seed submission script

* add nullable safety to prevent bad access for non-existent user and enable display of raw item userId

* rename db operation with db prefix and add reset to task list

* enable simple pagination and page size

* use page param for future prs

* fix spacing for earlier stripe task and remove accidental import statement

* revert item summary changes

* address pr comments

* lint and var name fix

* do not error when user does not exist

* revert component changes and add users and scores to db to self-contain script - also add tool to print kv

* update README.md

* fix bad checkout from forked main vs upstream main

* add avatar url for dummy users with guest profile pic from gravatar

README.md

@@ -83,6 +83,10 @@ stripe listen --forward-to localhost:8000/api/stripe-webhooks --events=customer.
 
 4. Copy the webhook signing secret to [.env](.env) as `STRIPE_WEBHOOK_SECRET`.
 
+> Note: You can use
+> [Stripe's test credit cards](https://stripe.com/docs/testing) to make test
+> payments while in Stripe's test mode.
+
 ### Running the Server
 
 Finally, start the server by running:
@@ -94,9 +98,34 @@ deno task start
 Go to [http://localhost:8000](http://localhost:8000) to begin playing with your
 new SaaS app.
 
-> Note: You can use
-> [Stripe's test credit cards](https://stripe.com/docs/testing) to make test
-> payments while in Stripe's test mode.
+### Bootstrapping your local Database (Optional)
+
+If the home page is feeling a little empty, run
+
+```
+deno task db:seed
+```
+
+On execution, this script will fetch 20 (customizable) of the top HN posts using
+the [HackerNews API](https://github.com/HackerNews/API) to populate your home
+page.
+
+To see all the values in your local Deno KV database, run
+
+```
+deno task db:dump
+```
+
+And all kv pairs will be logged to stdout
+
+To reset your Deno KV database, run
+
+```
+deno task db:reset
+```
+
+Since this operation is not recoverable, you will be prompted to confirm
+deletion before proceeding.
 
 ## Customization
 

deno.json

@@ -1,7 +1,10 @@
 {
   "lock": false,
   "tasks": {
-    "init:stripe": "deno run --allow-read --allow-env --allow-net tools/init_stripe.ts ",
+    "init:stripe": "deno run --allow-read --allow-env --allow-net tools/init_stripe.ts",
+    "db:dump": "deno run --allow-read --allow-env --unstable tools/dump_kv.ts",
+    "db:seed": "deno run --allow-read --allow-env --allow-net --unstable tools/seed_submissions.ts",
+    "db:reset": "deno run --allow-read --allow-env --unstable tools/reset_kv.ts",
     "start": "deno run --unstable -A --watch=static/,routes/ dev.ts",
     "test": "deno test -A --unstable",
     "check:license": "deno run --allow-read --allow-write tools/check_license.ts",

routes/index.tsx

@@ -17,6 +17,7 @@ import {
 interface HomePageData extends State {
   users: User[];
   items: Item[];
+  cursor?: string;
   areVoted: boolean[];
 }
 
@@ -33,23 +34,28 @@ export function compareScore(a: Item, b: Item) {
 }
 
 export const handler: Handlers<HomePageData, State> = {
-  async GET(_req, ctx) {
+  async GET(req, ctx) {
     /** @todo Add pagination functionality */
-    const items = (await getAllItems({ limit: 10 })).sort(compareScore);
+    const start = new URL(req.url).searchParams.get("page") || undefined;
+    const { items, cursor } = await getAllItems({ limit: 10, cursor: start });
+    items.sort(compareScore);
     const users = await getUsersByIds(items.map((item) => item.userId));
     let votedItemIds: string[] = [];
     if (ctx.state.sessionId) {
       const sessionUser = await getUserBySessionId(ctx.state.sessionId!);
-      votedItemIds = await getVotedItemIdsByUser(sessionUser!.id);
+      if (sessionUser) {
+        votedItemIds = await getVotedItemIdsByUser(sessionUser!.id);
+      }
     }
-
     /** @todo Optimise */
     const areVoted = items.map((item) => votedItemIds.includes(item.id));
-    return ctx.render({ ...ctx.state, items, users, areVoted });
+    return ctx.render({ ...ctx.state, items, cursor, users, areVoted });
   },
 };
 
 export default function HomePage(props: PageProps<HomePageData>) {
+  const nextPageUrl = new URL(props.url);
+  nextPageUrl.searchParams.set("page", props.data.cursor || "");
   return (
     <>
       <Head href={props.url.href} />
@@ -62,6 +68,11 @@ export default function HomePage(props: PageProps<HomePageData>) {
               user={props.data.users[index]}
             />
           ))}
+          {props.data?.cursor && (
+            <div class="mt-4 text-gray-500">
+              <a href={nextPageUrl.toString()}>More</a>
+            </div>
+          )}
         </div>
       </Layout>
     </>

tools/dump_kv.ts

@@ -0,0 +1,18 @@
+// Copyright 2023 the Deno authors. All rights reserved. MIT license.
+// Description: Prints kv to stdout
+// Usage: deno run -A --unstable tools/dump_kv.ts
+import { kv } from "@/utils/db.ts";
+
+export async function dumpKv() {
+  const iter = kv.list({ prefix: [] });
+  const items = [];
+  for await (const res of iter) {
+    items.push({ [res.key.toString()]: res.value });
+  }
+  console.log(`${JSON.stringify(items, null, 2)}`);
+}
+
+if (import.meta.main) {
+  await dumpKv();
+  await kv.close();
+}

tools/seed_submissions.ts

@@ -0,0 +1,106 @@
+// Copyright 2023 the Deno authors. All rights reserved. MIT license.
+// Description: Seeds the kv db with Hacker News stories
+import { createItem, createUser, type Item, kv } from "@/utils/db.ts";
+
+// Reference: https://github.com/HackerNews/API
+const API_BASE_URL = `https://hacker-news.firebaseio.com/v0`;
+
+interface Story {
+  id: number;
+  score: number;
+  time: number; // Unix seconds
+  by: string;
+  title: string;
+  url: string;
+}
+
+function* batchify<T>(arr: T[], n = 5): Generator<T[], void> {
+  for (let i = 0; i < arr.length; i += n) {
+    yield arr.slice(i, i + n);
+  }
+}
+
+// Fetches the top 500 HN stories to seed the db
+async function fetchTopStoryIds() {
+  const resp = await fetch(`${API_BASE_URL}/topstories.json`);
+  if (!resp.ok) {
+    console.error(`Failed to fetchTopStoryIds - status: ${resp.status}`);
+    return;
+  }
+  return await resp.json();
+}
+
+async function fetchStory(id: number | string) {
+  const resp = await fetch(`${API_BASE_URL}/item/${id}.json`);
+  if (!resp.ok) {
+    console.error(`Failed to fetchStory (${id}) - status: ${resp.status}`);
+    return;
+  }
+  return await resp.json();
+}
+
+async function fetchTopStories(limit = 10) {
+  const ids = await fetchTopStoryIds();
+  if (!(ids && ids.length)) {
+    console.error(`No ids to fetch!`);
+    return;
+  }
+  const filtered: [number] = ids.slice(0, limit);
+  const stories: Story[] = [];
+  for (const batch of batchify(filtered)) {
+    stories.push(...(await Promise.all(batch.map((id) => fetchStory(id))))
+      .filter((v) => Boolean(v)) as Story[]);
+  }
+  return stories;
+}
+
+async function createItemWithScore(item: Item) {
+  const res = await createItem(item);
+  return await kv.set(["items", res!.id], {
+    ...res,
+    score: item.score,
+    createdAt: item.createdAt,
+  });
+}
+
+async function seedSubmissions(stories: Story[]) {
+  const items = stories.map(({ by: userId, title, url, score, time }) => {
+    return {
+      userId,
+      title,
+      url,
+      score,
+      createdAt: new Date(time * 1000),
+    } as Item;
+  }).filter(({ url }) => url);
+  for (const batch of batchify(items)) {
+    await Promise.all(batch.map((item) => createItemWithScore(item)));
+  }
+  return items;
+}
+
+async function main(limit = 20) {
+  const stories = await fetchTopStories(limit);
+  if (!(stories && stories.length)) {
+    console.error(`No stories to seed!`);
+    return;
+  }
+  const items = await seedSubmissions(stories);
+
+  // Create dummy users to ensure each post has a corresponding user
+  for (const batch of batchify(items)) {
+    await Promise.allSettled(batch.map(({ userId: id }) =>
+      createUser({
+        id, // id must match userId for post
+        login: id,
+        avatarUrl: "https://www.gravatar.com/avatar/?d=mp&s=64",
+        stripeCustomerId: crypto.randomUUID(), // unique per userId
+        sessionId: crypto.randomUUID(), // unique per userId
+      }) // ignore errors if dummy user already exists
+    ));
+  }
+}
+
+if (import.meta.main) {
+  await main();
+}

utils/db.ts

@@ -43,7 +43,10 @@ export async function getAllItems(options?: Deno.KvListOptions) {
   const iter = await kv.list<Item>({ prefix: ["items"] }, options);
   const items = [];
   for await (const res of iter) items.push(res.value);
-  return items;
+  return {
+    items,
+    cursor: iter.cursor,
+  };
 }
 
 export async function getItemById(id: string) {