From 148dc85b981ef2b132677e1a7764c697cf0f316c Mon Sep 17 00:00:00 2001 From: Zach Zeleznick Date: Sat, 20 May 2023 18:36:19 -0700 Subject: [PATCH 01/14] run simple seed submission script --- deno.json | 3 +- tools/seed_submissions.ts | 81 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 tools/seed_submissions.ts diff --git a/deno.json b/deno.json index a5135cc10..449e22d93 100644 --- a/deno.json +++ b/deno.json @@ -2,6 +2,7 @@ "lock": false, "tasks": { "init:stripe": "deno run --allow-read --allow-env --allow-net tools/init_stripe.ts ", + "init:submissions": "deno run --allow-read --allow-env --allow-net --unstable tools/seed_submissions.ts", "start": "deno run --unstable -A --watch=static/,routes/ dev.ts", "test": "deno test -A --unstable", "check:license": "deno run --allow-read --allow-write tools/check_license.ts", @@ -12,4 +13,4 @@ "jsx": "react-jsx", "jsxImportSource": "preact" } -} +} \ No newline at end of file diff --git a/tools/seed_submissions.ts b/tools/seed_submissions.ts new file mode 100644 index 000000000..18f04361f --- /dev/null +++ b/tools/seed_submissions.ts @@ -0,0 +1,81 @@ +// Copyright 2023 the Deno authors. All rights reserved. MIT license. + +import { createItem } from "@/utils/db.ts"; + +// Reference: https://github.com/HackerNews/API +const API_BASE_URL = `https://hacker-news.firebaseio.com/v0` + +interface Story { + id: number; + score: number; + time: number; + by: string; + title: string; + url: string; +} + +// Fetch the top 500 HN stories to seed the db +const fetchTopStoryIds = async () => { + const resp = await fetch(`${API_BASE_URL}/topstories.json`); + if (!resp.ok) { + console.error(`Failed to fetchTopStoryIds - status: ${resp.status}`) + return + } + return await resp.json(); +} + +const fetchStory = async (id: number | string) => { + const resp = await fetch(`${API_BASE_URL}/item/${id}.json`); + if (!resp.ok) { + console.error(`Failed to fetchStory (${id}) - status: ${resp.status}`) + return + } + return await resp.json(); +} + +function* batchify(arr: T[], n = 5): Generator { + for (let i = 0; i < arr.length; i += n) { + yield arr.slice(i, i + n); + } +} + +const fetchTopStories = async (limit = 10) => { + const ids = await fetchTopStoryIds(); + if (!(ids && ids.length)) { + console.error(`No ids to fetch!`) + return + } + const filtered: [number] = ids.slice(0, limit); + const stories: Story[] = []; + for (const batch of batchify(filtered)) { + stories.push(...(await Promise.all(batch.map(id => fetchStory(id)))) + .filter(v => Boolean(v)) as Story[]) + } + return stories +} + +const seedSubmissions = async (stories: Story[]) => { + const items = stories.map(({ by: userId, title, url }) => { + return { userId, title, url } + }) + for (const batch of batchify(items)) { + await Promise.all(batch.map(item => createItem(item))) + } +} + +async function main(limit = 10) { + const start = performance.now(); + const stories = await fetchTopStories(limit); + console.log(`Fetching ${limit} stories took ${Math.floor(performance.now() - start)} ms`); + if (!(stories && stories.length)) { + console.error(`No stories to seed!`) + return + } + const seedStart = performance.now(); + await seedSubmissions(stories); + console.log(`Submitting ${stories.length} stories took ${Math.floor(performance.now() - seedStart)} ms`); +} + +if (import.meta.main) { + await main(); +} From 1d0bb48f6bd1b7d483a1504e684cea1f13439066 Mon Sep 17 00:00:00 2001 From: Zach Zeleznick Date: Sat, 20 May 2023 18:37:04 -0700 Subject: [PATCH 02/14] add nullable safety to prevent bad access for non-existent user and enable display of raw item userId --- components/ItemSummary.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/ItemSummary.tsx b/components/ItemSummary.tsx index ef8d1cceb..fa8f5c7f3 100644 --- a/components/ItemSummary.tsx +++ b/components/ItemSummary.tsx @@ -39,7 +39,7 @@ export default function ItemSummary(props: ItemSummaryProps) {

- {props.user.login}{" "} + {props.user?.login || props.item?.userId}{" "} {props.user?.isSubscribed && ( 🦕{" "} )} From 0dda90a3dcb61db2c45511219b1a3a5b665daf4a Mon Sep 17 00:00:00 2001 From: Zach Zeleznick Date: Sat, 20 May 2023 19:47:47 -0700 Subject: [PATCH 03/14] rename db operation with db prefix and add reset to task list --- deno.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deno.json b/deno.json index 449e22d93..d067aa565 100644 --- a/deno.json +++ b/deno.json @@ -2,7 +2,8 @@ "lock": false, "tasks": { "init:stripe": "deno run --allow-read --allow-env --allow-net tools/init_stripe.ts ", - "init:submissions": "deno run --allow-read --allow-env --allow-net --unstable tools/seed_submissions.ts", + "db:seed": "deno run --allow-read --allow-env --allow-net --unstable tools/seed_submissions.ts", + "db:reset": "deno run --allow-read --allow-env --unstable tools/reset_kv.ts", "start": "deno run --unstable -A --watch=static/,routes/ dev.ts", "test": "deno test -A --unstable", "check:license": "deno run --allow-read --allow-write tools/check_license.ts", From db69bf56a8d967719ec277d71231eb141b4907e5 Mon Sep 17 00:00:00 2001 From: Zach Zeleznick Date: Sat, 20 May 2023 19:48:15 -0700 Subject: [PATCH 04/14] enable simple pagination and page size --- components/ItemSummary.tsx | 11 ++++++++++- routes/index.tsx | 20 ++++++++++++++++---- tools/seed_submissions.ts | 10 ++-------- utils/db.ts | 20 +++++++++++++++++--- 4 files changed, 45 insertions(+), 16 deletions(-) diff --git a/components/ItemSummary.tsx b/components/ItemSummary.tsx index fa8f5c7f3..a3d26a530 100644 --- a/components/ItemSummary.tsx +++ b/components/ItemSummary.tsx @@ -20,7 +20,16 @@ export interface ItemSummaryProps { isVoted: boolean; } +const extractHost = (url: string) => { + try { + return new URL(url).host + } catch { + return + } +} + export default function ItemSummary(props: ItemSummaryProps) { + const host = extractHost(props.item.url) || '' return (

- {new URL(props.item.url).host} ↗ + {host} ↗

diff --git a/routes/index.tsx b/routes/index.tsx index 9f133da58..8be8f24f3 100644 --- a/routes/index.tsx +++ b/routes/index.tsx @@ -17,6 +17,7 @@ import { interface HomePageData extends State { users: User[]; items: Item[]; + cursor?: string; areVoted: boolean[]; } @@ -33,23 +34,29 @@ export function compareScore(a: Item, b: Item) { } export const handler: Handlers = { - async GET(_req, ctx) { + async GET(req, ctx) { /** @todo Add pagination functionality */ - const items = (await getAllItems({ limit: 10 })).sort(compareScore); + const { searchParams } = new URL(req.url); + const limit = Math.min(30, ~~(searchParams.get("limit") || 10)); + const start = decodeURI(searchParams.get("start") || ''); + const { items: itemsRaw, cursor } = await getAllItems({ limit, cursor: start }); + const items = itemsRaw.sort(compareScore); const users = await getUsersByIds(items.map((item) => item.userId)); let votedItemIds: string[] = []; if (ctx.state.sessionId) { const sessionUser = await getUserBySessionId(ctx.state.sessionId!); votedItemIds = await getVotedItemIdsByUser(sessionUser!.id); } - + console.log(`cursor: ${cursor}`); /** @todo Optimise */ const areVoted = items.map((item) => votedItemIds.includes(item.id)); - return ctx.render({ ...ctx.state, items, users, areVoted }); + return ctx.render({ ...ctx.state, items, cursor, users, areVoted }); }, }; export default function HomePage(props: PageProps) { + const nextUrl = new URL(props.url); + nextUrl.searchParams.set('start', props.data?.cursor || ''); return ( <> @@ -62,6 +69,11 @@ export default function HomePage(props: PageProps) { user={props.data.users[index]} /> ))} + {props.data?.cursor && ( +

+ More +
+ )}
diff --git a/tools/seed_submissions.ts b/tools/seed_submissions.ts index 18f04361f..553af85e4 100644 --- a/tools/seed_submissions.ts +++ b/tools/seed_submissions.ts @@ -1,6 +1,6 @@ // Copyright 2023 the Deno authors. All rights reserved. MIT license. -import { createItem } from "@/utils/db.ts"; +import { batchify, createItem } from "@/utils/db.ts"; // Reference: https://github.com/HackerNews/API const API_BASE_URL = `https://hacker-news.firebaseio.com/v0` @@ -33,12 +33,6 @@ const fetchStory = async (id: number | string) => { return await resp.json(); } -function* batchify(arr: T[], n = 5): Generator { - for (let i = 0; i < arr.length; i += n) { - yield arr.slice(i, i + n); - } -} - const fetchTopStories = async (limit = 10) => { const ids = await fetchTopStoryIds(); if (!(ids && ids.length)) { @@ -77,5 +71,5 @@ async function main(limit = 10) { } if (import.meta.main) { - await main(); + await main(50); } diff --git a/utils/db.ts b/utils/db.ts index a90539ed0..aa4465575 100644 --- a/utils/db.ts +++ b/utils/db.ts @@ -1,4 +1,5 @@ // Copyright 2023 the Deno authors. All rights reserved. MIT license. +import { resolve } from "https://deno.land/std@0.173.0/path/win32.ts"; import { AssertionError } from "https://deno.land/std@0.186.0/testing/asserts.ts"; export const kv = await Deno.openKv(); @@ -43,7 +44,10 @@ export async function getAllItems(options?: Deno.KvListOptions) { const iter = await kv.list({ prefix: ["items"] }, options); const items = []; for await (const res of iter) items.push(res.value); - return items; + return { + items, + cursor: iter.cursor, + }; } export async function getItemById(id: string) { @@ -409,6 +413,16 @@ export async function deleteUserBySession(sessionId: string) { export async function getUsersByIds(ids: string[]) { const keys = ids.map((id) => ["users", id]); - const res = await kv.getMany(keys); - return res.map((entry) => entry.value!); + // NOTE: limit of 10 for getMany or `TypeError: too many ranges (max 10)` + const users: User[] = []; + for (const batch of batchify(keys, 10)) { + users.push(...(await kv.getMany(batch)).map((entry) => entry.value!)) + } + return users +} + +export function* batchify(arr: T[], n = 5): Generator { + for (let i = 0; i < arr.length; i += n) { + yield arr.slice(i, i + n); + } } From 58907ab115ea686a25729d3cae9da088b8f7c6b3 Mon Sep 17 00:00:00 2001 From: Zach Zeleznick Date: Sat, 20 May 2023 19:56:37 -0700 Subject: [PATCH 05/14] use page param for future prs --- routes/index.tsx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/routes/index.tsx b/routes/index.tsx index 8be8f24f3..2cd61999a 100644 --- a/routes/index.tsx +++ b/routes/index.tsx @@ -38,7 +38,7 @@ export const handler: Handlers = { /** @todo Add pagination functionality */ const { searchParams } = new URL(req.url); const limit = Math.min(30, ~~(searchParams.get("limit") || 10)); - const start = decodeURI(searchParams.get("start") || ''); + const start = decodeURI(searchParams.get("page") || ''); const { items: itemsRaw, cursor } = await getAllItems({ limit, cursor: start }); const items = itemsRaw.sort(compareScore); const users = await getUsersByIds(items.map((item) => item.userId)); @@ -47,7 +47,6 @@ export const handler: Handlers = { const sessionUser = await getUserBySessionId(ctx.state.sessionId!); votedItemIds = await getVotedItemIdsByUser(sessionUser!.id); } - console.log(`cursor: ${cursor}`); /** @todo Optimise */ const areVoted = items.map((item) => votedItemIds.includes(item.id)); return ctx.render({ ...ctx.state, items, cursor, users, areVoted }); @@ -56,7 +55,7 @@ export const handler: Handlers = { export default function HomePage(props: PageProps) { const nextUrl = new URL(props.url); - nextUrl.searchParams.set('start', props.data?.cursor || ''); + nextUrl.searchParams.set('page', props.data?.cursor || ''); return ( <> From 7e35c2269620ede5f43114938a86089c6e4431af Mon Sep 17 00:00:00 2001 From: Zach Zeleznick Date: Sat, 20 May 2023 19:58:51 -0700 Subject: [PATCH 06/14] fix spacing for earlier stripe task and remove accidental import statement --- deno.json | 2 +- utils/db.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/deno.json b/deno.json index d067aa565..b09102cbc 100644 --- a/deno.json +++ b/deno.json @@ -1,7 +1,7 @@ { "lock": false, "tasks": { - "init:stripe": "deno run --allow-read --allow-env --allow-net tools/init_stripe.ts ", + "init:stripe": "deno run --allow-read --allow-env --allow-net tools/init_stripe.ts", "db:seed": "deno run --allow-read --allow-env --allow-net --unstable tools/seed_submissions.ts", "db:reset": "deno run --allow-read --allow-env --unstable tools/reset_kv.ts", "start": "deno run --unstable -A --watch=static/,routes/ dev.ts", diff --git a/utils/db.ts b/utils/db.ts index aa4465575..be8050a57 100644 --- a/utils/db.ts +++ b/utils/db.ts @@ -1,5 +1,4 @@ // Copyright 2023 the Deno authors. All rights reserved. MIT license. -import { resolve } from "https://deno.land/std@0.173.0/path/win32.ts"; import { AssertionError } from "https://deno.land/std@0.186.0/testing/asserts.ts"; export const kv = await Deno.openKv(); From fb2622b79c01a409a47c0debbdb22555e22a44c7 Mon Sep 17 00:00:00 2001 From: Zach Zeleznick Date: Mon, 22 May 2023 09:42:09 -0700 Subject: [PATCH 07/14] revert item summary changes --- components/ItemSummary.tsx | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/components/ItemSummary.tsx b/components/ItemSummary.tsx index a3d26a530..ef8d1cceb 100644 --- a/components/ItemSummary.tsx +++ b/components/ItemSummary.tsx @@ -20,16 +20,7 @@ export interface ItemSummaryProps { isVoted: boolean; } -const extractHost = (url: string) => { - try { - return new URL(url).host - } catch { - return - } -} - export default function ItemSummary(props: ItemSummaryProps) { - const host = extractHost(props.item.url) || '' return (
- {host} ↗ + {new URL(props.item.url).host} ↗

- {props.user?.login || props.item?.userId}{" "} + {props.user.login}{" "} {props.user?.isSubscribed && ( 🦕{" "} )} From 9ac0be18260af6a9e32693bf9e57bf485164ac31 Mon Sep 17 00:00:00 2001 From: Zach Zeleznick Date: Mon, 22 May 2023 09:59:36 -0700 Subject: [PATCH 08/14] address pr comments --- routes/index.tsx | 14 ++++++-------- tools/seed_submissions.ts | 31 ++++++++++++++++++++----------- utils/db.ts | 14 ++------------ 3 files changed, 28 insertions(+), 31 deletions(-) diff --git a/routes/index.tsx b/routes/index.tsx index 2cd61999a..a2f1600c6 100644 --- a/routes/index.tsx +++ b/routes/index.tsx @@ -36,16 +36,14 @@ export function compareScore(a: Item, b: Item) { export const handler: Handlers = { async GET(req, ctx) { /** @todo Add pagination functionality */ - const { searchParams } = new URL(req.url); - const limit = Math.min(30, ~~(searchParams.get("limit") || 10)); - const start = decodeURI(searchParams.get("page") || ''); - const { items: itemsRaw, cursor } = await getAllItems({ limit, cursor: start }); - const items = itemsRaw.sort(compareScore); + const start = new URL(req.url).searchParams.get("page") || undefined; + const { items, cursor } = await getAllItems({ limit: 10, cursor: start }); + items.sort(compareScore); const users = await getUsersByIds(items.map((item) => item.userId)); let votedItemIds: string[] = []; if (ctx.state.sessionId) { const sessionUser = await getUserBySessionId(ctx.state.sessionId!); - votedItemIds = await getVotedItemIdsByUser(sessionUser!.id); + if (sessionUser) votedItemIds = await getVotedItemIdsByUser(sessionUser!.id); } /** @todo Optimise */ const areVoted = items.map((item) => votedItemIds.includes(item.id)); @@ -54,8 +52,8 @@ export const handler: Handlers = { }; export default function HomePage(props: PageProps) { - const nextUrl = new URL(props.url); - nextUrl.searchParams.set('page', props.data?.cursor || ''); + const nextPageUrl = new URL(props.url); + nextPageUrl.searchParams.set('page', props.data.cursor || ''); return ( <> diff --git a/tools/seed_submissions.ts b/tools/seed_submissions.ts index 553af85e4..28137123e 100644 --- a/tools/seed_submissions.ts +++ b/tools/seed_submissions.ts @@ -1,6 +1,6 @@ // Copyright 2023 the Deno authors. All rights reserved. MIT license. -import { batchify, createItem } from "@/utils/db.ts"; +import { createItem } from "@/utils/db.ts"; // Reference: https://github.com/HackerNews/API const API_BASE_URL = `https://hacker-news.firebaseio.com/v0` @@ -14,8 +14,15 @@ interface Story { url: string; } +function* batchify(arr: T[], n = 5): Generator { + for (let i = 0; i < arr.length; i += n) { + yield arr.slice(i, i + n); + } +} + + // Fetch the top 500 HN stories to seed the db -const fetchTopStoryIds = async () => { +async function fetchTopStoryIds() { const resp = await fetch(`${API_BASE_URL}/topstories.json`); if (!resp.ok) { console.error(`Failed to fetchTopStoryIds - status: ${resp.status}`) @@ -24,7 +31,7 @@ const fetchTopStoryIds = async () => { return await resp.json(); } -const fetchStory = async (id: number | string) => { +async function fetchStory(id: number | string) { const resp = await fetch(`${API_BASE_URL}/item/${id}.json`); if (!resp.ok) { console.error(`Failed to fetchStory (${id}) - status: ${resp.status}`) @@ -33,7 +40,7 @@ const fetchStory = async (id: number | string) => { return await resp.json(); } -const fetchTopStories = async (limit = 10) => { +async function fetchTopStories(limit = 10) { const ids = await fetchTopStoryIds(); if (!(ids && ids.length)) { console.error(`No ids to fetch!`) @@ -48,28 +55,30 @@ const fetchTopStories = async (limit = 10) => { return stories } -const seedSubmissions = async (stories: Story[]) => { +async function seedSubmissions(stories: Story[]) { const items = stories.map(({ by: userId, title, url }) => { return { userId, title, url } + }).filter(({ url }) => { + try { + return Boolean(new URL(url).host) + } catch { + return + } }) for (const batch of batchify(items)) { await Promise.all(batch.map(item => createItem(item))) } } -async function main(limit = 10) { - const start = performance.now(); +async function main(limit = 20) { const stories = await fetchTopStories(limit); - console.log(`Fetching ${limit} stories took ${Math.floor(performance.now() - start)} ms`); if (!(stories && stories.length)) { console.error(`No stories to seed!`) return } - const seedStart = performance.now(); await seedSubmissions(stories); - console.log(`Submitting ${stories.length} stories took ${Math.floor(performance.now() - seedStart)} ms`); } if (import.meta.main) { - await main(50); + await main(); } diff --git a/utils/db.ts b/utils/db.ts index be8050a57..9116c1001 100644 --- a/utils/db.ts +++ b/utils/db.ts @@ -412,16 +412,6 @@ export async function deleteUserBySession(sessionId: string) { export async function getUsersByIds(ids: string[]) { const keys = ids.map((id) => ["users", id]); - // NOTE: limit of 10 for getMany or `TypeError: too many ranges (max 10)` - const users: User[] = []; - for (const batch of batchify(keys, 10)) { - users.push(...(await kv.getMany(batch)).map((entry) => entry.value!)) - } - return users -} - -export function* batchify(arr: T[], n = 5): Generator { - for (let i = 0; i < arr.length; i += n) { - yield arr.slice(i, i + n); - } + const res = await kv.getMany(keys); + return res.map((entry) => entry.value!); } From 2bf0a0bfc933102aa764fd1a36f4996c5284c79c Mon Sep 17 00:00:00 2001 From: Zach Zeleznick Date: Mon, 22 May 2023 10:02:08 -0700 Subject: [PATCH 09/14] lint and var name fix --- deno.json | 2 +- routes/index.tsx | 8 +-- tools/seed_submissions.ts | 105 +++++++++++++++++++------------------- 3 files changed, 58 insertions(+), 57 deletions(-) diff --git a/deno.json b/deno.json index b09102cbc..3780b6f00 100644 --- a/deno.json +++ b/deno.json @@ -14,4 +14,4 @@ "jsx": "react-jsx", "jsxImportSource": "preact" } -} \ No newline at end of file +} diff --git a/routes/index.tsx b/routes/index.tsx index a2f1600c6..ba49cd4c6 100644 --- a/routes/index.tsx +++ b/routes/index.tsx @@ -43,7 +43,9 @@ export const handler: Handlers = { let votedItemIds: string[] = []; if (ctx.state.sessionId) { const sessionUser = await getUserBySessionId(ctx.state.sessionId!); - if (sessionUser) votedItemIds = await getVotedItemIdsByUser(sessionUser!.id); + if (sessionUser) { + votedItemIds = await getVotedItemIdsByUser(sessionUser!.id); + } } /** @todo Optimise */ const areVoted = items.map((item) => votedItemIds.includes(item.id)); @@ -53,7 +55,7 @@ export const handler: Handlers = { export default function HomePage(props: PageProps) { const nextPageUrl = new URL(props.url); - nextPageUrl.searchParams.set('page', props.data.cursor || ''); + nextPageUrl.searchParams.set("page", props.data.cursor || ""); return ( <> @@ -68,7 +70,7 @@ export default function HomePage(props: PageProps) { ))} {props.data?.cursor && (

- More + More
)}
diff --git a/tools/seed_submissions.ts b/tools/seed_submissions.ts index 28137123e..73194eced 100644 --- a/tools/seed_submissions.ts +++ b/tools/seed_submissions.ts @@ -3,82 +3,81 @@ import { createItem } from "@/utils/db.ts"; // Reference: https://github.com/HackerNews/API -const API_BASE_URL = `https://hacker-news.firebaseio.com/v0` +const API_BASE_URL = `https://hacker-news.firebaseio.com/v0`; interface Story { - id: number; - score: number; - time: number; - by: string; - title: string; - url: string; + id: number; + score: number; + time: number; + by: string; + title: string; + url: string; } function* batchify(arr: T[], n = 5): Generator { - for (let i = 0; i < arr.length; i += n) { - yield arr.slice(i, i + n); - } + for (let i = 0; i < arr.length; i += n) { + yield arr.slice(i, i + n); + } } - // Fetch the top 500 HN stories to seed the db async function fetchTopStoryIds() { - const resp = await fetch(`${API_BASE_URL}/topstories.json`); - if (!resp.ok) { - console.error(`Failed to fetchTopStoryIds - status: ${resp.status}`) - return - } - return await resp.json(); + const resp = await fetch(`${API_BASE_URL}/topstories.json`); + if (!resp.ok) { + console.error(`Failed to fetchTopStoryIds - status: ${resp.status}`); + return; + } + return await resp.json(); } async function fetchStory(id: number | string) { - const resp = await fetch(`${API_BASE_URL}/item/${id}.json`); - if (!resp.ok) { - console.error(`Failed to fetchStory (${id}) - status: ${resp.status}`) - return - } - return await resp.json(); + const resp = await fetch(`${API_BASE_URL}/item/${id}.json`); + if (!resp.ok) { + console.error(`Failed to fetchStory (${id}) - status: ${resp.status}`); + return; + } + return await resp.json(); } async function fetchTopStories(limit = 10) { - const ids = await fetchTopStoryIds(); - if (!(ids && ids.length)) { - console.error(`No ids to fetch!`) - return - } - const filtered: [number] = ids.slice(0, limit); - const stories: Story[] = []; - for (const batch of batchify(filtered)) { - stories.push(...(await Promise.all(batch.map(id => fetchStory(id)))) - .filter(v => Boolean(v)) as Story[]) - } - return stories + const ids = await fetchTopStoryIds(); + if (!(ids && ids.length)) { + console.error(`No ids to fetch!`); + return; + } + const filtered: [number] = ids.slice(0, limit); + const stories: Story[] = []; + for (const batch of batchify(filtered)) { + stories.push(...(await Promise.all(batch.map((id) => fetchStory(id)))) + .filter((v) => Boolean(v)) as Story[]); + } + return stories; } async function seedSubmissions(stories: Story[]) { - const items = stories.map(({ by: userId, title, url }) => { - return { userId, title, url } - }).filter(({ url }) => { - try { - return Boolean(new URL(url).host) - } catch { - return - } - }) - for (const batch of batchify(items)) { - await Promise.all(batch.map(item => createItem(item))) + const items = stories.map(({ by: userId, title, url }) => { + return { userId, title, url }; + }).filter(({ url }) => { + try { + return Boolean(new URL(url).host); + } catch { + return; } + }); + for (const batch of batchify(items)) { + await Promise.all(batch.map((item) => createItem(item))); + } } async function main(limit = 20) { - const stories = await fetchTopStories(limit); - if (!(stories && stories.length)) { - console.error(`No stories to seed!`) - return - } - await seedSubmissions(stories); + const stories = await fetchTopStories(limit); + if (!(stories && stories.length)) { + console.error(`No stories to seed!`); + return; + } + await seedSubmissions(stories); } if (import.meta.main) { - await main(); + await main(); } From 641537b7ed8cde44f7547c07f443c54e51280521 Mon Sep 17 00:00:00 2001 From: Zach Zeleznick Date: Mon, 22 May 2023 16:18:56 -0700 Subject: [PATCH 10/14] do not error when user does not exist --- components/ItemSummary.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/ItemSummary.tsx b/components/ItemSummary.tsx index ef8d1cceb..68122459b 100644 --- a/components/ItemSummary.tsx +++ b/components/ItemSummary.tsx @@ -39,7 +39,7 @@ export default function ItemSummary(props: ItemSummaryProps) {

- {props.user.login}{" "} + {props.user?.login || props.item.userId}{" "} {props.user?.isSubscribed && ( 🦕{" "} )} From 7970c9a34e9e3a52d3d563c69bf307ba2db752cf Mon Sep 17 00:00:00 2001 From: Zach Zeleznick Date: Tue, 23 May 2023 20:56:02 -0700 Subject: [PATCH 11/14] revert component changes and add users and scores to db to self-contain script - also add tool to print kv --- components/ItemSummary.tsx | 2 +- deno.json | 1 + tools/dump_kv.ts | 18 +++++++++++++ tools/seed_submissions.ts | 53 +++++++++++++++++++++++++++----------- 4 files changed, 58 insertions(+), 16 deletions(-) create mode 100644 tools/dump_kv.ts diff --git a/components/ItemSummary.tsx b/components/ItemSummary.tsx index 42a32abde..07a307037 100644 --- a/components/ItemSummary.tsx +++ b/components/ItemSummary.tsx @@ -5,7 +5,7 @@ import UserPostedAt from "./UserPostedAt.tsx"; export interface ItemSummaryProps { item: Item; - user?: User; + user: User; isVoted: boolean; } diff --git a/deno.json b/deno.json index 3780b6f00..4e772a478 100644 --- a/deno.json +++ b/deno.json @@ -2,6 +2,7 @@ "lock": false, "tasks": { "init:stripe": "deno run --allow-read --allow-env --allow-net tools/init_stripe.ts", + "db:dump": "deno run --allow-read --allow-env --unstable tools/dump_kv.ts", "db:seed": "deno run --allow-read --allow-env --allow-net --unstable tools/seed_submissions.ts", "db:reset": "deno run --allow-read --allow-env --unstable tools/reset_kv.ts", "start": "deno run --unstable -A --watch=static/,routes/ dev.ts", diff --git a/tools/dump_kv.ts b/tools/dump_kv.ts new file mode 100644 index 000000000..6981c7969 --- /dev/null +++ b/tools/dump_kv.ts @@ -0,0 +1,18 @@ +// Copyright 2023 the Deno authors. All rights reserved. MIT license. +// Description: Prints kv to stdout +// Usage: deno run -A --unstable tools/dump_kv.ts +import { kv } from "@/utils/db.ts"; + +export async function dumpKv() { + const iter = kv.list({ prefix: [] }); + const items = []; + for await (const res of iter) { + items.push({ [res.key.toString()]: res.value }); + } + console.log(`${JSON.stringify(items, null, 2)}`); +} + +if (import.meta.main) { + await dumpKv(); + await kv.close(); +} diff --git a/tools/seed_submissions.ts b/tools/seed_submissions.ts index 73194eced..d78033e98 100644 --- a/tools/seed_submissions.ts +++ b/tools/seed_submissions.ts @@ -1,6 +1,6 @@ // Copyright 2023 the Deno authors. All rights reserved. MIT license. - -import { createItem } from "@/utils/db.ts"; +// Description: Seeds the kv db with Hacker News stories +import { createItem, createUser, type Item, kv } from "@/utils/db.ts"; // Reference: https://github.com/HackerNews/API const API_BASE_URL = `https://hacker-news.firebaseio.com/v0`; @@ -8,7 +8,7 @@ const API_BASE_URL = `https://hacker-news.firebaseio.com/v0`; interface Story { id: number; score: number; - time: number; + time: number; // Unix seconds by: string; title: string; url: string; @@ -20,7 +20,7 @@ function* batchify(arr: T[], n = 5): Generator { } } -// Fetch the top 500 HN stories to seed the db +// Fetches the top 500 HN stories to seed the db async function fetchTopStoryIds() { const resp = await fetch(`${API_BASE_URL}/topstories.json`); if (!resp.ok) { @@ -54,19 +54,29 @@ async function fetchTopStories(limit = 10) { return stories; } -async function seedSubmissions(stories: Story[]) { - const items = stories.map(({ by: userId, title, url }) => { - return { userId, title, url }; - }).filter(({ url }) => { - try { - return Boolean(new URL(url).host); - } catch { - return; - } +async function createItemWithScore(item: Item) { + const res = await createItem(item); + return await kv.set(["items", res!.id], { + ...res, + score: item.score, + createdAt: item.createdAt, }); +} + +async function seedSubmissions(stories: Story[]) { + const items = stories.map(({ by: userId, title, url, score, time }) => { + return { + userId, + title, + url, + score, + createdAt: new Date(time * 1000), + } as Item; + }).filter(({ url }) => url); for (const batch of batchify(items)) { - await Promise.all(batch.map((item) => createItem(item))); + await Promise.all(batch.map((item) => createItemWithScore(item))); } + return items; } async function main(limit = 20) { @@ -75,7 +85,20 @@ async function main(limit = 20) { console.error(`No stories to seed!`); return; } - await seedSubmissions(stories); + const items = await seedSubmissions(stories); + + // Create dummy users to ensure each post has a corresponding user + for (const batch of batchify(items)) { + await Promise.allSettled(batch.map(({ userId: id }) => + createUser({ + id, // id must match userId for post + login: id, + avatarUrl: "", + stripeCustomerId: crypto.randomUUID(), // unique per userId + sessionId: crypto.randomUUID(), // unique per userId + }) // ignore errors if dummy user already exists + )); + } } if (import.meta.main) { From 7d583820a503dc336fcf783b8c1c876725985194 Mon Sep 17 00:00:00 2001 From: Zach Zeleznick Date: Tue, 23 May 2023 21:22:46 -0700 Subject: [PATCH 12/14] update README.md --- README.md | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e661a8f9b..dbb0705a4 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,10 @@ stripe listen --forward-to localhost:8000/api/stripe-webhooks --events=customer. 4. Copy the webhook signing secret to [.env](.env) as `STRIPE_WEBHOOK_SECRET`. +> Note: You can use +> [Stripe's test credit cards](https://stripe.com/docs/testing) to make test +> payments while in Stripe's test mode. + ### Running the Server Finally, start the server by running: @@ -94,9 +98,34 @@ deno task start Go to [http://localhost:8000](http://localhost:8000) to begin playing with your new SaaS app. -> Note: You can use -> [Stripe's test credit cards](https://stripe.com/docs/testing) to make test -> payments while in Stripe's test mode. +### Bootstrapping your local Database (Optional) + +If the home page is feeling a little empty, run + +``` +deno task db:seed +``` + +On execution, this script will fetch 20 (customizable) of the top HN posts using +the [HackerNews API](https://github.com/HackerNews/API) to populate your home +page. + +To see all the values in your local Deno KV database, run + +``` +deno task db:dump +``` + +And all kv pairs will be logged to stdout + +To reset your Deno KV database, run + +``` +deno task db:reset +``` + +Since this operation is not recoverable, you will be prompted to confirm +deletion before proceeding. ## Customization From d2175c57d79bb7b6ac1228e50db96aee081867d7 Mon Sep 17 00:00:00 2001 From: Zach Zeleznick Date: Tue, 23 May 2023 21:26:03 -0700 Subject: [PATCH 13/14] fix bad checkout from forked main vs upstream main --- components/UserPostedAt.tsx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/components/UserPostedAt.tsx b/components/UserPostedAt.tsx index 523a07bb5..642bbc1fd 100644 --- a/components/UserPostedAt.tsx +++ b/components/UserPostedAt.tsx @@ -7,6 +7,12 @@ export default function UserPostedAt( ) { return (

+ {props.user.login} {props.user.login}{" "} {props.user.isSubscribed && ( 🦕{" "} From fe31191469040b03d24bee0c8711751991fe8dd0 Mon Sep 17 00:00:00 2001 From: Zach Zeleznick Date: Tue, 23 May 2023 21:30:04 -0700 Subject: [PATCH 14/14] add avatar url for dummy users with guest profile pic from gravatar --- tools/seed_submissions.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/seed_submissions.ts b/tools/seed_submissions.ts index d78033e98..e744ce98b 100644 --- a/tools/seed_submissions.ts +++ b/tools/seed_submissions.ts @@ -93,7 +93,7 @@ async function main(limit = 20) { createUser({ id, // id must match userId for post login: id, - avatarUrl: "", + avatarUrl: "https://www.gravatar.com/avatar/?d=mp&s=64", stripeCustomerId: crypto.randomUUID(), // unique per userId sessionId: crypto.randomUUID(), // unique per userId }) // ignore errors if dummy user already exists