[ECO-2537] Add parcel cache implementation

src/typescript/frontend/package.json

@@ -106,6 +106,7 @@
     "submodule": "./submodule.sh",
     "test": "pnpm run test:e2e",
     "test:e2e": "playwright test --project=firefox",
+    "test:unit": "playwright test --project=unit",
     "vercel-install": "./submodule.sh && pnpm i"
   },
   "version": "0.0.1-alpha"

src/typescript/frontend/playwright.config.js

@@ -56,11 +56,14 @@ export default defineConfig({
       use: { ...devices["Desktop Firefox"] },
       dependencies: ["setup"],
     },
-
     {
       name: "webkit",
       use: { ...devices["Desktop Safari"] },
       dependencies: ["setup"],
     },
+    {
+      name: "unit",
+      testMatch: /.*\.unit\.ts/,
+    },
   ],
 });

src/typescript/frontend/src/app/candlesticks/route.ts

@@ -1,110 +1,35 @@
-// cspell:word timespan
-
-import { type AnyNumberString, getPeriodStartTimeFromTime, toPeriod } from "@sdk/index";
+import { type Period, toPeriod } from "@sdk/index";
 import { parseInt } from "lodash";
 import { type NextRequest } from "next/server";
 import {
   type CandlesticksSearchParams,
-  type GetCandlesticksParams,
   getPeriodDurationSeconds,
   HISTORICAL_CACHE_DURATION,
-  indexToParcelEndDate,
-  indexToParcelStartDate,
   isValidCandlesticksSearchParams,
-  jsonStrAppend,
   NORMAL_CACHE_DURATION,
-  PARCEL_SIZE,
-  toIndex,
 } from "./utils";
-import { unstable_cache } from "next/cache";
 import { getLatestProcessedEmojicoinTimestamp } from "@sdk/indexer-v2/queries/utils";
-import { parseJSON, stringifyJSON } from "utils";
-import { fetchMarketRegistration, fetchPeriodicEventsSince } from "@/queries/market";
-
-/**
- * @property `data` the stringified version of {@link CandlesticksDataType}.
- * @property `count` the number of rows returned.
- */
-type GetCandlesticksResponse = {
-  data: string;
-  count: number;
-};
-
-type CandlesticksDataType = Awaited<ReturnType<typeof fetchPeriodicEventsSince>>;
-
-const getCandlesticks = async (params: GetCandlesticksParams) => {
-  const { marketID, index, period } = params;
-
-  const start = indexToParcelStartDate(index, period);
-
-  const periodDurationMilliseconds = getPeriodDurationSeconds(period) * 1000;
-  const timespan = periodDurationMilliseconds * PARCEL_SIZE;
-  const end = new Date(start.getTime() + timespan);
-
-  // PARCEL_SIZE determines the max number of rows, so we don't need to pass a `LIMIT` value.
-  // `start` and `end` determine the level of pagination, so no need to specify `offset` either.
-  const data = await fetchPeriodicEventsSince({
-    marketID,
-    period,
-    start,
-    end,
+import { fetchPeriodicEventsTo, tryFetchMarketRegistration } from "@/queries/market";
+import { Parcel } from "lib/parcel";
+import { stringifyJSON } from "utils";
+
+type CandlesticksDataType = Awaited<ReturnType<typeof fetchPeriodicEventsTo>>;
+
+const getCandlesticksParcel = async (
+  { to, count }: { to: number; count: number },
+  query: { marketID: number; period: Period }
+) => {
+  const endDate = new Date(to * 1000);
+
+  const data = await fetchPeriodicEventsTo({
+    ...query,
+    end: endDate,
+    amount: count,
   });
 
-  return {
-    data: stringifyJSON(data),
-    count: data.length,
-  };
+  return data;
 };
 
-/**
- * Returns the market registration event for a market if it exists.
- *
- * If it doesn't exist, it throws an error so that the value isn't cached in the
- * `unstable_cache` call.
- *
- * @see {@link getCachedMarketRegistrationMs}
- */
-const getMarketRegistrationMs = async (marketID: AnyNumberString) =>
-  fetchMarketRegistration({ marketID }).then((res) => {
-    if (res) {
-      return Number(res.market.time / 1000n);
-    }
-    throw new Error("Market is not yet registered.");
-  });
-
-const getCachedMarketRegistrationMs = unstable_cache(
-  getMarketRegistrationMs,
-  ["market-registrations"],
-  {
-    revalidate: HISTORICAL_CACHE_DURATION,
-  }
-);
-
-/**
- * Fetch all of the parcels of candlesticks that have completely ended.
- * The only difference between this and {@link getNormalCachedCandlesticks} is the cache tag and
- * thus how long the data is cached for.
- */
-const getHistoricCachedCandlesticks = unstable_cache(getCandlesticks, ["candlesticks-historic"], {
-  revalidate: HISTORICAL_CACHE_DURATION,
-});
-
-/**
- * Fetch all candlestick parcels that haven't completed yet.
- * The only difference between this and {@link getHistoricCachedCandlesticks} is the cache tag and
- * thus how long the data is cached for.
- */
-const getNormalCachedCandlesticks = unstable_cache(getCandlesticks, ["candlesticks"], {
-  revalidate: NORMAL_CACHE_DURATION,
-});
-
-const getCachedLatestProcessedEmojicoinTimestamp = unstable_cache(
-  getLatestProcessedEmojicoinTimestamp,
-  ["processor-timestamp"],
-  { revalidate: 5 }
-);
-
-/* eslint-disable-next-line import/no-unused-modules */
 export async function GET(request: NextRequest) {
   const searchParams = request.nextUrl.searchParams;
   const params: CandlesticksSearchParams = {
@@ -122,70 +47,23 @@ export async function GET(request: NextRequest) {
   const to = parseInt(params.to);
   const period = toPeriod(params.period);
   const countBack = parseInt(params.countBack);
-  const numParcels = parseInt(params.amount);
-
-  const index = toIndex(to, period);
-
-  // Ensure that the last start date as calculated per the search params is valid.
-  // This is specifically the last parcel's start date- aka the last parcel's first candlestick's
-  // start time.
-  const lastParcelStartDate = indexToParcelStartDate(index + numParcels - 1, period);
-  if (lastParcelStartDate > new Date()) {
-    return new Response("The last parcel's start date cannot be later than the current time.", {
-      status: 400,
-    });
-  }
-
-  let data: string = "[]";
 
-  const processorTimestamp = new Date(await getCachedLatestProcessedEmojicoinTimestamp());
-
-  let totalCount = 0;
-  let i = 0;
+  const queryHelper = new Parcel<CandlesticksDataType[number]>({
+    parcelSize: 500,
+    currentRevalidate: NORMAL_CACHE_DURATION,
+    historicRevalidate: HISTORICAL_CACHE_DURATION,
+    fetchHistoricThreshold: () => getLatestProcessedEmojicoinTimestamp().then((r) => r.getTime()),
+    fetchFirst: () => tryFetchMarketRegistration(marketID),
+    cacheKey: "candlesticks",
+    getKey: (s) => Number(s.periodicMetadata.startTime / 1000n / 1000n),
+    fetchFn: (params) => getCandlesticksParcel(params, { marketID, period }),
+    step: getPeriodDurationSeconds(period),
+  });
 
-  let registrationPeriodBoundaryStart: Date;
   try {
-    registrationPeriodBoundaryStart = await getCachedMarketRegistrationMs(marketID).then(
-      (time) => new Date(Number(getPeriodStartTimeFromTime(time, period)))
-    );
-  } catch {
-    return new Response("Market has not been registered yet.", { status: 400 });
-  }
-
-  while (totalCount <= countBack) {
-    const localIndex = index - i;
-    const endDate = indexToParcelEndDate(localIndex, period);
-    let res: GetCandlesticksResponse;
-    if (endDate < processorTimestamp) {
-      res = await getHistoricCachedCandlesticks({
-        marketID,
-        index: localIndex,
-        period,
-      });
-    } else {
-      res = await getNormalCachedCandlesticks({
-        marketID,
-        index: localIndex,
-        period,
-      });
-    }
-
-    if (i == 0) {
-      const parsed = parseJSON<CandlesticksDataType>(res.data);
-      const filtered = parsed.filter(
-        (val) => val.periodicMetadata.startTime < BigInt(to) * 1_000_000n
-      );
-      totalCount += filtered.length;
-      data = jsonStrAppend(data, stringifyJSON(filtered));
-    } else {
-      totalCount += res.count;
-      data = jsonStrAppend(data, res.data);
-    }
-    if (endDate < registrationPeriodBoundaryStart) {
-      break;
-    }
-    i++;
+    const data = await queryHelper.getData(to, countBack);
+    return new Response(stringifyJSON(data));
+  } catch (e) {
+    return new Response(e as string, { status: 400 });
   }
-
-  return new Response(data);
 }

src/typescript/frontend/src/app/candlesticks/utils.ts

@@ -1,47 +1,9 @@
 import { isPeriod, type Period, PeriodDuration, periodEnumToRawDuration } from "@sdk/index";
 import { isNumber } from "utils";
 
-/**
- * Parcel size is the amount of candlestick periods that will be in a single parcel.
- * That is, a parcel for 1m candlesticks will be `PARCEL_SIZE` minutes of time.
- *
- * Note that this is *NOT* the number of candlesticks in the database- as there may be gaps in the
- * on-chain data (and thus the database).
- *
- * More specifically, each parcel will have anywhere from 0 to PARCEL_SIZE number of candlesticks
- * and will always span `PARCEL_SIZE` candlesticks/periods worth of time.
- */
-export const PARCEL_SIZE = 500;
-
-export const indexToParcelStartDate = (index: number, period: Period): Date =>
-  new Date((PARCEL_SIZE * (index * periodEnumToRawDuration(period))) / 1000);
-export const indexToParcelEndDate = (index: number, period: Period): Date =>
-  new Date((PARCEL_SIZE * ((index + 1) * periodEnumToRawDuration(period))) / 1000);
-
 export const getPeriodDurationSeconds = (period: Period) =>
   (periodEnumToRawDuration(period) / PeriodDuration.PERIOD_1M) * 60;
 
-export const toIndex = (end: number, period: Period): number => {
-  const periodDuration = getPeriodDurationSeconds(period);
-  const parcelDuration = periodDuration * PARCEL_SIZE;
-
-  const index = Math.floor(end / parcelDuration);
-
-  return index;
-};
-
-export const jsonStrAppend = (a: string, b: string): string => {
-  if (a === "[]") return b;
-  if (b === "[]") return a;
-  return `${a.substring(0, a.length - 1)},${b.substring(1)}`;
-};
-
-export type GetCandlesticksParams = {
-  marketID: number;
-  index: number;
-  period: Period;
-};
-
 /**
  * The search params used in the `GET` request at `candlesticks/api`.
  *

src/typescript/frontend/src/app/chats/route.ts

@@ -0,0 +1,56 @@
+import { fetchChatEvents, tryFetchFirstChatEvent } from "@/queries/market";
+import { Parcel } from "lib/parcel";
+import type { NextRequest } from "next/server";
+import { isNumber, stringifyJSON } from "utils";
+
+type ChatSearchParams = {
+  marketID: string | null;
+  toMarketNonce: string | null;
+};
+
+export type ValidChatSearchParams = {
+  marketID: string;
+  toMarketNonce: string;
+};
+
+const isValidChatSearchParams = (params: ChatSearchParams): params is ValidChatSearchParams => {
+  const { marketID, toMarketNonce } = params;
+  // prettier-ignore
+  return (
+    marketID !== null && isNumber(marketID) &&
+    toMarketNonce !== null && isNumber(toMarketNonce)
+  );
+};
+
+type Chat = Awaited<ReturnType<typeof fetchChatEvents>>[number];
+
+export async function GET(request: NextRequest) {
+  const searchParams = request.nextUrl.searchParams;
+  const params: ChatSearchParams = {
+    marketID: searchParams.get("marketID"),
+    toMarketNonce: searchParams.get("toMarketNonce"),
+  };
+
+  if (!isValidChatSearchParams(params)) {
+    return new Response("Invalid chat search params.", { status: 400 });
+  }
+
+  const marketID = Number(params.marketID);
+  const toMarketNonce = Number(params.toMarketNonce);
+
+  const queryHelper = new Parcel<Chat>({
+    parcelSize: 20,
+    currentRevalidate: 5,
+    historicRevalidate: 365 * 24 * 60 * 60,
+    fetchHistoricThreshold: () =>
+      fetchChatEvents({ marketID, amount: 1 }).then((r) => Number(r[0].market.marketNonce)),
+    fetchFirst: () => tryFetchFirstChatEvent(marketID),
+    cacheKey: "chats",
+    getKey: (s) => Number(s.market.marketNonce),
+    fetchFn: ({ to, count }) => fetchChatEvents({ marketID, toMarketNonce: to, amount: count }),
+  });
+
+  const res = await queryHelper.getData(toMarketNonce, 50);
+
+  return new Response(stringifyJSON(res));
+}

src/typescript/frontend/src/app/market/[market]/page.tsx

@@ -76,8 +76,8 @@ const EmojicoinPage = async (params: EmojicoinPageProps) => {
     const marketAddress = getMarketAddress(emojis);
 
     const [chats, swaps, marketView] = await Promise.all([
-      fetchChatEvents({ marketID, pageSize: EVENTS_ON_PAGE_LOAD }),
-      fetchSwapEvents({ marketID, pageSize: EVENTS_ON_PAGE_LOAD }),
+      fetchChatEvents({ marketID, amount: EVENTS_ON_PAGE_LOAD }),
+      fetchSwapEvents({ marketID, amount: EVENTS_ON_PAGE_LOAD }),
       wrappedCachedContractMarketView(marketAddress.toString()),
     ]);