Skip to content

Commit

Permalink
Add unit tests for plarcel.ts
Browse files Browse the repository at this point in the history
  • Loading branch information
CRBl69 committed Dec 13, 2024
1 parent 293efc6 commit 1353121
Showing 6 changed files with 238 additions and 29 deletions.
1 change: 1 addition & 0 deletions src/typescript/frontend/package.json
Original file line number Diff line number Diff line change
@@ -106,6 +106,7 @@
"submodule": "./submodule.sh",
"test": "pnpm run test:e2e",
"test:e2e": "playwright test --project=firefox",
"test:unit": "playwright test --project=unit",
"vercel-install": "./submodule.sh && pnpm i"
},
"version": "0.0.1-alpha"
5 changes: 4 additions & 1 deletion src/typescript/frontend/playwright.config.js
Original file line number Diff line number Diff line change
@@ -56,11 +56,14 @@ export default defineConfig({
use: { ...devices["Desktop Firefox"] },
dependencies: ["setup"],
},

{
name: "webkit",
use: { ...devices["Desktop Safari"] },
dependencies: ["setup"],
},
{
name: "unit",
testMatch: /.*\.unit\.ts/,
},
],
});
57 changes: 31 additions & 26 deletions src/typescript/frontend/src/lib/parcel.ts
Original file line number Diff line number Diff line change
@@ -3,7 +3,7 @@ import { parseJSON, stringifyJSON } from "utils";

export type ParcelQueryParameters = {
count: number;
/** Inclusive. */
/** Exclusive. */
to: number;
};

@@ -134,6 +134,7 @@ export class Parcel<S> {
fetchFirst,
getKey,
step = 1,
cacheFn = unstable_cache,
}: {
/** How many events are stored in one parcel. */
parcelSize: number;
@@ -153,24 +154,26 @@ export class Parcel<S> {
getKey: (s: S) => number;
/** The spacing between keys. 1 by default. */
step?: number;
/** The caching method. Override for testing. */
cacheFn?: <T>(fn: (...args: any[]) => Promise<T>, keys: string[], options: {revalidate: number}) => ((...args: any[]) => Promise<T>)

Check warning on line 158 in src/typescript/frontend/src/lib/parcel.ts

GitHub Actions / pre-commit

Unexpected any. Specify a different type

Check warning on line 158 in src/typescript/frontend/src/lib/parcel.ts

GitHub Actions / pre-commit

Unexpected any. Specify a different type
}) {
this._parcelSize = parcelSize;
this._currentFetch = unstable_cache(
this._currentFetch = cacheFn(
(params: ParcelQueryParameters) => cachedWrapper(params, fetchFn, getKey),
["parcel", cacheKey, "current", parcelSize.toString()],
{ revalidate: currentRevalidate }
);
this._historicFetch = unstable_cache(
this._historicFetch = cacheFn(
(params: ParcelQueryParameters) => cachedWrapper(params, fetchFn, getKey),
["parcel", cacheKey, "historic", parcelSize.toString()],
{ revalidate: historicRevalidate }
);
this._fetchHistoricThreshold = unstable_cache(
this._fetchHistoricThreshold = cacheFn(
fetchHistoricThreshold,
["parcel", cacheKey, "threshold"],
{ revalidate: 2 }
);
this._fetchFirst = unstable_cache(
this._fetchFirst = cacheFn(
fetchFirst,
["parcel", cacheKey, "first"],
{ revalidate: 365 * 24 * 60 * 60 }
@@ -194,14 +197,15 @@ export class Parcel<S> {
return this.toKey(this.toParcelNumber(key));
}

/** Get the parcel start key from a key in that parcel. */
/** Get the parcel end key from a key in that parcel. */
private parcelEndKey(key: number): number {
return this.parcelStartKey(key) + this._parcelSize * this._step;
}

/** Get `count` events starting from `to` (inclusive) backwards. */
/** Get `count` events starting from `to` (exclusive) backwards. */
async getData(to: number, count: number): Promise<S[]> {
let first: number;

try {
first = await this._fetchFirst();
} catch (e) {
@@ -211,36 +215,37 @@ export class Parcel<S> {
);
return [];
}
if (to < first) {

// Because there are no events before the first event.
// `<=` is used here as `to` is exclusive.
if (to <= first) {
return [];
}

let parcel: CachedWrapperReturn;
const historicThreshold = await this._fetchHistoricThreshold();

const rightmostParcelNumber = this.toParcelNumber(to);
const rightmostParcelEndKey = this.parcelEndKey(rightmostParcelNumber);
let events: S[] = [];

if (rightmostParcelEndKey > historicThreshold) {
parcel = await this._currentFetch({ to: rightmostParcelEndKey, count: this._parcelSize });
} else {
parcel = await this._historicFetch({ to: rightmostParcelEndKey, count: this._parcelSize });
}
// While we haven't gotten `count` events AND there are still events.
while (events.length < count && to > first) {
// End key of parcel we're trying to query.
const endKey = this.parcelEndKey(to - 1);
const params = { to: endKey, count: this._parcelSize };

let events = parseJSON<S[]>(parcel.stringifiedData).filter((s) => this._getKey(s) <= to);
let parcel = await (endKey > historicThreshold ? this._currentFetch(params) : this._historicFetch(params));

Check failure on line 235 in src/typescript/frontend/src/lib/parcel.ts

GitHub Actions / pre-commit

'parcel' is never reassigned. Use 'const' instead

const parsedData = parseJSON<S[]>(parcel.stringifiedData)

// If `events` is empty, we want all events before `to`, else we want all events before the earliest event from `events`.
const newEvents: S[] = parsedData
.filter(e => this._getKey(e) < (events.length === 0 ? to : this._getKey(events[events.length - 1])));

while (events.length < count && parcel.first > first) {
const endKey = this.parcelEndKey(parcel.first - 1);
parcel = await this._historicFetch({
to: endKey,
count: this._parcelSize,
});
const newEvents: S[] = (parseJSON(parcel.stringifiedData) as S[])
.filter(e => this._getKey(e) < this._getKey(events[events.length - 1]));
events = [...events, ...newEvents];

to = parcel.first;
}

return events;
return events.slice(0, count);
}
}

199 changes: 199 additions & 0 deletions src/typescript/frontend/tests/e2e/parcel.unit.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,199 @@
import { expect, test } from "@playwright/test";
import { Parcel } from "../../src/lib/parcel";
import { parseJSON, stringifyJSON } from "utils";

test("test normal parcel", async () => {
// Store reads in this array to check cache hits.
let reads: {count: number, to: number}[] = [];
// Parcel where every key has an event.
const parcelData = [
0, 1, 2, 3, 4,
6, 6, 7, 8, 9,
11, 11, 12, 13, 14,
15, 16, 17, 18, 19,
];
const cache = new Map<string, {expire: number, value: string}>();
const helper = new Parcel<number>({
parcelSize: 5,
cacheKey: "test-parcels",
fetchFn: async (params) => {
const { to, count } = params;
reads.push(params);
const valid = parcelData.filter(v => v < to)
return valid.slice(Math.max(valid.length - count, 0), valid.length).toReversed();
},
currentRevalidate: 5,
historicRevalidate: 1000000,
fetchHistoricThreshold: async () => 10,
fetchFirst: async () => 0,
getKey: (s) => s,
cacheFn: (fn, keys, {revalidate}) => {
return async (...params) => {
const cacheKey = stringifyJSON([...keys, ...params]);
const cachedValue = cache.get(cacheKey);
if (cachedValue && cachedValue.expire * 1000 > new Date().getTime()) {
return parseJSON(cachedValue.value);
} else {
const res = await fn(...params);
cache.set(cacheKey, {expire: new Date().getTime() + revalidate * 1000, value: stringifyJSON(res)});
return res;
}
};
}
});

const data = await helper.getData(20, 6);
expect(data).toStrictEqual([19, 18, 17, 16, 15, 14]);
expect(reads).toStrictEqual([{count: 5, to: 20}, {count: 5, to: 15}]);
});

test("test parcel with missing data", async () => {
// Store reads in this array to check cache hits.
let reads: {count: number, to: number}[] = [];
// Parcel where not every key has an event.
const parcelData = [
// Parcel 1.
0, 1, 2, 4,
// Parcel 2. Also contains part of parcel 1 (4, 2 and 1) in order to have 5 events.
5, 8,
// Parcel 3. Also contains parcel 2 and part of parcel 1 (4) in order to have 5 events.
12, 14,
// Parcel 4. Also contains parcel 3 and part of parcel 2 (8) in order to have 5 events.
15, 18,
];
const cache = new Map<string, {expire: number, value: string}>();
const helper = new Parcel<number>({
parcelSize: 5,
cacheKey: "test-parcels-2",
fetchFn: async (params) => {
const { to, count } = params;
reads.push(params);
const valid = parcelData.filter(v => v < to)
return valid.slice(Math.max(valid.length - count, 0), valid.length).toReversed();
},
currentRevalidate: 5,
historicRevalidate: 1000000,
fetchHistoricThreshold: async () => 10,
fetchFirst: async () => 0,
getKey: (s) => s,
cacheFn: (fn, keys, {revalidate}) => {
return async (...params) => {
const cacheKey = stringifyJSON([...keys, ...params]);
const cachedValue = cache.get(cacheKey);
if (cachedValue && cachedValue.expire * 1000 > new Date().getTime()) {
return parseJSON(cachedValue.value);
} else {
const res = await fn(...params);
cache.set(cacheKey, {expire: new Date().getTime() + revalidate * 1000, value: stringifyJSON(res)});
return res;
}
};
}
});

// Get 6 events starting from 20 exclusive.
const data1 = await helper.getData(20, 6);
expect(data1).toStrictEqual([18, 15, 14, 12, 8, 5]);
// Only parcel 4 and 2 should see a read since parcel 3 is stored inside parcel 4.
expect(reads).toStrictEqual([{count: 5, to: 20}, {count: 5, to: 10}]);

reads = [];
// Get all events.
const data2 = await helper.getData(20, 20);
expect(data2).toStrictEqual([18, 15, 14, 12, 8, 5, 4, 2, 1, 0]);
// Only parcel 1 should see a read since the other data is already read.
expect(reads).toStrictEqual([{count: 5, to: 5}]);

// Here parcel 3 is never read. This is normal since it's contained in parcel 4.
// Theoretically, it might see a read if the query starts in parcel 3 (e.g.: getData(13, 5)).
// But in real world usage, this should not happened, as the clients do not request random data like that.
// They will always start to query from the most recent event, and then in reverse chronological order from there.
});

test("test exclusiveness", async () => {
let reads: {count: number, to: number}[] = [];
const parcelData = [
0, 1, 2, 3, 4,
6, 6, 7, 8, 9,
11, 11, 12, 13, 14,
15, 16, 17, 18, 19,
];
const cache = new Map<string, {expire: number, value: string}>();
const helper = new Parcel<number>({
parcelSize: 5,
cacheKey: "test-parcels",
fetchFn: async (params) => {
const { to, count } = params;
reads.push(params);
const valid = parcelData.filter(v => v < to)
return valid.slice(Math.max(valid.length - count, 0), valid.length).toReversed();
},
currentRevalidate: 5,
historicRevalidate: 1000000,
fetchHistoricThreshold: async () => 10,
fetchFirst: async () => 0,
getKey: (s) => s,
cacheFn: (fn, keys, {revalidate}) => {
return async (...params) => {
const cacheKey = stringifyJSON([...keys, ...params]);
const cachedValue = cache.get(cacheKey);
if (cachedValue && cachedValue.expire * 1000 > new Date().getTime()) {
return parseJSON(cachedValue.value);
} else {
const res = await fn(...params);
cache.set(cacheKey, {expire: new Date().getTime() + revalidate * 1000, value: stringifyJSON(res)});
return res;
}
};
}
});

const data = await helper.getData(5, 1);
expect(data).toStrictEqual([4]);
});

test("test request 0", async () => {
let reads: {count: number, to: number}[] = [];
const parcelData = [
0, 1, 2, 3, 4,
6, 6, 7, 8, 9,
11, 11, 12, 13, 14,
15, 16, 17, 18, 19,
];
const cache = new Map<string, {expire: number, value: string}>();
const helper = new Parcel<number>({
parcelSize: 5,
cacheKey: "test-parcels",
fetchFn: async (params) => {
const { to, count } = params;
reads.push(params);
const valid = parcelData.filter(v => v < to)
return valid.slice(Math.max(valid.length - count, 0), valid.length).toReversed();
},
currentRevalidate: 5,
historicRevalidate: 1000000, fetchHistoricThreshold: async () => 10,
fetchFirst: async () => 0,
getKey: (s) => s,
cacheFn: (fn, keys, {revalidate}) => {
return async (...params) => {
const cacheKey = stringifyJSON([...keys, ...params]);
const cachedValue = cache.get(cacheKey);
if (cachedValue && cachedValue.expire * 1000 > new Date().getTime()) {
return parseJSON(cachedValue.value);
} else {
const res = await fn(...params);
cache.set(cacheKey, {expire: new Date().getTime() + revalidate * 1000, value: stringifyJSON(res)});
return res;
}
};
}
});

const data1 = await helper.getData(5, 0);
expect(data1).toStrictEqual([]);
expect(reads).toStrictEqual([]);

const data2 = await helper.getData(0, 1);
expect(data2).toStrictEqual([]);
expect(reads).toStrictEqual([]);
});
1 change: 1 addition & 0 deletions src/typescript/package.json
Original file line number Diff line number Diff line change
@@ -43,6 +43,7 @@
"test:debug": "FETCH_DEBUG=true pnpm run load-env:test -- turbo run test --force",
"test:frontend": " pnpm run load-env:test -- turbo run test --filter @econia-labs/emojicoin-frontend --log-prefix none",
"test:frontend:e2e": "pnpm run load-env:test -- turbo run test:e2e --filter @econia-labs/emojicoin-frontend --log-prefix none",
"test:frontend:unit": "pnpm run load-env:test -- turbo run test:unit --filter @econia-labs/emojicoin-frontend --log-prefix none",
"test:sdk": "pnpm run load-env:test -- turbo run test --filter @econia-labs/emojicoin-sdk --log-prefix none",
"test:sdk:e2e": "pnpm run load-env:test -- turbo run test:e2e --filter @econia-labs/emojicoin-sdk --force --log-prefix none",
"test:sdk:parallel": "pnpm run load-env:test -- turbo run test:sdk:parallel --filter @econia-labs/emojicoin-sdk --force --log-prefix none",
4 changes: 2 additions & 2 deletions src/typescript/sdk/src/indexer-v2/queries/app/market.ts
Original file line number Diff line number Diff line change
@@ -36,7 +36,7 @@ const selectSwapsByMarketID = ({
.from(TableName.SwapEvents)
.select("*")
.eq("market_id", marketID)
.lte("market_nonce", toMarketNonce)
.lt("market_nonce", toMarketNonce)
.order("market_nonce", ORDER_BY[order])
.limit(amount);
}
@@ -82,7 +82,7 @@ const selectChatsByMarketID = ({
.from(TableName.ChatEvents)
.select("*")
.eq("market_id", marketID)
.lte("market_nonce", toMarketNonce)
.lt("market_nonce", toMarketNonce)
.order("market_nonce", ORDER_BY[order])
.limit(amount);
}

0 comments on commit 1353121

Please sign in to comment.