Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: update got to use built-in caching #5090

Merged
merged 3 commits into from
Jan 4, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/config/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@
"winston": "^3.8.2"
},
"devDependencies": {
"@esm2cjs/got": "^12.4.1",
"@esm2cjs/got": "^12.5.3",
"@microsoft/api-extractor": "*",
"@types/fs-extra": "^9.0.13",
"@types/jest": "^29.0.2",
Expand Down
2 changes: 1 addition & 1 deletion packages/zwave-js/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@
"dependencies": {
"@alcalzone/jsonl-db": "^2.5.3",
"@alcalzone/pak": "^0.8.1",
"@esm2cjs/got": "^12.4.1",
"@esm2cjs/got": "^12.5.3",
"@esm2cjs/p-queue": "^7.3.0",
"@sentry/integrations": "^7.12.1",
"@sentry/node": "^7.12.1",
Expand Down
117 changes: 14 additions & 103 deletions packages/zwave-js/src/lib/controller/FirmwareUpdateService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,109 +13,18 @@ import { formatId } from "@zwave-js/shared";
import crypto from "crypto";
import type { FirmwareUpdateFileInfo, FirmwareUpdateInfo } from "./_Types";

const serviceURL = "https://firmware.zwave-js.io";
function serviceURL(): string {
return process.env.ZWAVEJS_FW_SERVICE_URL || "https://firmware.zwave-js.io";
}

const DOWNLOAD_TIMEOUT = 60000;
// const MAX_FIRMWARE_SIZE = 10 * 1024 * 1024; // 10MB should be enough for any conceivable Z-Wave chip

const MAX_CACHE_SECONDS = 60 * 60 * 24; // Cache for a day at max
const CLEAN_CACHE_INTERVAL_MS = 60 * 60 * 1000; // Remove stale entries from the cache every hour

const requestCache = new Map<string, CachedRequest<unknown>>();
interface CachedRequest<T> {
response: T;
staleDate: number;
}
const requestCache = new Map();

// Queue requests to the firmware update service. Only allow few parallel requests so we can make some use of the cache.
const requestQueue = new PQueue({ concurrency: 2 });

let cleanCacheTimeout: NodeJS.Timeout | undefined;
function cleanCache() {
if (cleanCacheTimeout) {
clearTimeout(cleanCacheTimeout);
cleanCacheTimeout = undefined;
}

const now = Date.now();
for (const [key, cached] of requestCache) {
if (cached.staleDate < now) {
requestCache.delete(key);
}
}

if (requestCache.size > 0) {
cleanCacheTimeout = setTimeout(
cleanCache,
CLEAN_CACHE_INTERVAL_MS,
).unref();
}
}

async function cachedGot<T>(config: OptionsOfTextResponseBody): Promise<T> {
// Replaces got's built-in cache functionality because it depends on an outdated version of
// cacheable-request (<8.3.1), which does not distinguish between POSTs with different bodies

const hash = crypto
.createHash("sha256")
.update(JSON.stringify(config.json))
.digest("hex");
const cacheKey = `${config.method}:${config.url!.toString()}:${hash}`;

// Return cached requests if they are not stale yet
if (requestCache.has(cacheKey)) {
const cached = requestCache.get(cacheKey)!;
if (cached.staleDate > Date.now()) {
return cached.response as T;
}
}

const response = await got(config);
const responseJson = JSON.parse(response.body) as T;

// Check if we can cache the response
if (response.statusCode === 200 && response.headers["cache-control"]) {
const cacheControl = response.headers["cache-control"]!;

let maxAge: number | undefined;
const maxAgeMatch = cacheControl.match(/max-age=(\d+)/);
if (maxAgeMatch) {
maxAge = Math.max(0, parseInt(maxAgeMatch[1], 10));
}

if (maxAge) {
let currentAge: number;
if (response.headers.age) {
currentAge = parseInt(response.headers.age, 10);
} else if (response.headers.date) {
currentAge =
(Date.now() - Date.parse(response.headers.date)) / 1000;
} else {
currentAge = 0;
}
currentAge = Math.max(0, currentAge);

if (maxAge > currentAge) {
requestCache.set(cacheKey, {
response: responseJson,
staleDate:
Date.now() +
Math.min(MAX_CACHE_SECONDS, maxAge - currentAge) * 1000,
});
}
}
}

// Regularly clean the cache
if (!cleanCacheTimeout) {
cleanCacheTimeout = setTimeout(
cleanCache,
CLEAN_CACHE_INTERVAL_MS,
).unref();
}

return responseJson;
}

export interface GetAvailableFirmwareUpdateOptions {
userAgent: string;
apiKey?: string;
Expand Down Expand Up @@ -162,6 +71,7 @@ export function getAvailableFirmwareUpdates(
): Promise<FirmwareUpdateInfo[]> {
const headers: Headers = {
"User-Agent": options.userAgent,
"Content-Type": "application/json",
};
if (options.apiKey) {
headers["X-API-Key"] = options.apiKey;
Expand All @@ -180,19 +90,20 @@ export function getAvailableFirmwareUpdates(

const config: OptionsOfTextResponseBody = {
method: "POST",
url: `${serviceURL}/api/${
url: `${serviceURL()}/api/${
options.includePrereleases ? "v3" : "v1"
}/updates`,
json: body,
// TODO: Re-enable this in favor of cachedGot when fixed
// cache: requestCache,
// cacheOptions: {
// shared: false,
// },
cache: requestCache,
cacheOptions: {
shared: false,
},
headers,
};

return requestQueue.add(() => cachedGot(config));
return requestQueue.add(() => {
return got(config).json();
});
}

export async function downloadFirmwareUpdate(
Expand Down
40 changes: 40 additions & 0 deletions test/firmware-update.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
import assert from "assert";
import { getAvailableFirmwareUpdates } from "../packages/zwave-js/src/lib/controller/FirmwareUpdateService";

async function main() {
const get1 = () =>
getAvailableFirmwareUpdates(
{
manufacturerId: 0x0063,
productType: 0x4952,
productId: 0x3138,
firmwareVersion: "1.0",
},
{
userAgent: "TEST",
},
);

const get2 = () =>
getAvailableFirmwareUpdates(
{
manufacturerId: 0x0063,
productType: 0x4952,
productId: 0x3131,
firmwareVersion: "1.0",
},
{
userAgent: "TEST",
},
);

const upd1 = await get1();
const upd2 = await get2();

console.dir(upd1);
console.dir(upd2);

assert.notDeepStrictEqual(upd1, upd2);
}

void main();
3 changes: 3 additions & 0 deletions test/run.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
import { wait as _wait } from "alcalzone-shared/async";
import os from "os";
import path from "path";
import "reflect-metadata";
import { Driver } from "zwave-js";

const wait = _wait;

Check notice

Code scanning / CodeQL

Unused variable, import, function or class

Unused variable wait.

process.on("unhandledRejection", (_r) => {
debugger;
});
Expand Down
Loading