Add file caching utility function

One of the things that this tool will do is to hit the GitHub API and
request all of the repositories under the MetaMask organization. We
don't want to do this every time the tool is run, though, or else we
might get rate limited. To prevent this, we can cache the response data
from the API in a file.

This `fetchOrPopulateFileCache` function being introduced here (which we
will use in a later commit) makes that possible.

.eslintrc.js

@@ -6,7 +6,10 @@ module.exports = {
   overrides: [
     {
       files: ['*.ts'],
-      extends: ['@metamask/eslint-config-typescript'],
+      extends: [
+        '@metamask/eslint-config-typescript',
+        '@metamask/eslint-config-nodejs',
+      ],
     },
 
     {

jest.config.js

@@ -22,7 +22,7 @@ module.exports = {
   collectCoverage: true,
 
   // An array of glob patterns indicating a set of files for which coverage information should be collected
-  collectCoverageFrom: ['./src/**/*.ts'],
+  collectCoverageFrom: ['./src/**/*.ts', '!./src/logging-utils.ts'],
 
   // The directory where Jest should output its coverage files
   coverageDirectory: 'coverage',

package.json

@@ -47,6 +47,9 @@
     "test": "jest && jest-it-up",
     "test:watch": "jest --watch"
   },
+  "dependencies": {
+    "@metamask/utils": "^8.2.0"
+  },
   "devDependencies": {
     "@lavamoat/allow-scripts": "^2.3.1",
     "@lavamoat/preinstall-always-fail": "^1.0.0",

src/constants.ts

@@ -0,0 +1,4 @@
+/**
+ * The number of milliseconds in an hour, used as a cache age.
+ */
+export const ONE_HOUR = 60 * 60 * 1000;

src/fetch-or-populate-file-cache.test.ts

@@ -0,0 +1,261 @@
+import { readJsonFile, writeJsonFile } from '@metamask/utils/node';
+import path from 'path';
+
+import { fetchOrPopulateFileCache } from './fetch-or-populate-file-cache';
+import { withinSandbox, fakeDateOnly } from '../tests/helpers';
+
+describe('fetchOrPopulateFileCache', () => {
+  beforeEach(() => {
+    fakeDateOnly();
+  });
+
+  afterEach(() => {
+    jest.useRealTimers();
+  });
+
+  describe('if the given file does not already exist', () => {
+    it('saves the return value of the given function in the file as JSON along with its created time', async () => {
+      jest.setSystemTime(new Date('2023-01-01T00:00:00Z'));
+
+      await withinSandbox(async ({ directoryPath: sandboxDirectoryPath }) => {
+        const filePath = path.join(sandboxDirectoryPath, 'cache');
+        const data = { foo: 'bar' };
+
+        await fetchOrPopulateFileCache({
+          filePath,
+          getDataToCache: () => data,
+        });
+
+        const cache = await readJsonFile(filePath);
+        expect(cache).toStrictEqual({
+          ctime: '2023-01-01T00:00:00.000Z',
+          data,
+        });
+      });
+    });
+
+    it('returns the data that was cached', async () => {
+      await withinSandbox(async ({ directoryPath: sandboxDirectoryPath }) => {
+        const filePath = path.join(sandboxDirectoryPath, 'cache');
+        const dataToCache = { foo: 'bar' };
+
+        const cachedData = await fetchOrPopulateFileCache({
+          filePath,
+          getDataToCache: () => dataToCache,
+        });
+
+        expect(cachedData).toStrictEqual(dataToCache);
+      });
+    });
+  });
+
+  describe('if the given file already exists', () => {
+    describe('and no explicit max age is given', () => {
+      describe('and it was created less than an hour ago', () => {
+        it('does not overwrite the cache', async () => {
+          jest.setSystemTime(new Date('2023-01-01T00:00:00Z'));
+
+          await withinSandbox(
+            async ({ directoryPath: sandboxDirectoryPath }) => {
+              const filePath = path.join(sandboxDirectoryPath, 'cache');
+              const data = { foo: 'bar' };
+              await writeJsonFile(filePath, {
+                ctime: new Date('2023-01-01T00:30:00Z').toISOString(),
+                data,
+              });
+
+              await fetchOrPopulateFileCache({
+                filePath,
+                getDataToCache: () => data,
+              });
+
+              const cache = await readJsonFile(filePath);
+              expect(cache).toStrictEqual({
+                ctime: '2023-01-01T00:30:00.000Z',
+                data,
+              });
+            },
+          );
+        });
+
+        it('returns the data in the file', async () => {
+          jest.setSystemTime(new Date('2023-01-01T00:00:00Z'));
+
+          await withinSandbox(
+            async ({ directoryPath: sandboxDirectoryPath }) => {
+              const filePath = path.join(sandboxDirectoryPath, 'cache');
+              const data = { foo: 'bar' };
+              await writeJsonFile(filePath, {
+                ctime: new Date('2023-01-01T00:30:00Z').toISOString(),
+                data,
+              });
+
+              const cachedData = await fetchOrPopulateFileCache({
+                filePath,
+                getDataToCache: () => data,
+              });
+
+              expect(cachedData).toStrictEqual(data);
+            },
+          );
+        });
+      });
+
+      describe('and it was created more than an hour ago', () => {
+        it('overwrites the cache', async () => {
+          jest.setSystemTime(new Date('2023-01-01T00:00:00Z'));
+
+          await withinSandbox(
+            async ({ directoryPath: sandboxDirectoryPath }) => {
+              const filePath = path.join(sandboxDirectoryPath, 'cache');
+              const dataToCache = { foo: 'bar' };
+              await writeJsonFile(filePath, {
+                ctime: new Date('2023-01-01T01:00:01Z').toISOString(),
+                data: dataToCache,
+              });
+
+              await fetchOrPopulateFileCache({
+                filePath,
+                getDataToCache: () => dataToCache,
+              });
+
+              const cache = await readJsonFile(filePath);
+              expect(cache).toStrictEqual({
+                ctime: '2023-01-01T01:00:01.000Z',
+                data: dataToCache,
+              });
+            },
+          );
+        });
+
+        it('returns the data in the file', async () => {
+          jest.setSystemTime(new Date('2023-01-01T00:00:00Z'));
+
+          await withinSandbox(
+            async ({ directoryPath: sandboxDirectoryPath }) => {
+              const filePath = path.join(sandboxDirectoryPath, 'cache');
+              const dataToCache = { foo: 'bar' };
+              await writeJsonFile(filePath, {
+                ctime: new Date('2023-01-01T01:00:01Z').toISOString(),
+                data: dataToCache,
+              });
+
+              const cachedData = await fetchOrPopulateFileCache({
+                filePath,
+                getDataToCache: () => dataToCache,
+              });
+
+              expect(cachedData).toStrictEqual(dataToCache);
+            },
+          );
+        });
+      });
+    });
+
+    describe('and a max age is given', () => {
+      describe('and it was created less than <max age> seconds ago', () => {
+        it('does not overwrite the cache', async () => {
+          jest.setSystemTime(new Date('2023-01-01T00:00:00Z'));
+
+          await withinSandbox(
+            async ({ directoryPath: sandboxDirectoryPath }) => {
+              const filePath = path.join(sandboxDirectoryPath, 'cache');
+              const data = { foo: 'bar' };
+              await writeJsonFile(filePath, {
+                ctime: new Date('2023-01-01T00:00:04Z').toISOString(),
+                data,
+              });
+
+              await fetchOrPopulateFileCache({
+                filePath,
+                getDataToCache: () => data,
+                maxAge: 5000,
+              });
+
+              const cache = await readJsonFile(filePath);
+              expect(cache).toStrictEqual({
+                ctime: '2023-01-01T00:00:04.000Z',
+                data,
+              });
+            },
+          );
+        });
+
+        it('returns the data in the file', async () => {
+          jest.setSystemTime(new Date('2023-01-01T00:00:00Z'));
+
+          await withinSandbox(
+            async ({ directoryPath: sandboxDirectoryPath }) => {
+              const filePath = path.join(sandboxDirectoryPath, 'cache');
+              const data = { foo: 'bar' };
+              await writeJsonFile(filePath, {
+                ctime: new Date('2023-01-01T00:00:04Z').toISOString(),
+                data,
+              });
+
+              const cachedData = await fetchOrPopulateFileCache({
+                filePath,
+                getDataToCache: () => data,
+                maxAge: 5000,
+              });
+
+              expect(cachedData).toStrictEqual(data);
+            },
+          );
+        });
+      });
+
+      describe('and it was created more than an hour ago', () => {
+        it('overwrites the cache', async () => {
+          jest.setSystemTime(new Date('2023-01-01T00:00:00Z'));
+
+          await withinSandbox(
+            async ({ directoryPath: sandboxDirectoryPath }) => {
+              const filePath = path.join(sandboxDirectoryPath, 'cache');
+              const dataToCache = { foo: 'bar' };
+              await writeJsonFile(filePath, {
+                ctime: new Date('2023-01-01T00:00:06Z').toISOString(),
+                data: dataToCache,
+              });
+
+              await fetchOrPopulateFileCache({
+                filePath,
+                getDataToCache: () => dataToCache,
+                maxAge: 5000,
+              });
+
+              const cache = await readJsonFile(filePath);
+              expect(cache).toStrictEqual({
+                ctime: '2023-01-01T00:00:06.000Z',
+                data: dataToCache,
+              });
+            },
+          );
+        });
+
+        it('returns the data in the file', async () => {
+          jest.setSystemTime(new Date('2023-01-01T00:00:00Z'));
+
+          await withinSandbox(
+            async ({ directoryPath: sandboxDirectoryPath }) => {
+              const filePath = path.join(sandboxDirectoryPath, 'cache');
+              const dataToCache = { foo: 'bar' };
+              await writeJsonFile(filePath, {
+                ctime: new Date('2023-01-01T00:00:06Z').toISOString(),
+                data: dataToCache,
+              });
+
+              const cachedData = await fetchOrPopulateFileCache({
+                filePath,
+                getDataToCache: () => dataToCache,
+                maxAge: 5000,
+              });
+
+              expect(cachedData).toStrictEqual(dataToCache);
+            },
+          );
+        });
+      });
+    });
+  });
+});

src/fetch-or-populate-file-cache.ts

@@ -0,0 +1,77 @@
+import type { Json } from '@metamask/utils/node';
+import { fileExists, readJsonFile, writeJsonFile } from '@metamask/utils/node';
+
+import { ONE_HOUR } from './constants';
+import { createModuleLogger, projectLogger } from './logging-utils';
+
+const log = createModuleLogger(projectLogger, 'fetch-or-populate-file-cache');
+
+/**
+ * The data stored in the cache file.
+ */
+type FileCache<Data extends Json> = {
+  /**
+   * When the data was stored.
+   */
+  ctime: string;
+  /**
+   * The cached data.
+   */
+  data: Data;
+};
+
+/**
+ * How long to cache data retrieved from an API (to prevent rate limiting).
+ *
+ * Equal to 1 hour.
+ */
+const DEFAULT_MAX_AGE = ONE_HOUR;
+
+/**
+ * Avoids rate limits when making requests to an API by consulting a file cache.
+ *
+ * Reads the given cache file and returns the data within it if it exists and is
+ * fresh enough; otherwise runs the given function and saves its return value to
+ * the file.
+ *
+ * @param args - The arguments to this function.
+ * @param args.filePath - The path to the file where the data should be saved.
+ * @param args.getDataToCache - A function to get the data that should be cached
+ * if the cache does not exist or is stale.
+ * @param args.maxAge - The amount of time (in milliseconds) that the cache is
+ * considered "fresh". Affects subsequent calls: if `fetchOrPopulateFileCache`
+ * is called again with the same file path within the duration specified here,
+ * `getDataToCache` will not get called again, otherwise it will. Defaults to 1
+ * hour.
+ */
+export async function fetchOrPopulateFileCache<Data extends Json>({
+  filePath,
+  maxAge = DEFAULT_MAX_AGE,
+  getDataToCache,
+}: {
+  filePath: string;
+  maxAge?: number;
+  getDataToCache: () => Data | Promise<Data>;
+}): Promise<Data> {
+  const now = new Date();
+
+  if (await fileExists(filePath)) {
+    const cache = await readJsonFile<FileCache<Data>>(filePath);
+    const createdDate = new Date(cache.ctime);
+
+    if (now.getTime() - createdDate.getTime() <= maxAge) {
+      log(`Reusing fresh cached data under ${filePath}`);
+      return cache.data;
+    }
+  }
+
+  log(
+    `Cache does not exist or is stale; preparing data to write to ${filePath}`,
+  );
+  const dataToCache = await getDataToCache();
+  await writeJsonFile(filePath, {
+    ctime: now.toISOString(),
+    data: dataToCache,
+  });
+  return dataToCache;
+}