Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Core: Remove dependence on file-system-cache #29256

Merged
merged 15 commits into from
Oct 7, 2024
Merged
Show file tree
Hide file tree
Changes from 12 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions code/__mocks__/fs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ export const realpathSync = vi.fn();
export const readdir = vi.fn();
export const readdirSync = vi.fn();
export const readlinkSync = vi.fn();
export const mkdirSync = vi.fn();

export default {
__setMockFiles,
Expand All @@ -29,4 +30,5 @@ export default {
readdir,
readdirSync,
readlinkSync,
mkdirSync,
};
4 changes: 2 additions & 2 deletions code/builders/builder-webpack5/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ const starter: StarterFunction = async function* starterGeneratorFn({
}

yield;
const modulesCount = (await options.cache?.get('modulesCount').catch(() => {})) || 1000;
const modulesCount = await options.cache?.get('modulesCount', 1000);
let totalModules: number;
let value = 0;

Expand All @@ -147,7 +147,7 @@ const starter: StarterFunction = async function* starterGeneratorFn({
const progress = { value, message: message.charAt(0).toUpperCase() + message.slice(1) };
if (message === 'building') {
// arg3 undefined in webpack5
const counts = (arg3 && arg3.match(/(\d+)\/(\d+)/)) || [];
const counts = (arg3 && arg3.match(/entries (\d+)\/(\d+)/)) || [];
const complete = parseInt(counts[1], 10);
const total = parseInt(counts[2], 10);
if (!Number.isNaN(complete) && !Number.isNaN(total)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ export default async (
docsOptions,
entries,
nonNormalizedStories,
modulesCount = 1000,
modulesCount,
build,
tagsOptions,
] = await Promise.all([
Expand All @@ -86,7 +86,7 @@ export default async (
presets.apply('docs'),
presets.apply<string[]>('entries', []),
presets.apply('stories', []),
options.cache?.get('modulesCount').catch(() => {}),
options.cache?.get('modulesCount', 1000),
options.presets.apply('build'),
presets.apply('tags', {}),
]);
Expand Down
1 change: 0 additions & 1 deletion code/core/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -365,7 +365,6 @@
"express": "^4.19.2",
"fd-package-json": "^1.2.0",
"fetch-retry": "^6.0.0",
"file-system-cache": "^2.4.4",
"find-cache-dir": "^5.0.0",
"find-up": "^7.0.0",
"flush-promises": "^1.0.2",
Expand Down
1 change: 1 addition & 0 deletions code/core/src/cli/detect.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ vi.mock('fs', () => ({
readdirSync: vi.fn(),
readlinkSync: vi.fn(),
default: vi.fn(),
mkdirSync: vi.fn(),
}));

vi.mock('@storybook/core/node-logger');
Expand Down
6 changes: 4 additions & 2 deletions code/core/src/cli/dev.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,16 @@ export const dev = async (cliOptions: CLIOptions) => {

const packageJson = await findPackage(__dirname);
invariant(packageJson, 'Failed to find the closest package.json file.');
type Options = Parameters<typeof buildDevStandalone>[0];

const options = {
...cliOptions,
configDir: cliOptions.configDir || './.storybook',
configType: 'DEVELOPMENT',
ignorePreview: !!cliOptions.previewUrl && !cliOptions.forceBuildPreview,
cache,
cache: cache as any,
packageJson,
} as Parameters<typeof buildDevStandalone>[0];
} as Options;

await withTelemetry(
'dev',
Expand Down
160 changes: 153 additions & 7 deletions code/core/src/common/utils/file-cache.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,157 @@
import * as fsc from 'file-system-cache';
import { createHash, randomBytes } from 'node:crypto';
import { mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
ndelangen marked this conversation as resolved.
Show resolved Hide resolved
import { readFile, readdir, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';

// @ts-expect-error (needed due to it's use of `exports.default`)
const Cache = (fsc.default.default || fsc.default) as typeof fsc.default;
interface FileSystemCacheOptions {
ns?: string;
prefix?: string;
hash_alg?: string;
basePath?: string;
ttl?: number;
}

interface CacheItem {
key: string;
content?: any;
value?: any;
}
Comment on lines +18 to +19
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

style: Consider using a more specific type than 'any' for content and value


interface CacheSetOptions {
ttl?: number;
encoding?: BufferEncoding;
}

export class FileSystemCache {
private prefix: string;

private hash_alg: string;

private cache_dir: string;

private ttl: number;

constructor(options: FileSystemCacheOptions = {}) {
this.prefix = (options.ns || options.prefix || '') + '-';
this.hash_alg = options.hash_alg || 'md5';
this.cache_dir =
options.basePath || join(tmpdir(), randomBytes(15).toString('base64').replace(/\//g, '-'));
this.ttl = options.ttl || 0;
createHash(this.hash_alg); // Verifies hash algorithm is available
mkdirSync(this.cache_dir, { recursive: true });
}

private generateHash(name: string): string {
return join(this.cache_dir, this.prefix + createHash(this.hash_alg).update(name).digest('hex'));
}

private isExpired(parsed: { ttl?: number }, now: number): boolean {
return parsed.ttl != null && now > parsed.ttl;
}

private parseCacheData<T>(data: string, fallback: T | null): T | null {
const parsed = JSON.parse(data);
return this.isExpired(parsed, Date.now()) ? fallback : (parsed.content as T);
}

export type Options = Parameters<typeof Cache>['0'];
export type FileSystemCache = ReturnType<typeof Cache>;
private parseSetData<T>(key: string, data: T, opts: CacheSetOptions = {}): string {
const ttl = opts.ttl ?? this.ttl;
return JSON.stringify({ key, content: data, ...(ttl && { ttl: Date.now() + ttl * 1000 }) });
}

public async get<T = any>(name: string, fallback?: T): Promise<T> {
try {
const data = await readFile(this.generateHash(name), 'utf8');
return this.parseCacheData(data, fallback) as T;
} catch {
return fallback as T;
}
}

public getSync<T>(name: string, fallback?: T): T {
try {
const data = readFileSync(this.generateHash(name), 'utf8');
return this.parseCacheData(data, fallback) as T;
} catch {
return fallback as T;
}
}

public async set<T>(
name: string,
data: T,
orgOpts: CacheSetOptions | number = {}
): Promise<void> {
const opts: CacheSetOptions = typeof orgOpts === 'number' ? { ttl: orgOpts } : orgOpts;
await writeFile(this.generateHash(name), this.parseSetData(name, data, opts), {
encoding: opts.encoding || 'utf8',
});
}

public setSync<T>(name: string, data: T, orgOpts: CacheSetOptions | number = {}): void {
const opts: CacheSetOptions = typeof orgOpts === 'number' ? { ttl: orgOpts } : orgOpts;
writeFileSync(this.generateHash(name), this.parseSetData(name, data, opts), {
encoding: opts.encoding || 'utf8',
});
}

public async setMany(items: CacheItem[], options?: CacheSetOptions): Promise<void> {
await Promise.all(items.map((item) => this.set(item.key, item.content ?? item.value, options)));
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

style: This could potentially cause performance issues with large numbers of items. Consider batching


public setManySync(items: CacheItem[], options?: CacheSetOptions): void {
items.forEach((item) => this.setSync(item.key, item.content ?? item.value, options));
}

public async remove(name: string): Promise<void> {
await rm(this.generateHash(name), { force: true });
}

public removeSync(name: string): void {
rmSync(this.generateHash(name), { force: true });
}

public async clear(): Promise<void> {
const files = await readdir(this.cache_dir);
await Promise.all(
files
.filter((f) => f.startsWith(this.prefix))
.map((f) => rm(join(this.cache_dir, f), { force: true }))
);
}

public clearSync(): void {
readdirSync(this.cache_dir)
.filter((f) => f.startsWith(this.prefix))
.forEach((f) => rmSync(join(this.cache_dir, f), { force: true }));
}

public async getAll(): Promise<CacheItem[]> {
const now = Date.now();
const files = await readdir(this.cache_dir);
const items = await Promise.all(
files
.filter((f) => f.startsWith(this.prefix))
.map((f) => readFile(join(this.cache_dir, f), 'utf8'))
);
return items
.map((data) => JSON.parse(data))
.filter((entry) => entry.content && !this.isExpired(entry, now));
}
Comment on lines +140 to +141
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

style: This approach might be inefficient for large caches. Consider implementing pagination or streaming


public async load(): Promise<{ files: CacheItem[] }> {
const res = await this.getAll();
return {
files: res.map((entry) => ({
path: this.generateHash(entry.key),
value: entry.content,
key: entry.key,
})),
};
}
}

export function createFileSystemCache(options: Options): FileSystemCache {
return Cache(options);
export function createFileSystemCache(options: FileSystemCacheOptions): FileSystemCache {
return new FileSystemCache(options);
}
2 changes: 1 addition & 1 deletion code/core/src/types/modules/core-common.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
/* eslint-disable @typescript-eslint/naming-convention */
import type { Router } from 'express';
import type { FileSystemCache } from 'file-system-cache';
// should be node:http, but that caused the ui/manager to fail to build, might be able to switch this back once ui/manager is in the core
import type { Server } from 'http';
import type * as telejson from 'telejson';
import type { PackageJson as PackageJsonFromTypeFest } from 'type-fest';

import type { FileSystemCache } from '../../common/utils/file-cache';
import type { Indexer, StoriesEntry } from './indexer';

/** ⚠️ This file contains internal WIP types they MUST NOT be exported outside this package for now! */
Expand Down
Loading
Loading