Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor: use file paths in cache keys #1125

Merged
merged 5 commits into from
May 10, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/polyfill/fsPoly.ts
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ export default class FsPoly {
return FsPoly.DRIVES
.filter((drive) => drive.available > 0)
.map((drive) => drive.mounted)
.filter((mountPath) => mountPath !== '/')
// Sort by mount points with the deepest number of subdirectories first
.sort((a, b) => b.split(/[\\/]/).length - a.split(/[\\/]/).length);
}
Expand Down
33 changes: 27 additions & 6 deletions src/types/files/fileCache.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { Stats } from 'node:fs';
import { Semaphore } from 'async-mutex';

import Constants from '../../constants.js';
import FsPoly from '../../polyfill/fsPoly.js';
import Cache from '../cache.js';
import Archive from './archives/archive.js';
Expand All @@ -19,7 +20,7 @@ enum ValueType {
}

export default class FileCache {
private static readonly VERSION = 2;
private static readonly VERSION = 3;

private static cache: Cache<CacheValue> | Promise<Cache<CacheValue>> = new Cache<CacheValue>();

Expand All @@ -35,6 +36,7 @@ export default class FileCache {
fileFlushMillis: 30_000,
})
.load()
// Cleanup the loaded cache file
.then(async (cache) => {
// Delete keys from old cache versions
await Promise.all([...Array.from({ length: FileCache.VERSION }).keys()].slice(1)
Expand All @@ -43,6 +45,25 @@ export default class FileCache {
return cache.delete(keyRegex);
}));
return cache;
})
.then(async (cache) => {
// Delete keys for deleted files
const disks = FsPoly.disksSync();
const semaphore = new Semaphore(Constants.MAX_FS_THREADS);
await Promise.all([...cache.keys()].map(async (cacheKey) => {
const cacheKeyFilePath = cacheKey.split('|')[1];
if (!disks.some((disk) => cacheKeyFilePath.startsWith(disk))) {
// Don't delete the key if it's for a disk that isn't mounted right now
return;
}
await semaphore.runExclusive(async () => {
if (!await FsPoly.exists(cacheKeyFilePath)) {
// If the file no longer exists then delete its key from the cache
await cache.delete(cacheKeyFilePath);
}
});
}));
return cache;
});
}

Expand All @@ -56,7 +77,7 @@ export default class FileCache {

// NOTE(cemmer): we're explicitly not catching ENOENT errors here, we want it to bubble up
const stats = await FsPoly.stat(filePath);
const cacheKey = this.getCacheKey(stats, ValueType.FILE);
const cacheKey = this.getCacheKey(filePath, ValueType.FILE);

// NOTE(cemmer): we're using the cache as a mutex here, so even if this function is called
// multiple times concurrently, entries will only be fetched once.
Expand Down Expand Up @@ -109,7 +130,7 @@ export default class FileCache {

// NOTE(cemmer): we're explicitly not catching ENOENT errors here, we want it to bubble up
const stats = await FsPoly.stat(archive.getFilePath());
const cacheKey = this.getCacheKey(stats, ValueType.ARCHIVE_ENTRIES);
const cacheKey = this.getCacheKey(archive.getFilePath(), ValueType.ARCHIVE_ENTRIES);

// NOTE(cemmer): we're using the cache as a mutex here, so even if this function is called
// multiple times concurrently, entries will only be fetched once.
Expand Down Expand Up @@ -161,7 +182,7 @@ export default class FileCache {
})));
}

private static getCacheKey(stats: Stats, valueType: ValueType): string {
return `V${FileCache.VERSION}|${stats.ino}|${valueType}`;
private static getCacheKey(filePath: string, valueType: ValueType): string {
return `V${FileCache.VERSION}|${filePath}|${valueType}`;
}
}
Loading