Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: persistent cache between runs (webpack@5 only) #541

Merged
merged 6 commits into from
Oct 22, 2020
Merged

feat: persistent cache between runs (webpack@5 only) #541

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
3139c04
feat: persistent cache between runs (webpack@5 only)
alexander-akait Oct 21, 2020
7487458
test: fix
alexander-akait Oct 21, 2020
d7bff39
test: fix
alexander-akait Oct 21, 2020
9a11643
refactor: code
alexander-akait Oct 21, 2020
2477837
refactor: code
alexander-akait Oct 22, 2020
4b29447
refactor: code
alexander-akait Oct 22, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ npm-debug.log*
/reports
/node_modules
/test/fixtures/\[special\$directory\]
/test/outputs

.DS_Store
Thumbs.db
Expand Down
225 changes: 173 additions & 52 deletions src/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,57 @@ class CopyPlugin {
this.options = options.options || {};
}

static async createSnapshot(compilation, dependency) {
if (!compilation.fileSystemInfo) {
return;
}

// eslint-disable-next-line consistent-return
return new Promise((resolve, reject) => {
compilation.fileSystemInfo.createSnapshot(
// eslint-disable-next-line no-undefined
undefined,
[dependency],
[],
[],
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can pass undefined here, that saves a bit memory.

null,
(error, snapshot) => {
if (error) {
reject(error);

return;
}

resolve(snapshot);
}
);
});
}

static async checkSnapshotValid(compilation, snapshot) {
if (!compilation.fileSystemInfo) {
return;
}

// eslint-disable-next-line consistent-return
return new Promise((resolve, reject) => {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://nodejs.org/api/util.html#util_util_promisify_original

const checkSnapshotValid = util.promisify((compilation, snapshot, callback) =>
  compilation.fileSystemInfo.checkSnapshotValid(snapshot, callback)
);
const createSnapshot = util.promisify((compilation, dependency, callback) =>
  compilation.fileSystemInfo.createSnapshot(Date.now(), [dependency], undefined, undefined, null, callback)
);

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Passing an undefined startTime, would assume worse case.

compilation.fileSystemInfo.checkSnapshotValid(
snapshot,
(error, isValid) => {
if (error) {
reject(error);

return;
}

resolve(isValid);
}
);
});
}

// eslint-disable-next-line class-methods-use-this
async runPattern(compiler, compilation, logger, inputPattern) {
static async runPattern(compiler, compilation, logger, cache, inputPattern) {
const pattern =
typeof inputPattern === 'string'
? { from: inputPattern }
Expand All @@ -49,7 +98,6 @@ class CopyPlugin {
pattern.to = path.normalize(
typeof pattern.to !== 'undefined' ? pattern.to : ''
);

pattern.context = path.normalize(
typeof pattern.context !== 'undefined'
? !path.isAbsolute(pattern.context)
Expand Down Expand Up @@ -266,64 +314,129 @@ class CopyPlugin {
compilation.fileDependencies.add(file.absoluteFrom);
}

logger.debug(`reading "${file.absoluteFrom}" to write to assets`);
let itemCache;
let source;

let data;
// TODO logger
if (cache) {
let snapshot;

try {
data = await readFile(inputFileSystem, file.absoluteFrom);
} catch (error) {
compilation.errors.push(error);
try {
snapshot = await CopyPlugin.createSnapshot(
compilation,
file.absoluteFrom
);
} catch (error) {
compilation.errors.push(error);

return;
}
return;
}

if (snapshot) {
let isValidSnapshot;

if (pattern.transform) {
logger.log(`transforming content for "${file.absoluteFrom}"`);

if (pattern.cacheTransform) {
const cacheDirectory = pattern.cacheTransform.directory
? pattern.cacheTransform.directory
: typeof pattern.cacheTransform === 'string'
? pattern.cacheTransform
: findCacheDir({ name: 'copy-webpack-plugin' }) || os.tmpdir();
let defaultCacheKeys = {
version,
transform: pattern.transform,
contentHash: crypto.createHash('md4').update(data).digest('hex'),
};

if (typeof pattern.cacheTransform.keys === 'function') {
defaultCacheKeys = await pattern.cacheTransform.keys(
defaultCacheKeys,
file.absoluteFrom
try {
isValidSnapshot = await CopyPlugin.checkSnapshotValid(
compilation,
snapshot
);
} else {
defaultCacheKeys = {
...defaultCacheKeys,
...pattern.cacheTransform.keys,
};
} catch (error) {
compilation.errors.push(error);

return;
}

const cacheKeys = serialize(defaultCacheKeys);
itemCache = cache.getItemCache(file.relativeFrom, null);

try {
const result = await cacache.get(cacheDirectory, cacheKeys);
if (isValidSnapshot) {
try {
source = await itemCache.getPromise();
} catch (error) {
compilation.errors.push(error);

logger.debug(
`getting cached transformation for "${file.absoluteFrom}"`
);
return;
}
}
}
}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

checkSnapshotValid on a fresh created snapshot will always be true. You need to call checkSnapshotValid on a snapshot that was created in the last run of webpack. The snapshot need to be stored and restored in and from the cache.

Here is some pseudo code:

// 1. check cache first, validate snapshot in cache entry
cacheEntry = await cache.getPromise(file.relativeFrom, null);
if (cacheEntry !== undefined) {
  const { snapshot, source } = cacheEntry;
  if (await checkSnapshotValid(snapshot)) {
    // return early when cache entry can be used
    return source;
  }
}
// 2. create a new snapshot
const snapshot = await createSnapshot(file.relativeFrom);
// 3. create new data
const source = await createNewSource(file.relativeFrom);
// 4. store both in cache
await cache.storePromise(file.relativeFrom, null, { source, snapshot });
// 5. return new data
return source;

You are in the good position that you can call 2. before 3. Usually that's not possible, because dependencies are not known before execution. In that case one would do this:

// 2a.
const startTime = Date.now();
// 2b. create new data
const source = await createNewSource(file.relativeFrom);
// 3. create a new snapshot
const snapshot = await createSnapshot(file.relativeFrom, startTime);


if (!source) {
logger.debug(`reading "${file.absoluteFrom}" to write to assets`);

let data;

try {
data = await readFile(inputFileSystem, file.absoluteFrom);
} catch (error) {
compilation.errors.push(error);

return;
}

if (pattern.transform) {
logger.log(`transforming content for "${file.absoluteFrom}"`);

if (pattern.cacheTransform) {
const cacheDirectory = pattern.cacheTransform.directory
? pattern.cacheTransform.directory
: typeof pattern.cacheTransform === 'string'
? pattern.cacheTransform
: findCacheDir({ name: 'copy-webpack-plugin' }) || os.tmpdir();
let defaultCacheKeys = {
version,
transform: pattern.transform,
contentHash: crypto
.createHash('md4')
.update(data)
.digest('hex'),
};

({ data } = result);
} catch (_ignoreError) {
if (typeof pattern.cacheTransform.keys === 'function') {
defaultCacheKeys = await pattern.cacheTransform.keys(
defaultCacheKeys,
file.absoluteFrom
);
} else {
defaultCacheKeys = {
...defaultCacheKeys,
...pattern.cacheTransform.keys,
};
}

const cacheKeys = serialize(defaultCacheKeys);

try {
const result = await cacache.get(cacheDirectory, cacheKeys);

logger.debug(
`getting cached transformation for "${file.absoluteFrom}"`
);

({ data } = result);
} catch (_ignoreError) {
data = await pattern.transform(data, file.absoluteFrom);

logger.debug(
`caching transformation for "${file.absoluteFrom}"`
);

await cacache.put(cacheDirectory, cacheKeys, data);
}
} else {
data = await pattern.transform(data, file.absoluteFrom);
}
}

source = new RawSource(data);

logger.debug(`caching transformation for "${file.absoluteFrom}"`);
if (itemCache) {
try {
await itemCache.storePromise(source);
} catch (error) {
compilation.errors.push(error);

await cacache.put(cacheDirectory, cacheKeys, data);
return;
}
} else {
data = await pattern.transform(data, file.absoluteFrom);
}
}

Expand All @@ -349,7 +462,7 @@ class CopyPlugin {
{ resourcePath: file.absoluteFrom },
file.webpackTo,
{
content: data,
content: source.source(),
context: pattern.context,
}
);
Expand All @@ -374,7 +487,7 @@ class CopyPlugin {
}

// eslint-disable-next-line no-param-reassign
file.data = data;
file.source = source;
// eslint-disable-next-line no-param-reassign
file.targetPath = normalizePath(file.webpackTo);
// eslint-disable-next-line no-param-reassign
Expand All @@ -392,6 +505,10 @@ class CopyPlugin {

compiler.hooks.thisCompilation.tap(pluginName, (compilation) => {
const logger = compilation.getLogger('copy-webpack-plugin');
// eslint-disable-next-line no-undefined
const cache = compilation.getCache
? compilation.getCache('CopyWebpackPlugin')
: undefined;

compilation.hooks.additionalAssets.tapAsync(
'copy-webpack-plugin',
Expand All @@ -404,7 +521,13 @@ class CopyPlugin {
assets = await Promise.all(
this.patterns.map((item) =>
limit(async () =>
this.runPattern(compiler, compilation, logger, item)
CopyPlugin.runPattern(
compiler,
compilation,
logger,
cache,
item
)
)
)
);
Expand All @@ -426,12 +549,10 @@ class CopyPlugin {
absoluteFrom,
targetPath,
webpackTo,
data,
source,
force,
} = asset;

const source = new RawSource(data);

// For old version webpack 4
/* istanbul ignore if */
if (typeof compilation.emitAsset !== 'function') {
Expand Down
70 changes: 67 additions & 3 deletions test/CopyPlugin.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import path from 'path';

import webpack from 'webpack';
import del from 'del';

import CopyPlugin from '../src';

Expand Down Expand Up @@ -633,9 +634,72 @@ describe('CopyPlugin', () => {
.then(done)
.catch(done);
});
});

describe('cache', () => {
it('should work with the "memory" cache', async () => {
const compiler = getCompiler({
cache: {
type: 'memory',
},
});

new CopyPlugin({
patterns: [
{
from: path.resolve(__dirname, './fixtures/directory'),
},
],
}).apply(compiler);

const { stats } = await compile(compiler);

if (webpack.version[0] === '4') {
expect(
Object.keys(stats.compilation.assets).filter(
(assetName) => stats.compilation.assets[assetName].emitted
).length
).toBe(5);
} else {
expect(stats.compilation.emittedAssets.size).toBe(5);
}

expect(readAssets(compiler, stats)).toMatchSnapshot('assets');
expect(stats.compilation.errors).toMatchSnapshot('errors');
expect(stats.compilation.warnings).toMatchSnapshot('warnings');

await new Promise(async (resolve) => {
const { stats: newStats } = await compile(compiler);

if (webpack.version[0] === '4') {
expect(
Object.keys(newStats.compilation.assets).filter(
(assetName) => newStats.compilation.assets[assetName].emitted
).length
).toBe(4);
} else {
expect(newStats.compilation.emittedAssets.size).toBe(0);
}

it('should work and do not emit unchanged assets', async () => {
const compiler = getCompiler();
expect(readAssets(compiler, newStats)).toMatchSnapshot('assets');
expect(newStats.compilation.errors).toMatchSnapshot('errors');
expect(newStats.compilation.warnings).toMatchSnapshot('warnings');

resolve();
});
});

it('should work with the "filesystem" cache', async () => {
const cacheDirectory = path.resolve(__dirname, './outputs/.cache');

await del(cacheDirectory);

const compiler = getCompiler({
cache: {
type: 'filesystem',
cacheDirectory,
},
});

new CopyPlugin({
patterns: [
Expand Down Expand Up @@ -671,7 +735,7 @@ describe('CopyPlugin', () => {
).length
).toBe(4);
} else {
expect(newStats.compilation.emittedAssets.size).toBe(4);
expect(newStats.compilation.emittedAssets.size).toBe(0);
}

expect(readAssets(compiler, newStats)).toMatchSnapshot('assets');
Expand Down
Loading