Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: persistent cache between runs (webpack@5 only) #541

Merged
merged 6 commits into from
Oct 22, 2020
Merged

feat: persistent cache between runs (webpack@5 only) #541

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
3139c04
feat: persistent cache between runs (webpack@5 only)
alexander-akait Oct 21, 2020
7487458
test: fix
alexander-akait Oct 21, 2020
d7bff39
test: fix
alexander-akait Oct 21, 2020
9a11643
refactor: code
alexander-akait Oct 21, 2020
2477837
refactor: code
alexander-akait Oct 22, 2020
4b29447
refactor: code
alexander-akait Oct 22, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ npm-debug.log*
/reports
/node_modules
/test/fixtures/\[special\$directory\]
/test/outputs

.DS_Store
Thumbs.db
Expand Down
208 changes: 155 additions & 53 deletions src/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,58 @@ class CopyPlugin {
this.options = options.options || {};
}

static async createSnapshot(compilation, startTime, dependency) {
if (!compilation.fileSystemInfo) {
return;
}

// eslint-disable-next-line consistent-return
return new Promise((resolve, reject) => {
compilation.fileSystemInfo.createSnapshot(
startTime,
[dependency],
// eslint-disable-next-line no-undefined
undefined,
// eslint-disable-next-line no-undefined
undefined,
null,
(error, snapshot) => {
if (error) {
reject(error);

return;
}

resolve(snapshot);
}
);
});
}

static async checkSnapshotValid(compilation, snapshot) {
if (!compilation.fileSystemInfo) {
return;
}

// eslint-disable-next-line consistent-return
return new Promise((resolve, reject) => {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://nodejs.org/api/util.html#util_util_promisify_original

const checkSnapshotValid = util.promisify((compilation, snapshot, callback) =>
  compilation.fileSystemInfo.checkSnapshotValid(snapshot, callback)
);
const createSnapshot = util.promisify((compilation, dependency, callback) =>
  compilation.fileSystemInfo.createSnapshot(Date.now(), [dependency], undefined, undefined, null, callback)
);

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Passing an undefined startTime, would assume worse case.

compilation.fileSystemInfo.checkSnapshotValid(
snapshot,
(error, isValid) => {
if (error) {
reject(error);

return;
}

resolve(isValid);
}
);
});
}

// eslint-disable-next-line class-methods-use-this
async runPattern(compiler, compilation, logger, inputPattern) {
static async runPattern(compiler, compilation, logger, cache, inputPattern) {
const pattern =
typeof inputPattern === 'string'
? { from: inputPattern }
Expand All @@ -49,7 +99,6 @@ class CopyPlugin {
pattern.to = path.normalize(
typeof pattern.to !== 'undefined' ? pattern.to : ''
);

pattern.context = path.normalize(
typeof pattern.context !== 'undefined'
? !path.isAbsolute(pattern.context)
Expand Down Expand Up @@ -266,64 +315,109 @@ class CopyPlugin {
compilation.fileDependencies.add(file.absoluteFrom);
}

logger.debug(`reading "${file.absoluteFrom}" to write to assets`);
let source;

let data;
if (cache) {
const cacheEntry = await cache.getPromise(file.relativeFrom, null);

try {
data = await readFile(inputFileSystem, file.absoluteFrom);
} catch (error) {
compilation.errors.push(error);
if (cacheEntry) {
const isValidSnapshot = await CopyPlugin.checkSnapshotValid(
compilation,
cacheEntry.snapshot
);

return;
if (isValidSnapshot) {
({ source } = cacheEntry);
}
}
}

if (pattern.transform) {
logger.log(`transforming content for "${file.absoluteFrom}"`);

if (pattern.cacheTransform) {
const cacheDirectory = pattern.cacheTransform.directory
? pattern.cacheTransform.directory
: typeof pattern.cacheTransform === 'string'
? pattern.cacheTransform
: findCacheDir({ name: 'copy-webpack-plugin' }) || os.tmpdir();
let defaultCacheKeys = {
version,
transform: pattern.transform,
contentHash: crypto.createHash('md4').update(data).digest('hex'),
};

if (typeof pattern.cacheTransform.keys === 'function') {
defaultCacheKeys = await pattern.cacheTransform.keys(
defaultCacheKeys,
file.absoluteFrom
);
} else {
defaultCacheKeys = {
...defaultCacheKeys,
...pattern.cacheTransform.keys,
if (!source) {
let startTime;

if (cache) {
startTime = Date.now();
}

logger.debug(`reading "${file.absoluteFrom}" to write to assets`);

let data;

try {
data = await readFile(inputFileSystem, file.absoluteFrom);
} catch (error) {
compilation.errors.push(error);

return;
}

if (pattern.transform) {
logger.log(`transforming content for "${file.absoluteFrom}"`);

if (pattern.cacheTransform) {
const cacheDirectory = pattern.cacheTransform.directory
? pattern.cacheTransform.directory
: typeof pattern.cacheTransform === 'string'
? pattern.cacheTransform
: findCacheDir({ name: 'copy-webpack-plugin' }) || os.tmpdir();
let defaultCacheKeys = {
version,
transform: pattern.transform,
contentHash: crypto
.createHash('md4')
.update(data)
.digest('hex'),
};
}

const cacheKeys = serialize(defaultCacheKeys);
if (typeof pattern.cacheTransform.keys === 'function') {
defaultCacheKeys = await pattern.cacheTransform.keys(
defaultCacheKeys,
file.absoluteFrom
);
} else {
defaultCacheKeys = {
...defaultCacheKeys,
...pattern.cacheTransform.keys,
};
}

try {
const result = await cacache.get(cacheDirectory, cacheKeys);
const cacheKeys = serialize(defaultCacheKeys);

logger.debug(
`getting cached transformation for "${file.absoluteFrom}"`
);
try {
const result = await cacache.get(cacheDirectory, cacheKeys);

({ data } = result);
} catch (_ignoreError) {
data = await pattern.transform(data, file.absoluteFrom);
logger.debug(
`getting cached transformation for "${file.absoluteFrom}"`
);

logger.debug(`caching transformation for "${file.absoluteFrom}"`);
({ data } = result);
} catch (_ignoreError) {
data = await pattern.transform(data, file.absoluteFrom);

logger.debug(
`caching transformation for "${file.absoluteFrom}"`
);

await cacache.put(cacheDirectory, cacheKeys, data);
await cacache.put(cacheDirectory, cacheKeys, data);
}
} else {
data = await pattern.transform(data, file.absoluteFrom);
}
} else {
data = await pattern.transform(data, file.absoluteFrom);
}

source = new RawSource(data);

if (cache) {
const snapshot = await CopyPlugin.createSnapshot(
compilation,
startTime,
file.relativeFrom
);

await cache.storePromise(file.relativeFrom, null, {
source,
snapshot,
});
}
}

Expand All @@ -349,7 +443,7 @@ class CopyPlugin {
{ resourcePath: file.absoluteFrom },
file.webpackTo,
{
content: data,
content: source.source(),
context: pattern.context,
}
);
Expand All @@ -374,7 +468,7 @@ class CopyPlugin {
}

// eslint-disable-next-line no-param-reassign
file.data = data;
file.source = source;
// eslint-disable-next-line no-param-reassign
file.targetPath = normalizePath(file.webpackTo);
// eslint-disable-next-line no-param-reassign
Expand All @@ -392,6 +486,10 @@ class CopyPlugin {

compiler.hooks.thisCompilation.tap(pluginName, (compilation) => {
const logger = compilation.getLogger('copy-webpack-plugin');
const cache = compilation.getCache
? compilation.getCache('CopyWebpackPlugin')
: // eslint-disable-next-line no-undefined
undefined;

compilation.hooks.additionalAssets.tapAsync(
'copy-webpack-plugin',
Expand All @@ -404,7 +502,13 @@ class CopyPlugin {
assets = await Promise.all(
this.patterns.map((item) =>
limit(async () =>
this.runPattern(compiler, compilation, logger, item)
CopyPlugin.runPattern(
compiler,
compilation,
logger,
cache,
item
)
)
)
);
Expand All @@ -426,12 +530,10 @@ class CopyPlugin {
absoluteFrom,
targetPath,
webpackTo,
data,
source,
force,
} = asset;

const source = new RawSource(data);

// For old version webpack 4
/* istanbul ignore if */
if (typeof compilation.emitAsset !== 'function') {
Expand Down
70 changes: 67 additions & 3 deletions test/CopyPlugin.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import path from 'path';

import webpack from 'webpack';
import del from 'del';

import CopyPlugin from '../src';

Expand Down Expand Up @@ -633,9 +634,72 @@ describe('CopyPlugin', () => {
.then(done)
.catch(done);
});
});

describe('cache', () => {
it('should work with the "memory" cache', async () => {
const compiler = getCompiler({
cache: {
type: 'memory',
},
});

new CopyPlugin({
patterns: [
{
from: path.resolve(__dirname, './fixtures/directory'),
},
],
}).apply(compiler);

const { stats } = await compile(compiler);

if (webpack.version[0] === '4') {
expect(
Object.keys(stats.compilation.assets).filter(
(assetName) => stats.compilation.assets[assetName].emitted
).length
).toBe(5);
} else {
expect(stats.compilation.emittedAssets.size).toBe(5);
}

expect(readAssets(compiler, stats)).toMatchSnapshot('assets');
expect(stats.compilation.errors).toMatchSnapshot('errors');
expect(stats.compilation.warnings).toMatchSnapshot('warnings');

await new Promise(async (resolve) => {
const { stats: newStats } = await compile(compiler);

if (webpack.version[0] === '4') {
expect(
Object.keys(newStats.compilation.assets).filter(
(assetName) => newStats.compilation.assets[assetName].emitted
).length
).toBe(4);
} else {
expect(newStats.compilation.emittedAssets.size).toBe(0);
}

it('should work and do not emit unchanged assets', async () => {
const compiler = getCompiler();
expect(readAssets(compiler, newStats)).toMatchSnapshot('assets');
expect(newStats.compilation.errors).toMatchSnapshot('errors');
expect(newStats.compilation.warnings).toMatchSnapshot('warnings');

resolve();
});
});

it('should work with the "filesystem" cache', async () => {
const cacheDirectory = path.resolve(__dirname, './outputs/.cache');

await del(cacheDirectory);

const compiler = getCompiler({
cache: {
type: 'filesystem',
cacheDirectory,
},
});

new CopyPlugin({
patterns: [
Expand Down Expand Up @@ -671,7 +735,7 @@ describe('CopyPlugin', () => {
).length
).toBe(4);
} else {
expect(newStats.compilation.emittedAssets.size).toBe(4);
expect(newStats.compilation.emittedAssets.size).toBe(0);
}

expect(readAssets(compiler, newStats)).toMatchSnapshot('assets');
Expand Down
Loading