From d066cec84f35280b0edb469bc7612f580265ae8b Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Tue, 26 Oct 2021 23:32:04 +0200 Subject: [PATCH 1/3] fix(gatsby-plugin-sharp): pass input buffer instead of readStream when processing image jobs --- .../gatsby-plugin-sharp/src/gatsby-worker.js | 20 +- .../gatsby-plugin-sharp/src/process-file.js | 201 +++++++++--------- 2 files changed, 112 insertions(+), 109 deletions(-) diff --git a/packages/gatsby-plugin-sharp/src/gatsby-worker.js b/packages/gatsby-plugin-sharp/src/gatsby-worker.js index f1670de53070d..0b34c1a6579cb 100644 --- a/packages/gatsby-plugin-sharp/src/gatsby-worker.js +++ b/packages/gatsby-plugin-sharp/src/gatsby-worker.js @@ -20,17 +20,15 @@ exports.IMAGE_PROCESSING_JOB_NAME = `IMAGE_PROCESSING` */ const q = queue( async ({ inputPaths, outputDir, args }) => - Promise.all( - processFile( - inputPaths[0].path, - args.operations.map(operation => { - return { - outputPath: path.join(outputDir, operation.outputPath), - args: operation.args, - } - }), - args.pluginOptions - ) + processFile( + inputPaths[0].path, + args.operations.map(operation => { + return { + outputPath: path.join(outputDir, operation.outputPath), + args: operation.args, + } + }), + args.pluginOptions ), // When inside query workers, we only want to use the current core process.env.GATSBY_WORKER_POOL_WORKER ? 1 : Math.max(1, cpuCoreCount() - 1) diff --git a/packages/gatsby-plugin-sharp/src/process-file.js b/packages/gatsby-plugin-sharp/src/process-file.js index 2318cc904413a..e8ab67a094346 100644 --- a/packages/gatsby-plugin-sharp/src/process-file.js +++ b/packages/gatsby-plugin-sharp/src/process-file.js @@ -53,10 +53,13 @@ sharp.concurrency(1) * @param {String} file * @param {Transform[]} transforms */ -exports.processFile = (file, transforms, options = {}) => { +exports.processFile = async (file, transforms, options = {}) => { let pipeline try { - pipeline = !options.failOnError ? sharp({ failOnError: false }) : sharp() + const inputBuffer = await fs.readFile(file) + pipeline = !options.failOnError + ? sharp(inputBuffer, { failOnError: false }) + : sharp(inputBuffer) // Keep Metadata if (!options.stripMetadata) { @@ -67,108 +70,110 @@ exports.processFile = (file, transforms, options = {}) => { throw new SharpError(`Failed to load image ${file} into sharp.`, err) } - return transforms.map(async transform => { - try { - const { outputPath, args } = transform - debug(`Start processing ${outputPath}`) - await fs.ensureDir(path.dirname(outputPath)) - - const transformArgs = healOptions( - { defaultQuality: options.defaultQuality }, - args - ) - - let clonedPipeline = transforms.length > 1 ? pipeline.clone() : pipeline - - if (transformArgs.trim) { - clonedPipeline = clonedPipeline.trim(transformArgs.trim) - } - - if (!transformArgs.rotate) { - clonedPipeline = clonedPipeline.rotate() - } - - // Sharp only allows ints as height/width. Since both aren't always - // set, check first before trying to round them. - let roundedHeight = transformArgs.height - if (roundedHeight) { - roundedHeight = Math.round(roundedHeight) - } - - let roundedWidth = transformArgs.width - if (roundedWidth) { - roundedWidth = Math.round(roundedWidth) - } - - clonedPipeline - .resize(roundedWidth, roundedHeight, { - position: transformArgs.cropFocus, - fit: transformArgs.fit, - background: transformArgs.background, - }) - .png({ - compressionLevel: transformArgs.pngCompressionLevel, - adaptiveFiltering: false, - quality: transformArgs.pngQuality || transformArgs.quality, - force: transformArgs.toFormat === `png`, - }) - .webp({ - quality: transformArgs.webpQuality || transformArgs.quality, - force: transformArgs.toFormat === `webp`, - }) - .tiff({ - quality: transformArgs.quality, - force: transformArgs.toFormat === `tiff`, - }) - .avif({ - quality: transformArgs.quality, - force: transformArgs.toFormat === `avif`, - }) - .jpeg({ - mozjpeg: options.useMozJpeg, - quality: transformArgs.jpegQuality || transformArgs.quality, - progressive: transformArgs.jpegProgressive, - force: transformArgs.toFormat === `jpg`, - }) - - // grayscale - if (transformArgs.grayscale) { - clonedPipeline = clonedPipeline.grayscale() - } - - // rotate - if (transformArgs.rotate && transformArgs.rotate !== 0) { - clonedPipeline = clonedPipeline.rotate(transformArgs.rotate) - } + return Promise.all( + transforms.map(async transform => { + try { + const { outputPath, args } = transform + debug(`Start processing ${outputPath}`) + await fs.ensureDir(path.dirname(outputPath)) - // duotone - if (transformArgs.duotone) { - clonedPipeline = await duotone( - transformArgs.duotone, - transformArgs.toFormat, - clonedPipeline + const transformArgs = healOptions( + { defaultQuality: options.defaultQuality }, + args ) - } - try { - const buffer = await clonedPipeline.toBuffer() - await fs.writeFile(outputPath, buffer) + let clonedPipeline = transforms.length > 1 ? pipeline.clone() : pipeline + + if (transformArgs.trim) { + clonedPipeline = clonedPipeline.trim(transformArgs.trim) + } + + if (!transformArgs.rotate) { + clonedPipeline = clonedPipeline.rotate() + } + + // Sharp only allows ints as height/width. Since both aren't always + // set, check first before trying to round them. + let roundedHeight = transformArgs.height + if (roundedHeight) { + roundedHeight = Math.round(roundedHeight) + } + + let roundedWidth = transformArgs.width + if (roundedWidth) { + roundedWidth = Math.round(roundedWidth) + } + + clonedPipeline + .resize(roundedWidth, roundedHeight, { + position: transformArgs.cropFocus, + fit: transformArgs.fit, + background: transformArgs.background, + }) + .png({ + compressionLevel: transformArgs.pngCompressionLevel, + adaptiveFiltering: false, + quality: transformArgs.pngQuality || transformArgs.quality, + force: transformArgs.toFormat === `png`, + }) + .webp({ + quality: transformArgs.webpQuality || transformArgs.quality, + force: transformArgs.toFormat === `webp`, + }) + .tiff({ + quality: transformArgs.quality, + force: transformArgs.toFormat === `tiff`, + }) + .avif({ + quality: transformArgs.quality, + force: transformArgs.toFormat === `avif`, + }) + .jpeg({ + mozjpeg: options.useMozJpeg, + quality: transformArgs.jpegQuality || transformArgs.quality, + progressive: transformArgs.jpegProgressive, + force: transformArgs.toFormat === `jpg`, + }) + + // grayscale + if (transformArgs.grayscale) { + clonedPipeline = clonedPipeline.grayscale() + } + + // rotate + if (transformArgs.rotate && transformArgs.rotate !== 0) { + clonedPipeline = clonedPipeline.rotate(transformArgs.rotate) + } + + // duotone + if (transformArgs.duotone) { + clonedPipeline = await duotone( + transformArgs.duotone, + transformArgs.toFormat, + clonedPipeline + ) + } + + try { + const buffer = await clonedPipeline.toBuffer() + await fs.writeFile(outputPath, buffer) + } catch (err) { + throw new Error( + `Failed to write ${file} into ${outputPath}. (${err.message})` + ) + } } catch (err) { - throw new Error( - `Failed to write ${file} into ${outputPath}. (${err.message})` - ) - } - } catch (err) { - if (err instanceof SharpError) { - // rethrow - throw err - } + if (err instanceof SharpError) { + // rethrow + throw err + } - throw new SharpError(`Processing ${file} failed`, err) - } + throw new SharpError(`Processing ${file} failed`, err) + } - return transform - }) + return transform + }) + ) } exports.createArgsDigest = args => { From 4268caac5540b42b9ff706b630fc07f9b7f8c903 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Wed, 27 Oct 2021 00:15:08 +0200 Subject: [PATCH 2/3] fix tests --- packages/gatsby-plugin-sharp/src/__tests__/gatsby-worker.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/gatsby-plugin-sharp/src/__tests__/gatsby-worker.js b/packages/gatsby-plugin-sharp/src/__tests__/gatsby-worker.js index a527345205998..d14235643dba7 100644 --- a/packages/gatsby-plugin-sharp/src/__tests__/gatsby-worker.js +++ b/packages/gatsby-plugin-sharp/src/__tests__/gatsby-worker.js @@ -54,9 +54,9 @@ describe(`worker`, () => { }) it(`should fail a promise when image processing fails`, async () => { - processFile.mockImplementation(() => [ - Promise.reject(new Error(`transform failed`)), - ]) + processFile.mockImplementation(() => + Promise.reject(new Error(`transform failed`)) + ) const job = { inputPaths: [ From da65fc47996bafd05302f4f4f157802d25e1a71a Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Wed, 27 Oct 2021 00:26:35 +0200 Subject: [PATCH 3/3] actually drop using streams, doh --- packages/gatsby-plugin-sharp/src/process-file.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/gatsby-plugin-sharp/src/process-file.js b/packages/gatsby-plugin-sharp/src/process-file.js index e8ab67a094346..ebd7d33748016 100644 --- a/packages/gatsby-plugin-sharp/src/process-file.js +++ b/packages/gatsby-plugin-sharp/src/process-file.js @@ -65,7 +65,6 @@ exports.processFile = async (file, transforms, options = {}) => { if (!options.stripMetadata) { pipeline = pipeline.withMetadata() } - fs.createReadStream(file).pipe(pipeline) } catch (err) { throw new SharpError(`Failed to load image ${file} into sharp.`, err) }