Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(gatsby-plugin-sharp): pass input buffer instead of readStream when processing image jobs #33685

Merged
merged 3 commits into from
Oct 27, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions packages/gatsby-plugin-sharp/src/__tests__/gatsby-worker.js
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,9 @@ describe(`worker`, () => {
})

it(`should fail a promise when image processing fails`, async () => {
processFile.mockImplementation(() => [
Promise.reject(new Error(`transform failed`)),
])
processFile.mockImplementation(() =>
Promise.reject(new Error(`transform failed`))
)

const job = {
inputPaths: [
Expand Down
20 changes: 9 additions & 11 deletions packages/gatsby-plugin-sharp/src/gatsby-worker.js
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,15 @@ exports.IMAGE_PROCESSING_JOB_NAME = `IMAGE_PROCESSING`
*/
const q = queue(
async ({ inputPaths, outputDir, args }) =>
Promise.all(
processFile(
inputPaths[0].path,
args.operations.map(operation => {
return {
outputPath: path.join(outputDir, operation.outputPath),
args: operation.args,
}
}),
args.pluginOptions
)
processFile(
inputPaths[0].path,
args.operations.map(operation => {
return {
outputPath: path.join(outputDir, operation.outputPath),
args: operation.args,
}
}),
args.pluginOptions
),
// When inside query workers, we only want to use the current core
process.env.GATSBY_WORKER_POOL_WORKER ? 1 : Math.max(1, cpuCoreCount() - 1)
Expand Down
202 changes: 103 additions & 99 deletions packages/gatsby-plugin-sharp/src/process-file.js
Original file line number Diff line number Diff line change
Expand Up @@ -53,122 +53,126 @@ sharp.concurrency(1)
* @param {String} file
* @param {Transform[]} transforms
*/
exports.processFile = (file, transforms, options = {}) => {
exports.processFile = async (file, transforms, options = {}) => {
let pipeline
try {
pipeline = !options.failOnError ? sharp({ failOnError: false }) : sharp()
const inputBuffer = await fs.readFile(file)
pipeline = !options.failOnError
? sharp(inputBuffer, { failOnError: false })
: sharp(inputBuffer)

// Keep Metadata
if (!options.stripMetadata) {
pipeline = pipeline.withMetadata()
}
fs.createReadStream(file).pipe(pipeline)
} catch (err) {
throw new SharpError(`Failed to load image ${file} into sharp.`, err)
}

return transforms.map(async transform => {
try {
const { outputPath, args } = transform
debug(`Start processing ${outputPath}`)
await fs.ensureDir(path.dirname(outputPath))

const transformArgs = healOptions(
{ defaultQuality: options.defaultQuality },
args
)

let clonedPipeline = transforms.length > 1 ? pipeline.clone() : pipeline

if (transformArgs.trim) {
clonedPipeline = clonedPipeline.trim(transformArgs.trim)
}

if (!transformArgs.rotate) {
clonedPipeline = clonedPipeline.rotate()
}

// Sharp only allows ints as height/width. Since both aren't always
// set, check first before trying to round them.
let roundedHeight = transformArgs.height
if (roundedHeight) {
roundedHeight = Math.round(roundedHeight)
}

let roundedWidth = transformArgs.width
if (roundedWidth) {
roundedWidth = Math.round(roundedWidth)
}

clonedPipeline
.resize(roundedWidth, roundedHeight, {
position: transformArgs.cropFocus,
fit: transformArgs.fit,
background: transformArgs.background,
})
.png({
compressionLevel: transformArgs.pngCompressionLevel,
adaptiveFiltering: false,
quality: transformArgs.pngQuality || transformArgs.quality,
force: transformArgs.toFormat === `png`,
})
.webp({
quality: transformArgs.webpQuality || transformArgs.quality,
force: transformArgs.toFormat === `webp`,
})
.tiff({
quality: transformArgs.quality,
force: transformArgs.toFormat === `tiff`,
})
.avif({
quality: transformArgs.quality,
force: transformArgs.toFormat === `avif`,
})
.jpeg({
mozjpeg: options.useMozJpeg,
quality: transformArgs.jpegQuality || transformArgs.quality,
progressive: transformArgs.jpegProgressive,
force: transformArgs.toFormat === `jpg`,
})

// grayscale
if (transformArgs.grayscale) {
clonedPipeline = clonedPipeline.grayscale()
}

// rotate
if (transformArgs.rotate && transformArgs.rotate !== 0) {
clonedPipeline = clonedPipeline.rotate(transformArgs.rotate)
}
return Promise.all(
transforms.map(async transform => {
try {
const { outputPath, args } = transform
debug(`Start processing ${outputPath}`)
await fs.ensureDir(path.dirname(outputPath))

// duotone
if (transformArgs.duotone) {
clonedPipeline = await duotone(
transformArgs.duotone,
transformArgs.toFormat,
clonedPipeline
const transformArgs = healOptions(
{ defaultQuality: options.defaultQuality },
args
)
}

try {
const buffer = await clonedPipeline.toBuffer()
await fs.writeFile(outputPath, buffer)
let clonedPipeline = transforms.length > 1 ? pipeline.clone() : pipeline

if (transformArgs.trim) {
clonedPipeline = clonedPipeline.trim(transformArgs.trim)
}

if (!transformArgs.rotate) {
clonedPipeline = clonedPipeline.rotate()
}

// Sharp only allows ints as height/width. Since both aren't always
// set, check first before trying to round them.
let roundedHeight = transformArgs.height
if (roundedHeight) {
roundedHeight = Math.round(roundedHeight)
}

let roundedWidth = transformArgs.width
if (roundedWidth) {
roundedWidth = Math.round(roundedWidth)
}

clonedPipeline
.resize(roundedWidth, roundedHeight, {
position: transformArgs.cropFocus,
fit: transformArgs.fit,
background: transformArgs.background,
})
.png({
compressionLevel: transformArgs.pngCompressionLevel,
adaptiveFiltering: false,
quality: transformArgs.pngQuality || transformArgs.quality,
force: transformArgs.toFormat === `png`,
})
.webp({
quality: transformArgs.webpQuality || transformArgs.quality,
force: transformArgs.toFormat === `webp`,
})
.tiff({
quality: transformArgs.quality,
force: transformArgs.toFormat === `tiff`,
})
.avif({
quality: transformArgs.quality,
force: transformArgs.toFormat === `avif`,
})
.jpeg({
mozjpeg: options.useMozJpeg,
quality: transformArgs.jpegQuality || transformArgs.quality,
progressive: transformArgs.jpegProgressive,
force: transformArgs.toFormat === `jpg`,
})

// grayscale
if (transformArgs.grayscale) {
clonedPipeline = clonedPipeline.grayscale()
}

// rotate
if (transformArgs.rotate && transformArgs.rotate !== 0) {
clonedPipeline = clonedPipeline.rotate(transformArgs.rotate)
}

// duotone
if (transformArgs.duotone) {
clonedPipeline = await duotone(
transformArgs.duotone,
transformArgs.toFormat,
clonedPipeline
)
}

try {
const buffer = await clonedPipeline.toBuffer()
await fs.writeFile(outputPath, buffer)
} catch (err) {
throw new Error(
`Failed to write ${file} into ${outputPath}. (${err.message})`
)
}
} catch (err) {
throw new Error(
`Failed to write ${file} into ${outputPath}. (${err.message})`
)
}
} catch (err) {
if (err instanceof SharpError) {
// rethrow
throw err
}
if (err instanceof SharpError) {
// rethrow
throw err
}

throw new SharpError(`Processing ${file} failed`, err)
}
throw new SharpError(`Processing ${file} failed`, err)
}

return transform
})
return transform
})
)
}

exports.createArgsDigest = args => {
Expand Down