From e6d9eb37f95ad096febbf513d330c8e5b06299e2 Mon Sep 17 00:00:00 2001 From: Lennart Date: Fri, 3 Sep 2021 16:18:30 +0200 Subject: [PATCH] feat(gatsby-plugin-sharp): Use file streams instead of file paths (#33029) * copied changes * new changes * fix(sharp): retain order of operations and images * use toBuffer and writeFile instead write stream (at least for now) * revert making queueImageResizing async * fixing up too much of revert in previous commit Co-authored-by: Michal Piechowiak --- .../src/__tests__/trace-svg.js | 17 +++++++++++++++++ packages/gatsby-plugin-sharp/src/image-data.ts | 7 +++++-- packages/gatsby-plugin-sharp/src/index.js | 15 ++++++++++----- .../gatsby-plugin-sharp/src/process-file.js | 8 ++++---- packages/gatsby-plugin-sharp/src/trace-svg.js | 4 +++- packages/gatsby-plugin-sharp/src/utils.js | 7 ++++++- 6 files changed, 45 insertions(+), 13 deletions(-) diff --git a/packages/gatsby-plugin-sharp/src/__tests__/trace-svg.js b/packages/gatsby-plugin-sharp/src/__tests__/trace-svg.js index e77506aa96e06..ddb790fa5b049 100644 --- a/packages/gatsby-plugin-sharp/src/__tests__/trace-svg.js +++ b/packages/gatsby-plugin-sharp/src/__tests__/trace-svg.js @@ -6,6 +6,11 @@ jest.mock(`sharp`, () => { png: () => pipeline, jpeg: () => pipeline, toFile: (_, cb) => cb(), + on: () => pipeline, + once: () => pipeline, + write: () => pipeline, + end: () => pipeline, + emit: () => pipeline, } return pipeline } @@ -16,6 +21,18 @@ jest.mock(`sharp`, () => { return sharp }) +jest.mock(`fs-extra`, () => { + return { + ...jest.requireActual(`fs-extra`), + createReadStream: () => { + const stream = { + pipe: () => stream, + } + return stream + }, + } +}) + jest.mock(`potrace`, () => { const circleSvgString = `` return { diff --git a/packages/gatsby-plugin-sharp/src/image-data.ts b/packages/gatsby-plugin-sharp/src/image-data.ts index 26cff19d689d6..bd48443bed063 100644 --- a/packages/gatsby-plugin-sharp/src/image-data.ts +++ b/packages/gatsby-plugin-sharp/src/image-data.ts @@ -2,6 +2,7 @@ import { IGatsbyImageData, ISharpGatsbyImageArgs } from "gatsby-plugin-image" import { GatsbyCache, Node } from "gatsby" import { Reporter } from "gatsby/reporter" +import fs from "fs-extra" import { rgbToHex, calculateImageSizes, getSrcSet, getSizes } from "./utils" import { traceSVG, getImageSizeAsync, base64, batchQueueImageResizing } from "." import sharp from "./safe-sharp" @@ -15,7 +16,7 @@ const DEFAULT_BREAKPOINTS = [750, 1080, 1366, 1920] type ImageFormat = "jpg" | "png" | "webp" | "avif" | "" | "auto" export type FileNode = Node & { - absolutePath?: string + absolutePath: string extension: string } @@ -44,7 +45,9 @@ export async function getImageMetadata( } try { - const pipeline = sharp(file.absolutePath) + const pipeline = sharp() + + fs.createReadStream(file.absolutePath).pipe(pipeline) const { width, height, density, format } = await pipeline.metadata() diff --git a/packages/gatsby-plugin-sharp/src/index.js b/packages/gatsby-plugin-sharp/src/index.js index c9c3df1dbc069..0d701feff6ad3 100644 --- a/packages/gatsby-plugin-sharp/src/index.js +++ b/packages/gatsby-plugin-sharp/src/index.js @@ -267,13 +267,12 @@ async function generateBase64({ file, args = {}, reporter }) { }) let pipeline try { - pipeline = !options.failOnError - ? sharp(file.absolutePath, { failOnError: false }) - : sharp(file.absolutePath) + pipeline = !options.failOnError ? sharp({ failOnError: false }) : sharp() if (!options.rotate) { pipeline.rotate() } + fs.createReadStream(file.absolutePath).pipe(pipeline) } catch (err) { reportError(`Failed to process image ${file.absolutePath}`, err, reporter) return null @@ -413,7 +412,10 @@ async function getTracedSVG({ file, options, cache, reporter }) { async function stats({ file, reporter }) { let imgStats try { - imgStats = await sharp(file.absolutePath).stats() + const pipeline = sharp() + fs.createReadStream(file.absolutePath).pipe(pipeline) + + imgStats = await pipeline.stats() } catch (err) { reportError( `Failed to get stats for image ${file.absolutePath}`, @@ -450,7 +452,10 @@ async function fluid({ file, args = {}, reporter, cache }) { // images are intended to be displayed at their native resolution. let metadata try { - metadata = await sharp(file.absolutePath).metadata() + const pipeline = sharp() + fs.createReadStream(file.absolutePath).pipe(pipeline) + + metadata = await pipeline.metadata() } catch (err) { reportError( `Failed to retrieve metadata from image ${file.absolutePath}`, diff --git a/packages/gatsby-plugin-sharp/src/process-file.js b/packages/gatsby-plugin-sharp/src/process-file.js index 6b8ccc76fddfb..2318cc904413a 100644 --- a/packages/gatsby-plugin-sharp/src/process-file.js +++ b/packages/gatsby-plugin-sharp/src/process-file.js @@ -56,14 +56,13 @@ sharp.concurrency(1) exports.processFile = (file, transforms, options = {}) => { let pipeline try { - pipeline = !options.failOnError - ? sharp(file, { failOnError: false }) - : sharp(file) + pipeline = !options.failOnError ? sharp({ failOnError: false }) : sharp() // Keep Metadata if (!options.stripMetadata) { pipeline = pipeline.withMetadata() } + fs.createReadStream(file).pipe(pipeline) } catch (err) { throw new SharpError(`Failed to load image ${file} into sharp.`, err) } @@ -152,7 +151,8 @@ exports.processFile = (file, transforms, options = {}) => { } try { - await clonedPipeline.toFile(outputPath) + const buffer = await clonedPipeline.toBuffer() + await fs.writeFile(outputPath, buffer) } catch (err) { throw new Error( `Failed to write ${file} into ${outputPath}. (${err.message})` diff --git a/packages/gatsby-plugin-sharp/src/trace-svg.js b/packages/gatsby-plugin-sharp/src/trace-svg.js index f94ca2fe8dc36..b9a894d9f66d3 100644 --- a/packages/gatsby-plugin-sharp/src/trace-svg.js +++ b/packages/gatsby-plugin-sharp/src/trace-svg.js @@ -1,4 +1,5 @@ const { promisify } = require(`bluebird`) +const fs = require(`fs-extra`) const _ = require(`lodash`) const tmpDir = require(`os`).tmpdir() const path = require(`path`) @@ -17,11 +18,12 @@ exports.notMemoizedPrepareTraceSVGInputFile = async ({ }) => { let pipeline try { - pipeline = sharp(file.absolutePath) + pipeline = sharp() if (!options.rotate) { pipeline.rotate() } + fs.createReadStream(file.absolutePath).pipe(pipeline) } catch (err) { reportError(`Failed to process image ${file.absolutePath}`, err, reporter) return diff --git a/packages/gatsby-plugin-sharp/src/utils.js b/packages/gatsby-plugin-sharp/src/utils.js index 6d4e845dbe958..a4eed401ce0a5 100644 --- a/packages/gatsby-plugin-sharp/src/utils.js +++ b/packages/gatsby-plugin-sharp/src/utils.js @@ -1,4 +1,5 @@ import sharp from "./safe-sharp" +import fs from "fs-extra" export function rgbToHex(red, green, blue) { return `#${(blue | (green << 8) | (red << 16) | (1 << 24)) @@ -39,6 +40,7 @@ export function calculateImageSizes(args) { return [] } } + export function fixedImageSizes({ file, imgDimensions, @@ -288,7 +290,10 @@ export const getDominantColor = async absolutePath => { return dominantColor } - const pipeline = sharp(absolutePath) + const pipeline = sharp() + + fs.createReadStream(absolutePath).pipe(pipeline) + const { dominant } = await pipeline.stats() // Fallback in case sharp doesn't support dominant