diff --git a/packages/next/next-server/server/api-utils.ts b/packages/next/next-server/server/api-utils.ts index e4e3a1e44ef25..76eb4095c5822 100644 --- a/packages/next/next-server/server/api-utils.ts +++ b/packages/next/next-server/server/api-utils.ts @@ -1,8 +1,6 @@ import { IncomingMessage, ServerResponse } from 'http' import { parse } from 'next/dist/compiled/content-type' import { CookieSerializeOptions } from 'next/dist/compiled/cookie' -import generateETag from 'etag' -import fresh from 'next/dist/compiled/fresh' import getRawBody from 'raw-body' import { PageConfig } from 'next/types' import { Stream } from 'stream' @@ -10,6 +8,8 @@ import { isResSent, NextApiRequest, NextApiResponse } from '../lib/utils' import { decryptWithSecret, encryptWithSecret } from './crypto-utils' import { interopDefault } from './load-components' import { Params } from './router' +import { sendEtagResponse } from './send-payload' +import generateETag from 'etag' export type NextApiRequestCookies = { [key: string]: string } export type NextApiRequestQuery = { [key: string]: string | string[] } @@ -216,23 +216,6 @@ export function redirect( return res } -function sendEtagResponse( - req: NextApiRequest, - res: NextApiResponse, - body: string | Buffer -): boolean { - const etag = generateETag(body) - - if (fresh(req.headers, { etag })) { - res.statusCode = 304 - res.end() - return true - } - - res.setHeader('ETag', etag) - return false -} - /** * Send `any` body to response * @param req request object @@ -261,8 +244,8 @@ export function sendData( const isJSONLike = ['object', 'number', 'boolean'].includes(typeof body) const stringifiedBody = isJSONLike ? JSON.stringify(body) : body - - if (sendEtagResponse(req, res, stringifiedBody)) { + const etag = generateETag(stringifiedBody) + if (sendEtagResponse(req, res, etag)) { return } diff --git a/packages/next/next-server/server/image-optimizer.ts b/packages/next/next-server/server/image-optimizer.ts index e6f8d05936273..8832cb542e861 100644 --- a/packages/next/next-server/server/image-optimizer.ts +++ b/packages/next/next-server/server/image-optimizer.ts @@ -10,6 +10,7 @@ import { fileExists } from '../../lib/file-exists' // @ts-ignore no types for is-animated import isAnimated from 'next/dist/compiled/is-animated' import Stream from 'stream' +import { sendEtagResponse } from './send-payload' let sharp: typeof import('sharp') //const AVIF = 'image/avif' @@ -18,7 +19,7 @@ const PNG = 'image/png' const JPEG = 'image/jpeg' const GIF = 'image/gif' const SVG = 'image/svg+xml' -const CACHE_VERSION = 1 +const CACHE_VERSION = 2 const MODERN_TYPES = [/* AVIF, */ WEBP] const ANIMATABLE_TYPES = [WEBP, PNG, GIF] const VECTOR_TYPES = [SVG] @@ -143,17 +144,22 @@ export async function imageOptimizer( if (await fileExists(hashDir, 'directory')) { const files = await promises.readdir(hashDir) for (let file of files) { - const [filename, extension] = file.split('.') - const expireAt = Number(filename) + const [prefix, etag, extension] = file.split('.') + const expireAt = Number(prefix) const contentType = getContentType(extension) + const fsPath = join(hashDir, file) if (now < expireAt) { + res.setHeader('Cache-Control', 'public, max-age=0, must-revalidate') + if (sendEtagResponse(req, res, etag)) { + return { finished: true } + } if (contentType) { res.setHeader('Content-Type', contentType) } - createReadStream(join(hashDir, file)).pipe(res) + createReadStream(fsPath).pipe(res) return { finished: true } } else { - await promises.unlink(join(hashDir, file)) + await promises.unlink(fsPath) } } } @@ -223,8 +229,7 @@ export async function imageOptimizer( const animate = ANIMATABLE_TYPES.includes(upstreamType) && isAnimated(upstreamBuffer) if (vector || animate) { - res.setHeader('Content-Type', upstreamType) - res.end(upstreamBuffer) + sendResponse(req, res, upstreamType, upstreamBuffer) return { finished: true } } } @@ -248,10 +253,8 @@ export async function imageOptimizer( if (error.code === 'MODULE_NOT_FOUND') { error.message += '\n\nLearn more: https://err.sh/next.js/install-sharp' server.logError(error) - if (upstreamType) { - res.setHeader('Content-Type', upstreamType) - } - res.end(upstreamBuffer) + sendResponse(req, res, upstreamType, upstreamBuffer) + return { finished: true } } throw error } @@ -281,30 +284,46 @@ export async function imageOptimizer( const optimizedBuffer = await transformer.toBuffer() await promises.mkdir(hashDir, { recursive: true }) const extension = getExtension(contentType) - const filename = join(hashDir, `${expireAt}.${extension}`) + const etag = getHash([optimizedBuffer]) + const filename = join(hashDir, `${expireAt}.${etag}.${extension}`) await promises.writeFile(filename, optimizedBuffer) - res.setHeader('Content-Type', contentType) - res.end(optimizedBuffer) + sendResponse(req, res, contentType, optimizedBuffer) } catch (error) { - server.logError(error) - if (upstreamType) { - res.setHeader('Content-Type', upstreamType) - } - res.end(upstreamBuffer) + sendResponse(req, res, upstreamType, upstreamBuffer) } return { finished: true } } +function sendResponse( + req: IncomingMessage, + res: ServerResponse, + contentType: string | null, + buffer: Buffer +) { + const etag = getHash([buffer]) + res.setHeader('Cache-Control', 'public, max-age=0, must-revalidate') + if (sendEtagResponse(req, res, etag)) { + return + } + if (contentType) { + res.setHeader('Content-Type', contentType) + } + res.end(buffer) +} + function getSupportedMimeType(options: string[], accept = ''): string { const mimeType = mediaType(accept, options) return accept.includes(mimeType) ? mimeType : '' } -function getHash(items: (string | number | undefined)[]) { +function getHash(items: (string | number | Buffer)[]) { const hash = createHash('sha256') for (let item of items) { - hash.update(String(item)) + if (typeof item === 'number') hash.update(String(item)) + else { + hash.update(item) + } } // See https://en.wikipedia.org/wiki/Base64#Filenames return hash.digest('base64').replace(/\//g, '-') diff --git a/packages/next/next-server/server/send-payload.ts b/packages/next/next-server/server/send-payload.ts index c5dd4d7f449ac..a9868b42b99cd 100644 --- a/packages/next/next-server/server/send-payload.ts +++ b/packages/next/next-server/server/send-payload.ts @@ -26,17 +26,10 @@ export function sendPayload( } const etag = generateEtags ? generateETag(payload) : undefined - - if (fresh(req.headers, { etag })) { - res.statusCode = 304 - res.end() + if (sendEtagResponse(req, res, etag)) { return } - if (etag) { - res.setHeader('ETag', etag) - } - if (!res.getHeader('Content-Type')) { res.setHeader( 'Content-Type', @@ -72,3 +65,27 @@ export function sendPayload( } res.end(req.method === 'HEAD' ? null : payload) } + +export function sendEtagResponse( + req: IncomingMessage, + res: ServerResponse, + etag: string | undefined +): boolean { + if (etag) { + /** + * The server generating a 304 response MUST generate any of the + * following header fields that would have been sent in a 200 (OK) + * response to the same request: Cache-Control, Content-Location, Date, + * ETag, Expires, and Vary. https://tools.ietf.org/html/rfc7232#section-4.1 + */ + res.setHeader('ETag', etag) + } + + if (fresh(req.headers, { etag })) { + res.statusCode = 304 + res.end() + return true + } + + return false +} diff --git a/test/integration/image-optimizer/test/index.test.js b/test/integration/image-optimizer/test/index.test.js index 3eff52b1d8c86..957af45b3527d 100644 --- a/test/integration/image-optimizer/test/index.test.js +++ b/test/integration/image-optimizer/test/index.test.js @@ -55,6 +55,10 @@ function runTests({ w, isDev, domains }) { const res = await fetchViaHTTP(appPort, '/_next/image', query, {}) expect(res.status).toBe(200) expect(res.headers.get('content-type')).toContain('image/gif') + expect(res.headers.get('cache-control')).toBe( + 'public, max-age=0, must-revalidate' + ) + expect(res.headers.get('etag')).toBeTruthy() expect(isAnimated(await res.buffer())).toBe(true) }) @@ -63,6 +67,10 @@ function runTests({ w, isDev, domains }) { const res = await fetchViaHTTP(appPort, '/_next/image', query, {}) expect(res.status).toBe(200) expect(res.headers.get('content-type')).toContain('image/png') + expect(res.headers.get('cache-control')).toBe( + 'public, max-age=0, must-revalidate' + ) + expect(res.headers.get('etag')).toBeTruthy() expect(isAnimated(await res.buffer())).toBe(true) }) @@ -71,6 +79,10 @@ function runTests({ w, isDev, domains }) { const res = await fetchViaHTTP(appPort, '/_next/image', query, {}) expect(res.status).toBe(200) expect(res.headers.get('content-type')).toContain('image/webp') + expect(res.headers.get('cache-control')).toBe( + 'public, max-age=0, must-revalidate' + ) + expect(res.headers.get('etag')).toBeTruthy() expect(isAnimated(await res.buffer())).toBe(true) }) @@ -80,6 +92,10 @@ function runTests({ w, isDev, domains }) { const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) expect(res.status).toBe(200) expect(res.headers.get('Content-Type')).toContain('image/svg+xml') + expect(res.headers.get('cache-control')).toBe( + 'public, max-age=0, must-revalidate' + ) + expect(res.headers.get('etag')).toBeTruthy() const actual = await res.text() const expected = await fs.readFile( join(__dirname, '..', 'public', 'test.svg'), @@ -96,6 +112,10 @@ function runTests({ w, isDev, domains }) { const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) expect(res.status).toBe(200) expect(res.headers.get('Content-Type')).toContain('image/jpeg') + expect(res.headers.get('cache-control')).toBe( + 'public, max-age=0, must-revalidate' + ) + expect(res.headers.get('etag')).toBeTruthy() }) it('should maintain png format for old Safari', async () => { @@ -106,6 +126,10 @@ function runTests({ w, isDev, domains }) { const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) expect(res.status).toBe(200) expect(res.headers.get('Content-Type')).toContain('image/png') + expect(res.headers.get('cache-control')).toBe( + 'public, max-age=0, must-revalidate' + ) + expect(res.headers.get('etag')).toBeTruthy() }) it('should fail when url is missing', async () => { @@ -199,6 +223,10 @@ function runTests({ w, isDev, domains }) { const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) expect(res.status).toBe(200) expect(res.headers.get('Content-Type')).toBe('image/webp') + expect(res.headers.get('cache-control')).toBe( + 'public, max-age=0, must-revalidate' + ) + expect(res.headers.get('etag')).toBeTruthy() await expectWidth(res, w) }) @@ -208,6 +236,10 @@ function runTests({ w, isDev, domains }) { const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) expect(res.status).toBe(200) expect(res.headers.get('Content-Type')).toBe('image/png') + expect(res.headers.get('cache-control')).toBe( + 'public, max-age=0, must-revalidate' + ) + expect(res.headers.get('etag')).toBeTruthy() await expectWidth(res, w) }) @@ -217,6 +249,10 @@ function runTests({ w, isDev, domains }) { const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) expect(res.status).toBe(200) expect(res.headers.get('Content-Type')).toBe('image/png') + expect(res.headers.get('cache-control')).toBe( + 'public, max-age=0, must-revalidate' + ) + expect(res.headers.get('etag')).toBeTruthy() await expectWidth(res, w) }) @@ -226,6 +262,10 @@ function runTests({ w, isDev, domains }) { const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) expect(res.status).toBe(200) expect(res.headers.get('Content-Type')).toBe('image/gif') + expect(res.headers.get('cache-control')).toBe( + 'public, max-age=0, must-revalidate' + ) + expect(res.headers.get('etag')).toBeTruthy() await expectWidth(res, w) }) @@ -235,6 +275,10 @@ function runTests({ w, isDev, domains }) { const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) expect(res.status).toBe(200) expect(res.headers.get('Content-Type')).toBe('image/tiff') + expect(res.headers.get('cache-control')).toBe( + 'public, max-age=0, must-revalidate' + ) + expect(res.headers.get('etag')).toBeTruthy() await expectWidth(res, w) }) @@ -246,6 +290,10 @@ function runTests({ w, isDev, domains }) { const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) expect(res.status).toBe(200) expect(res.headers.get('Content-Type')).toBe('image/webp') + expect(res.headers.get('cache-control')).toBe( + 'public, max-age=0, must-revalidate' + ) + expect(res.headers.get('etag')).toBeTruthy() await expectWidth(res, w) }) @@ -257,6 +305,10 @@ function runTests({ w, isDev, domains }) { const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) expect(res.status).toBe(200) expect(res.headers.get('Content-Type')).toBe('image/webp') + expect(res.headers.get('cache-control')).toBe( + 'public, max-age=0, must-revalidate' + ) + expect(res.headers.get('etag')).toBeTruthy() await expectWidth(res, w) }) } @@ -310,6 +362,39 @@ function runTests({ w, isDev, domains }) { expect(json2).toStrictEqual(json1) }) + it('should set 304 status without body when etag matches if-none-match', async () => { + const query = { url: '/test.jpg', w, q: 80 } + const opts1 = { headers: { accept: 'image/webp' } } + + const res1 = await fetchViaHTTP(appPort, '/_next/image', query, opts1) + expect(res1.status).toBe(200) + expect(res1.headers.get('Content-Type')).toBe('image/webp') + expect(res1.headers.get('Cache-Control')).toBe( + 'public, max-age=0, must-revalidate' + ) + const etag = res1.headers.get('Etag') + expect(etag).toBeTruthy() + await expectWidth(res1, w) + + const opts2 = { headers: { accept: 'image/webp', 'if-none-match': etag } } + const res2 = await fetchViaHTTP(appPort, '/_next/image', query, opts2) + expect(res2.status).toBe(304) + expect(res2.headers.get('Content-Type')).toBeFalsy() + expect(res2.headers.get('Etag')).toBe(etag) + expect((await res2.buffer()).length).toBe(0) + + const query3 = { url: '/test.jpg', w, q: 25 } + const res3 = await fetchViaHTTP(appPort, '/_next/image', query3, opts2) + expect(res3.status).toBe(200) + expect(res3.headers.get('Content-Type')).toBe('image/webp') + expect(res3.headers.get('Cache-Control')).toBe( + 'public, max-age=0, must-revalidate' + ) + expect(res3.headers.get('Etag')).toBeTruthy() + expect(res3.headers.get('Etag')).not.toBe(etag) + await expectWidth(res3, w) + }) + it('should proxy-pass unsupported image types and should not cache file', async () => { const json1 = await fsToJson(imagesDir) expect(json1).toBeTruthy() @@ -319,6 +404,10 @@ function runTests({ w, isDev, domains }) { const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) expect(res.status).toBe(200) expect(res.headers.get('Content-Type')).toBe('image/bmp') + expect(res.headers.get('cache-control')).toBe( + 'public, max-age=0, must-revalidate' + ) + expect(res.headers.get('etag')).toBeTruthy() const json2 = await fsToJson(imagesDir) expect(json2).toStrictEqual(json1) @@ -330,6 +419,10 @@ function runTests({ w, isDev, domains }) { const res = await fetchViaHTTP(appPort, '/_next/image', query, opts) expect(res.status).toBe(200) expect(res.headers.get('Content-Type')).toBe('image/webp') + expect(res.headers.get('cache-control')).toBe( + 'public, max-age=0, must-revalidate' + ) + expect(res.headers.get('etag')).toBeTruthy() await expectWidth(res, 400) }) }