diff --git a/package.json b/package.json index e24b8545ad..e6d851ba5c 100644 --- a/package.json +++ b/package.json @@ -104,7 +104,7 @@ "ufo": "^0.8.3", "unenv": "^0.4.6", "unimport": "^0.1.8", - "unstorage": "^0.3.3", + "unstorage": "^0.4.0", "util": "^0.12.4" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e09f4e65a5..b4e5229819 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -88,7 +88,7 @@ importers: unbuild: ^0.7.4 unenv: ^0.4.6 unimport: ^0.1.8 - unstorage: ^0.3.3 + unstorage: ^0.4.0 util: ^0.12.4 vitepress: ^0.22.3 vitest: ^0.10.0 @@ -159,7 +159,7 @@ importers: ufo: 0.8.3 unenv: 0.4.6 unimport: 0.1.8_esbuild@0.14.38+rollup@2.71.1 - unstorage: 0.3.3 + unstorage: 0.4.0 util: 0.12.4 devDependencies: '@nuxtjs/eslint-config-typescript': 9.0.0_eslint@8.14.0 @@ -609,6 +609,10 @@ packages: resolution: {integrity: sha512-trnsAYxU3xnS1gPHPyU961coFyLkh4gAD/0zQ5mymY4yOZ+CYvsPqUbOFSw0aDM4y0tV7tiFxL/1XfXPNC6IPg==} dev: true + /@ioredis/commands/1.1.1: + resolution: {integrity: sha512-fsR4P/ROllzf/7lXYyElUJCheWdTJVJvOTps8v9IWKFATxR61ANOlnoPqhH099xYLrJGpc2ZQ28B3rMeUt5VQg==} + dev: false + /@istanbuljs/schema/0.1.3: resolution: {integrity: sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==} engines: {node: '>=8'} @@ -2181,8 +2185,8 @@ packages: resolution: {integrity: sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=} dev: false - /denque/1.5.1: - resolution: {integrity: sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==} + /denque/2.0.1: + resolution: {integrity: sha512-tfiWc6BQLXNLpNiR5iGd0Ocu3P3VpxfzFiqubLgMfhfOw9WyvgJBd46CClNn9k3qfbjvT//0cf7AlYRX/OslMQ==} engines: {node: '>=0.10'} dev: false @@ -3471,10 +3475,6 @@ packages: duplexer: 0.1.2 dev: false - /h3/0.3.9: - resolution: {integrity: sha512-C9MbuWVQ88mGb3lmqtp/iqXdALBo34oyjrVT1hx+FYRgcQReNXiKWvzXM6eQdpp4MWQkMGNnp4tlRBT7wsgmmg==} - dev: false - /h3/0.7.6: resolution: {integrity: sha512-OoxDWBBpGNAStSVCOQagN+3EKRbCSgwQKFIeu4p4XvsLvd4L9KaQV6GDY5H8ZDYHsWfnsx4KTa2eFwSw530jnQ==} dependencies: @@ -3673,18 +3673,16 @@ packages: has: 1.0.3 side-channel: 1.0.4 - /ioredis/4.28.5: - resolution: {integrity: sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==} - engines: {node: '>=6'} + /ioredis/5.0.4: + resolution: {integrity: sha512-qFJw3MnPNsJF1lcIOP3vztbsasOXK3nDdNAgjQj7t7/Bn/w10PGchTOpqylQNxjzPbLoYDu34LjeJtSWiKBntQ==} + engines: {node: '>=12.22.0'} dependencies: + '@ioredis/commands': 1.1.1 cluster-key-slot: 1.1.0 debug: 4.3.4 - denque: 1.5.1 + denque: 2.0.1 lodash.defaults: 4.2.0 - lodash.flatten: 4.4.0 lodash.isarguments: 3.1.0 - p-map: 2.1.0 - redis-commands: 1.7.0 redis-errors: 1.2.0 redis-parser: 3.0.0 standard-as-callback: 2.1.0 @@ -4731,11 +4729,6 @@ packages: p-limit: 3.1.0 dev: true - /p-map/2.1.0: - resolution: {integrity: sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==} - engines: {node: '>=6'} - dev: false - /p-try/1.0.0: resolution: {integrity: sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=} engines: {node: '>=4'} @@ -5015,10 +5008,6 @@ packages: strip-indent: 3.0.0 dev: true - /redis-commands/1.7.0: - resolution: {integrity: sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ==} - dev: false - /redis-errors/1.2.0: resolution: {integrity: sha1-62LSrbFeTq9GEMBK/hUpOEJQq60=} engines: {node: '>=4'} @@ -5716,10 +5705,6 @@ packages: hasBin: true dev: true - /ufo/0.7.11: - resolution: {integrity: sha512-IT3q0lPvtkqQ8toHQN/BkOi4VIqoqheqM1FnkNWT9y0G8B3xJhwnoKBu5OHx8zHDOvveQzfKuFowJ0VSARiIDg==} - dev: false - /ufo/0.8.3: resolution: {integrity: sha512-AIkk06G21y/P+NCatfU+1qldCmI0XCszZLn8AkuKotffF3eqCvlce0KuwM7ZemLE/my0GSYADOAeM5zDYWMB+A==} dev: false @@ -5851,19 +5836,19 @@ packages: webpack-virtual-modules: 0.4.3 dev: false - /unstorage/0.3.3: - resolution: {integrity: sha512-hBF0W+YTdu50I6m0In02H/Spq9DOmGymqu92Bo5+om/mW5SDvTn2Lu817TmZWXiOW40dCAJ8/+8ZbhVdXyBxqA==} + /unstorage/0.4.0: + resolution: {integrity: sha512-hbc4bKfeeYHlW3PrfJozSp1TQ2XFIm9NOxYzW3iFe5vs4bT1atYOfFgeD84IvParXiIWAr1qaR3Sf9QcGOxWHw==} dependencies: anymatch: 3.1.2 chokidar: 3.5.3 destr: 1.1.1 - h3: 0.3.9 - ioredis: 4.28.5 + h3: 0.7.6 + ioredis: 5.0.4 listhen: 0.2.11 mri: 1.2.0 ohmyfetch: 0.4.16 - ufo: 0.7.11 - ws: 8.5.0 + ufo: 0.8.3 + ws: 8.6.0 transitivePeerDependencies: - bufferutil - supports-color @@ -6150,6 +6135,20 @@ packages: optional: true utf-8-validate: optional: true + dev: true + + /ws/8.6.0: + resolution: {integrity: sha512-AzmM3aH3gk0aX7/rZLYvjdvZooofDu3fFOzGqcSnQ1tOcTWwhM/o+q++E8mAyVVIyUdajrkzWUGftaVSDLn1bw==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: ^5.0.2 + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + dev: false /xtend/4.0.2: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} diff --git a/src/build.ts b/src/build.ts index d965dc9ee6..3c70a57feb 100644 --- a/src/build.ts +++ b/src/build.ts @@ -1,5 +1,5 @@ import { promises as fsp } from 'fs' -import { relative, resolve, join } from 'pathe' +import { relative, resolve, join, dirname } from 'pathe' import * as rollup from 'rollup' import fse from 'fs-extra' import defu from 'defu' @@ -12,6 +12,7 @@ import { prettyPath, writeFile, isDirectory } from './utils' import { GLOB_SCAN_PATTERN, scanHandlers } from './scan' import type { Nitro } from './types' import { runtimeDir } from './dirs' +import { snapshotStorage } from './storage' export async function prepare (nitro: Nitro) { await prepareDir(nitro.options.output.dir) @@ -106,9 +107,32 @@ export async function writeTypes (nitro: Nitro) { } } +async function _snapshot (nitro: Nitro) { + if (!nitro.options.bundledStorage.length || + nitro.options.preset === 'nitro-prerender' + ) { + return + } + // TODO: Use virtual storage for server assets + const storageDir = resolve(nitro.options.buildDir, 'snapshot') + nitro.options.serverAssets.push({ + baseName: 'nitro:bundled', + dir: storageDir + }) + + const data = await snapshotStorage(nitro) + await Promise.all(Object.entries(data).map(async ([path, contents]) => { + if (typeof contents !== 'string') { contents = JSON.stringify(contents) } + const fsPath = join(storageDir, path.replace(/:/g, '/')) + await fsp.mkdir(dirname(fsPath), { recursive: true }) + await fsp.writeFile(fsPath, contents, 'utf8') + })) +} + async function _build (nitro: Nitro) { await scanHandlers(nitro) await writeTypes(nitro) + await _snapshot(nitro) nitro.logger.start('Building server...') const build = await rollup.rollup(nitro.options.rollupConfig).catch((error) => { diff --git a/src/nitro.ts b/src/nitro.ts index ea8b2573cb..c273c2cda9 100644 --- a/src/nitro.ts +++ b/src/nitro.ts @@ -58,17 +58,15 @@ export async function createNitro (config: NitroConfig = {}): Promise { nitro.options.virtual['#nitro'] = 'export * from "#imports"' } - // Dev-only storage - if (options.dev) { - const fsMounts = { - root: resolve(options.rootDir), - src: resolve(options.srcDir), - build: resolve(options.buildDir), - cache: resolve(options.buildDir, 'cache') - } - for (const p in fsMounts) { - options.storage[p] = options.storage[p] || { driver: 'fs', base: fsMounts[p] } - } + // Build-only storage + const fsMounts = { + root: resolve(options.rootDir), + src: resolve(options.srcDir), + build: resolve(options.buildDir), + cache: resolve(options.buildDir, 'cache') + } + for (const p in fsMounts) { + options.devStorage[p] = options.devStorage[p] || { driver: 'fs', base: fsMounts[p] } } return nitro diff --git a/src/options.ts b/src/options.ts index 4191c7620c..f8d15cd35b 100644 --- a/src/options.ts +++ b/src/options.ts @@ -30,6 +30,8 @@ const NitroDefaults: NitroConfig = { // Featueres experimental: {}, storage: {}, + devStorage: {}, + bundledStorage: [], publicAssets: [], serverAssets: [], plugins: [], @@ -107,6 +109,7 @@ export async function loadOptions (userConfig: NitroConfig = {}): Promise { rollupConfig.plugins.push(publicAssets(nitro)) // Storage - rollupConfig.plugins.push(storage({ - mounts: nitro.options.storage - })) + rollupConfig.plugins.push(storage(nitro)) // Handlers rollupConfig.plugins.push(handlers(() => { diff --git a/src/rollup/plugins/server-assets.ts b/src/rollup/plugins/server-assets.ts index cba8bd7f89..85a91d6a85 100644 --- a/src/rollup/plugins/server-assets.ts +++ b/src/rollup/plugins/server-assets.ts @@ -28,7 +28,7 @@ interface ResolvedAsset { export function serverAssets (nitro: Nitro): Plugin { // Development: Use filesystem - if (nitro.options.dev) { + if (nitro.options.dev || nitro.options.preset === 'nitro-prerender') { return virtual({ '#internal/nitro/virtual/server-assets': getAssetsDev(nitro) }) } diff --git a/src/rollup/plugins/storage.ts b/src/rollup/plugins/storage.ts index f1a84f5551..ee58965bb2 100644 --- a/src/rollup/plugins/storage.ts +++ b/src/rollup/plugins/storage.ts @@ -1,42 +1,45 @@ import virtual from '@rollup/plugin-virtual' import { serializeImportName } from '../../utils' - -export interface StorageMounts { - [path: string]: { - driver: 'fs' | 'http' | 'memory' | 'redis' | 'cloudflare-kv', - [option: string]: any - } -} +import { builtinDrivers } from '../../storage' +import type { Nitro, StorageMounts } from '../../types' export interface StorageOptions { mounts: StorageMounts } -const drivers = { - fs: 'unstorage/drivers/fs', - http: 'unstorage/drivers/http', - memory: 'unstorage/drivers/memory', - redis: 'unstorage/drivers/redis', - 'cloudflare-kv': 'unstorage/drivers/cloudflare-kv' -} - -export function storage (opts: StorageOptions) { +export function storage (nitro: Nitro) { const mounts: { path: string, driver: string, opts: object }[] = [] - for (const path in opts.mounts) { - const mount = opts.mounts[path] + for (const path in nitro.options.storage) { + const mount = nitro.options.storage[path] mounts.push({ path, - driver: drivers[mount.driver] || mount.driver, + driver: builtinDrivers[mount.driver] || mount.driver, opts: mount }) } const driverImports = Array.from(new Set(mounts.map(m => m.driver))) + const bundledStorageCode = ` +import overlay from 'unstorage/drivers/overlay' +import memory from 'unstorage/drivers/memory' + +const bundledStorage = ${JSON.stringify(nitro.options.bundledStorage)} +for (const base of bundledStorage) { + storage.mount(base, overlay({ + layers: [ + memory(), + // TODO + // prefixStorage(storage, base), + prefixStorage(storage, '/assets/nitro/bundled' + base) + ] + })) +}` + return virtual({ '#internal/nitro/virtual/storage': ` -import { createStorage } from 'unstorage' +import { createStorage, prefixStorage } from 'unstorage' import { assets } from '#internal/nitro/virtual/server-assets' ${driverImports.map(i => `import ${serializeImportName(i)} from '${i}'`).join('\n')} @@ -48,6 +51,8 @@ export const useStorage = () => storage storage.mount('/assets', assets) ${mounts.map(m => `storage.mount('${m.path}', ${serializeImportName(m.driver)}(${JSON.stringify(m.opts)}))`).join('\n')} + +${nitro.options.bundledStorage.length ? bundledStorageCode : ''} ` }) } diff --git a/src/storage.ts b/src/storage.ts new file mode 100644 index 0000000000..83cc56ed97 --- /dev/null +++ b/src/storage.ts @@ -0,0 +1,44 @@ +import { createStorage as _createStorage } from 'unstorage' +import type { Nitro } from './types' + +export const builtinDrivers = { + fs: 'unstorage/drivers/fs', + http: 'unstorage/drivers/http', + memory: 'unstorage/drivers/memory', + redis: 'unstorage/drivers/redis', + 'cloudflare-kv': 'unstorage/drivers/cloudflare-kv' +} + +export async function createStorage (nitro: Nitro) { + const storage = _createStorage() + + const mounts = { + ...nitro.options.storage, + ...nitro.options.devStorage + } + + for (const [path, opts] of Object.entries(mounts)) { + const driver = await import(builtinDrivers[opts.driver] || opts.driver) + .then(r => r.default || r) + storage.mount(path, driver(opts)) + } + + return storage +} + +export async function snapshotStorage (nitro: Nitro) { + const storage = await createStorage(nitro) + const data: Record = {} + + const allKeys = Array.from(new Set(await Promise.all( + nitro.options.bundledStorage.map(base => storage.getKeys(base)) + ).then(r => r.flat()))) + + await Promise.all(allKeys.map(async (key) => { + data[key] = await storage.getItem(key) + })) + + await storage.dispose() + + return data +} diff --git a/src/types/nitro.ts b/src/types/nitro.ts index 16d63079d6..e2bf151522 100644 --- a/src/types/nitro.ts +++ b/src/types/nitro.ts @@ -6,7 +6,6 @@ import type { NestedHooks, Hookable } from 'hookable' import type { Consola, LogLevel } from 'consola' import { WatchOptions } from 'chokidar' import type { NodeExternalsOptions } from '../rollup/plugins/externals' -import type { StorageMounts } from '../rollup/plugins/storage' import type { RollupConfig } from '../rollup/config' import type { Options as EsbuildOptions } from '../rollup/plugins/esbuild' import { NitroErrorHandler, NitroDevEventHandler, NitroEventHandler } from './handler' @@ -29,6 +28,13 @@ export interface NitroHooks { 'close': () => HookResult } +export interface StorageMounts { + [path: string]: { + driver: 'fs' | 'http' | 'memory' | 'redis' | 'cloudflare-kv', + [option: string]: any + } +} + type DeepPartial = T extends Record ? { [P in keyof T]?: DeepPartial | T[P] } : T export type NitroPreset = NitroConfig | (() => NitroConfig) @@ -93,6 +99,8 @@ export interface NitroOptions { // Features storage: StorageMounts + devStorage: StorageMounts + bundledStorage: string[] timing: boolean renderer: string serveStatic: boolean