diff --git a/.github/workflows/docker-hub.yml b/.github/workflows/docker-hub.yml index 1f1f220..08353e6 100644 --- a/.github/workflows/docker-hub.yml +++ b/.github/workflows/docker-hub.yml @@ -2,7 +2,7 @@ name: Docker Image CI on: push: - branches: [ master ] + branches: [ master, next ] jobs: build: diff --git a/CHANGELOG.md b/CHANGELOG.md index fda9449..f871336 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,43 @@ All notable changes to this project will be documented in this file. Dates are d Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog). +#### [v1.11.0-4](https://github.com/bangbang93/openbmclapi/compare/v1.11.0-3...v1.11.0-4) + +- fix: 修正异常上报 [`6c5e3b8`](https://github.com/bangbang93/openbmclapi/commit/6c5e3b8c1cf8f79a447c55806a9eb0bdededef1b) +- feat: 移除noopen参数 [`fecb72a`](https://github.com/bangbang93/openbmclapi/commit/fecb72ad5a75730697212bdfd914714c4adc2e29) + +#### [v1.11.0-3](https://github.com/bangbang93/openbmclapi/compare/v1.11.0-2...v1.11.0-3) + +> 18 June 2024 + +- refactor: 简化上报的错误 [`03400c5`](https://github.com/bangbang93/openbmclapi/commit/03400c543cd73d6abe64481cecc12a221aef93c1) +- Release 1.11.0-3 [`750b711`](https://github.com/bangbang93/openbmclapi/commit/750b711e467b3f992672bcbf3c4c2072b79e45b0) + +#### [v1.11.0-2](https://github.com/bangbang93/openbmclapi/compare/v1.11.0-1...v1.11.0-2) + +> 18 June 2024 + +- fix: 修正错误上报 [`ba027be`](https://github.com/bangbang93/openbmclapi/commit/ba027be00f11e1fab76aa974cd625c37b3c3079b) +- Release 1.11.0-2 [`84ca717`](https://github.com/bangbang93/openbmclapi/commit/84ca717eda91a2f2735ec02465d162dec0224750) + +#### [v1.11.0-1](https://github.com/bangbang93/openbmclapi/compare/v1.11.0-0...v1.11.0-1) + +> 18 June 2024 + +- fix: 应该在下载失败的时候就上报了，不应该等到重试失败 [`1f933f8`](https://github.com/bangbang93/openbmclapi/commit/1f933f89489aa808a7d02f983845c25389eb4de1) +- Release 1.11.0-1 [`ca5fa71`](https://github.com/bangbang93/openbmclapi/commit/ca5fa715391721f327079089480d38115fc62a97) + +#### [v1.11.0-0](https://github.com/bangbang93/openbmclapi/compare/v1.10.10...v1.11.0-0) + +> 18 June 2024 + +- feat: 下载错误时上报主控 [`19149d5`](https://github.com/bangbang93/openbmclapi/commit/19149d50d8a8cc90b72a8989d17dd95eeb9b3289) +- Release 1.11.0-0 [`ff77a85`](https://github.com/bangbang93/openbmclapi/commit/ff77a85530b643a72e1cbc5d60f8aaabc451121f) + #### [v1.10.10](https://github.com/bangbang93/openbmclapi/compare/v1.10.9...v1.10.10) +> 13 June 2024 + - feat: 同时执行gc与启用 [`#77`](https://github.com/bangbang93/openbmclapi/pull/77) - feat: clean outdated files after enabled [`#75`](https://github.com/bangbang93/openbmclapi/pull/75) - feat: 简化emit异步写法 [`1a6c0bb`](https://github.com/bangbang93/openbmclapi/commit/1a6c0bb0cc436a9070d44719ada550708bad2237) diff --git a/package-lock.json b/package-lock.json index edc188a..7a57b5e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "openbmclapi", - "version": "1.10.10", + "version": "1.11.0-4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "openbmclapi", - "version": "1.10.10", + "version": "1.11.0-4", "license": "MIT", "dependencies": { "@bangbang93/utils": "^6.9.1", @@ -23,6 +23,7 @@ "fs-extra": "^8.1.0", "got": "^14.2.0", "http2-express-bridge": "^1.0.7", + "json-stringify-safe": "^5.0.1", "keyv": "^4.5.4", "keyv-file": "^0.3.0", "lodash-es": "^4.17.21", @@ -35,6 +36,7 @@ "pino-pretty": "^10.3.1", "pretty-bytes": "^6.1.1", "range-parser": "^1.2.1", + "serialize-error": "^11.0.3", "socket.io-client": "^4.7.4", "tail": "^2.2.6", "webdav": "^5.3.1", @@ -52,6 +54,7 @@ "@types/dotenv": "^6.1.1", "@types/express": "^4.17.13", "@types/fs-extra": "^8.0.0", + "@types/json-stringify-safe": "^5.0.3", "@types/lodash-es": "^4.17.7", "@types/morgan": "^1.7.36", "@types/ms": "^0.7.30", @@ -1095,6 +1098,12 @@ "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", "dev": true }, + "node_modules/@types/json-stringify-safe": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/@types/json-stringify-safe/-/json-stringify-safe-5.0.3.tgz", + "integrity": "sha512-oNOjRxLfPeYbBSQ60maucaFNqbslVOPU4WWs5t/sHvAh6tyo/CThXSG+E24tEzkgh/fzvxyDrYdOJufgeNy1sQ==", + "dev": true + }, "node_modules/@types/lodash": { "version": "4.14.195", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.195.tgz", @@ -6276,6 +6285,31 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" }, + "node_modules/serialize-error": { + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-11.0.3.tgz", + "integrity": "sha512-2G2y++21dhj2R7iHAdd0FIzjGwuKZld+7Pl/bTU6YIkrC2ZMbVUjm+luj6A6V34Rv9XfKJDKpTWu9W4Gse1D9g==", + "dependencies": { + "type-fest": "^2.12.2" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/serialize-error/node_modules/type-fest": { + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-2.19.0.tgz", + "integrity": "sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==", + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/serve-static": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", @@ -7896,6 +7930,12 @@ "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", "dev": true }, + "@types/json-stringify-safe": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/@types/json-stringify-safe/-/json-stringify-safe-5.0.3.tgz", + "integrity": "sha512-oNOjRxLfPeYbBSQ60maucaFNqbslVOPU4WWs5t/sHvAh6tyo/CThXSG+E24tEzkgh/fzvxyDrYdOJufgeNy1sQ==", + "dev": true + }, "@types/lodash": { "version": "4.14.195", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.195.tgz", @@ -11594,6 +11634,21 @@ } } }, + "serialize-error": { + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-11.0.3.tgz", + "integrity": "sha512-2G2y++21dhj2R7iHAdd0FIzjGwuKZld+7Pl/bTU6YIkrC2ZMbVUjm+luj6A6V34Rv9XfKJDKpTWu9W4Gse1D9g==", + "requires": { + "type-fest": "^2.12.2" + }, + "dependencies": { + "type-fest": { + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-2.19.0.tgz", + "integrity": "sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==" + } + } + }, "serve-static": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", diff --git a/package.json b/package.json index d9d4e7b..2bb89a3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "openbmclapi", - "version": "1.10.10", + "version": "1.11.0-4", "description": "bmclapi@home", "bin": "dist/openbmclapi.js", "private": true, @@ -39,6 +39,7 @@ "fs-extra": "^8.1.0", "got": "^14.2.0", "http2-express-bridge": "^1.0.7", + "json-stringify-safe": "^5.0.1", "keyv": "^4.5.4", "keyv-file": "^0.3.0", "lodash-es": "^4.17.21", @@ -51,6 +52,7 @@ "pino-pretty": "^10.3.1", "pretty-bytes": "^6.1.1", "range-parser": "^1.2.1", + "serialize-error": "^11.0.3", "socket.io-client": "^4.7.4", "tail": "^2.2.6", "webdav": "^5.3.1", @@ -65,6 +67,7 @@ "@types/dotenv": "^6.1.1", "@types/express": "^4.17.13", "@types/fs-extra": "^8.0.0", + "@types/json-stringify-safe": "^5.0.3", "@types/lodash-es": "^4.17.7", "@types/morgan": "^1.7.36", "@types/ms": "^0.7.30", diff --git a/src/cluster.ts b/src/cluster.ts index b81d7e7..b36e873 100644 --- a/src/cluster.ts +++ b/src/cluster.ts @@ -7,11 +7,12 @@ import express, {type NextFunction, type Request, type Response} from 'express' import {readFileSync} from 'fs' import fse from 'fs-extra' import {mkdtemp, open, readFile, rm} from 'fs/promises' -import got, {type Got, HTTPError} from 'got' +import got, {type Got, HTTPError, RequestError} from 'got' import {createServer, Server} from 'http' import {createSecureServer} from 'http2' import http2Express from 'http2-express-bridge' import {Agent as HttpsAgent} from 'https' +import stringifySafe from 'json-stringify-safe' import {template, toString} from 'lodash-es' import morgan from 'morgan' import ms from 'ms' @@ -169,35 +170,25 @@ export class Cluster { return } logger.info(`mismatch ${missingFiles.length} files, start syncing`) - if (process.env.FORCE_NOOPEN) { - syncConfig = { - concurrency: 1, - source: 'center', - } - } logger.info(syncConfig, '同步策略') const multibar = new MultiBar({ format: ' {bar} | {filename} | {value}/{total}', - noTTYOutput: !process.stdout.isTTY, + noTTYOutput: true, notTTYSchedule: ms('10s'), }) const totalBar = multibar.create(missingFiles.length, 0, {filename: '总文件数'}) const parallel = syncConfig.concurrency - const noopen = syncConfig.source === 'center' ? '1' : '' let hasError = false await pMap( missingFiles, async (file) => { const bar = multibar.create(file.size, 0, {filename: file.path}) try { - const res = await pRetry( - () => { + await pRetry( + async () => { bar.update(0) - return this.got + const res = await this.got .get(file.path.substring(1), { - searchParams: { - noopen, - }, retry: { limit: 0, }, @@ -205,29 +196,48 @@ export class Cluster { .on('downloadProgress', (progress) => { bar.update(progress.transferred) }) + + const isFileCorrect = validateFile(res.body, file.hash) + if (!isFileCorrect) { + throw new RequestError(`文件${file.path}校验失败`, new Error(`文件${file.path}校验失败`), res.request) + } + await this.storage.writeFile(hashToFilename(file.hash), res.body, file) }, { retries: 10, - onFailedAttempt: (e) => { - if (e.cause instanceof HTTPError) { + onFailedAttempt: async (e) => { + if (e instanceof HTTPError) { logger.debug( - {redirectUrls: e.cause.response.redirectUrls}, - `下载文件${file.path}失败: ${e.cause.response.statusCode}`, + {redirectUrls: e.response.redirectUrls}, + `下载文件${file.path}失败: ${e.response.statusCode}`, ) - logger.trace({err: e}, toString(e.cause.response.body)) + logger.trace({err: e}, toString(e.response.body)) } else { logger.debug({err: e}, `下载文件${file.path}失败，正在重试`) } + + if (e instanceof RequestError) { + const redirectUrls = e.response?.redirectUrls + if (redirectUrls?.length) { + const urls = [ + new URL(file.path, this.prefixUrl).toString(), + ...redirectUrls.map((e) => e.toString()), + ] + await this.got + .post('openbmclapi/report', { + json: { + urls, + error: stringifySafe({message: e.message}), + }, + }) + .catch((e) => { + logger.error(e, '上报重定向失败') + }) + } + } }, }, ) - const isFileCorrect = validateFile(res.body, file.hash) - if (!isFileCorrect) { - hasError = true - logger.error({redirectUrls: res.redirectUrls}, `文件${file.path}校验失败`) - return - } - await this.storage.writeFile(hashToFilename(file.hash), res.body, file) } catch (e) { hasError = true if (e instanceof HTTPError) { diff --git a/src/storage/alist-webdav.storage.ts b/src/storage/alist-webdav.storage.ts index b1be090..970cc03 100644 --- a/src/storage/alist-webdav.storage.ts +++ b/src/storage/alist-webdav.storage.ts @@ -4,17 +4,46 @@ import Keyv from 'keyv' import {KeyvFile} from 'keyv-file' import ms from 'ms' import {join} from 'path' +import {z} from 'zod' +import {fromZodError} from 'zod-validation-error' import {WebdavStorage} from './webdav.storage.js' +const storageConfigSchema = WebdavStorage.configSchema.extend({ + cacheTtl: z.union([z.string().optional(), z.number().int()]).default('1h'), +}) + export class AlistWebdavStorage extends WebdavStorage { - protected readonly redirectUrlCache = new Keyv({ - namespace: 'redirectUrl', - ttl: ms('1h'), - store: new KeyvFile({ - filename: join(process.cwd(), 'cache', 'redirectUrl.json'), - writeDelay: ms('1m'), - }), - }) + public readonly configSchema = storageConfigSchema + + protected readonly redirectUrlCache: Keyv + protected readonly storageConfig: z.infer + + constructor(storageConfig: unknown) { + super(storageConfig) + try { + this.storageConfig = this.configSchema.parse(storageConfig) + } catch (e) { + if (e instanceof z.ZodError) { + throw new Error('alist存储选项无效', {cause: fromZodError(e)}) + } else { + throw new Error('alist存储选项无效', {cause: e}) + } + } + let ttl: number + if (typeof this.storageConfig.cacheTtl === 'string') { + ttl = ms(this.storageConfig.cacheTtl) + } else { + ttl = this.storageConfig.cacheTtl + } + this.redirectUrlCache = new Keyv({ + namespace: 'redirectUrl', + ttl, + store: new KeyvFile({ + filename: join(process.cwd(), 'cache', 'redirectUrl.json'), + writeDelay: ms('1m'), + }), + }) + } public async express(hashPath: string, req: Request, res: Response): Promise<{bytes: number; hits: number}> { if (this.emptyFiles.has(hashPath)) { diff --git a/src/storage/webdav.storage.ts b/src/storage/webdav.storage.ts index 7976f2f..6f37ab3 100644 --- a/src/storage/webdav.storage.ts +++ b/src/storage/webdav.storage.ts @@ -19,6 +19,7 @@ const storageConfigSchema = z.object({ }) export class WebdavStorage implements IStorage { + public static readonly configSchema = storageConfigSchema protected readonly client: WebDAVClient protected readonly storageConfig: z.infer protected readonly basePath: string