From 64f32f0020f4a9da68576862fbaaadf2f1d7f96d Mon Sep 17 00:00:00 2001 From: Daniel Lando Date: Mon, 4 Mar 2024 14:32:42 +0100 Subject: [PATCH] fix: create custom logs cleanup function (#3610) --- api/lib/logger.ts | 165 +++++++++++++++++++++++++++++++++++++++- package-lock.json | 55 +++++++------- package.json | 1 + test/lib/logger.test.ts | 5 ++ 4 files changed, 194 insertions(+), 32 deletions(-) diff --git a/api/lib/logger.ts b/api/lib/logger.ts index ce2a158f02..aea668200c 100644 --- a/api/lib/logger.ts +++ b/api/lib/logger.ts @@ -1,10 +1,15 @@ -import DailyRotateFile from '@zwave-js/winston-daily-rotate-file' +import DailyRotateFile, { + DailyRotateFileTransportOptions, +} from '@zwave-js/winston-daily-rotate-file' import { ensureDirSync } from 'fs-extra' import winston from 'winston' import { logsDir, storeDir } from '../config/app' import { GatewayConfig } from './Gateway' import { DeepPartial, joinPath } from './utils' import * as path from 'path' +import { readdir, stat, unlink } from 'fs/promises' +import { Stats } from 'fs' +import escapeStringRegexp from 'escape-string-regexp' const { format, transports, addColors } = winston const { combine, timestamp, label, printf, colorize, splat } = format @@ -121,7 +126,7 @@ export function customTransports(config: LoggerConfig): winston.transport[] { level: config.level, }) } else { - fileTransport = new DailyRotateFile({ + const options: DailyRotateFileTransportOptions = { filename: config.filePath, auditFile: joinPath(logsDir, 'zui-logs.audit.json'), datePattern: 'YYYY-MM-DD', @@ -134,7 +139,10 @@ export function customTransports(config: LoggerConfig): winston.transport[] { maxSize: process.env.ZUI_LOG_MAXSIZE || '50m', level: config.level, format: customFormat(config, true), - }) + } + fileTransport = new DailyRotateFile(options) + + setupCleanJob(options) } transportsList.push(fileTransport) @@ -177,9 +185,160 @@ export function module(module: string): ModuleLogger { * Setup all loggers starting from config */ export function setupAll(config: DeepPartial) { + stopCleanJob() + logContainer.loggers.forEach((logger: ModuleLogger) => { logger.setup(config) }) } +let cleanJob: NodeJS.Timeout + +export function setupCleanJob(settings: DailyRotateFileTransportOptions) { + if (cleanJob) { + return + } + + let maxFilesMs: number + let maxFiles: number + let maxSizeBytes: number + + const logger = module('LOGGER') + + // convert maxFiles to milliseconds + if (settings.maxFiles !== undefined) { + const matches = settings.maxFiles.toString().match(/(\d+)([dhm])/) + + if (settings.maxFiles) { + const value = parseInt(matches[1]) + const unit = matches[2] + switch (unit) { + case 'd': + maxFilesMs = value * 24 * 60 * 60 * 1000 + break + case 'h': + maxFilesMs = value * 60 * 60 * 1000 + break + case 'm': + maxFilesMs = value * 60 * 1000 + break + } + } else { + maxFiles = Number(settings.maxFiles) + } + } + + if (settings.maxSize !== undefined) { + // convert maxSize to bytes + const matches2 = settings.maxSize.toString().match(/(\d+)([kmg])/) + if (matches2) { + const value = parseInt(matches2[1]) + const unit = matches2[2] + switch (unit) { + case 'k': + maxSizeBytes = value * 1024 + break + case 'm': + maxSizeBytes = value * 1024 * 1024 + break + case 'g': + maxSizeBytes = value * 1024 * 1024 * 1024 + break + } + } else { + maxSizeBytes = Number(settings.maxSize) + } + } + + // clean up old log files based on maxFiles and maxSize + + const filePathRegExp = new RegExp( + escapeStringRegexp(path.basename(settings.filename)).replace( + /%DATE%/g, + '(.*)', + ), + ) + + const logsDir = path.dirname(settings.filename) + + const deleteFile = async (filePath: string) => { + logger.info(`Deleting log file: ${filePath}`) + return unlink(filePath).catch((e) => { + if (e.code !== 'ENOENT') { + logger.error(`Error deleting log file: ${filePath}`, e) + } + }) + } + + const clean = async () => { + try { + logger.info('Cleaning up log files...') + const files = await readdir(logsDir) + const logFiles = files.filter( + (file) => + file !== settings.symlinkName && filePathRegExp.test(file), + ) + + const fileStats = await Promise.allSettled<{ + file: string + stats: Stats + }>( + logFiles.map(async (file) => ({ + file, + stats: await stat(path.join(logsDir, file)), + })), + ) + + const logFilesStats: { + file: string + stats: Stats + }[] = [] + + for (const res of fileStats) { + if (res.status === 'fulfilled') { + logFilesStats.push(res.value) + } else { + logger.error('Error getting file stats:', res.reason) + } + } + + logFilesStats.sort((a, b) => a.stats.mtimeMs - b.stats.mtimeMs) + + // sort by mtime + + let totalSize = 0 + let deletedFiles = 0 + for (const { file, stats } of logFilesStats) { + const filePath = path.join(logsDir, file) + totalSize += stats.size + + // last modified time in milliseconds + const fileMs = stats.mtimeMs + + const shouldDelete = + (maxSizeBytes && totalSize > maxSizeBytes) || + (maxFiles && logFiles.length - deletedFiles > maxFiles) || + (maxFilesMs && fileMs && Date.now() - fileMs > maxFilesMs) + + if (shouldDelete) { + await deleteFile(filePath) + deletedFiles++ + } + } + } catch (e) { + logger.error('Error cleaning up log files:', e) + } + } + + cleanJob = setInterval(clean, 60 * 60 * 1000) + clean().catch(() => {}) +} + +export function stopCleanJob() { + if (cleanJob) { + clearInterval(cleanJob) + cleanJob = undefined + } +} + export default logContainer.loggers diff --git a/package-lock.json b/package-lock.json index 95e9b3da85..0225173344 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,6 +24,7 @@ "cronstrue": "^2.32.0", "csurf": "^1.11.0", "dotenv": "^16.3.1", + "escape-string-regexp": "^4.0.0", "express": "^4.18.2", "express-rate-limit": "^7.1.1", "express-session": "^1.17.3", @@ -6051,6 +6052,15 @@ "node": ">=4" } }, + "node_modules/chalk/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, "node_modules/chardet": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz", @@ -7984,12 +7994,14 @@ "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" }, "node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true, + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", "engines": { - "node": ">=0.8.0" + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/escodegen": { @@ -8446,18 +8458,6 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "dev": true }, - "node_modules/eslint/node_modules/escape-string-regexp": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", - "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", - "dev": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/eslint/node_modules/globals": { "version": "13.23.0", "resolved": "https://registry.npmjs.org/globals/-/globals-13.23.0.tgz", @@ -12338,18 +12338,6 @@ "node": ">=6" } }, - "node_modules/mocha/node_modules/escape-string-regexp": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", - "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", - "dev": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/mocha/node_modules/glob": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", @@ -19921,6 +19909,15 @@ "node": ">=0.10.0" } }, + "node_modules/yargonaut/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, "node_modules/yargonaut/node_modules/strip-ansi": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", diff --git a/package.json b/package.json index ab218f79c8..ef4a4c809d 100644 --- a/package.json +++ b/package.json @@ -75,6 +75,7 @@ "cronstrue": "^2.32.0", "csurf": "^1.11.0", "dotenv": "^16.3.1", + "escape-string-regexp": "^4.0.0", "express": "^4.18.2", "express-rate-limit": "^7.1.1", "express-session": "^1.17.3", diff --git a/test/lib/logger.test.ts b/test/lib/logger.test.ts index 6e19dba4bb..631f0d9ddc 100644 --- a/test/lib/logger.test.ts +++ b/test/lib/logger.test.ts @@ -8,6 +8,7 @@ import { sanitizedConfig, module, setupAll, + stopCleanJob, } from '../../api/lib/logger' import winston from 'winston' @@ -23,6 +24,10 @@ describe('logger.js', () => { let logger1: ModuleLogger let logger2: ModuleLogger + afterEach(() => { + stopCleanJob() + }) + describe('sanitizedConfig()', () => { it('should set undefined config object to defaults', () => { const cfg = sanitizedConfig('-', undefined)