fix: create custom logs cleanup function (#3610)

api/lib/logger.ts

@@ -1,10 +1,15 @@
-import DailyRotateFile from '@zwave-js/winston-daily-rotate-file'
+import DailyRotateFile, {
+	DailyRotateFileTransportOptions,
+} from '@zwave-js/winston-daily-rotate-file'
 import { ensureDirSync } from 'fs-extra'
 import winston from 'winston'
 import { logsDir, storeDir } from '../config/app'
 import { GatewayConfig } from './Gateway'
 import { DeepPartial, joinPath } from './utils'
 import * as path from 'path'
+import { readdir, stat, unlink } from 'fs/promises'
+import { Stats } from 'fs'
+import escapeStringRegexp from 'escape-string-regexp'
 
 const { format, transports, addColors } = winston
 const { combine, timestamp, label, printf, colorize, splat } = format
@@ -121,7 +126,7 @@ export function customTransports(config: LoggerConfig): winston.transport[] {
 				level: config.level,
 			})
 		} else {
-			fileTransport = new DailyRotateFile({
+			const options: DailyRotateFileTransportOptions = {
 				filename: config.filePath,
 				auditFile: joinPath(logsDir, 'zui-logs.audit.json'),
 				datePattern: 'YYYY-MM-DD',
@@ -134,7 +139,10 @@ export function customTransports(config: LoggerConfig): winston.transport[] {
 				maxSize: process.env.ZUI_LOG_MAXSIZE || '50m',
 				level: config.level,
 				format: customFormat(config, true),
-			})
+			}
+			fileTransport = new DailyRotateFile(options)
+
+			setupCleanJob(options)
 		}
 
 		transportsList.push(fileTransport)
@@ -177,9 +185,160 @@ export function module(module: string): ModuleLogger {
  * Setup all loggers starting from config
  */
 export function setupAll(config: DeepPartial<GatewayConfig>) {
+	stopCleanJob()
+
 	logContainer.loggers.forEach((logger: ModuleLogger) => {
 		logger.setup(config)
 	})
 }
 
+let cleanJob: NodeJS.Timeout
+
+export function setupCleanJob(settings: DailyRotateFileTransportOptions) {
+	if (cleanJob) {
+		return
+	}
+
+	let maxFilesMs: number
+	let maxFiles: number
+	let maxSizeBytes: number
+
+	const logger = module('LOGGER')
+
+	// convert maxFiles to milliseconds
+	if (settings.maxFiles !== undefined) {
+		const matches = settings.maxFiles.toString().match(/(\d+)([dhm])/)
+
+		if (settings.maxFiles) {
+			const value = parseInt(matches[1])
+			const unit = matches[2]
+			switch (unit) {
+				case 'd':
+					maxFilesMs = value * 24 * 60 * 60 * 1000
+					break
+				case 'h':
+					maxFilesMs = value * 60 * 60 * 1000
+					break
+				case 'm':
+					maxFilesMs = value * 60 * 1000
+					break
+			}
+		} else {
+			maxFiles = Number(settings.maxFiles)
+		}
+	}
+
+	if (settings.maxSize !== undefined) {
+		// convert maxSize to bytes
+		const matches2 = settings.maxSize.toString().match(/(\d+)([kmg])/)
+		if (matches2) {
+			const value = parseInt(matches2[1])
+			const unit = matches2[2]
+			switch (unit) {
+				case 'k':
+					maxSizeBytes = value * 1024
+					break
+				case 'm':
+					maxSizeBytes = value * 1024 * 1024
+					break
+				case 'g':
+					maxSizeBytes = value * 1024 * 1024 * 1024
+					break
+			}
+		} else {
+			maxSizeBytes = Number(settings.maxSize)
+		}
+	}
+
+	// clean up old log files based on maxFiles and maxSize
+
+	const filePathRegExp = new RegExp(
+		escapeStringRegexp(path.basename(settings.filename)).replace(
+			/%DATE%/g,
+			'(.*)',
+		),
+	)
+
+	const logsDir = path.dirname(settings.filename)
+
+	const deleteFile = async (filePath: string) => {
+		logger.info(`Deleting log file: ${filePath}`)
+		return unlink(filePath).catch((e) => {
+			if (e.code !== 'ENOENT') {
+				logger.error(`Error deleting log file: ${filePath}`, e)
+			}
+		})
+	}
+
+	const clean = async () => {
+		try {
+			logger.info('Cleaning up log files...')
+			const files = await readdir(logsDir)
+			const logFiles = files.filter(
+				(file) =>
+					file !== settings.symlinkName && filePathRegExp.test(file),
+			)
+
+			const fileStats = await Promise.allSettled<{
+				file: string
+				stats: Stats
+			}>(
+				logFiles.map(async (file) => ({
+					file,
+					stats: await stat(path.join(logsDir, file)),
+				})),
+			)
+
+			const logFilesStats: {
+				file: string
+				stats: Stats
+			}[] = []
+
+			for (const res of fileStats) {
+				if (res.status === 'fulfilled') {
+					logFilesStats.push(res.value)
+				} else {
+					logger.error('Error getting file stats:', res.reason)
+				}
+			}
+
+			logFilesStats.sort((a, b) => a.stats.mtimeMs - b.stats.mtimeMs)
+
+			// sort by mtime
+
+			let totalSize = 0
+			let deletedFiles = 0
+			for (const { file, stats } of logFilesStats) {
+				const filePath = path.join(logsDir, file)
+				totalSize += stats.size
+
+				// last modified time in milliseconds
+				const fileMs = stats.mtimeMs
+
+				const shouldDelete =
+					(maxSizeBytes && totalSize > maxSizeBytes) ||
+					(maxFiles && logFiles.length - deletedFiles > maxFiles) ||
+					(maxFilesMs && fileMs && Date.now() - fileMs > maxFilesMs)
+
+				if (shouldDelete) {
+					await deleteFile(filePath)
+					deletedFiles++
+				}
+			}
+		} catch (e) {
+			logger.error('Error cleaning up log files:', e)
+		}
+	}
+
+	cleanJob = setInterval(clean, 60 * 60 * 1000)
+	clean().catch(() => {})
+}
+
+export function stopCleanJob() {
+	if (cleanJob) {
+		clearInterval(cleanJob)
+		cleanJob = undefined
+	}
+}
+
 export default logContainer.loggers

package.json

@@ -75,6 +75,7 @@
     "cronstrue": "^2.32.0",
     "csurf": "^1.11.0",
     "dotenv": "^16.3.1",
+    "escape-string-regexp": "^4.0.0",
     "express": "^4.18.2",
     "express-rate-limit": "^7.1.1",
     "express-session": "^1.17.3",

test/lib/logger.test.ts

@@ -8,6 +8,7 @@ import {
 	sanitizedConfig,
 	module,
 	setupAll,
+	stopCleanJob,
 } from '../../api/lib/logger'
 import winston from 'winston'
 
@@ -23,6 +24,10 @@ describe('logger.js', () => {
 	let logger1: ModuleLogger
 	let logger2: ModuleLogger
 
+	afterEach(() => {
+		stopCleanJob()
+	})
+
 	describe('sanitizedConfig()', () => {
 		it('should set undefined config object to defaults', () => {
 			const cfg = sanitizedConfig('-', undefined)