feat: Workflow History pruning and prune time settings (#7343)

Github issue / Community forum post (link here to close automatically):

packages/cli/src/License.ts

@@ -244,6 +244,12 @@ export class License {
 		return this.getFeatureValue(LICENSE_QUOTAS.VARIABLES_LIMIT) ?? UNLIMITED_LICENSE_QUOTA;
 	}
 
+	getWorkflowHistoryPruneLimit() {
+		return (
+			this.getFeatureValue(LICENSE_QUOTAS.WORKFLOW_HISTORY_PRUNE_LIMIT) ?? UNLIMITED_LICENSE_QUOTA
+		);
+	}
+
 	getPlanName(): string {
 		return this.getFeatureValue('planName') ?? 'Community';
 	}

packages/cli/src/Server.ts

@@ -178,7 +178,11 @@ import { JwtService } from './services/jwt.service';
 import { RoleService } from './services/role.service';
 import { UserService } from './services/user.service';
 import { OrchestrationController } from './controllers/orchestration.controller';
-import { isWorkflowHistoryEnabled } from './workflows/workflowHistory/workflowHistoryHelper.ee';
+import {
+	getWorkflowHistoryLicensePruneTime,
+	getWorkflowHistoryPruneTime,
+	isWorkflowHistoryEnabled,
+} from './workflows/workflowHistory/workflowHistoryHelper.ee';
 import { WorkflowHistoryController } from './workflows/workflowHistory/workflowHistory.controller.ee';
 
 const exec = promisify(callbackExec);
@@ -350,6 +354,10 @@ export class Server extends AbstractServer {
 			ai: {
 				enabled: config.getEnv('ai.enabled'),
 			},
+			workflowHistory: {
+				pruneTime: -1,
+				licensePruneTime: -1,
+			},
 		};
 	}
 
@@ -496,6 +504,13 @@ export class Server extends AbstractServer {
 			this.frontendSettings.variables.limit = getVariablesLimit();
 		}
 
+		if (isWorkflowHistoryEnabled()) {
+			Object.assign(this.frontendSettings.workflowHistory, {
+				pruneTime: getWorkflowHistoryPruneTime(),
+				licensePruneTime: getWorkflowHistoryLicensePruneTime(),
+			});
+		}
+
 		if (config.get('nodes.packagesMissing').length > 0) {
 			this.frontendSettings.missingPackages = true;
 		}

packages/cli/src/commands/BaseCommand.ts

@@ -23,6 +23,7 @@ import { License } from '@/License';
 import { ExternalSecretsManager } from '@/ExternalSecrets/ExternalSecretsManager.ee';
 import { initExpressionEvaluator } from '@/ExpressionEvalator';
 import { generateHostInstanceId } from '../databases/utils/generators';
+import { WorkflowHistoryManager } from '@/workflows/workflowHistory/workflowHistoryManager.ee';
 
 export abstract class BaseCommand extends Command {
 	protected logger = LoggerProxy.init(getLogger());
@@ -161,6 +162,10 @@ export abstract class BaseCommand extends Command {
 		await secretsManager.init();
 	}
 
+	initWorkflowHistory() {
+		Container.get(WorkflowHistoryManager).init();
+	}
+
 	async finally(error: Error | undefined) {
 		if (inTest || this.id === 'start') return;
 		if (Db.connectionState.connected) {

packages/cli/src/commands/start.ts

@@ -216,6 +216,7 @@ export class Start extends BaseCommand {
 		await this.initBinaryDataService();
 		await this.initExternalHooks();
 		await this.initExternalSecrets();
+		this.initWorkflowHistory();
 
 		if (!config.getEnv('endpoints.disableUi')) {
 			await this.generateStaticAssets();

packages/cli/src/config/schema.ts

@@ -1227,4 +1227,20 @@ export const schema = {
 			env: 'N8N_SOURCECONTROL_DEFAULT_SSH_KEY_TYPE',
 		},
 	},
+
+	workflowHistory: {
+		enabled: {
+			doc: 'Whether to save workflow history versions',
+			format: Boolean,
+			default: true,
+			env: 'N8N_WORKFLOW_HISTORY_ENABLED',
+		},
+
+		pruneTime: {
+			doc: 'Time (in hours) to keep workflow history versions for',
+			format: Number,
+			default: -1,
+			env: 'N8N_WORKFLOW_HISTORY_PRUNE_TIME',
+		},
+	},
 };

packages/cli/src/constants.ts

@@ -87,6 +87,7 @@ export const LICENSE_QUOTAS = {
 	TRIGGER_LIMIT: 'quota:activeWorkflows',
 	VARIABLES_LIMIT: 'quota:maxVariables',
 	USERS_LIMIT: 'quota:users',
+	WORKFLOW_HISTORY_PRUNE_LIMIT: 'quota:workflowHistoryPrune',
 } as const;
 export const UNLIMITED_LICENSE_QUOTA = -1;
 

packages/cli/src/workflows/workflowHistory/constants.ts

@@ -0,0 +1,3 @@
+import { TIME } from '@/constants';
+
+export const WORKFLOW_HISTORY_PRUNE_INTERVAL = 1 * TIME.HOUR;

packages/cli/src/workflows/workflowHistory/workflowHistoryHelper.ee.ts

@@ -1,4 +1,5 @@
 import { License } from '@/License';
+import config from '@/config';
 import Container from 'typedi';
 
 export function isWorkflowHistoryLicensed() {
@@ -7,5 +8,28 @@ export function isWorkflowHistoryLicensed() {
 }
 
 export function isWorkflowHistoryEnabled() {
-	return isWorkflowHistoryLicensed();
+	return isWorkflowHistoryLicensed() && config.getEnv('workflowHistory.enabled');
+}
+
+export function getWorkflowHistoryLicensePruneTime() {
+	return Container.get(License).getWorkflowHistoryPruneLimit();
+}
+
+// Time in hours
+export function getWorkflowHistoryPruneTime(): number {
+	const licenseTime = Container.get(License).getWorkflowHistoryPruneLimit();
+	const configTime = config.getEnv('workflowHistory.pruneTime');
+
+	// License is infinite and config time is infinite
+	if (licenseTime === -1) {
+		return configTime;
+	}
+
+	// License is not infinite but config is, use license time
+	if (configTime === -1) {
+		return licenseTime;
+	}
+
+	// Return the smallest of the license or config if not infinite
+	return Math.min(configTime, licenseTime);
 }

packages/cli/src/workflows/workflowHistory/workflowHistoryManager.ee.ts

@@ -0,0 +1,45 @@
+import { WorkflowHistoryRepository } from '@/databases/repositories';
+import { Service } from 'typedi';
+import { WORKFLOW_HISTORY_PRUNE_INTERVAL } from './constants';
+import { getWorkflowHistoryPruneTime, isWorkflowHistoryEnabled } from './workflowHistoryHelper.ee';
+import { DateTime } from 'luxon';
+import { LessThan } from 'typeorm';
+
+@Service()
+export class WorkflowHistoryManager {
+	pruneTimer?: NodeJS.Timeout;
+
+	constructor(private workflowHistoryRepo: WorkflowHistoryRepository) {}
+
+	init() {
+		if (this.pruneTimer !== undefined) {
+			clearInterval(this.pruneTimer);
+		}
+
+		this.pruneTimer = setInterval(async () => this.prune(), WORKFLOW_HISTORY_PRUNE_INTERVAL);
+	}
+
+	shutdown() {
+		if (this.pruneTimer !== undefined) {
+			clearInterval(this.pruneTimer);
+			this.pruneTimer = undefined;
+		}
+	}
+
+	async prune() {
+		if (!isWorkflowHistoryEnabled()) {
+			return;
+		}
+
+		const pruneHours = getWorkflowHistoryPruneTime();
+		// No prune time set
+		if (pruneHours === -1) {
+			return;
+		}
+		const pruneDateTime = DateTime.now().minus({ hours: pruneHours }).toJSDate();
+
+		await this.workflowHistoryRepo.delete({
+			createdAt: LessThan(pruneDateTime),
+		});
+	}
+}

packages/cli/test/integration/shared/testDb.ts

@@ -610,6 +610,22 @@ export async function createWorkflowHistoryItem(
 	});
 }
 
+export async function createManyWorkflowHistoryItems(
+	workflowId: string,
+	count: number,
+	time?: Date,
+) {
+	const baseTime = (time ?? new Date()).valueOf();
+	return Promise.all(
+		[...Array(count)].map(async (_, i) =>
+			createWorkflowHistoryItem(workflowId, {
+				createdAt: new Date(baseTime + i),
+				updatedAt: new Date(baseTime + i),
+			}),
+		),
+	);
+}
+
 // ----------------------------------
 //        connection options
 // ----------------------------------