feat(core): Workflow Execution Statistics

packages/cli/src/Db.ts

@@ -202,6 +202,7 @@ export async function init(
 	collections.Settings = linkRepository(entities.Settings);
 	collections.InstalledPackages = linkRepository(entities.InstalledPackages);
 	collections.InstalledNodes = linkRepository(entities.InstalledNodes);
+	collections.WorkflowStatistics = linkRepository(entities.WorkflowStatistics);
 
 	isInitialized = true;
 

packages/cli/src/Interfaces.ts

@@ -40,6 +40,7 @@ import type { SharedWorkflow } from './databases/entities/SharedWorkflow';
 import type { TagEntity } from './databases/entities/TagEntity';
 import type { User } from './databases/entities/User';
 import type { WorkflowEntity } from './databases/entities/WorkflowEntity';
+import { WorkflowStatistics } from './databases/entities/WorkflowStatistics';
 
 export interface IActivationError {
 	time: number;
@@ -83,6 +84,7 @@ export interface IDatabaseCollections {
 	Settings: Repository<Settings>;
 	InstalledPackages: Repository<InstalledPackages>;
 	InstalledNodes: Repository<InstalledNodes>;
+	WorkflowStatistics: Repository<WorkflowStatistics>;
 }
 
 export interface IWebhookDb {

packages/cli/src/databases/entities/WorkflowEntity.ts

@@ -18,6 +18,7 @@ import {
 	CreateDateColumn,
 	Entity,
 	Index,
+	JoinColumn,
 	JoinTable,
 	ManyToMany,
 	OneToMany,
@@ -30,6 +31,7 @@ import { DatabaseType, IWorkflowDb } from '../..';
 import { TagEntity } from './TagEntity';
 import { SharedWorkflow } from './SharedWorkflow';
 import { objectRetriever, sqlite } from '../utils/transformers';
+import { WorkflowStatistics } from './WorkflowStatistics';
 
 function resolveDataType(dataType: string) {
 	const dbType = config.getEnv('database.type');
@@ -124,6 +126,16 @@ export class WorkflowEntity implements IWorkflowDb {
 	@OneToMany(() => SharedWorkflow, (sharedWorkflow) => sharedWorkflow.workflow)
 	shared: SharedWorkflow[];
 
+	@OneToMany(
+		() => WorkflowStatistics,
+		(workflowStatistics: WorkflowStatistics) => workflowStatistics.workflow,
+	)
+	@JoinColumn({ referencedColumnName: 'workflow' })
+	statistics: WorkflowStatistics[];
+
+	@Column()
+	dataLoaded: boolean;
+
 	@Column({
 		type: config.getEnv('database.type') === 'sqlite' ? 'text' : 'json',
 		nullable: true,

packages/cli/src/databases/entities/WorkflowStatistics.ts

@@ -0,0 +1,44 @@
+/* eslint-disable import/no-cycle */
+import { Column, Entity, RelationId, ManyToOne, PrimaryColumn, UpdateDateColumn } from 'typeorm';
+import * as config from '../../../config';
+import { DatabaseType } from '../..';
+import { WorkflowEntity } from './WorkflowEntity';
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
+function getTimestampSyntax() {
+	const dbType = config.getEnv('database.type');
+
+	const map: { [key in DatabaseType]: string } = {
+		sqlite: "STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')",
+		postgresdb: 'CURRENT_TIMESTAMP(3)',
+		mysqldb: 'CURRENT_TIMESTAMP(3)',
+		mariadb: 'CURRENT_TIMESTAMP(3)',
+	};
+
+	return map[dbType];
+}
+
+@Entity()
+export class WorkflowStatistics {
+	@Column()
+	count: number;
+
+	@UpdateDateColumn({
+		precision: 3,
+		default: () => getTimestampSyntax(),
+		onUpdate: getTimestampSyntax(),
+	})
+	latestEvent: Date;
+
+	@PrimaryColumn({ length: 128 })
+	name: string;
+
+	@ManyToOne(() => WorkflowEntity, (workflow) => workflow.shared, {
+		primary: true,
+		onDelete: 'CASCADE',
+	})
+	workflow: WorkflowEntity;
+
+	@RelationId((workflowStatistics: WorkflowStatistics) => workflowStatistics.workflow)
+	workflowId: number;
+}

packages/cli/src/databases/entities/index.ts

@@ -12,6 +12,7 @@ import { SharedWorkflow } from './SharedWorkflow';
 import { SharedCredentials } from './SharedCredentials';
 import { InstalledPackages } from './InstalledPackages';
 import { InstalledNodes } from './InstalledNodes';
+import { WorkflowStatistics } from './WorkflowStatistics';
 
 export const entities = {
 	CredentialsEntity,
@@ -26,4 +27,5 @@ export const entities = {
 	SharedCredentials,
 	InstalledPackages,
 	InstalledNodes,
+	WorkflowStatistics,
 };

packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts

@@ -0,0 +1,66 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+import { logMigrationEnd, logMigrationStart } from '../../utils/migrationHelpers';
+import config from '../../../../config';
+
+export class WorkflowStatistics1664196174002 implements MigrationInterface {
+	name = 'WorkflowStatistics1664196174002';
+
+	async up(queryRunner: QueryRunner): Promise<void> {
+		logMigrationStart(this.name);
+
+		const tablePrefix = config.getEnv('database.tablePrefix');
+
+		await queryRunner.query(
+			`CREATE TABLE ${tablePrefix}workflow_statistics (
+				count INTEGER DEFAULT 0,
+				latestEvent DATETIME,
+				name VARCHAR(128) NOT NULL,
+				workflowId INTEGER,
+				PRIMARY KEY(workflowId, name),
+				FOREIGN KEY(workflowId) REFERENCES ${tablePrefix}workflow_entity(id) ON DELETE CASCADE
+			)`,
+		);
+
+		// Add dataLoaded column to workflow table
+		await queryRunner.query(
+			`ALTER TABLE ${tablePrefix}workflow_entity ADD COLUMN dataLoaded BOOLEAN DEFAULT false`,
+		);
+
+		// Fetch data from executions table to populate statistics table
+		await queryRunner.query(
+			`INSERT INTO ${tablePrefix}workflow_statistics (count, latestEvent, name, workflowId)
+			SELECT
+			  COUNT("id") as "count",
+				COALESCE(MAX("stoppedAt"), MAX("startedAt")) as "latestEvent",
+				CASE WHEN "finished" = true THEN 'production_success' ELSE 'production_error' END as "name",
+				CAST ("workflowId" AS UNSIGNED) AS "workflowId"
+			FROM ${tablePrefix}execution_entity
+			WHERE "workflowId" IS NOT NULL
+				AND mode != 'manual'
+			GROUP BY "workflowId", "finished"
+			ORDER BY "workflowId";`,
+		);
+		await queryRunner.query(
+			`INSERT INTO ${tablePrefix}workflow_statistics (count, latestEvent, name, workflowId)
+			SELECT
+			  COUNT("id") as "count",
+				COALESCE(MAX("stoppedAt"), MAX("startedAt")) as "latestEvent",
+				CASE WHEN "finished" = true THEN 'production_success' ELSE 'production_error' END as "name",
+				CAST ("workflowId" AS UNSIGNED) AS "workflowId"
+			FROM ${tablePrefix}execution_entity
+			WHERE "workflowId" IS NOT NULL
+				AND mode = 'manual'
+			GROUP BY "workflowId", "finished"
+			ORDER BY "workflowId";`,
+		);
+
+		logMigrationEnd(this.name);
+	}
+
+	async down(queryRunner: QueryRunner): Promise<void> {
+		const tablePrefix = config.getEnv('database.tablePrefix');
+
+		await queryRunner.query(`DROP TABLE "${tablePrefix}workflow_statistics"`);
+		await queryRunner.query(`ALTER TABLE ${tablePrefix}workflow_entity DROP COLUMN dataLoaded`);
+	}
+}

packages/cli/src/databases/migrations/mysqldb/index.ts

@@ -20,6 +20,7 @@ import { IntroducePinData1654090101303 } from './1654090101303-IntroducePinData'
 import { AddNodeIds1658932910559 } from './1658932910559-AddNodeIds';
 import { AddJsonKeyPinData1659895550980 } from './1659895550980-AddJsonKeyPinData';
 import { CreateCredentialsUserRole1660062385367 } from './1660062385367-CreateCredentialsUserRole';
+import { WorkflowStatistics1664196174002 } from './1664196174002-WorkflowStatistics';
 
 export const mysqlMigrations = [
 	InitialMigration1588157391238,
@@ -44,4 +45,5 @@ export const mysqlMigrations = [
 	AddNodeIds1658932910559,
 	AddJsonKeyPinData1659895550980,
 	CreateCredentialsUserRole1660062385367,
+	WorkflowStatistics1664196174002,
 ];

packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts

@@ -0,0 +1,75 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+import { logMigrationEnd, logMigrationStart } from '../../utils/migrationHelpers';
+import config from '../../../../config';
+
+export class WorkflowStatistics1664196174001 implements MigrationInterface {
+	name = 'WorkflowStatistics1664196174001';
+
+	async up(queryRunner: QueryRunner): Promise<void> {
+		logMigrationStart(this.name);
+
+		let tablePrefix = config.getEnv('database.tablePrefix');
+		const schema = config.getEnv('database.postgresdb.schema');
+		if (schema) {
+			tablePrefix = schema + '.' + tablePrefix;
+		}
+
+		// Create statistics table
+		await queryRunner.query(
+			`CREATE TABLE ${tablePrefix}workflow_statistics (
+				"count" INTEGER DEFAULT 0,
+				"latestEvent" DATE,
+				"name" VARCHAR(128) NOT NULL,
+				"workflowId" INTEGER,
+				PRIMARY KEY("workflowId", "name"),
+				FOREIGN KEY("workflowId") REFERENCES ${tablePrefix}workflow_entity("id") ON DELETE CASCADE
+			)`,
+		);
+
+		// Add dataLoaded column to workflow table
+		await queryRunner.query(
+			`ALTER TABLE ${tablePrefix}workflow_entity ADD COLUMN "dataLoaded" BOOLEAN DEFAULT false;`,
+		);
+
+		// Fetch data from executions table to populate statistics table
+		await queryRunner.query(
+			`INSERT INTO ${tablePrefix}workflow_statistics ("count", "latestEvent", "name", "workflowId")
+			SELECT
+			  COUNT("id") as "count",
+				COALESCE(MAX("stoppedAt"), MAX("startedAt")) as "latestEvent",
+				CASE WHEN "finished" = true THEN 'production_success' ELSE 'production_error' END as "name",
+				CAST ("workflowId" AS INTEGER) AS "workflowId"
+			FROM ${tablePrefix}execution_entity
+			WHERE "workflowId" IS NOT NULL
+				AND mode != 'manual'
+			GROUP BY "workflowId", "finished"
+			ORDER BY "workflowId";`,
+		);
+		await queryRunner.query(
+			`INSERT INTO ${tablePrefix}workflow_statistics ("count", "latestEvent", "name", "workflowId")
+			SELECT
+			  COUNT("id") as "count",
+				COALESCE(MAX("stoppedAt"), MAX("startedAt")) as "latestEvent",
+				CASE WHEN "finished" = true THEN 'production_success' ELSE 'production_error' END as "name",
+				CAST ("workflowId" AS INTEGER) AS "workflowId"
+			FROM ${tablePrefix}execution_entity
+			WHERE "workflowId" IS NOT NULL
+				AND mode == 'manual'
+			GROUP BY "workflowId", "finished"
+			ORDER BY "workflowId";`,
+		);
+
+		logMigrationEnd(this.name);
+	}
+
+	async down(queryRunner: QueryRunner): Promise<void> {
+		let tablePrefix = config.getEnv('database.tablePrefix');
+		const schema = config.getEnv('database.postgresdb.schema');
+		if (schema) {
+			tablePrefix = schema + '.' + tablePrefix;
+		}
+
+		await queryRunner.query(`DROP TABLE ${tablePrefix}workflow_statistics`);
+		await queryRunner.query(`ALTER TABLE ${tablePrefix}workflow_entity DROP COLUMN dataLoaded`);
+	}
+}

packages/cli/src/databases/migrations/postgresdb/index.ts

@@ -18,6 +18,7 @@ import { IntroducePinData1654090467022 } from './1654090467022-IntroducePinData'
 import { AddNodeIds1658932090381 } from './1658932090381-AddNodeIds';
 import { AddJsonKeyPinData1659902242948 } from './1659902242948-AddJsonKeyPinData';
 import { CreateCredentialsUserRole1660062385367 } from './1660062385367-CreateCredentialsUserRole';
+import { WorkflowStatistics1664196174001 } from './1664196174001-WorkflowStatistics';
 
 export const postgresMigrations = [
 	InitialMigration1587669153312,
@@ -40,4 +41,5 @@ export const postgresMigrations = [
 	CreateCredentialsUserRole1660062385367,
 	AddNodeIds1658932090381,
 	AddJsonKeyPinData1659902242948,
+	WorkflowStatistics1664196174001,
 ];

packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts

@@ -0,0 +1,68 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+import { logMigrationEnd, logMigrationStart } from '../../utils/migrationHelpers';
+import config from '../../../../config';
+
+export class WorkflowStatistics1664196174000 implements MigrationInterface {
+	name = 'WorkflowStatistics1664196174000';
+
+	async up(queryRunner: QueryRunner): Promise<void> {
+		logMigrationStart(this.name);
+
+		const tablePrefix = config.getEnv('database.tablePrefix');
+
+		await queryRunner.query(
+			`CREATE TABLE \`${tablePrefix}workflow_statistics\` (
+				"count" INTEGER DEFAULT 0,
+				"latestEvent" DATETIME,
+				"name" VARCHAR(128) NOT NULL,
+				"workflowId" INTEGER,
+				PRIMARY KEY("workflowId", "name"),
+				FOREIGN KEY("workflowId") REFERENCES \`${tablePrefix}workflow_entity\`("id") ON DELETE CASCADE
+			)`,
+		);
+
+		// Add dataLoaded column to workflow table
+		await queryRunner.query(
+			`ALTER TABLE \`${tablePrefix}workflow_entity\` ADD COLUMN "dataLoaded" BOOLEAN DEFAULT false`,
+		);
+
+		// Fetch data from executions table to populate statistics table
+		await queryRunner.query(
+			`INSERT INTO \`${tablePrefix}workflow_statistics\` ("count", "latestEvent", "name", "workflowId")
+			SELECT
+			  COUNT("id") as "count",
+				IFNULL(MAX("stoppedAt"), MAX("startedAt")) as "latestEvent",
+				IIF("finished" == 1, 'production_success', 'production_error') as "name",
+				"workflowId"
+			FROM \`${tablePrefix}execution_entity\`
+			WHERE "workflowId" IS NOT NULL
+				AND mode != 'manual'
+			GROUP BY "workflowId", "finished"
+			ORDER BY "workflowId";`,
+		);
+		await queryRunner.query(
+			`INSERT INTO \`${tablePrefix}workflow_statistics\` ("count", "latestEvent", "name", "workflowId")
+			SELECT
+				COUNT("id") as "count",
+				IFNULL(MAX("stoppedAt"), MAX("startedAt")) as "latestEvent",
+				IIF("finished" == 1, 'production_success', 'production_error') as "name",
+				"workflowId"
+			FROM \`${tablePrefix}execution_entity\`
+			WHERE "workflowId" IS NOT NULL
+				AND mode == 'manual'
+			GROUP BY "workflowId", "finished"
+			ORDER BY "workflowId";`,
+		);
+
+		logMigrationEnd(this.name);
+	}
+
+	async down(queryRunner: QueryRunner): Promise<void> {
+		const tablePrefix = config.getEnv('database.tablePrefix');
+
+		await queryRunner.query(`DROP TABLE "${tablePrefix}workflow_statistics"`);
+		await queryRunner.query(
+			`ALTER TABLE \`${tablePrefix}workflow_entity\` DROP COLUMN "dataLoaded"`,
+		);
+	}
+}

packages/cli/src/databases/migrations/sqlite/index.ts

@@ -17,6 +17,7 @@ import { IntroducePinData1654089251344 } from './1654089251344-IntroducePinData'
 import { AddNodeIds1658930531669 } from './1658930531669-AddNodeIds';
 import { AddJsonKeyPinData1659888469333 } from './1659888469333-AddJsonKeyPinData';
 import { CreateCredentialsUserRole1660062385367 } from './1660062385367-CreateCredentialsUserRole';
+import { WorkflowStatistics1664196174000 } from './1664196174000-WorkflowStatistics';
 
 const sqliteMigrations = [
 	InitialMigration1588102412422,
@@ -38,6 +39,7 @@ const sqliteMigrations = [
 	AddNodeIds1658930531669,
 	AddJsonKeyPinData1659888469333,
 	CreateCredentialsUserRole1660062385367,
+	WorkflowStatistics1664196174000,
 ];
 
 export { sqliteMigrations };