From 42753a3fbbbaefd500811d72b76e87e572e5efa3 Mon Sep 17 00:00:00 2001 From: freyamade Date: Mon, 26 Sep 2022 17:10:52 +0100 Subject: [PATCH 01/49] :tada: - Initial attempt at putting a DB table together --- .../src/databases/entities/WorkflowEntity.ts | 6 +++ .../databases/entities/WorkflowStatistics.ts | 40 +++++++++++++++++++ .../1664196174000-WorkflowStatistics.ts | 34 ++++++++++++++++ 3 files changed, 80 insertions(+) create mode 100644 packages/cli/src/databases/entities/WorkflowStatistics.ts create mode 100644 packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts diff --git a/packages/cli/src/databases/entities/WorkflowEntity.ts b/packages/cli/src/databases/entities/WorkflowEntity.ts index cbc7c5269d898..33c0efbfc3ecf 100644 --- a/packages/cli/src/databases/entities/WorkflowEntity.ts +++ b/packages/cli/src/databases/entities/WorkflowEntity.ts @@ -18,6 +18,7 @@ import { CreateDateColumn, Entity, Index, + JoinColumn, JoinTable, ManyToMany, OneToMany, @@ -30,6 +31,7 @@ import { DatabaseType, IWorkflowDb } from '../..'; import { TagEntity } from './TagEntity'; import { SharedWorkflow } from './SharedWorkflow'; import { objectRetriever, sqlite } from '../utils/transformers'; +import { WorkflowStatistics } from './WorkflowStatistics'; function resolveDataType(dataType: string) { const dbType = config.getEnv('database.type'); @@ -124,6 +126,10 @@ export class WorkflowEntity implements IWorkflowDb { @OneToMany(() => SharedWorkflow, (sharedWorkflow) => sharedWorkflow.workflow) shared: SharedWorkflow[]; + @OneToMany(() => WorkflowStatistics, (workflowStatistics: WorkflowStatistics) => workflowStatistics.workflow) + @JoinColumn({ referencedColumnName: 'workflow' }) + statistics: WorkflowStatistics[]; + @Column({ type: config.getEnv('database.type') === 'sqlite' ? 'text' : 'json', nullable: true, diff --git a/packages/cli/src/databases/entities/WorkflowStatistics.ts b/packages/cli/src/databases/entities/WorkflowStatistics.ts new file mode 100644 index 0000000000000..64e887f2fb1b8 --- /dev/null +++ b/packages/cli/src/databases/entities/WorkflowStatistics.ts @@ -0,0 +1,40 @@ +/* eslint-disable import/no-cycle */ +import { Column, Entity, JoinColumn, ManyToOne, PrimaryColumn, UpdateDateColumn } from 'typeorm'; +import * as config from '../../../config'; +import { DatabaseType } from '../..'; +import { WorkflowEntity } from './WorkflowEntity'; + +// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types +function getTimestampSyntax() { + const dbType = config.getEnv('database.type'); + + const map: { [key in DatabaseType]: string } = { + sqlite: "STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')", + postgresdb: 'CURRENT_TIMESTAMP(3)', + mysqldb: 'CURRENT_TIMESTAMP(3)', + mariadb: 'CURRENT_TIMESTAMP(3)', + }; + + return map[dbType]; +} + +@Entity() +export class WorkflowStatistics { + @Column() + count: number; + + @UpdateDateColumn({ + precision: 3, + default: () => getTimestampSyntax(), + onUpdate: getTimestampSyntax(), + }) + latestEvent: Date; + + @PrimaryColumn({ length: 128 }) + name: string; + + @PrimaryColumn() + @ManyToOne(() => WorkflowEntity, (workflowEntity: WorkflowEntity) => workflowEntity.statistics) + @JoinColumn({ name: 'workflow', referencedColumnName: 'id' }) + workflow: WorkflowEntity; +} diff --git a/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts new file mode 100644 index 0000000000000..8b2cb5b63948a --- /dev/null +++ b/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts @@ -0,0 +1,34 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; +import { logMigrationEnd, logMigrationStart } from '../../utils/migrationHelpers'; +import config from '../../../../config'; + +export class WorkflowStatistics1664196174000 implements MigrationInterface { + name = 'WorkflowStatistics1664196174000'; + + async up(queryRunner: QueryRunner): Promise { + logMigrationStart(this.name); + + const tablePrefix = config.getEnv('database.tablePrefix'); + + await queryRunner.query( + `CREATE TABLE \`${tablePrefix}workflow_statistics\` ( + "count" INTEGER DEFAULT 0, + "latestEvent" DATETIME NOT NULL, + "name" VARCHAR(128) NOT NULL, + "workflow" INTEGER, + PRIMARY KEY("workflow", "name"), + FOREIGN KEY("workflow") REFERENCES \`${tablePrefix}workflow_entity\`("id") + )`, + ); + + // TODO - Prepop these keys / values + + logMigrationEnd(this.name); + } + + async down(queryRunner: QueryRunner): Promise { + const tablePrefix = config.getEnv('database.tablePrefix'); + + await queryRunner.query(`DROP TABLE "${tablePrefix}workflow_entity"`); + } +} From 3c7fc283b3a7432829d43ba652175a33669f8ba7 Mon Sep 17 00:00:00 2001 From: freyamade Date: Tue, 27 Sep 2022 09:42:24 +0100 Subject: [PATCH 02/49] :sparkles: - Add statistics to DB.collections Should allow for tests to run on PR --- packages/cli/src/Db.ts | 1 + packages/cli/src/Interfaces.ts | 2 ++ packages/cli/src/databases/entities/index.ts | 2 ++ .../migrations/sqlite/1664196174000-WorkflowStatistics.ts | 2 +- 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/Db.ts b/packages/cli/src/Db.ts index 514734ae5aea9..6d24ee57c1780 100644 --- a/packages/cli/src/Db.ts +++ b/packages/cli/src/Db.ts @@ -196,6 +196,7 @@ export async function init( collections.Settings = linkRepository(entities.Settings); collections.InstalledPackages = linkRepository(entities.InstalledPackages); collections.InstalledNodes = linkRepository(entities.InstalledNodes); + collections.WorkflowStatistics = linkRepository(entities.WorkflowStatistics); isInitialized = true; diff --git a/packages/cli/src/Interfaces.ts b/packages/cli/src/Interfaces.ts index 0b92a8d1cb534..b0f3b178629bd 100644 --- a/packages/cli/src/Interfaces.ts +++ b/packages/cli/src/Interfaces.ts @@ -39,6 +39,7 @@ import type { SharedWorkflow } from './databases/entities/SharedWorkflow'; import type { TagEntity } from './databases/entities/TagEntity'; import type { User } from './databases/entities/User'; import type { WorkflowEntity } from './databases/entities/WorkflowEntity'; +import { WorkflowStatistics } from './databases/entities/WorkflowStatistics'; export interface IActivationError { time: number; @@ -75,6 +76,7 @@ export interface IDatabaseCollections { Settings: Repository; InstalledPackages: Repository; InstalledNodes: Repository; + WorkflowStatistics: Repository; } export interface IWebhookDb { diff --git a/packages/cli/src/databases/entities/index.ts b/packages/cli/src/databases/entities/index.ts index 97628e8ef805f..31bc4b4675740 100644 --- a/packages/cli/src/databases/entities/index.ts +++ b/packages/cli/src/databases/entities/index.ts @@ -12,6 +12,7 @@ import { SharedWorkflow } from './SharedWorkflow'; import { SharedCredentials } from './SharedCredentials'; import { InstalledPackages } from './InstalledPackages'; import { InstalledNodes } from './InstalledNodes'; +import { WorkflowStatistics } from './WorkflowStatistics'; export const entities = { CredentialsEntity, @@ -26,4 +27,5 @@ export const entities = { SharedCredentials, InstalledPackages, InstalledNodes, + WorkflowStatistics, }; diff --git a/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts index 8b2cb5b63948a..43abd8076b6c1 100644 --- a/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts +++ b/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts @@ -17,7 +17,7 @@ export class WorkflowStatistics1664196174000 implements MigrationInterface { "name" VARCHAR(128) NOT NULL, "workflow" INTEGER, PRIMARY KEY("workflow", "name"), - FOREIGN KEY("workflow") REFERENCES \`${tablePrefix}workflow_entity\`("id") + FOREIGN KEY("workflow") REFERENCES \`${tablePrefix}workflow_entity\`("id") ON DELETE CASCADE )`, ); From d78d9acdab11e6eb40403a5b4b3a6830d4b2d37a Mon Sep 17 00:00:00 2001 From: freyamade Date: Tue, 27 Sep 2022 10:45:35 +0100 Subject: [PATCH 03/49] :art: - actually run the statistics sqlite migration --- packages/cli/src/databases/migrations/sqlite/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/cli/src/databases/migrations/sqlite/index.ts b/packages/cli/src/databases/migrations/sqlite/index.ts index 4e14548d193d9..3b10aeb88f187 100644 --- a/packages/cli/src/databases/migrations/sqlite/index.ts +++ b/packages/cli/src/databases/migrations/sqlite/index.ts @@ -17,6 +17,7 @@ import { IntroducePinData1654089251344 } from './1654089251344-IntroducePinData' import { AddNodeIds1658930531669 } from './1658930531669-AddNodeIds'; import { AddJsonKeyPinData1659888469333 } from './1659888469333-AddJsonKeyPinData'; import { CreateCredentialsUserRole1660062385367 } from './1660062385367-CreateCredentialsUserRole'; +import { WorkflowStatistics1664196174000 } from './1664196174000-WorkflowStatistics'; const sqliteMigrations = [ InitialMigration1588102412422, @@ -38,6 +39,7 @@ const sqliteMigrations = [ AddNodeIds1658930531669, AddJsonKeyPinData1659888469333, CreateCredentialsUserRole1660062385367, + WorkflowStatistics1664196174000, ]; export { sqliteMigrations }; From 10deae812884624a0e160ccb1d36664512d5a70e Mon Sep 17 00:00:00 2001 From: freyamade Date: Tue, 27 Sep 2022 12:21:08 +0100 Subject: [PATCH 04/49] :sparkles: - add postgres migration --- .../1664196174001-WorkflowStatistics.ts | 34 +++++++++++++++++++ .../databases/migrations/postgresdb/index.ts | 2 ++ 2 files changed, 36 insertions(+) create mode 100644 packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts diff --git a/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts new file mode 100644 index 0000000000000..5e894d35ac6d8 --- /dev/null +++ b/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts @@ -0,0 +1,34 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; +import { logMigrationEnd, logMigrationStart } from '../../utils/migrationHelpers'; +import config from '../../../../config'; + +export class WorkflowStatistics1664196174001 implements MigrationInterface { + name = 'WorkflowStatistics1664196174001'; + + async up(queryRunner: QueryRunner): Promise { + logMigrationStart(this.name); + + const tablePrefix = config.getEnv('database.tablePrefix'); + + await queryRunner.query( + `CREATE TABLE \`${tablePrefix}workflow_statistics\` ( + "count" INTEGER DEFAULT 0, + "latestEvent" DATETIME NOT NULL, + "name" VARCHAR(128) NOT NULL, + "workflow" INTEGER, + PRIMARY KEY("workflow", "name"), + FOREIGN KEY("workflow") REFERENCES \`${tablePrefix}workflow_entity\`("id") ON DELETE CASCADE + )`, + ); + + // TODO - Prepop these keys / values + + logMigrationEnd(this.name); + } + + async down(queryRunner: QueryRunner): Promise { + const tablePrefix = config.getEnv('database.tablePrefix'); + + await queryRunner.query(`DROP TABLE "${tablePrefix}workflow_entity"`); + } +} diff --git a/packages/cli/src/databases/migrations/postgresdb/index.ts b/packages/cli/src/databases/migrations/postgresdb/index.ts index 491393ea9c95b..d663f03810a8c 100644 --- a/packages/cli/src/databases/migrations/postgresdb/index.ts +++ b/packages/cli/src/databases/migrations/postgresdb/index.ts @@ -18,6 +18,7 @@ import { IntroducePinData1654090467022 } from './1654090467022-IntroducePinData' import { AddNodeIds1658932090381 } from './1658932090381-AddNodeIds'; import { AddJsonKeyPinData1659902242948 } from './1659902242948-AddJsonKeyPinData'; import { CreateCredentialsUserRole1660062385367 } from './1660062385367-CreateCredentialsUserRole'; +import { WorkflowStatistics1664196174001 } from './1664196174001-WorkflowStatistics'; export const postgresMigrations = [ InitialMigration1587669153312, @@ -40,4 +41,5 @@ export const postgresMigrations = [ CreateCredentialsUserRole1660062385367, AddNodeIds1658932090381, AddJsonKeyPinData1659902242948, + WorkflowStatistics1664196174001, ]; From f98516c9a871a0e64c65b852f13cfb56d56c3d2d Mon Sep 17 00:00:00 2001 From: freyamade Date: Tue, 27 Sep 2022 12:35:59 +0100 Subject: [PATCH 05/49] :bug: - Fixed wrong table in down migrations --- .../1664196174001-WorkflowStatistics.ts | 16 ++++++++-------- .../sqlite/1664196174000-WorkflowStatistics.ts | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts index 5e894d35ac6d8..3ec8d53d7837c 100644 --- a/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts +++ b/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts @@ -11,13 +11,13 @@ export class WorkflowStatistics1664196174001 implements MigrationInterface { const tablePrefix = config.getEnv('database.tablePrefix'); await queryRunner.query( - `CREATE TABLE \`${tablePrefix}workflow_statistics\` ( - "count" INTEGER DEFAULT 0, - "latestEvent" DATETIME NOT NULL, - "name" VARCHAR(128) NOT NULL, - "workflow" INTEGER, - PRIMARY KEY("workflow", "name"), - FOREIGN KEY("workflow") REFERENCES \`${tablePrefix}workflow_entity\`("id") ON DELETE CASCADE + `CREATE TABLE ${tablePrefix}workflow_statistics ( + count INTEGER DEFAULT 0, + latestEvent DATETIME NOT NULL, + name VARCHAR(128) NOT NULL, + workflow INTEGER, + PRIMARY KEY(workflow, name), + FOREIGN KEY(workflow) REFERENCES ${tablePrefix}workflow_entity(id) ON DELETE CASCADE )`, ); @@ -29,6 +29,6 @@ export class WorkflowStatistics1664196174001 implements MigrationInterface { async down(queryRunner: QueryRunner): Promise { const tablePrefix = config.getEnv('database.tablePrefix'); - await queryRunner.query(`DROP TABLE "${tablePrefix}workflow_entity"`); + await queryRunner.query(`DROP TABLE ${tablePrefix}workflow_statistics`); } } diff --git a/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts index 43abd8076b6c1..7a7af2cec73b4 100644 --- a/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts +++ b/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts @@ -29,6 +29,6 @@ export class WorkflowStatistics1664196174000 implements MigrationInterface { async down(queryRunner: QueryRunner): Promise { const tablePrefix = config.getEnv('database.tablePrefix'); - await queryRunner.query(`DROP TABLE "${tablePrefix}workflow_entity"`); + await queryRunner.query(`DROP TABLE "${tablePrefix}workflow_statistics"`); } } From 1da308f8aa86e0214fd45b2086f36686f142587f Mon Sep 17 00:00:00 2001 From: freyamade Date: Tue, 27 Sep 2022 12:36:47 +0100 Subject: [PATCH 06/49] :sparkles: - Added mysql migration --- .../1664196174002-WorkflowStatistics.ts | 34 +++++++++++++++++++ .../src/databases/migrations/mysqldb/index.ts | 2 ++ 2 files changed, 36 insertions(+) create mode 100644 packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts diff --git a/packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts new file mode 100644 index 0000000000000..cdc4033d880eb --- /dev/null +++ b/packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts @@ -0,0 +1,34 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; +import { logMigrationEnd, logMigrationStart } from '../../utils/migrationHelpers'; +import config from '../../../../config'; + +export class WorkflowStatistics1664196174002 implements MigrationInterface { + name = 'WorkflowStatistics1664196174002'; + + async up(queryRunner: QueryRunner): Promise { + logMigrationStart(this.name); + + const tablePrefix = config.getEnv('database.tablePrefix'); + + await queryRunner.query( + `CREATE TABLE ${tablePrefix}workflow_statistics ( + count INTEGER DEFAULT 0, + latestEvent DATETIME NOT NULL, + name VARCHAR(128) NOT NULL, + workflow INTEGER, + PRIMARY KEY(workflow, name), + FOREIGN KEY(workflow) REFERENCES ${tablePrefix}workflow_entity(id) ON DELETE CASCADE + )`, + ); + + // TODO - Prepop these keys / values + + logMigrationEnd(this.name); + } + + async down(queryRunner: QueryRunner): Promise { + const tablePrefix = config.getEnv('database.tablePrefix'); + + await queryRunner.query(`DROP TABLE "${tablePrefix}workflow_statistics"`); + } +} diff --git a/packages/cli/src/databases/migrations/mysqldb/index.ts b/packages/cli/src/databases/migrations/mysqldb/index.ts index 691009b2e129a..aa5ddfb21821e 100644 --- a/packages/cli/src/databases/migrations/mysqldb/index.ts +++ b/packages/cli/src/databases/migrations/mysqldb/index.ts @@ -20,6 +20,7 @@ import { IntroducePinData1654090101303 } from './1654090101303-IntroducePinData' import { AddNodeIds1658932910559 } from './1658932910559-AddNodeIds'; import { AddJsonKeyPinData1659895550980 } from './1659895550980-AddJsonKeyPinData'; import { CreateCredentialsUserRole1660062385367 } from './1660062385367-CreateCredentialsUserRole'; +import { WorkflowStatistics1664196174002 } from './1664196174002-WorkflowStatistics'; export const mysqlMigrations = [ InitialMigration1588157391238, @@ -44,4 +45,5 @@ export const mysqlMigrations = [ AddNodeIds1658932910559, AddJsonKeyPinData1659895550980, CreateCredentialsUserRole1660062385367, + WorkflowStatistics1664196174002, ]; From 35a24a4fd5cb0553898cebcae36a33eb09782411 Mon Sep 17 00:00:00 2001 From: freyamade Date: Tue, 27 Sep 2022 12:48:20 +0100 Subject: [PATCH 07/49] :bug: - Postgres migration tested --- .../migrations/postgresdb/1664196174001-WorkflowStatistics.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts index 3ec8d53d7837c..ed431ca276b8a 100644 --- a/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts +++ b/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts @@ -13,7 +13,7 @@ export class WorkflowStatistics1664196174001 implements MigrationInterface { await queryRunner.query( `CREATE TABLE ${tablePrefix}workflow_statistics ( count INTEGER DEFAULT 0, - latestEvent DATETIME NOT NULL, + latestEvent DATE NOT NULL, name VARCHAR(128) NOT NULL, workflow INTEGER, PRIMARY KEY(workflow, name), From e5098a44d2f555d281badda198f21520517a3d93 Mon Sep 17 00:00:00 2001 From: freyamade Date: Tue, 27 Sep 2022 13:22:13 +0100 Subject: [PATCH 08/49] :sparkles: - Add dataLoaded flag to workflow table --- packages/cli/src/databases/entities/WorkflowEntity.ts | 8 +++++++- .../mysqldb/1664196174002-WorkflowStatistics.ts | 6 ++++++ .../postgresdb/1664196174001-WorkflowStatistics.ts | 6 ++++++ .../migrations/sqlite/1664196174000-WorkflowStatistics.ts | 8 ++++++++ 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/databases/entities/WorkflowEntity.ts b/packages/cli/src/databases/entities/WorkflowEntity.ts index 33c0efbfc3ecf..99a462e07352c 100644 --- a/packages/cli/src/databases/entities/WorkflowEntity.ts +++ b/packages/cli/src/databases/entities/WorkflowEntity.ts @@ -126,10 +126,16 @@ export class WorkflowEntity implements IWorkflowDb { @OneToMany(() => SharedWorkflow, (sharedWorkflow) => sharedWorkflow.workflow) shared: SharedWorkflow[]; - @OneToMany(() => WorkflowStatistics, (workflowStatistics: WorkflowStatistics) => workflowStatistics.workflow) + @OneToMany( + () => WorkflowStatistics, + (workflowStatistics: WorkflowStatistics) => workflowStatistics.workflow, + ) @JoinColumn({ referencedColumnName: 'workflow' }) statistics: WorkflowStatistics[]; + @Column() + dataLoaded: boolean; + @Column({ type: config.getEnv('database.type') === 'sqlite' ? 'text' : 'json', nullable: true, diff --git a/packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts index cdc4033d880eb..02fa9c565ce59 100644 --- a/packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts +++ b/packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts @@ -21,6 +21,11 @@ export class WorkflowStatistics1664196174002 implements MigrationInterface { )`, ); + // Add dataLoaded column to workflow table + await queryRunner.query( + `ALTER TABLE ${tablePrefix}workflow_entity ADD COLUMN dataLoaded BOOLEAN DEFAULT false`, + ); + // TODO - Prepop these keys / values logMigrationEnd(this.name); @@ -30,5 +35,6 @@ export class WorkflowStatistics1664196174002 implements MigrationInterface { const tablePrefix = config.getEnv('database.tablePrefix'); await queryRunner.query(`DROP TABLE "${tablePrefix}workflow_statistics"`); + await queryRunner.query(`ALTER TABLE ${tablePrefix}workflow_entity DROP COLUMN dataLoaded`); } } diff --git a/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts index ed431ca276b8a..90bcf08a0745e 100644 --- a/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts +++ b/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts @@ -21,6 +21,11 @@ export class WorkflowStatistics1664196174001 implements MigrationInterface { )`, ); + // Add dataLoaded column to workflow table + await queryRunner.query( + `ALTER TABLE ${tablePrefix}workflow_entity ADD COLUMN dataLoaded BOOLEAN DEFAULT false`, + ); + // TODO - Prepop these keys / values logMigrationEnd(this.name); @@ -30,5 +35,6 @@ export class WorkflowStatistics1664196174001 implements MigrationInterface { const tablePrefix = config.getEnv('database.tablePrefix'); await queryRunner.query(`DROP TABLE ${tablePrefix}workflow_statistics`); + await queryRunner.query(`ALTER TABLE ${tablePrefix}workflow_entity DROP COLUMN dataLoaded`); } } diff --git a/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts index 7a7af2cec73b4..32d862ebddcc5 100644 --- a/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts +++ b/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts @@ -21,6 +21,11 @@ export class WorkflowStatistics1664196174000 implements MigrationInterface { )`, ); + // Add dataLoaded column to workflow table + await queryRunner.query( + `ALTER TABLE \`${tablePrefix}workflow_entity\` ADD COLUMN "dataLoaded" BOOLEAN DEFAULT false`, + ); + // TODO - Prepop these keys / values logMigrationEnd(this.name); @@ -30,5 +35,8 @@ export class WorkflowStatistics1664196174000 implements MigrationInterface { const tablePrefix = config.getEnv('database.tablePrefix'); await queryRunner.query(`DROP TABLE "${tablePrefix}workflow_statistics"`); + await queryRunner.query( + `ALTER TABLE \`${tablePrefix}workflow_entity\` DROP COLUMN "dataLoaded"`, + ); } } From 613bc02a61d37f389ada98f51f6067c304ebc116 Mon Sep 17 00:00:00 2001 From: freyamade Date: Thu, 6 Oct 2022 14:52:52 +0100 Subject: [PATCH 09/49] :art: - Make latestEvent nullable --- .../migrations/mysqldb/1664196174002-WorkflowStatistics.ts | 2 +- .../migrations/postgresdb/1664196174001-WorkflowStatistics.ts | 2 +- .../migrations/sqlite/1664196174000-WorkflowStatistics.ts | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts index 02fa9c565ce59..535b9f2d74d28 100644 --- a/packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts +++ b/packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts @@ -13,7 +13,7 @@ export class WorkflowStatistics1664196174002 implements MigrationInterface { await queryRunner.query( `CREATE TABLE ${tablePrefix}workflow_statistics ( count INTEGER DEFAULT 0, - latestEvent DATETIME NOT NULL, + latestEvent DATETIME, name VARCHAR(128) NOT NULL, workflow INTEGER, PRIMARY KEY(workflow, name), diff --git a/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts index 90bcf08a0745e..58289bdffcefa 100644 --- a/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts +++ b/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts @@ -13,7 +13,7 @@ export class WorkflowStatistics1664196174001 implements MigrationInterface { await queryRunner.query( `CREATE TABLE ${tablePrefix}workflow_statistics ( count INTEGER DEFAULT 0, - latestEvent DATE NOT NULL, + latestEvent DATE, name VARCHAR(128) NOT NULL, workflow INTEGER, PRIMARY KEY(workflow, name), diff --git a/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts index 32d862ebddcc5..ebfb1f130fd37 100644 --- a/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts +++ b/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts @@ -13,7 +13,7 @@ export class WorkflowStatistics1664196174000 implements MigrationInterface { await queryRunner.query( `CREATE TABLE \`${tablePrefix}workflow_statistics\` ( "count" INTEGER DEFAULT 0, - "latestEvent" DATETIME NOT NULL, + "latestEvent" DATETIME, "name" VARCHAR(128) NOT NULL, "workflow" INTEGER, PRIMARY KEY("workflow", "name"), @@ -26,7 +26,7 @@ export class WorkflowStatistics1664196174000 implements MigrationInterface { `ALTER TABLE \`${tablePrefix}workflow_entity\` ADD COLUMN "dataLoaded" BOOLEAN DEFAULT false`, ); - // TODO - Prepop these keys / values + // Need to logMigrationEnd(this.name); } From 563ff7739e92e989657b56e399d548ffcee1f6a0 Mon Sep 17 00:00:00 2001 From: freyamade Date: Thu, 6 Oct 2022 15:38:53 +0100 Subject: [PATCH 10/49] :sparkles: - Load current execution counts during migration --- .../1664196174002-WorkflowStatistics.ts | 39 +++++++++++++++++-- .../1664196174001-WorkflowStatistics.ts | 39 +++++++++++++++++-- .../1664196174000-WorkflowStatistics.ts | 39 +++++++++++++++++-- 3 files changed, 105 insertions(+), 12 deletions(-) diff --git a/packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts index 535b9f2d74d28..71477576234e6 100644 --- a/packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts +++ b/packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts @@ -15,9 +15,9 @@ export class WorkflowStatistics1664196174002 implements MigrationInterface { count INTEGER DEFAULT 0, latestEvent DATETIME, name VARCHAR(128) NOT NULL, - workflow INTEGER, - PRIMARY KEY(workflow, name), - FOREIGN KEY(workflow) REFERENCES ${tablePrefix}workflow_entity(id) ON DELETE CASCADE + workflowId INTEGER, + PRIMARY KEY(workflowId, name), + FOREIGN KEY(workflowId) REFERENCES ${tablePrefix}workflow_entity(id) ON DELETE CASCADE )`, ); @@ -26,7 +26,38 @@ export class WorkflowStatistics1664196174002 implements MigrationInterface { `ALTER TABLE ${tablePrefix}workflow_entity ADD COLUMN dataLoaded BOOLEAN DEFAULT false`, ); - // TODO - Prepop these keys / values + // Need to loop through all workflows, and then make the insertions + const workflows = await queryRunner.query(`SELECT id FROM ${tablePrefix}workflow_entity`) + workflows.forEach(async (workflow: { id: number }) => { + // Run a query for each workflow to count executions + await queryRunner.query( + `INSERT INTO ${tablePrefix}workflow_statistics (workflowId, name, count, latestEvent) VALUES + ( + ${workflow.id}, + 'production_success', + (SELECT COUNT(*) FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 1 AND mode != 'manual'), + (SELECT startedAt FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 1 AND mode != 'manual' ORDER BY startedAt DESC) + ), + ( + ${workflow.id}, + 'production_error', + (SELECT COUNT(*) FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 0 AND mode != 'manual'), + (SELECT startedAt FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 0 AND mode != 'manual' ORDER BY startedAt DESC) + ), + ( + ${workflow.id}, + 'manual_success', + (SELECT COUNT(*) FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 1 AND mode == 'manual'), + (SELECT startedAt FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 1 AND mode == 'manual' ORDER BY startedAt DESC) + ), + ( + ${workflow.id}, + 'manual_error', + (SELECT COUNT(*) FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 0 AND mode == 'manual'), + (SELECT startedAt FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 0 AND mode == 'manual' ORDER BY startedAt DESC) + );`, + ) + }); logMigrationEnd(this.name); } diff --git a/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts index 58289bdffcefa..cce3b88a1e69d 100644 --- a/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts +++ b/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts @@ -15,9 +15,9 @@ export class WorkflowStatistics1664196174001 implements MigrationInterface { count INTEGER DEFAULT 0, latestEvent DATE, name VARCHAR(128) NOT NULL, - workflow INTEGER, - PRIMARY KEY(workflow, name), - FOREIGN KEY(workflow) REFERENCES ${tablePrefix}workflow_entity(id) ON DELETE CASCADE + workflowId INTEGER, + PRIMARY KEY(workflowId, name), + FOREIGN KEY(workflowId) REFERENCES ${tablePrefix}workflow_entity(id) ON DELETE CASCADE )`, ); @@ -26,7 +26,38 @@ export class WorkflowStatistics1664196174001 implements MigrationInterface { `ALTER TABLE ${tablePrefix}workflow_entity ADD COLUMN dataLoaded BOOLEAN DEFAULT false`, ); - // TODO - Prepop these keys / values + // Need to loop through all workflows, and then make the insertions + const workflows = await queryRunner.query(`SELECT id FROM ${tablePrefix}workflow_entity`) + workflows.forEach(async (workflow: { id: number }) => { + // Run a query for each workflow to count executions + await queryRunner.query( + `INSERT INTO ${tablePrefix}workflow_statistics (workflowId, name, count, latestEvent) VALUES + ( + ${workflow.id}, + 'production_success', + (SELECT COUNT(*) FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 1 AND mode != 'manual'), + (SELECT startedAt FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 1 AND mode != 'manual' ORDER BY startedAt DESC) + ), + ( + ${workflow.id}, + 'production_error', + (SELECT COUNT(*) FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 0 AND mode != 'manual'), + (SELECT startedAt FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 0 AND mode != 'manual' ORDER BY startedAt DESC) + ), + ( + ${workflow.id}, + 'manual_success', + (SELECT COUNT(*) FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 1 AND mode == 'manual'), + (SELECT startedAt FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 1 AND mode == 'manual' ORDER BY startedAt DESC) + ), + ( + ${workflow.id}, + 'manual_error', + (SELECT COUNT(*) FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 0 AND mode == 'manual'), + (SELECT startedAt FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 0 AND mode == 'manual' ORDER BY startedAt DESC) + );`, + ) + }); logMigrationEnd(this.name); } diff --git a/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts index ebfb1f130fd37..0e7921b82a644 100644 --- a/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts +++ b/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts @@ -15,9 +15,9 @@ export class WorkflowStatistics1664196174000 implements MigrationInterface { "count" INTEGER DEFAULT 0, "latestEvent" DATETIME, "name" VARCHAR(128) NOT NULL, - "workflow" INTEGER, - PRIMARY KEY("workflow", "name"), - FOREIGN KEY("workflow") REFERENCES \`${tablePrefix}workflow_entity\`("id") ON DELETE CASCADE + "workflowId" INTEGER, + PRIMARY KEY("workflowId", "name"), + FOREIGN KEY("workflowId") REFERENCES \`${tablePrefix}workflow_entity\`("id") ON DELETE CASCADE )`, ); @@ -26,7 +26,38 @@ export class WorkflowStatistics1664196174000 implements MigrationInterface { `ALTER TABLE \`${tablePrefix}workflow_entity\` ADD COLUMN "dataLoaded" BOOLEAN DEFAULT false`, ); - // Need to + // Need to loop through all workflows, and then make the insertions + const workflows = await queryRunner.query(`SELECT id FROM \`${tablePrefix}workflow_entity\``) + workflows.forEach(async (workflow: { id: number }) => { + // Run a query for each workflow to count executions + await queryRunner.query( + `INSERT INTO \`${tablePrefix}workflow_statistics\` (workflowId, name, count, latestEvent) VALUES + ( + ${workflow.id}, + 'production_success', + (SELECT COUNT(*) FROM \`${tablePrefix}execution_entity\` WHERE workflowId = ${workflow.id} AND finished = 1 AND mode != 'manual'), + (SELECT startedAt FROM \`${tablePrefix}execution_entity\` WHERE workflowId = ${workflow.id} AND finished = 1 AND mode != 'manual' ORDER BY startedAt DESC) + ), + ( + ${workflow.id}, + 'production_error', + (SELECT COUNT(*) FROM \`${tablePrefix}execution_entity\` WHERE workflowId = ${workflow.id} AND finished = 0 AND mode != 'manual'), + (SELECT startedAt FROM \`${tablePrefix}execution_entity\` WHERE workflowId = ${workflow.id} AND finished = 0 AND mode != 'manual' ORDER BY startedAt DESC) + ), + ( + ${workflow.id}, + 'manual_success', + (SELECT COUNT(*) FROM \`${tablePrefix}execution_entity\` WHERE workflowId = ${workflow.id} AND finished = 1 AND mode == 'manual'), + (SELECT startedAt FROM \`${tablePrefix}execution_entity\` WHERE workflowId = ${workflow.id} AND finished = 1 AND mode == 'manual' ORDER BY startedAt DESC) + ), + ( + ${workflow.id}, + 'manual_error', + (SELECT COUNT(*) FROM \`${tablePrefix}execution_entity\` WHERE workflowId = ${workflow.id} AND finished = 0 AND mode == 'manual'), + (SELECT startedAt FROM \`${tablePrefix}execution_entity\` WHERE workflowId = ${workflow.id} AND finished = 0 AND mode == 'manual' ORDER BY startedAt DESC) + );`, + ) + }); logMigrationEnd(this.name); } From 0ff421eb50935c9456e076eb3e67d86d00fa30b4 Mon Sep 17 00:00:00 2001 From: freyamade Date: Mon, 10 Oct 2022 10:20:56 +0100 Subject: [PATCH 11/49] :bug: - Fixed issue with postgres migrations --- .../databases/entities/WorkflowStatistics.ts | 12 ++++--- .../1664196174001-WorkflowStatistics.ts | 31 ++++++++++++------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/packages/cli/src/databases/entities/WorkflowStatistics.ts b/packages/cli/src/databases/entities/WorkflowStatistics.ts index 64e887f2fb1b8..110a9a1390922 100644 --- a/packages/cli/src/databases/entities/WorkflowStatistics.ts +++ b/packages/cli/src/databases/entities/WorkflowStatistics.ts @@ -1,5 +1,5 @@ /* eslint-disable import/no-cycle */ -import { Column, Entity, JoinColumn, ManyToOne, PrimaryColumn, UpdateDateColumn } from 'typeorm'; +import { Column, Entity, RelationId, ManyToOne, PrimaryColumn, UpdateDateColumn } from 'typeorm'; import * as config from '../../../config'; import { DatabaseType } from '../..'; import { WorkflowEntity } from './WorkflowEntity'; @@ -33,8 +33,12 @@ export class WorkflowStatistics { @PrimaryColumn({ length: 128 }) name: string; - @PrimaryColumn() - @ManyToOne(() => WorkflowEntity, (workflowEntity: WorkflowEntity) => workflowEntity.statistics) - @JoinColumn({ name: 'workflow', referencedColumnName: 'id' }) + @ManyToOne(() => WorkflowEntity, (workflow) => workflow.shared, { + primary: true, + onDelete: 'CASCADE', + }) workflow: WorkflowEntity; + + @RelationId((workflowStatistics: WorkflowStatistics) => workflowStatistics.workflow) + workflowId: number; } diff --git a/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts index cce3b88a1e69d..4a0166da65c9a 100644 --- a/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts +++ b/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts @@ -8,26 +8,31 @@ export class WorkflowStatistics1664196174001 implements MigrationInterface { async up(queryRunner: QueryRunner): Promise { logMigrationStart(this.name); - const tablePrefix = config.getEnv('database.tablePrefix'); + let tablePrefix = config.getEnv('database.tablePrefix'); + const schema = config.getEnv('database.postgresdb.schema'); + if (schema) { + tablePrefix = schema + '.' + tablePrefix; + } + // Create statistics table await queryRunner.query( `CREATE TABLE ${tablePrefix}workflow_statistics ( - count INTEGER DEFAULT 0, - latestEvent DATE, - name VARCHAR(128) NOT NULL, - workflowId INTEGER, - PRIMARY KEY(workflowId, name), - FOREIGN KEY(workflowId) REFERENCES ${tablePrefix}workflow_entity(id) ON DELETE CASCADE + "count" INTEGER DEFAULT 0, + "latestEvent" DATE, + "name" VARCHAR(128) NOT NULL, + "workflowId" INTEGER, + PRIMARY KEY("workflowId", "name"), + FOREIGN KEY("workflowId") REFERENCES ${tablePrefix}workflow_entity("id") ON DELETE CASCADE )`, ); // Add dataLoaded column to workflow table await queryRunner.query( - `ALTER TABLE ${tablePrefix}workflow_entity ADD COLUMN dataLoaded BOOLEAN DEFAULT false`, + `ALTER TABLE ${tablePrefix}workflow_entity ADD COLUMN "dataLoaded" BOOLEAN DEFAULT false;`, ); // Need to loop through all workflows, and then make the insertions - const workflows = await queryRunner.query(`SELECT id FROM ${tablePrefix}workflow_entity`) + const workflows = await queryRunner.query(`SELECT id FROM ${tablePrefix}workflow_entity`); workflows.forEach(async (workflow: { id: number }) => { // Run a query for each workflow to count executions await queryRunner.query( @@ -56,14 +61,18 @@ export class WorkflowStatistics1664196174001 implements MigrationInterface { (SELECT COUNT(*) FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 0 AND mode == 'manual'), (SELECT startedAt FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 0 AND mode == 'manual' ORDER BY startedAt DESC) );`, - ) + ); }); logMigrationEnd(this.name); } async down(queryRunner: QueryRunner): Promise { - const tablePrefix = config.getEnv('database.tablePrefix'); + let tablePrefix = config.getEnv('database.tablePrefix'); + const schema = config.getEnv('database.postgresdb.schema'); + if (schema) { + tablePrefix = schema + '.' + tablePrefix; + } await queryRunner.query(`DROP TABLE ${tablePrefix}workflow_statistics`); await queryRunner.query(`ALTER TABLE ${tablePrefix}workflow_entity DROP COLUMN dataLoaded`); From b1ad9d024e5d2522d0bc86ddc54a3aab44f1635e Mon Sep 17 00:00:00 2001 From: freyamade Date: Mon, 10 Oct 2022 11:53:19 +0100 Subject: [PATCH 12/49] :art: - Better insertion queries for initial stats --- .../1664196174002-WorkflowStatistics.ts | 59 +++++++++---------- .../1664196174001-WorkflowStatistics.ts | 59 +++++++++---------- .../1664196174000-WorkflowStatistics.ts | 59 +++++++++---------- 3 files changed, 81 insertions(+), 96 deletions(-) diff --git a/packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts index 71477576234e6..9eef1f7fbae87 100644 --- a/packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts +++ b/packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts @@ -26,38 +26,33 @@ export class WorkflowStatistics1664196174002 implements MigrationInterface { `ALTER TABLE ${tablePrefix}workflow_entity ADD COLUMN dataLoaded BOOLEAN DEFAULT false`, ); - // Need to loop through all workflows, and then make the insertions - const workflows = await queryRunner.query(`SELECT id FROM ${tablePrefix}workflow_entity`) - workflows.forEach(async (workflow: { id: number }) => { - // Run a query for each workflow to count executions - await queryRunner.query( - `INSERT INTO ${tablePrefix}workflow_statistics (workflowId, name, count, latestEvent) VALUES - ( - ${workflow.id}, - 'production_success', - (SELECT COUNT(*) FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 1 AND mode != 'manual'), - (SELECT startedAt FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 1 AND mode != 'manual' ORDER BY startedAt DESC) - ), - ( - ${workflow.id}, - 'production_error', - (SELECT COUNT(*) FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 0 AND mode != 'manual'), - (SELECT startedAt FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 0 AND mode != 'manual' ORDER BY startedAt DESC) - ), - ( - ${workflow.id}, - 'manual_success', - (SELECT COUNT(*) FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 1 AND mode == 'manual'), - (SELECT startedAt FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 1 AND mode == 'manual' ORDER BY startedAt DESC) - ), - ( - ${workflow.id}, - 'manual_error', - (SELECT COUNT(*) FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 0 AND mode == 'manual'), - (SELECT startedAt FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 0 AND mode == 'manual' ORDER BY startedAt DESC) - );`, - ) - }); + // Fetch data from executions table to populate statistics table + await queryRunner.query( + `INSERT INTO ${tablePrefix}workflow_statistics (count, latestEvent, name, workflowId) + SELECT + COUNT("id") as "count", + COALESCE(MAX("stoppedAt"), MAX("startedAt")) as "latestEvent", + CASE WHEN "finished" = true THEN 'production_success' ELSE 'production_error' END as "name", + CAST ("workflowId" AS UNSIGNED) AS "workflowId" + FROM ${tablePrefix}execution_entity + WHERE "workflowId" IS NOT NULL + AND mode != 'manual' + GROUP BY "workflowId", "finished" + ORDER BY "workflowId";`, + ); + await queryRunner.query( + `INSERT INTO ${tablePrefix}workflow_statistics (count, latestEvent, name, workflowId) + SELECT + COUNT("id") as "count", + COALESCE(MAX("stoppedAt"), MAX("startedAt")) as "latestEvent", + CASE WHEN "finished" = true THEN 'production_success' ELSE 'production_error' END as "name", + CAST ("workflowId" AS UNSIGNED) AS "workflowId" + FROM ${tablePrefix}execution_entity + WHERE "workflowId" IS NOT NULL + AND mode = 'manual' + GROUP BY "workflowId", "finished" + ORDER BY "workflowId";`, + ); logMigrationEnd(this.name); } diff --git a/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts index 4a0166da65c9a..a069c1e1e3169 100644 --- a/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts +++ b/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts @@ -31,38 +31,33 @@ export class WorkflowStatistics1664196174001 implements MigrationInterface { `ALTER TABLE ${tablePrefix}workflow_entity ADD COLUMN "dataLoaded" BOOLEAN DEFAULT false;`, ); - // Need to loop through all workflows, and then make the insertions - const workflows = await queryRunner.query(`SELECT id FROM ${tablePrefix}workflow_entity`); - workflows.forEach(async (workflow: { id: number }) => { - // Run a query for each workflow to count executions - await queryRunner.query( - `INSERT INTO ${tablePrefix}workflow_statistics (workflowId, name, count, latestEvent) VALUES - ( - ${workflow.id}, - 'production_success', - (SELECT COUNT(*) FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 1 AND mode != 'manual'), - (SELECT startedAt FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 1 AND mode != 'manual' ORDER BY startedAt DESC) - ), - ( - ${workflow.id}, - 'production_error', - (SELECT COUNT(*) FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 0 AND mode != 'manual'), - (SELECT startedAt FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 0 AND mode != 'manual' ORDER BY startedAt DESC) - ), - ( - ${workflow.id}, - 'manual_success', - (SELECT COUNT(*) FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 1 AND mode == 'manual'), - (SELECT startedAt FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 1 AND mode == 'manual' ORDER BY startedAt DESC) - ), - ( - ${workflow.id}, - 'manual_error', - (SELECT COUNT(*) FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 0 AND mode == 'manual'), - (SELECT startedAt FROM ${tablePrefix}execution_entity WHERE workflowId = ${workflow.id} AND finished = 0 AND mode == 'manual' ORDER BY startedAt DESC) - );`, - ); - }); + // Fetch data from executions table to populate statistics table + await queryRunner.query( + `INSERT INTO ${tablePrefix}workflow_statistics ("count", "latestEvent", "name", "workflowId") + SELECT + COUNT("id") as "count", + COALESCE(MAX("stoppedAt"), MAX("startedAt")) as "latestEvent", + CASE WHEN "finished" = true THEN 'production_success' ELSE 'production_error' END as "name", + CAST ("workflowId" AS INTEGER) AS "workflowId" + FROM ${tablePrefix}execution_entity + WHERE "workflowId" IS NOT NULL + AND mode != 'manual' + GROUP BY "workflowId", "finished" + ORDER BY "workflowId";`, + ); + await queryRunner.query( + `INSERT INTO ${tablePrefix}workflow_statistics ("count", "latestEvent", "name", "workflowId") + SELECT + COUNT("id") as "count", + COALESCE(MAX("stoppedAt"), MAX("startedAt")) as "latestEvent", + CASE WHEN "finished" = true THEN 'production_success' ELSE 'production_error' END as "name", + CAST ("workflowId" AS INTEGER) AS "workflowId" + FROM ${tablePrefix}execution_entity + WHERE "workflowId" IS NOT NULL + AND mode == 'manual' + GROUP BY "workflowId", "finished" + ORDER BY "workflowId";`, + ); logMigrationEnd(this.name); } diff --git a/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts index 0e7921b82a644..6a7ae215aff3f 100644 --- a/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts +++ b/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts @@ -26,38 +26,33 @@ export class WorkflowStatistics1664196174000 implements MigrationInterface { `ALTER TABLE \`${tablePrefix}workflow_entity\` ADD COLUMN "dataLoaded" BOOLEAN DEFAULT false`, ); - // Need to loop through all workflows, and then make the insertions - const workflows = await queryRunner.query(`SELECT id FROM \`${tablePrefix}workflow_entity\``) - workflows.forEach(async (workflow: { id: number }) => { - // Run a query for each workflow to count executions - await queryRunner.query( - `INSERT INTO \`${tablePrefix}workflow_statistics\` (workflowId, name, count, latestEvent) VALUES - ( - ${workflow.id}, - 'production_success', - (SELECT COUNT(*) FROM \`${tablePrefix}execution_entity\` WHERE workflowId = ${workflow.id} AND finished = 1 AND mode != 'manual'), - (SELECT startedAt FROM \`${tablePrefix}execution_entity\` WHERE workflowId = ${workflow.id} AND finished = 1 AND mode != 'manual' ORDER BY startedAt DESC) - ), - ( - ${workflow.id}, - 'production_error', - (SELECT COUNT(*) FROM \`${tablePrefix}execution_entity\` WHERE workflowId = ${workflow.id} AND finished = 0 AND mode != 'manual'), - (SELECT startedAt FROM \`${tablePrefix}execution_entity\` WHERE workflowId = ${workflow.id} AND finished = 0 AND mode != 'manual' ORDER BY startedAt DESC) - ), - ( - ${workflow.id}, - 'manual_success', - (SELECT COUNT(*) FROM \`${tablePrefix}execution_entity\` WHERE workflowId = ${workflow.id} AND finished = 1 AND mode == 'manual'), - (SELECT startedAt FROM \`${tablePrefix}execution_entity\` WHERE workflowId = ${workflow.id} AND finished = 1 AND mode == 'manual' ORDER BY startedAt DESC) - ), - ( - ${workflow.id}, - 'manual_error', - (SELECT COUNT(*) FROM \`${tablePrefix}execution_entity\` WHERE workflowId = ${workflow.id} AND finished = 0 AND mode == 'manual'), - (SELECT startedAt FROM \`${tablePrefix}execution_entity\` WHERE workflowId = ${workflow.id} AND finished = 0 AND mode == 'manual' ORDER BY startedAt DESC) - );`, - ) - }); + // Fetch data from executions table to populate statistics table + await queryRunner.query( + `INSERT INTO \`${tablePrefix}workflow_statistics\` ("count", "latestEvent", "name", "workflowId") + SELECT + COUNT("id") as "count", + IFNULL(MAX("stoppedAt"), MAX("startedAt")) as "latestEvent", + IIF("finished" == 1, 'production_success', 'production_error') as "name", + "workflowId" + FROM \`${tablePrefix}execution_entity\` + WHERE "workflowId" IS NOT NULL + AND mode != 'manual' + GROUP BY "workflowId", "finished" + ORDER BY "workflowId";`, + ); + await queryRunner.query( + `INSERT INTO \`${tablePrefix}workflow_statistics\` ("count", "latestEvent", "name", "workflowId") + SELECT + COUNT("id") as "count", + IFNULL(MAX("stoppedAt"), MAX("startedAt")) as "latestEvent", + IIF("finished" == 1, 'production_success', 'production_error') as "name", + "workflowId" + FROM \`${tablePrefix}execution_entity\` + WHERE "workflowId" IS NOT NULL + AND mode == 'manual' + GROUP BY "workflowId", "finished" + ORDER BY "workflowId";`, + ); logMigrationEnd(this.name); } From 268862f1e14639b977cc5724ae8ec3953481218e Mon Sep 17 00:00:00 2001 From: freyamade Date: Mon, 10 Oct 2022 14:52:15 +0100 Subject: [PATCH 13/49] :art: - Remove initial counts of executions from migration --- .../1664196174002-WorkflowStatistics.ts | 28 ------------------- .../1664196174001-WorkflowStatistics.ts | 28 ------------------- .../1664196174000-WorkflowStatistics.ts | 28 ------------------- 3 files changed, 84 deletions(-) diff --git a/packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts index 9eef1f7fbae87..262df435c4d84 100644 --- a/packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts +++ b/packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts @@ -26,34 +26,6 @@ export class WorkflowStatistics1664196174002 implements MigrationInterface { `ALTER TABLE ${tablePrefix}workflow_entity ADD COLUMN dataLoaded BOOLEAN DEFAULT false`, ); - // Fetch data from executions table to populate statistics table - await queryRunner.query( - `INSERT INTO ${tablePrefix}workflow_statistics (count, latestEvent, name, workflowId) - SELECT - COUNT("id") as "count", - COALESCE(MAX("stoppedAt"), MAX("startedAt")) as "latestEvent", - CASE WHEN "finished" = true THEN 'production_success' ELSE 'production_error' END as "name", - CAST ("workflowId" AS UNSIGNED) AS "workflowId" - FROM ${tablePrefix}execution_entity - WHERE "workflowId" IS NOT NULL - AND mode != 'manual' - GROUP BY "workflowId", "finished" - ORDER BY "workflowId";`, - ); - await queryRunner.query( - `INSERT INTO ${tablePrefix}workflow_statistics (count, latestEvent, name, workflowId) - SELECT - COUNT("id") as "count", - COALESCE(MAX("stoppedAt"), MAX("startedAt")) as "latestEvent", - CASE WHEN "finished" = true THEN 'production_success' ELSE 'production_error' END as "name", - CAST ("workflowId" AS UNSIGNED) AS "workflowId" - FROM ${tablePrefix}execution_entity - WHERE "workflowId" IS NOT NULL - AND mode = 'manual' - GROUP BY "workflowId", "finished" - ORDER BY "workflowId";`, - ); - logMigrationEnd(this.name); } diff --git a/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts index a069c1e1e3169..73f0f5c6298f4 100644 --- a/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts +++ b/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts @@ -31,34 +31,6 @@ export class WorkflowStatistics1664196174001 implements MigrationInterface { `ALTER TABLE ${tablePrefix}workflow_entity ADD COLUMN "dataLoaded" BOOLEAN DEFAULT false;`, ); - // Fetch data from executions table to populate statistics table - await queryRunner.query( - `INSERT INTO ${tablePrefix}workflow_statistics ("count", "latestEvent", "name", "workflowId") - SELECT - COUNT("id") as "count", - COALESCE(MAX("stoppedAt"), MAX("startedAt")) as "latestEvent", - CASE WHEN "finished" = true THEN 'production_success' ELSE 'production_error' END as "name", - CAST ("workflowId" AS INTEGER) AS "workflowId" - FROM ${tablePrefix}execution_entity - WHERE "workflowId" IS NOT NULL - AND mode != 'manual' - GROUP BY "workflowId", "finished" - ORDER BY "workflowId";`, - ); - await queryRunner.query( - `INSERT INTO ${tablePrefix}workflow_statistics ("count", "latestEvent", "name", "workflowId") - SELECT - COUNT("id") as "count", - COALESCE(MAX("stoppedAt"), MAX("startedAt")) as "latestEvent", - CASE WHEN "finished" = true THEN 'production_success' ELSE 'production_error' END as "name", - CAST ("workflowId" AS INTEGER) AS "workflowId" - FROM ${tablePrefix}execution_entity - WHERE "workflowId" IS NOT NULL - AND mode == 'manual' - GROUP BY "workflowId", "finished" - ORDER BY "workflowId";`, - ); - logMigrationEnd(this.name); } diff --git a/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts index 6a7ae215aff3f..8da8e0a55b8a8 100644 --- a/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts +++ b/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts @@ -26,34 +26,6 @@ export class WorkflowStatistics1664196174000 implements MigrationInterface { `ALTER TABLE \`${tablePrefix}workflow_entity\` ADD COLUMN "dataLoaded" BOOLEAN DEFAULT false`, ); - // Fetch data from executions table to populate statistics table - await queryRunner.query( - `INSERT INTO \`${tablePrefix}workflow_statistics\` ("count", "latestEvent", "name", "workflowId") - SELECT - COUNT("id") as "count", - IFNULL(MAX("stoppedAt"), MAX("startedAt")) as "latestEvent", - IIF("finished" == 1, 'production_success', 'production_error') as "name", - "workflowId" - FROM \`${tablePrefix}execution_entity\` - WHERE "workflowId" IS NOT NULL - AND mode != 'manual' - GROUP BY "workflowId", "finished" - ORDER BY "workflowId";`, - ); - await queryRunner.query( - `INSERT INTO \`${tablePrefix}workflow_statistics\` ("count", "latestEvent", "name", "workflowId") - SELECT - COUNT("id") as "count", - IFNULL(MAX("stoppedAt"), MAX("startedAt")) as "latestEvent", - IIF("finished" == 1, 'production_success', 'production_error') as "name", - "workflowId" - FROM \`${tablePrefix}execution_entity\` - WHERE "workflowId" IS NOT NULL - AND mode == 'manual' - GROUP BY "workflowId", "finished" - ORDER BY "workflowId";`, - ); - logMigrationEnd(this.name); } From 4f401ff33af7f51eef90858865e72969a055afdb Mon Sep 17 00:00:00 2001 From: freyamade Date: Thu, 13 Oct 2022 12:20:37 +0100 Subject: [PATCH 14/49] :art: - Implemented change comments --- .../cli/src/databases/entities/WorkflowStatistics.ts | 9 ++++++++- .../postgresdb/1664196174001-WorkflowStatistics.ts | 9 ++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/cli/src/databases/entities/WorkflowStatistics.ts b/packages/cli/src/databases/entities/WorkflowStatistics.ts index 110a9a1390922..aaa3c58c3acbc 100644 --- a/packages/cli/src/databases/entities/WorkflowStatistics.ts +++ b/packages/cli/src/databases/entities/WorkflowStatistics.ts @@ -18,6 +18,13 @@ function getTimestampSyntax() { return map[dbType]; } +export enum StatisticsNames { + productionSuccess = 'production_success', + productionError = 'production_error', + manualSuccess = 'manual_success', + manualError = 'manual_error', +} + @Entity() export class WorkflowStatistics { @Column() @@ -31,7 +38,7 @@ export class WorkflowStatistics { latestEvent: Date; @PrimaryColumn({ length: 128 }) - name: string; + name: StatisticsNames; @ManyToOne(() => WorkflowEntity, (workflow) => workflow.shared, { primary: true, diff --git a/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts index 73f0f5c6298f4..86d09605aa696 100644 --- a/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts +++ b/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts @@ -1,5 +1,5 @@ import { MigrationInterface, QueryRunner } from 'typeorm'; -import { logMigrationEnd, logMigrationStart } from '../../utils/migrationHelpers'; +import { getTablePrefix, logMigrationEnd, logMigrationStart } from '../../utils/migrationHelpers'; import config from '../../../../config'; export class WorkflowStatistics1664196174001 implements MigrationInterface { @@ -7,12 +7,7 @@ export class WorkflowStatistics1664196174001 implements MigrationInterface { async up(queryRunner: QueryRunner): Promise { logMigrationStart(this.name); - - let tablePrefix = config.getEnv('database.tablePrefix'); - const schema = config.getEnv('database.postgresdb.schema'); - if (schema) { - tablePrefix = schema + '.' + tablePrefix; - } + const tablePrefix = getTablePrefix(); // Create statistics table await queryRunner.query( From c1c0aa636e743d3cab595bee35e3cd209e641d2b Mon Sep 17 00:00:00 2001 From: freyamade Date: Thu, 13 Oct 2022 16:31:45 +0100 Subject: [PATCH 15/49] :sparkles: - Add new eventEmitter to core --- packages/cli/src/index.ts | 2 ++ packages/core/src/EventEmitter.ts | 5 +++++ packages/core/src/index.ts | 3 ++- 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 packages/core/src/EventEmitter.ts diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index c98040e648738..0d6f08ab2af99 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -26,6 +26,8 @@ import * as WebhookServer from './WebhookServer'; import * as WorkflowExecuteAdditionalData from './WorkflowExecuteAdditionalData'; import * as WorkflowHelpers from './WorkflowHelpers'; +import './events'; + export { ActiveExecutions, ActiveWorkflowRunner, diff --git a/packages/core/src/EventEmitter.ts b/packages/core/src/EventEmitter.ts new file mode 100644 index 0000000000000..b04cfe9b77890 --- /dev/null +++ b/packages/core/src/EventEmitter.ts @@ -0,0 +1,5 @@ +import EventEmitter from 'events'; + +class N8NEventEmitter extends EventEmitter {} + +export const eventEmitter = new N8NEventEmitter(); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 6a327177ccc5c..6e62c62f428b5 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,4 +1,5 @@ /* eslint-disable import/no-cycle */ +import { eventEmitter } from './EventEmitter'; import * as NodeExecuteFunctions from './NodeExecuteFunctions'; import * as UserSettings from './UserSettings'; @@ -18,4 +19,4 @@ export * from './LoadNodeParameterOptions'; export * from './LoadNodeListSearch'; export * from './NodeExecuteFunctions'; export * from './WorkflowExecute'; -export { NodeExecuteFunctions, UserSettings }; +export { eventEmitter, NodeExecuteFunctions, UserSettings }; From e13fe4258a7a72f33acbac45c4b6055d316d737f Mon Sep 17 00:00:00 2001 From: freyamade Date: Thu, 13 Oct 2022 16:32:47 +0100 Subject: [PATCH 16/49] :sparkles: - Add workflow finished event handler --- .../databases/entities/WorkflowStatistics.ts | 1 + packages/cli/src/events/WorkflowStatistics.ts | 41 +++++++++++++++++++ packages/cli/src/events/index.ts | 1 + 3 files changed, 43 insertions(+) create mode 100644 packages/cli/src/events/WorkflowStatistics.ts create mode 100644 packages/cli/src/events/index.ts diff --git a/packages/cli/src/databases/entities/WorkflowStatistics.ts b/packages/cli/src/databases/entities/WorkflowStatistics.ts index aaa3c58c3acbc..8aca130b76ded 100644 --- a/packages/cli/src/databases/entities/WorkflowStatistics.ts +++ b/packages/cli/src/databases/entities/WorkflowStatistics.ts @@ -47,5 +47,6 @@ export class WorkflowStatistics { workflow: WorkflowEntity; @RelationId((workflowStatistics: WorkflowStatistics) => workflowStatistics.workflow) + @PrimaryColumn() workflowId: number; } diff --git a/packages/cli/src/events/WorkflowStatistics.ts b/packages/cli/src/events/WorkflowStatistics.ts new file mode 100644 index 0000000000000..781d5e8826c3d --- /dev/null +++ b/packages/cli/src/events/WorkflowStatistics.ts @@ -0,0 +1,41 @@ +import { eventEmitter } from 'n8n-core'; +import { IRun, IWorkflowBase } from 'n8n-workflow'; +import { Db } from '..'; +import { StatisticsNames } from '../databases/entities/WorkflowStatistics'; + +eventEmitter.on('saveWorkflowStatistics', async (workflowData: IWorkflowBase, runData: IRun) => { + // Determine the name of the statistic + const finished = runData.finished ? runData.finished : false; + const manual = runData.mode === 'manual'; + let name: StatisticsNames; + + if (finished) { + if (manual) name = StatisticsNames.manualSuccess; + else name = StatisticsNames.productionSuccess; + } else { + if (manual) name = StatisticsNames.manualError; + else name = StatisticsNames.productionError; + } + + // Get the workflow id + let workflowId: number; + try { + workflowId = parseInt(workflowData.id as string, 10); + } catch (error) { + console.error(`Error ${error as string} when casting workflow ID to a number`); + return; + } + + // Try insertion and if it fails due to key conflicts then update the existing entry instead + try { + console.log(await Db.collections.WorkflowStatistics.insert({ count: 1, name, workflowId })); + + // TODO - Add first production run code here + } catch (error) { + // Do we just assume it's a conflict error? If there is any other sort of error in the DB it should trigger here too + await Db.collections.WorkflowStatistics.update( + { workflowId, name }, + { count: () => '"count" + 1' }, + ); + } +}); diff --git a/packages/cli/src/events/index.ts b/packages/cli/src/events/index.ts new file mode 100644 index 0000000000000..b468d302e25a1 --- /dev/null +++ b/packages/cli/src/events/index.ts @@ -0,0 +1 @@ +import './WorkflowStatistics'; From 87adfcd60799c56db86480ce6a66496ba3e8052c Mon Sep 17 00:00:00 2001 From: freyamade Date: Thu, 13 Oct 2022 16:33:00 +0100 Subject: [PATCH 17/49] :sparkles: - Trigger workflow finish events --- packages/cli/src/WorkflowExecuteAdditionalData.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/WorkflowExecuteAdditionalData.ts b/packages/cli/src/WorkflowExecuteAdditionalData.ts index 0dddb3735aefe..1d07c26e6cfaa 100644 --- a/packages/cli/src/WorkflowExecuteAdditionalData.ts +++ b/packages/cli/src/WorkflowExecuteAdditionalData.ts @@ -16,7 +16,7 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ /* eslint-disable func-names */ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ -import { BinaryDataManager, UserSettings, WorkflowExecute } from 'n8n-core'; +import { BinaryDataManager, eventEmitter, UserSettings, WorkflowExecute } from 'n8n-core'; import { IDataObject, @@ -642,6 +642,8 @@ function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks { this.retryOf, ); } + } finally { + eventEmitter.emit('saveWorkflowStatistics', this.workflowData, fullRunData); } }, ], @@ -735,6 +737,8 @@ function hookFunctionsSaveWorker(): IWorkflowExecuteHooks { this.executionId, this.retryOf, ); + } finally { + eventEmitter.emit('saveWorkflowStatistics', this.workflowData, fullRunData); } }, ], From 46e0df1c958832286804a5d04b72e19fc4ffc0f7 Mon Sep 17 00:00:00 2001 From: freyamade Date: Mon, 17 Oct 2022 13:10:41 +0100 Subject: [PATCH 18/49] :art: - Replaced magic strings with key/value pair --- .../cli/src/WorkflowExecuteAdditionalData.ts | 6 +- packages/cli/src/events/WorkflowStatistics.ts | 69 ++++++++++--------- packages/core/src/EventEmitter.ts | 12 +++- 3 files changed, 54 insertions(+), 33 deletions(-) diff --git a/packages/cli/src/WorkflowExecuteAdditionalData.ts b/packages/cli/src/WorkflowExecuteAdditionalData.ts index f40c0992eea4f..361c35b72b41d 100644 --- a/packages/cli/src/WorkflowExecuteAdditionalData.ts +++ b/packages/cli/src/WorkflowExecuteAdditionalData.ts @@ -643,7 +643,11 @@ function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks { ); } } finally { - eventEmitter.emit('saveWorkflowStatistics', this.workflowData, fullRunData); + eventEmitter.emit( + eventEmitter.types.workflowExecutionCompleted, + this.workflowData, + fullRunData, + ); } }, ], diff --git a/packages/cli/src/events/WorkflowStatistics.ts b/packages/cli/src/events/WorkflowStatistics.ts index 781d5e8826c3d..e20662a644941 100644 --- a/packages/cli/src/events/WorkflowStatistics.ts +++ b/packages/cli/src/events/WorkflowStatistics.ts @@ -3,39 +3,46 @@ import { IRun, IWorkflowBase } from 'n8n-workflow'; import { Db } from '..'; import { StatisticsNames } from '../databases/entities/WorkflowStatistics'; -eventEmitter.on('saveWorkflowStatistics', async (workflowData: IWorkflowBase, runData: IRun) => { - // Determine the name of the statistic - const finished = runData.finished ? runData.finished : false; - const manual = runData.mode === 'manual'; - let name: StatisticsNames; +eventEmitter.on( + eventEmitter.types.workflowExecutionCompleted, + async (workflowData: IWorkflowBase, runData: IRun) => { + // Determine the name of the statistic + const finished = runData.finished ? runData.finished : false; + const manual = runData.mode === 'manual'; + let name: StatisticsNames; - if (finished) { - if (manual) name = StatisticsNames.manualSuccess; - else name = StatisticsNames.productionSuccess; - } else { - if (manual) name = StatisticsNames.manualError; - else name = StatisticsNames.productionError; - } + if (finished) { + if (manual) name = StatisticsNames.manualSuccess; + else name = StatisticsNames.productionSuccess; + } else { + if (manual) name = StatisticsNames.manualError; + else name = StatisticsNames.productionError; + } - // Get the workflow id - let workflowId: number; - try { - workflowId = parseInt(workflowData.id as string, 10); - } catch (error) { - console.error(`Error ${error as string} when casting workflow ID to a number`); - return; - } + // Get the workflow id + let workflowId: number; + try { + workflowId = parseInt(workflowData.id as string, 10); + } catch (error) { + console.error(`Error ${error as string} when casting workflow ID to a number`); + return; + } - // Try insertion and if it fails due to key conflicts then update the existing entry instead - try { - console.log(await Db.collections.WorkflowStatistics.insert({ count: 1, name, workflowId })); + // Try insertion and if it fails due to key conflicts then update the existing entry instead + try { + await Db.collections.WorkflowStatistics.insert({ count: 1, name, workflowId }); - // TODO - Add first production run code here - } catch (error) { - // Do we just assume it's a conflict error? If there is any other sort of error in the DB it should trigger here too - await Db.collections.WorkflowStatistics.update( - { workflowId, name }, - { count: () => '"count" + 1' }, - ); - } + // TODO - Add first production run code here + } catch (error) { + // Do we just assume it's a conflict error? If there is any other sort of error in the DB it should trigger here too + await Db.collections.WorkflowStatistics.update( + { workflowId, name }, + { count: () => '"count" + 1' }, + ); + } + }, +); + +eventEmitter.on(eventEmitter.types.nodeFetchedData, (nodeName, runData, execData) => { + console.log(nodeName, runData, execData); }); diff --git a/packages/core/src/EventEmitter.ts b/packages/core/src/EventEmitter.ts index b04cfe9b77890..a5e71633564bc 100644 --- a/packages/core/src/EventEmitter.ts +++ b/packages/core/src/EventEmitter.ts @@ -1,5 +1,15 @@ import EventEmitter from 'events'; -class N8NEventEmitter extends EventEmitter {} +interface EventTypes { + nodeFetchedData: string; + workflowExecutionCompleted: string; +} + +class N8NEventEmitter extends EventEmitter { + types: EventTypes = { + nodeFetchedData: 'nodeFetchedData', + workflowExecutionCompleted: 'workflowExecutionCompleted', + }; +} export const eventEmitter = new N8NEventEmitter(); From e98c15e7cd8051562d75a719a54845f418e3ad6a Mon Sep 17 00:00:00 2001 From: freyamade Date: Thu, 20 Oct 2022 09:47:44 +0100 Subject: [PATCH 19/49] :sparkles: - Add execution counts endpoint to internal api --- packages/cli/src/Interfaces.ts | 7 ++ packages/cli/src/Server.ts | 6 ++ packages/cli/src/api/workflowStats.api.ts | 83 +++++++++++++++++++++++ 3 files changed, 96 insertions(+) create mode 100644 packages/cli/src/api/workflowStats.api.ts diff --git a/packages/cli/src/Interfaces.ts b/packages/cli/src/Interfaces.ts index a0c53da2be575..4c2a8311301e4 100644 --- a/packages/cli/src/Interfaces.ts +++ b/packages/cli/src/Interfaces.ts @@ -724,6 +724,13 @@ export interface IWorkflowExecuteProcess { workflowExecute: WorkflowExecute; } +export interface IWorkflowStatisticsCounts { + productionSuccess: number; + productionError: number; + manualSuccess: number; + manualError: number; +} + export type WhereClause = Record; // ---------------------------------- diff --git a/packages/cli/src/Server.ts b/packages/cli/src/Server.ts index eb5da9492c86e..78d82300851e3 100644 --- a/packages/cli/src/Server.ts +++ b/packages/cli/src/Server.ts @@ -152,6 +152,7 @@ import glob from 'fast-glob'; import { ResponseError } from './ResponseHelper'; import { toHttpNodeParameters } from './CurlConverterHelper'; +import { workflowStatsController } from './api/workflowStats.api'; require('body-parser-xml')(bodyParser); @@ -776,6 +777,11 @@ class App { // ---------------------------------------- this.app.use(`/${this.restEndpoint}/workflows`, workflowsController); + // ---------------------------------------- + // Workflow Statistics + // ---------------------------------------- + this.app.use(`/${this.restEndpoint}/workflow-stats`, workflowStatsController); + // ---------------------------------------- // Tags // ---------------------------------------- diff --git a/packages/cli/src/api/workflowStats.api.ts b/packages/cli/src/api/workflowStats.api.ts new file mode 100644 index 0000000000000..57621af88aaaa --- /dev/null +++ b/packages/cli/src/api/workflowStats.api.ts @@ -0,0 +1,83 @@ +import express from 'express'; +import { LoggerProxy } from 'n8n-workflow'; +import { In } from 'typeorm'; +import { ResponseHelper, IWorkflowStatisticsCounts, Db } from '..'; +import { StatisticsNames } from '../databases/entities/WorkflowStatistics'; +import { getLogger } from '../Logger'; +import { ExecutionRequest } from '../requests'; + +export const workflowStatsController = express.Router(); + +/** + * Initialise Logger if needed + */ +workflowStatsController.use((req, res, next) => { + try { + LoggerProxy.getInstance(); + } catch (error) { + LoggerProxy.init(getLogger()); + } + next(); +}); + +// Helper function that validates the ID, throws an error if not valud +async function checkWorkflowId(workflowId: string): Promise { + const workflow = await Db.collections.Workflow.findOne(workflowId); + if (!workflow) { + throw new ResponseHelper.ResponseError( + `The workflow with the ID "${workflowId}" does not exist.`, + 404, + 404, + ); + } +} + +// Base endpoint is /workflow-stats +workflowStatsController.get( + '/:id/counts/', + ResponseHelper.send(async (req: ExecutionRequest.Get): Promise => { + // Get counts from DB + const workflowId = req.params.id; + console.log(workflowId); + + // Check that the id is valid + await checkWorkflowId(workflowId); + + // Find the stats for this workflow + const stats = await Db.collections.WorkflowStatistics.find({ + select: ['count', 'name'], + where: { + workflowId, + }, + }); + + const data: IWorkflowStatisticsCounts = { + productionSuccess: 0, + productionError: 0, + manualSuccess: 0, + manualError: 0, + }; + + // There will be a maximum of 4 stats (currently) + stats.forEach(({ count, name }) => { + switch (name) { + case StatisticsNames.manualError: + data.manualError = count; + break; + + case StatisticsNames.manualSuccess: + data.manualSuccess = count; + break; + + case StatisticsNames.productionError: + data.productionError = count; + break; + + case StatisticsNames.productionSuccess: + data.productionSuccess = count; + } + }); + + return data; + }), +); From f1296abbdb29d5c9faaeca5ba54062af48af6c2b Mon Sep 17 00:00:00 2001 From: freyamade Date: Thu, 20 Oct 2022 16:33:57 +0100 Subject: [PATCH 20/49] :sparkles: - Added timestamp endpoint --- packages/cli/src/Interfaces.ts | 7 +++ packages/cli/src/api/workflowStats.api.ts | 59 +++++++++++++++++++++-- 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/Interfaces.ts b/packages/cli/src/Interfaces.ts index 4c2a8311301e4..871394ca7eb8c 100644 --- a/packages/cli/src/Interfaces.ts +++ b/packages/cli/src/Interfaces.ts @@ -731,6 +731,13 @@ export interface IWorkflowStatisticsCounts { manualError: number; } +export interface IWorkflowStatisticsTimestamps { + productionSuccess: Date | null; + productionError: Date | null; + manualSuccess: Date | null; + manualError: Date | null; +} + export type WhereClause = Record; // ---------------------------------- diff --git a/packages/cli/src/api/workflowStats.api.ts b/packages/cli/src/api/workflowStats.api.ts index 57621af88aaaa..5d8409af0bc33 100644 --- a/packages/cli/src/api/workflowStats.api.ts +++ b/packages/cli/src/api/workflowStats.api.ts @@ -1,7 +1,6 @@ import express from 'express'; import { LoggerProxy } from 'n8n-workflow'; -import { In } from 'typeorm'; -import { ResponseHelper, IWorkflowStatisticsCounts, Db } from '..'; +import { Db, IWorkflowStatisticsCounts, IWorkflowStatisticsTimestamps, ResponseHelper } from '..'; import { StatisticsNames } from '../databases/entities/WorkflowStatistics'; import { getLogger } from '../Logger'; import { ExecutionRequest } from '../requests'; @@ -32,13 +31,14 @@ async function checkWorkflowId(workflowId: string): Promise { } } -// Base endpoint is /workflow-stats +/** + * GET /workflow-stats/:id/counts/ + */ workflowStatsController.get( '/:id/counts/', ResponseHelper.send(async (req: ExecutionRequest.Get): Promise => { // Get counts from DB const workflowId = req.params.id; - console.log(workflowId); // Check that the id is valid await checkWorkflowId(workflowId); @@ -81,3 +81,54 @@ workflowStatsController.get( return data; }), ); + +/** + * GET /workflow-stats/:id/times/ + */ +workflowStatsController.get( + '/:id/times/', + ResponseHelper.send(async (req: ExecutionRequest.Get): Promise => { + // Get times from DB + const workflowId = req.params.id; + + // Check that the id is valid + await checkWorkflowId(workflowId); + + // Find the stats for this workflow + const stats = await Db.collections.WorkflowStatistics.find({ + select: ['latestEvent', 'name'], + where: { + workflowId, + }, + }); + + const data: IWorkflowStatisticsTimestamps = { + productionSuccess: null, + productionError: null, + manualSuccess: null, + manualError: null, + }; + + // There will be a maximum of 4 stats (currently) + stats.forEach(({ latestEvent, name }) => { + switch (name) { + case StatisticsNames.manualError: + data.manualError = latestEvent; + break; + + case StatisticsNames.manualSuccess: + data.manualSuccess = latestEvent; + break; + + case StatisticsNames.productionError: + data.productionError = latestEvent; + break; + + case StatisticsNames.productionSuccess: + data.productionSuccess = latestEvent; + } + }); + + return data; + }), +); From d593e43afdc5992f3509c5c03c4fef276988ea48 Mon Sep 17 00:00:00 2001 From: freyamade Date: Thu, 20 Oct 2022 16:43:55 +0100 Subject: [PATCH 21/49] :bug: - fix issue where new workflows had null for dataLoaded --- packages/cli/src/databases/entities/WorkflowEntity.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/databases/entities/WorkflowEntity.ts b/packages/cli/src/databases/entities/WorkflowEntity.ts index 99a462e07352c..e31f5183069d0 100644 --- a/packages/cli/src/databases/entities/WorkflowEntity.ts +++ b/packages/cli/src/databases/entities/WorkflowEntity.ts @@ -133,7 +133,7 @@ export class WorkflowEntity implements IWorkflowDb { @JoinColumn({ referencedColumnName: 'workflow' }) statistics: WorkflowStatistics[]; - @Column() + @Column({ default: false }) dataLoaded: boolean; @Column({ From d7b2bafdc2391b6bff201b0c6d5d024128838ea2 Mon Sep 17 00:00:00 2001 From: freyamade Date: Thu, 20 Oct 2022 16:44:12 +0100 Subject: [PATCH 22/49] :sparkles: - Add dataLoaded endpoint --- packages/cli/src/Interfaces.ts | 4 +++ packages/cli/src/api/workflowStats.api.ts | 32 +++++++++++++++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/Interfaces.ts b/packages/cli/src/Interfaces.ts index 871394ca7eb8c..1a2b26ea8da91 100644 --- a/packages/cli/src/Interfaces.ts +++ b/packages/cli/src/Interfaces.ts @@ -731,6 +731,10 @@ export interface IWorkflowStatisticsCounts { manualError: number; } +export interface IWorkflowStatisticsDataLoaded { + dataLoaded: boolean; +} + export interface IWorkflowStatisticsTimestamps { productionSuccess: Date | null; productionError: Date | null; diff --git a/packages/cli/src/api/workflowStats.api.ts b/packages/cli/src/api/workflowStats.api.ts index 5d8409af0bc33..aedf284a81f24 100644 --- a/packages/cli/src/api/workflowStats.api.ts +++ b/packages/cli/src/api/workflowStats.api.ts @@ -1,6 +1,13 @@ import express from 'express'; import { LoggerProxy } from 'n8n-workflow'; -import { Db, IWorkflowStatisticsCounts, IWorkflowStatisticsTimestamps, ResponseHelper } from '..'; +import { + Db, + IWorkflowStatisticsCounts, + IWorkflowStatisticsDataLoaded, + IWorkflowStatisticsTimestamps, + ResponseHelper, +} from '..'; +import { WorkflowEntity } from '../databases/entities/WorkflowEntity'; import { StatisticsNames } from '../databases/entities/WorkflowStatistics'; import { getLogger } from '../Logger'; import { ExecutionRequest } from '../requests'; @@ -20,7 +27,7 @@ workflowStatsController.use((req, res, next) => { }); // Helper function that validates the ID, throws an error if not valud -async function checkWorkflowId(workflowId: string): Promise { +async function checkWorkflowId(workflowId: string): Promise { const workflow = await Db.collections.Workflow.findOne(workflowId); if (!workflow) { throw new ResponseHelper.ResponseError( @@ -29,6 +36,7 @@ async function checkWorkflowId(workflowId: string): Promise { 404, ); } + return workflow; } /** @@ -132,3 +140,23 @@ workflowStatsController.get( return data; }), ); + +/** + * GET /workflow-stats/:id/data-loaded/ + */ +workflowStatsController.get( + '/:id/data-loaded/', + ResponseHelper.send(async (req: ExecutionRequest.Get): Promise => { + // Get flag + const workflowId = req.params.id; + + // Get the corresponding workflow + const workflow = await checkWorkflowId(workflowId); + + const data: IWorkflowStatisticsDataLoaded = { + dataLoaded: workflow.dataLoaded, + }; + + return data; + }), +); From e6714598a968d3dd197c304fa2d6bb7d8bfb465f Mon Sep 17 00:00:00 2001 From: freyamade Date: Fri, 28 Oct 2022 10:39:07 +0100 Subject: [PATCH 23/49] :sparkles: - Add nodefetcheddata event handling For some weird reason this doesn't work on node executions currently. It did when it was first implemented and then it stopped. --- packages/cli/src/WebhookHelpers.ts | 3 +- packages/cli/src/events/WorkflowStatistics.ts | 24 +++++++++++-- packages/core/src/NodeExecuteFunctions.ts | 36 ++++++++++++++----- 3 files changed, 51 insertions(+), 12 deletions(-) diff --git a/packages/cli/src/WebhookHelpers.ts b/packages/cli/src/WebhookHelpers.ts index d50b86243f3d5..31571c12293ce 100644 --- a/packages/cli/src/WebhookHelpers.ts +++ b/packages/cli/src/WebhookHelpers.ts @@ -18,7 +18,7 @@ import express from 'express'; // eslint-disable-next-line import/no-extraneous-dependencies import { get } from 'lodash'; -import { BINARY_ENCODING, BinaryDataManager, NodeExecuteFunctions } from 'n8n-core'; +import { BINARY_ENCODING, BinaryDataManager, NodeExecuteFunctions, eventEmitter } from 'n8n-core'; import { createDeferredPromise, @@ -239,6 +239,7 @@ export async function executeWebhook( NodeExecuteFunctions, executionMode, ); + eventEmitter.emit(eventEmitter.types.nodeFetchedData, workflow.id); } catch (err) { // Send error response to webhook caller const errorMessage = 'Workflow Webhook Error: Workflow could not be started!'; diff --git a/packages/cli/src/events/WorkflowStatistics.ts b/packages/cli/src/events/WorkflowStatistics.ts index e20662a644941..5c716394d5d72 100644 --- a/packages/cli/src/events/WorkflowStatistics.ts +++ b/packages/cli/src/events/WorkflowStatistics.ts @@ -43,6 +43,26 @@ eventEmitter.on( }, ); -eventEmitter.on(eventEmitter.types.nodeFetchedData, (nodeName, runData, execData) => { - console.log(nodeName, runData, execData); +eventEmitter.on(eventEmitter.types.nodeFetchedData, async (workflowId: string) => { + // Get the workflow id + console.log('event', workflowId); + let id: number; + try { + id = parseInt(workflowId, 10); + } catch (error) { + console.error(`Error ${error as string} when casting workflow ID to a number`); + return; + } + + // Update only if necessary + console.log('update db'); + const response = await Db.collections.Workflow.update( + { id, dataLoaded: false }, + { dataLoaded: true }, + ); + + // If response.affected is 1 then we know this was the first time data was loaded into the workflow; do posthog event here + if (response.affected) { + console.log('posthog'); + } }); diff --git a/packages/core/src/NodeExecuteFunctions.ts b/packages/core/src/NodeExecuteFunctions.ts index d52a24e7e64e7..7aeb3c86aeb1c 100644 --- a/packages/core/src/NodeExecuteFunctions.ts +++ b/packages/core/src/NodeExecuteFunctions.ts @@ -93,6 +93,7 @@ import url, { URL, URLSearchParams } from 'url'; import { BinaryDataManager } from './BinaryDataManager'; // eslint-disable-next-line import/no-cycle import { + eventEmitter, ICredentialTestFunctions, IHookFunctions, ILoadOptionsFunctions, @@ -543,6 +544,7 @@ function digestAuthAxiosConfig( } async function proxyRequestToAxios( + workflow: Workflow, uriOrObject: string | IDataObject, options?: IDataObject, // tslint:disable-next-line:no-any @@ -620,6 +622,7 @@ async function proxyRequestToAxios( body = undefined; } } + eventEmitter.emit(eventEmitter.types.nodeFetchedData, workflow.id); resolve({ body, headers: response.headers, @@ -636,6 +639,7 @@ async function proxyRequestToAxios( body = undefined; } } + eventEmitter.emit(eventEmitter.types.nodeFetchedData, workflow.id); resolve(body); } }) @@ -1446,7 +1450,7 @@ export async function requestWithAuthentication( node, additionalData.timezone, ); - return await proxyRequestToAxios(requestOptions as IDataObject); + return await proxyRequestToAxios(workflow, requestOptions as IDataObject); } catch (error) { try { if (credentialsDecrypted !== undefined) { @@ -1472,7 +1476,7 @@ export async function requestWithAuthentication( additionalData.timezone, ); // retry the request - return await proxyRequestToAxios(requestOptions as IDataObject); + return await proxyRequestToAxios(workflow, requestOptions as IDataObject); } } throw error; @@ -1928,7 +1932,9 @@ export function getExecutePollFunctions( mimeType, ); }, - request: proxyRequestToAxios, + request: async (uriOrObject: string | IDataObject, options?: IDataObject | undefined) => { + return proxyRequestToAxios(workflow, uriOrObject, options); + }, async requestWithAuthentication( this: IAllExecuteFunctions, credentialsType: string, @@ -2093,7 +2099,9 @@ export function getExecuteTriggerFunctions( mimeType, ); }, - request: proxyRequestToAxios, + request: async (uriOrObject: string | IDataObject, options?: IDataObject | undefined) => { + return proxyRequestToAxios(workflow, uriOrObject, options); + }, async requestOAuth2( this: IAllExecuteFunctions, credentialsType: string, @@ -2361,7 +2369,9 @@ export function getExecuteFunctions( ): Promise { return getBinaryDataBuffer.call(this, inputData, itemIndex, propertyName, inputIndex); }, - request: proxyRequestToAxios, + request: async (uriOrObject: string | IDataObject, options?: IDataObject | undefined) => { + return proxyRequestToAxios(workflow, uriOrObject, options); + }, async requestOAuth2( this: IAllExecuteFunctions, credentialsType: string, @@ -2586,7 +2596,9 @@ export function getExecuteSingleFunctions( mimeType, ); }, - request: proxyRequestToAxios, + request: async (uriOrObject: string | IDataObject, options?: IDataObject | undefined) => { + return proxyRequestToAxios(workflow, uriOrObject, options); + }, async requestOAuth2( this: IAllExecuteFunctions, credentialsType: string, @@ -2721,7 +2733,9 @@ export function getLoadOptionsFunctions( additionalCredentialOptions, ); }, - request: proxyRequestToAxios, + request: async (uriOrObject: string | IDataObject, options?: IDataObject | undefined) => { + return proxyRequestToAxios(workflow, uriOrObject, options); + }, async requestOAuth2( this: IAllExecuteFunctions, credentialsType: string, @@ -2867,7 +2881,9 @@ export function getExecuteHookFunctions( additionalCredentialOptions, ); }, - request: proxyRequestToAxios, + request: async (uriOrObject: string | IDataObject, options?: IDataObject | undefined) => { + return proxyRequestToAxios(workflow, uriOrObject, options); + }, async requestOAuth2( this: IAllExecuteFunctions, credentialsType: string, @@ -3054,7 +3070,9 @@ export function getExecuteWebhookFunctions( mimeType, ); }, - request: proxyRequestToAxios, + request: async (uriOrObject: string | IDataObject, options?: IDataObject | undefined) => { + return proxyRequestToAxios(workflow, uriOrObject, options); + }, async requestOAuth2( this: IAllExecuteFunctions, credentialsType: string, From 793d40d7790cb6c0f7a2551fbcccfe95b855edbc Mon Sep 17 00:00:00 2001 From: freyamade Date: Tue, 1 Nov 2022 12:04:42 +0000 Subject: [PATCH 24/49] :sparkles: - set data loaded flag --- .../cli/src/WorkflowExecuteAdditionalData.ts | 16 ++++++++++- packages/cli/src/WorkflowRunnerProcess.ts | 5 ++++ packages/cli/src/events/WorkflowStatistics.ts | 2 -- packages/core/src/NodeExecuteFunctions.ts | 27 ++++++++++--------- 4 files changed, 34 insertions(+), 16 deletions(-) diff --git a/packages/cli/src/WorkflowExecuteAdditionalData.ts b/packages/cli/src/WorkflowExecuteAdditionalData.ts index 361c35b72b41d..ebb09a776abea 100644 --- a/packages/cli/src/WorkflowExecuteAdditionalData.ts +++ b/packages/cli/src/WorkflowExecuteAdditionalData.ts @@ -651,6 +651,11 @@ function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks { } }, ], + nodeFetchedData: [ + async (workflowId: string) => { + eventEmitter.emit(eventEmitter.types.nodeFetchedData, workflowId); + }, + ], }; } @@ -742,10 +747,19 @@ function hookFunctionsSaveWorker(): IWorkflowExecuteHooks { this.retryOf, ); } finally { - eventEmitter.emit('saveWorkflowStatistics', this.workflowData, fullRunData); + eventEmitter.emit( + eventEmitter.types.workflowExecutionCompleted, + this.workflowData, + fullRunData, + ); } }, ], + nodeFetchedData: [ + async (workflowId: string) => { + eventEmitter.emit(eventEmitter.types.nodeFetchedData, workflowId); + }, + ], }; } diff --git a/packages/cli/src/WorkflowRunnerProcess.ts b/packages/cli/src/WorkflowRunnerProcess.ts index abf501b93db9a..9c32a9cfbe353 100644 --- a/packages/cli/src/WorkflowRunnerProcess.ts +++ b/packages/cli/src/WorkflowRunnerProcess.ts @@ -433,6 +433,11 @@ export class WorkflowRunnerProcess { await this.sendHookToParentProcess('workflowExecuteAfter', [fullRunData, newStaticData]); }, ], + nodeFetchedData: [ + async (workflowId: string) => { + await this.sendHookToParentProcess('nodeFetchedData', [workflowId]); + }, + ], }; const preExecuteFunctions = WorkflowExecuteAdditionalData.hookFunctionsPreExecute(); diff --git a/packages/cli/src/events/WorkflowStatistics.ts b/packages/cli/src/events/WorkflowStatistics.ts index 5c716394d5d72..fb5b938e12e14 100644 --- a/packages/cli/src/events/WorkflowStatistics.ts +++ b/packages/cli/src/events/WorkflowStatistics.ts @@ -45,7 +45,6 @@ eventEmitter.on( eventEmitter.on(eventEmitter.types.nodeFetchedData, async (workflowId: string) => { // Get the workflow id - console.log('event', workflowId); let id: number; try { id = parseInt(workflowId, 10); @@ -55,7 +54,6 @@ eventEmitter.on(eventEmitter.types.nodeFetchedData, async (workflowId: string) = } // Update only if necessary - console.log('update db'); const response = await Db.collections.Workflow.update( { id, dataLoaded: false }, { dataLoaded: true }, diff --git a/packages/core/src/NodeExecuteFunctions.ts b/packages/core/src/NodeExecuteFunctions.ts index 7aeb3c86aeb1c..cadf2ba6cb419 100644 --- a/packages/core/src/NodeExecuteFunctions.ts +++ b/packages/core/src/NodeExecuteFunctions.ts @@ -93,7 +93,7 @@ import url, { URL, URLSearchParams } from 'url'; import { BinaryDataManager } from './BinaryDataManager'; // eslint-disable-next-line import/no-cycle import { - eventEmitter, + // eventEmitter, ICredentialTestFunctions, IHookFunctions, ILoadOptionsFunctions, @@ -545,6 +545,7 @@ function digestAuthAxiosConfig( async function proxyRequestToAxios( workflow: Workflow, + additionalData: IWorkflowExecuteAdditionalData, uriOrObject: string | IDataObject, options?: IDataObject, // tslint:disable-next-line:no-any @@ -612,7 +613,7 @@ async function proxyRequestToAxios( return new Promise((resolve, reject) => { axiosPromise - .then((response) => { + .then(async (response) => { if (configObject.resolveWithFullResponse === true) { let body = response.data; if (response.data === '') { @@ -622,7 +623,7 @@ async function proxyRequestToAxios( body = undefined; } } - eventEmitter.emit(eventEmitter.types.nodeFetchedData, workflow.id); + await additionalData.hooks?.executeHookFunctions('nodeFetchedData', [workflow.id]); resolve({ body, headers: response.headers, @@ -639,7 +640,7 @@ async function proxyRequestToAxios( body = undefined; } } - eventEmitter.emit(eventEmitter.types.nodeFetchedData, workflow.id); + await additionalData.hooks?.executeHookFunctions('nodeFetchedData', [workflow.id]); resolve(body); } }) @@ -1450,7 +1451,7 @@ export async function requestWithAuthentication( node, additionalData.timezone, ); - return await proxyRequestToAxios(workflow, requestOptions as IDataObject); + return await proxyRequestToAxios(workflow, additionalData, requestOptions as IDataObject); } catch (error) { try { if (credentialsDecrypted !== undefined) { @@ -1476,7 +1477,7 @@ export async function requestWithAuthentication( additionalData.timezone, ); // retry the request - return await proxyRequestToAxios(workflow, requestOptions as IDataObject); + return await proxyRequestToAxios(workflow, additionalData, requestOptions as IDataObject); } } throw error; @@ -1933,7 +1934,7 @@ export function getExecutePollFunctions( ); }, request: async (uriOrObject: string | IDataObject, options?: IDataObject | undefined) => { - return proxyRequestToAxios(workflow, uriOrObject, options); + return proxyRequestToAxios(workflow, additionalData, uriOrObject, options); }, async requestWithAuthentication( this: IAllExecuteFunctions, @@ -2100,7 +2101,7 @@ export function getExecuteTriggerFunctions( ); }, request: async (uriOrObject: string | IDataObject, options?: IDataObject | undefined) => { - return proxyRequestToAxios(workflow, uriOrObject, options); + return proxyRequestToAxios(workflow, additionalData, uriOrObject, options); }, async requestOAuth2( this: IAllExecuteFunctions, @@ -2370,7 +2371,7 @@ export function getExecuteFunctions( return getBinaryDataBuffer.call(this, inputData, itemIndex, propertyName, inputIndex); }, request: async (uriOrObject: string | IDataObject, options?: IDataObject | undefined) => { - return proxyRequestToAxios(workflow, uriOrObject, options); + return proxyRequestToAxios(workflow, additionalData, uriOrObject, options); }, async requestOAuth2( this: IAllExecuteFunctions, @@ -2597,7 +2598,7 @@ export function getExecuteSingleFunctions( ); }, request: async (uriOrObject: string | IDataObject, options?: IDataObject | undefined) => { - return proxyRequestToAxios(workflow, uriOrObject, options); + return proxyRequestToAxios(workflow, additionalData, uriOrObject, options); }, async requestOAuth2( this: IAllExecuteFunctions, @@ -2734,7 +2735,7 @@ export function getLoadOptionsFunctions( ); }, request: async (uriOrObject: string | IDataObject, options?: IDataObject | undefined) => { - return proxyRequestToAxios(workflow, uriOrObject, options); + return proxyRequestToAxios(workflow, additionalData, uriOrObject, options); }, async requestOAuth2( this: IAllExecuteFunctions, @@ -2882,7 +2883,7 @@ export function getExecuteHookFunctions( ); }, request: async (uriOrObject: string | IDataObject, options?: IDataObject | undefined) => { - return proxyRequestToAxios(workflow, uriOrObject, options); + return proxyRequestToAxios(workflow, additionalData, uriOrObject, options); }, async requestOAuth2( this: IAllExecuteFunctions, @@ -3071,7 +3072,7 @@ export function getExecuteWebhookFunctions( ); }, request: async (uriOrObject: string | IDataObject, options?: IDataObject | undefined) => { - return proxyRequestToAxios(workflow, uriOrObject, options); + return proxyRequestToAxios(workflow, additionalData, uriOrObject, options); }, async requestOAuth2( this: IAllExecuteFunctions, From af9271125ba5e5bf13c2c78db786aae99b66939c Mon Sep 17 00:00:00 2001 From: freyamade Date: Wed, 2 Nov 2022 17:29:17 +0000 Subject: [PATCH 25/49] :sparkles: - workflow exec stats now have user id --- packages/cli/src/telemetry/index.ts | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/packages/cli/src/telemetry/index.ts b/packages/cli/src/telemetry/index.ts index 6ce3e3d54af50..b1ac69b748113 100644 --- a/packages/cli/src/telemetry/index.ts +++ b/packages/cli/src/telemetry/index.ts @@ -21,6 +21,7 @@ interface IExecutionsBuffer { manual_success?: IExecutionTrackData; prod_error?: IExecutionTrackData; prod_success?: IExecutionTrackData; + user_id: string | undefined; }; } @@ -81,11 +82,15 @@ export class Telemetry { } const allPromises = Object.keys(this.executionCountsBuffer).map(async (workflowId) => { - const promise = this.track('Workflow execution count', { - event_version: '2', - workflow_id: workflowId, - ...this.executionCountsBuffer[workflowId], - }); + const promise = this.track( + 'Workflow execution count', + { + event_version: '2', + workflow_id: workflowId, + ...this.executionCountsBuffer[workflowId], + }, + { withPostHog: true }, + ); return promise; }); @@ -100,7 +105,9 @@ export class Telemetry { const execTime = new Date(); const workflowId = properties.workflow_id; - this.executionCountsBuffer[workflowId] = this.executionCountsBuffer[workflowId] ?? {}; + this.executionCountsBuffer[workflowId] = this.executionCountsBuffer[workflowId] ?? { + user_id: properties.user_id, + }; const key: ExecutionTrackDataKey = `${properties.is_manual ? 'manual' : 'prod'}_${ properties.success ? 'success' : 'error' From 0830fb45a1dabfc3bade6f4f6b64f8a8f46499bc Mon Sep 17 00:00:00 2001 From: freyamade Date: Thu, 3 Nov 2022 16:55:51 +0000 Subject: [PATCH 26/49] :sparkles: - Add first prod success to posthog --- packages/cli/src/InternalHooks.ts | 10 ++++++++++ packages/cli/src/events/WorkflowStatistics.ts | 16 ++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/InternalHooks.ts b/packages/cli/src/InternalHooks.ts index 31ae7ca36e0fe..e2a64f2288cd5 100644 --- a/packages/cli/src/InternalHooks.ts +++ b/packages/cli/src/InternalHooks.ts @@ -478,4 +478,14 @@ export class InternalHooksClass implements IInternalHooksClass { }): Promise { return this.telemetry.track('cnr package deleted', updateData); } + + /** + * Execution Statistics + */ + async onFirstProductionWorkflowSuccess(data: { + user_id: string; + workflow_id: string | number; + }): Promise { + return this.telemetry.track('Workflow first prod success', data, { withPostHog: true }); + } } diff --git a/packages/cli/src/events/WorkflowStatistics.ts b/packages/cli/src/events/WorkflowStatistics.ts index fb5b938e12e14..52899992c0bd0 100644 --- a/packages/cli/src/events/WorkflowStatistics.ts +++ b/packages/cli/src/events/WorkflowStatistics.ts @@ -1,7 +1,8 @@ import { eventEmitter } from 'n8n-core'; import { IRun, IWorkflowBase } from 'n8n-workflow'; -import { Db } from '..'; +import { Db, InternalHooksManager } from '..'; import { StatisticsNames } from '../databases/entities/WorkflowStatistics'; +import { getWorkflowOwner } from '../UserManagement/UserManagementHelper'; eventEmitter.on( eventEmitter.types.workflowExecutionCompleted, @@ -32,7 +33,18 @@ eventEmitter.on( try { await Db.collections.WorkflowStatistics.insert({ count: 1, name, workflowId }); - // TODO - Add first production run code here + // If we're here we can check if we're sending the first production success metric + if (name !== StatisticsNames.productionSuccess) return; + + // Get the owner of the workflow so we can send the metric + const owner = await getWorkflowOwner(workflowId); + const metrics = { + user_id: owner.id, + workflow_id: workflowId, + }; + + // Send the metrics + await InternalHooksManager.getInstance().onFirstProductionWorkflowSuccess(metrics); } catch (error) { // Do we just assume it's a conflict error? If there is any other sort of error in the DB it should trigger here too await Db.collections.WorkflowStatistics.update( From 6f36b54227c6e1ef1ae270584702b3e152f587aa Mon Sep 17 00:00:00 2001 From: freyamade Date: Thu, 3 Nov 2022 18:46:20 +0000 Subject: [PATCH 27/49] :sparkles: - Pass node to nodeFetchedData event --- packages/cli/src/WebhookHelpers.ts | 3 ++- .../cli/src/WorkflowExecuteAdditionalData.ts | 8 +++---- packages/cli/src/WorkflowRunnerProcess.ts | 5 ++-- packages/cli/src/events/WorkflowStatistics.ts | 5 ++-- packages/core/src/NodeExecuteFunctions.ts | 23 ++++++++++--------- 5 files changed, 24 insertions(+), 20 deletions(-) diff --git a/packages/cli/src/WebhookHelpers.ts b/packages/cli/src/WebhookHelpers.ts index 31571c12293ce..73c143089a052 100644 --- a/packages/cli/src/WebhookHelpers.ts +++ b/packages/cli/src/WebhookHelpers.ts @@ -239,7 +239,8 @@ export async function executeWebhook( NodeExecuteFunctions, executionMode, ); - eventEmitter.emit(eventEmitter.types.nodeFetchedData, workflow.id); + console.log('webhook fetched data', workflowStartNode); + eventEmitter.emit(eventEmitter.types.nodeFetchedData, workflow.id, workflowStartNode); } catch (err) { // Send error response to webhook caller const errorMessage = 'Workflow Webhook Error: Workflow could not be started!'; diff --git a/packages/cli/src/WorkflowExecuteAdditionalData.ts b/packages/cli/src/WorkflowExecuteAdditionalData.ts index ebb09a776abea..b9fd6d0501c83 100644 --- a/packages/cli/src/WorkflowExecuteAdditionalData.ts +++ b/packages/cli/src/WorkflowExecuteAdditionalData.ts @@ -652,8 +652,8 @@ function hookFunctionsSave(parentProcessMode?: string): IWorkflowExecuteHooks { }, ], nodeFetchedData: [ - async (workflowId: string) => { - eventEmitter.emit(eventEmitter.types.nodeFetchedData, workflowId); + async (workflowId: string, node: INode) => { + eventEmitter.emit(eventEmitter.types.nodeFetchedData, workflowId, node); }, ], }; @@ -756,8 +756,8 @@ function hookFunctionsSaveWorker(): IWorkflowExecuteHooks { }, ], nodeFetchedData: [ - async (workflowId: string) => { - eventEmitter.emit(eventEmitter.types.nodeFetchedData, workflowId); + async (workflowId: string, node: INode) => { + eventEmitter.emit(eventEmitter.types.nodeFetchedData, workflowId, node); }, ], }; diff --git a/packages/cli/src/WorkflowRunnerProcess.ts b/packages/cli/src/WorkflowRunnerProcess.ts index 9c32a9cfbe353..e2a102b4f3fe2 100644 --- a/packages/cli/src/WorkflowRunnerProcess.ts +++ b/packages/cli/src/WorkflowRunnerProcess.ts @@ -16,6 +16,7 @@ import { IExecuteResponsePromiseData, IExecuteWorkflowInfo, ILogger, + INode, INodeExecutionData, INodeType, INodeTypeData, @@ -434,8 +435,8 @@ export class WorkflowRunnerProcess { }, ], nodeFetchedData: [ - async (workflowId: string) => { - await this.sendHookToParentProcess('nodeFetchedData', [workflowId]); + async (workflowId: string, node: INode) => { + await this.sendHookToParentProcess('nodeFetchedData', [workflowId, node]); }, ], }; diff --git a/packages/cli/src/events/WorkflowStatistics.ts b/packages/cli/src/events/WorkflowStatistics.ts index 52899992c0bd0..3c6130422dd91 100644 --- a/packages/cli/src/events/WorkflowStatistics.ts +++ b/packages/cli/src/events/WorkflowStatistics.ts @@ -1,5 +1,5 @@ import { eventEmitter } from 'n8n-core'; -import { IRun, IWorkflowBase } from 'n8n-workflow'; +import { INode, IRun, IWorkflowBase } from 'n8n-workflow'; import { Db, InternalHooksManager } from '..'; import { StatisticsNames } from '../databases/entities/WorkflowStatistics'; import { getWorkflowOwner } from '../UserManagement/UserManagementHelper'; @@ -55,7 +55,8 @@ eventEmitter.on( }, ); -eventEmitter.on(eventEmitter.types.nodeFetchedData, async (workflowId: string) => { +eventEmitter.on(eventEmitter.types.nodeFetchedData, async (workflowId: string, node: INode) => { + console.log(node); // Get the workflow id let id: number; try { diff --git a/packages/core/src/NodeExecuteFunctions.ts b/packages/core/src/NodeExecuteFunctions.ts index cadf2ba6cb419..c89a7ccbc4f61 100644 --- a/packages/core/src/NodeExecuteFunctions.ts +++ b/packages/core/src/NodeExecuteFunctions.ts @@ -546,6 +546,7 @@ function digestAuthAxiosConfig( async function proxyRequestToAxios( workflow: Workflow, additionalData: IWorkflowExecuteAdditionalData, + node: INode, uriOrObject: string | IDataObject, options?: IDataObject, // tslint:disable-next-line:no-any @@ -623,7 +624,7 @@ async function proxyRequestToAxios( body = undefined; } } - await additionalData.hooks?.executeHookFunctions('nodeFetchedData', [workflow.id]); + await additionalData.hooks?.executeHookFunctions('nodeFetchedData', [workflow.id, node]); resolve({ body, headers: response.headers, @@ -640,7 +641,7 @@ async function proxyRequestToAxios( body = undefined; } } - await additionalData.hooks?.executeHookFunctions('nodeFetchedData', [workflow.id]); + await additionalData.hooks?.executeHookFunctions('nodeFetchedData', [workflow.id, node]); resolve(body); } }) @@ -1451,7 +1452,7 @@ export async function requestWithAuthentication( node, additionalData.timezone, ); - return await proxyRequestToAxios(workflow, additionalData, requestOptions as IDataObject); + return await proxyRequestToAxios(workflow, additionalData, node, requestOptions as IDataObject); } catch (error) { try { if (credentialsDecrypted !== undefined) { @@ -1477,7 +1478,7 @@ export async function requestWithAuthentication( additionalData.timezone, ); // retry the request - return await proxyRequestToAxios(workflow, additionalData, requestOptions as IDataObject); + return await proxyRequestToAxios(workflow, additionalData, node, requestOptions as IDataObject); } } throw error; @@ -1934,7 +1935,7 @@ export function getExecutePollFunctions( ); }, request: async (uriOrObject: string | IDataObject, options?: IDataObject | undefined) => { - return proxyRequestToAxios(workflow, additionalData, uriOrObject, options); + return proxyRequestToAxios(workflow, additionalData, node, uriOrObject, options); }, async requestWithAuthentication( this: IAllExecuteFunctions, @@ -2101,7 +2102,7 @@ export function getExecuteTriggerFunctions( ); }, request: async (uriOrObject: string | IDataObject, options?: IDataObject | undefined) => { - return proxyRequestToAxios(workflow, additionalData, uriOrObject, options); + return proxyRequestToAxios(workflow, additionalData, node, uriOrObject, options); }, async requestOAuth2( this: IAllExecuteFunctions, @@ -2371,7 +2372,7 @@ export function getExecuteFunctions( return getBinaryDataBuffer.call(this, inputData, itemIndex, propertyName, inputIndex); }, request: async (uriOrObject: string | IDataObject, options?: IDataObject | undefined) => { - return proxyRequestToAxios(workflow, additionalData, uriOrObject, options); + return proxyRequestToAxios(workflow, additionalData, node, uriOrObject, options); }, async requestOAuth2( this: IAllExecuteFunctions, @@ -2598,7 +2599,7 @@ export function getExecuteSingleFunctions( ); }, request: async (uriOrObject: string | IDataObject, options?: IDataObject | undefined) => { - return proxyRequestToAxios(workflow, additionalData, uriOrObject, options); + return proxyRequestToAxios(workflow, additionalData, node, uriOrObject, options); }, async requestOAuth2( this: IAllExecuteFunctions, @@ -2735,7 +2736,7 @@ export function getLoadOptionsFunctions( ); }, request: async (uriOrObject: string | IDataObject, options?: IDataObject | undefined) => { - return proxyRequestToAxios(workflow, additionalData, uriOrObject, options); + return proxyRequestToAxios(workflow, additionalData, node, uriOrObject, options); }, async requestOAuth2( this: IAllExecuteFunctions, @@ -2883,7 +2884,7 @@ export function getExecuteHookFunctions( ); }, request: async (uriOrObject: string | IDataObject, options?: IDataObject | undefined) => { - return proxyRequestToAxios(workflow, additionalData, uriOrObject, options); + return proxyRequestToAxios(workflow, additionalData, node, uriOrObject, options); }, async requestOAuth2( this: IAllExecuteFunctions, @@ -3072,7 +3073,7 @@ export function getExecuteWebhookFunctions( ); }, request: async (uriOrObject: string | IDataObject, options?: IDataObject | undefined) => { - return proxyRequestToAxios(workflow, additionalData, uriOrObject, options); + return proxyRequestToAxios(workflow, additionalData, node, uriOrObject, options); }, async requestOAuth2( this: IAllExecuteFunctions, From 91bac53dc68f2a7ff033af8eef8799f5e664684b Mon Sep 17 00:00:00 2001 From: freyamade Date: Mon, 7 Nov 2022 14:34:38 +0000 Subject: [PATCH 28/49] :sparkles: - sending node fetch metrics to posthog --- packages/cli/src/InternalHooks.ts | 11 ++++++++ packages/cli/src/WebhookHelpers.ts | 1 - packages/cli/src/events/WorkflowStatistics.ts | 25 ++++++++++++++++--- packages/core/src/NodeExecuteFunctions.ts | 7 +++++- 4 files changed, 39 insertions(+), 5 deletions(-) diff --git a/packages/cli/src/InternalHooks.ts b/packages/cli/src/InternalHooks.ts index e2a64f2288cd5..60e66c6dcc25f 100644 --- a/packages/cli/src/InternalHooks.ts +++ b/packages/cli/src/InternalHooks.ts @@ -488,4 +488,15 @@ export class InternalHooksClass implements IInternalHooksClass { }): Promise { return this.telemetry.track('Workflow first prod success', data, { withPostHog: true }); } + + async onFirstWorkflowDataLoad(data: { + user_id: string; + workflow_id: string | number; + node_type: string; + node_id: string; + credential_type?: string; + credential_id?: string; + }): Promise { + return this.telemetry.track('Workflow first data fetched', data, { withPostHog: true }); + } } diff --git a/packages/cli/src/WebhookHelpers.ts b/packages/cli/src/WebhookHelpers.ts index 73c143089a052..80808712e85be 100644 --- a/packages/cli/src/WebhookHelpers.ts +++ b/packages/cli/src/WebhookHelpers.ts @@ -239,7 +239,6 @@ export async function executeWebhook( NodeExecuteFunctions, executionMode, ); - console.log('webhook fetched data', workflowStartNode); eventEmitter.emit(eventEmitter.types.nodeFetchedData, workflow.id, workflowStartNode); } catch (err) { // Send error response to webhook caller diff --git a/packages/cli/src/events/WorkflowStatistics.ts b/packages/cli/src/events/WorkflowStatistics.ts index 3c6130422dd91..71ab22979c45c 100644 --- a/packages/cli/src/events/WorkflowStatistics.ts +++ b/packages/cli/src/events/WorkflowStatistics.ts @@ -1,5 +1,5 @@ import { eventEmitter } from 'n8n-core'; -import { INode, IRun, IWorkflowBase } from 'n8n-workflow'; +import { INode, INodeCredentialsDetails, IRun, IWorkflowBase } from 'n8n-workflow'; import { Db, InternalHooksManager } from '..'; import { StatisticsNames } from '../databases/entities/WorkflowStatistics'; import { getWorkflowOwner } from '../UserManagement/UserManagementHelper'; @@ -56,7 +56,6 @@ eventEmitter.on( ); eventEmitter.on(eventEmitter.types.nodeFetchedData, async (workflowId: string, node: INode) => { - console.log(node); // Get the workflow id let id: number; try { @@ -74,6 +73,26 @@ eventEmitter.on(eventEmitter.types.nodeFetchedData, async (workflowId: string, n // If response.affected is 1 then we know this was the first time data was loaded into the workflow; do posthog event here if (response.affected) { - console.log('posthog'); + // Compile the metrics + const owner = await getWorkflowOwner(workflowId); + let metrics = { + user_id: owner.id, + workflow_id: workflowId, + node_type: node.type, + node_id: node.id, + }; + + // This is probably naive but I can't see a way for a node to have multiple credentials attached so.. + if (node.credentials) { + Object.entries(node.credentials).forEach(([credName, credDetails]) => { + metrics = Object.assign(metrics, { + credential_type: credName, + credential_id: credDetails.id, + }); + }); + } + + // Send metrics to posthog + await InternalHooksManager.getInstance().onFirstWorkflowDataLoad(metrics); } }); diff --git a/packages/core/src/NodeExecuteFunctions.ts b/packages/core/src/NodeExecuteFunctions.ts index c89a7ccbc4f61..56d4ca9707ee7 100644 --- a/packages/core/src/NodeExecuteFunctions.ts +++ b/packages/core/src/NodeExecuteFunctions.ts @@ -1478,7 +1478,12 @@ export async function requestWithAuthentication( additionalData.timezone, ); // retry the request - return await proxyRequestToAxios(workflow, additionalData, node, requestOptions as IDataObject); + return await proxyRequestToAxios( + workflow, + additionalData, + node, + requestOptions as IDataObject, + ); } } throw error; From c0a124b710ff24fc7862740021c68dad7808d81e Mon Sep 17 00:00:00 2001 From: freyamade Date: Tue, 8 Nov 2022 10:26:27 +0000 Subject: [PATCH 29/49] fix existing tests --- packages/cli/test/unit/Telemetry.test.ts | 68 ++++++++++++++---------- 1 file changed, 41 insertions(+), 27 deletions(-) diff --git a/packages/cli/test/unit/Telemetry.test.ts b/packages/cli/test/unit/Telemetry.test.ts index 888aec61664b8..101368e9b844e 100644 --- a/packages/cli/test/unit/Telemetry.test.ts +++ b/packages/cli/test/unit/Telemetry.test.ts @@ -335,38 +335,52 @@ describe('Telemetry', () => { expect(pulseSpy).toBeCalledTimes(1); expect(spyTrack).toHaveBeenCalledTimes(3); - expect(spyTrack).toHaveBeenNthCalledWith(1, 'Workflow execution count', { - event_version: '2', - workflow_id: '1', - manual_error: { - count: 2, - first: testDateTime, - }, - manual_success: { - count: 2, - first: testDateTime, - }, - prod_error: { - count: 2, - first: testDateTime, + expect(spyTrack).toHaveBeenNthCalledWith( + 1, + 'Workflow execution count', + { + event_version: '2', + workflow_id: '1', + user_id: undefined, + manual_error: { + count: 2, + first: testDateTime, + }, + manual_success: { + count: 2, + first: testDateTime, + }, + prod_error: { + count: 2, + first: testDateTime, + }, + prod_success: { + count: 2, + first: testDateTime, + }, }, - prod_success: { - count: 2, - first: testDateTime, + { withPostHog: true }, + ); + expect(spyTrack).toHaveBeenNthCalledWith( + 2, + 'Workflow execution count', + { + event_version: '2', + workflow_id: '2', + user_id: undefined, + prod_error: { + count: 2, + first: testDateTime, + }, }, - }); - expect(spyTrack).toHaveBeenNthCalledWith(2, 'Workflow execution count', { - event_version: '2', - workflow_id: '2', - prod_error: { - count: 2, - first: testDateTime, - }, - }); + { withPostHog: true }, + ); expect(spyTrack).toHaveBeenNthCalledWith(3, 'pulse'); expect(Object.keys(execBuffer).length).toBe(0); - jest.advanceTimersToNextTimer(); + // Adding a second step here because we believe PostHog may use timers for sending data + // and adding posthog to the above metric was causing the pulseSpy timer to not be ran + jest.advanceTimersToNextTimer(2); execBuffer = telemetry.getCountsBuffer(); expect(Object.keys(execBuffer).length).toBe(0); From b9dd2372a5cfbf4466331ac986cd21e9e743272e Mon Sep 17 00:00:00 2001 From: freyamade Date: Wed, 9 Nov 2022 14:42:34 +0000 Subject: [PATCH 30/49] :sparkles: - Added tests for event handler code --- packages/cli/src/events/WorkflowStatistics.ts | 138 +++++----- packages/cli/src/events/index.ts | 8 +- packages/cli/test/unit/Events.test.ts | 238 ++++++++++++++++++ 3 files changed, 315 insertions(+), 69 deletions(-) create mode 100644 packages/cli/test/unit/Events.test.ts diff --git a/packages/cli/src/events/WorkflowStatistics.ts b/packages/cli/src/events/WorkflowStatistics.ts index 71ab22979c45c..9fd890e67779d 100644 --- a/packages/cli/src/events/WorkflowStatistics.ts +++ b/packages/cli/src/events/WorkflowStatistics.ts @@ -1,65 +1,67 @@ -import { eventEmitter } from 'n8n-core'; -import { INode, INodeCredentialsDetails, IRun, IWorkflowBase } from 'n8n-workflow'; +import { INode, IRun, IWorkflowBase } from 'n8n-workflow'; import { Db, InternalHooksManager } from '..'; import { StatisticsNames } from '../databases/entities/WorkflowStatistics'; import { getWorkflowOwner } from '../UserManagement/UserManagementHelper'; -eventEmitter.on( - eventEmitter.types.workflowExecutionCompleted, - async (workflowData: IWorkflowBase, runData: IRun) => { - // Determine the name of the statistic - const finished = runData.finished ? runData.finished : false; - const manual = runData.mode === 'manual'; - let name: StatisticsNames; +export async function workflowExecutionCompleted( + workflowData: IWorkflowBase, + runData: IRun, +): Promise { + // Determine the name of the statistic + const finished = runData.finished ? runData.finished : false; + const manual = runData.mode === 'manual'; + let name: StatisticsNames; - if (finished) { - if (manual) name = StatisticsNames.manualSuccess; - else name = StatisticsNames.productionSuccess; - } else { - if (manual) name = StatisticsNames.manualError; - else name = StatisticsNames.productionError; - } + if (finished) { + if (manual) name = StatisticsNames.manualSuccess; + else name = StatisticsNames.productionSuccess; + } else { + if (manual) name = StatisticsNames.manualError; + else name = StatisticsNames.productionError; + } - // Get the workflow id - let workflowId: number; - try { - workflowId = parseInt(workflowData.id as string, 10); - } catch (error) { - console.error(`Error ${error as string} when casting workflow ID to a number`); - return; - } + // Get the workflow id + let workflowId: number; + try { + workflowId = parseInt(workflowData.id as string, 10); + if (isNaN(workflowId)) throw new Error('not a number'); + } catch (error) { + console.error(`Error "${error as string}" when casting workflow ID to a number`); + return; + } - // Try insertion and if it fails due to key conflicts then update the existing entry instead - try { - await Db.collections.WorkflowStatistics.insert({ count: 1, name, workflowId }); + // Try insertion and if it fails due to key conflicts then update the existing entry instead + try { + await Db.collections.WorkflowStatistics.insert({ count: 1, name, workflowId }); - // If we're here we can check if we're sending the first production success metric - if (name !== StatisticsNames.productionSuccess) return; + // If we're here we can check if we're sending the first production success metric + if (name !== StatisticsNames.productionSuccess) return; - // Get the owner of the workflow so we can send the metric - const owner = await getWorkflowOwner(workflowId); - const metrics = { - user_id: owner.id, - workflow_id: workflowId, - }; + // Get the owner of the workflow so we can send the metric + const owner = await getWorkflowOwner(workflowId); + const metrics = { + user_id: owner.id, + workflow_id: workflowId, + }; - // Send the metrics - await InternalHooksManager.getInstance().onFirstProductionWorkflowSuccess(metrics); - } catch (error) { - // Do we just assume it's a conflict error? If there is any other sort of error in the DB it should trigger here too - await Db.collections.WorkflowStatistics.update( - { workflowId, name }, - { count: () => '"count" + 1' }, - ); - } - }, -); + // Send the metrics + await InternalHooksManager.getInstance().onFirstProductionWorkflowSuccess(metrics); + } catch (error) { + console.error(error); + // Do we just assume it's a conflict error? If there is any other sort of error in the DB it should trigger here too + await Db.collections.WorkflowStatistics.update( + { workflowId, name }, + { count: () => '"count" + 1' }, + ); + } +} -eventEmitter.on(eventEmitter.types.nodeFetchedData, async (workflowId: string, node: INode) => { +export async function nodeFetchedData(workflowId: string, node: INode): Promise { // Get the workflow id let id: number; try { id = parseInt(workflowId, 10); + if (isNaN(id)) throw new Error('not a number'); } catch (error) { console.error(`Error ${error as string} when casting workflow ID to a number`); return; @@ -72,27 +74,27 @@ eventEmitter.on(eventEmitter.types.nodeFetchedData, async (workflowId: string, n ); // If response.affected is 1 then we know this was the first time data was loaded into the workflow; do posthog event here - if (response.affected) { - // Compile the metrics - const owner = await getWorkflowOwner(workflowId); - let metrics = { - user_id: owner.id, - workflow_id: workflowId, - node_type: node.type, - node_id: node.id, - }; + if (!response.affected) return; - // This is probably naive but I can't see a way for a node to have multiple credentials attached so.. - if (node.credentials) { - Object.entries(node.credentials).forEach(([credName, credDetails]) => { - metrics = Object.assign(metrics, { - credential_type: credName, - credential_id: credDetails.id, - }); - }); - } + // Compile the metrics + const owner = await getWorkflowOwner(workflowId); + let metrics = { + user_id: owner.id, + workflow_id: id, + node_type: node.type, + node_id: node.id, + }; - // Send metrics to posthog - await InternalHooksManager.getInstance().onFirstWorkflowDataLoad(metrics); + // This is probably naive but I can't see a way for a node to have multiple credentials attached so.. + if (node.credentials) { + Object.entries(node.credentials).forEach(([credName, credDetails]) => { + metrics = Object.assign(metrics, { + credential_type: credName, + credential_id: credDetails.id, + }); + }); } -}); + + // Send metrics to posthog + await InternalHooksManager.getInstance().onFirstWorkflowDataLoad(metrics); +} diff --git a/packages/cli/src/events/index.ts b/packages/cli/src/events/index.ts index b468d302e25a1..35f6c5d8d9745 100644 --- a/packages/cli/src/events/index.ts +++ b/packages/cli/src/events/index.ts @@ -1 +1,7 @@ -import './WorkflowStatistics'; +import { eventEmitter } from 'n8n-core'; +import { nodeFetchedData, workflowExecutionCompleted } from './WorkflowStatistics'; + +// Check for undefined as during testing these functions end up undefined for some reason +if (nodeFetchedData) eventEmitter.on(eventEmitter.types.nodeFetchedData, nodeFetchedData); +if (workflowExecutionCompleted) + eventEmitter.on(eventEmitter.types.workflowExecutionCompleted, workflowExecutionCompleted); diff --git a/packages/cli/test/unit/Events.test.ts b/packages/cli/test/unit/Events.test.ts new file mode 100644 index 0000000000000..3d56e4113e3da --- /dev/null +++ b/packages/cli/test/unit/Events.test.ts @@ -0,0 +1,238 @@ +import config from '../../config'; +import { InternalHooksManager } from '../../src'; +import { nodeFetchedData, workflowExecutionCompleted } from '../../src/events/WorkflowStatistics'; +import { WorkflowExecuteMode } from 'n8n-workflow'; + +const FAKE_USER_ID = 'abcde-fghij'; + +const mockedFirstProductionWorkflowSuccess = jest.fn((...args) => {}); +const mockedFirstWorkflowDataLoad = jest.fn((...args) => {}); +const mockedError = jest.spyOn(console, 'error'); + +jest.spyOn(InternalHooksManager, 'getInstance').mockImplementation((...args) => { + const actual = jest.requireActual('../../src/InternalHooks'); + return { + ...actual, + onFirstProductionWorkflowSuccess: mockedFirstProductionWorkflowSuccess, + onFirstWorkflowDataLoad: mockedFirstWorkflowDataLoad, + }; +}); +jest.mock('../../src/Db', () => { + return { + collections: { + Workflow: { + update: jest.fn(({ id, dataLoaded }, updateArgs) => { + if (id === 1) return { affected: 1 }; + return { affected: 0 }; + }), + }, + WorkflowStatistics: { + insert: jest.fn(({ count, name, workflowId }) => { + if (workflowId === -1) throw new Error('test error'); + return null; + }), + update: jest.fn((...args) => {}), + }, + }, + }; +}); +jest.mock('../../src/UserManagement/UserManagementHelper', () => { + return { + getWorkflowOwner: jest.fn((workflowId) => { + return { id: FAKE_USER_ID }; + }), + }; +}); + +describe('Events', () => { + beforeAll(() => { + config.set('diagnostics.enabled', true); + config.set('deployment.type', 'n8n-testing'); + }); + + afterAll(() => { + jest.clearAllTimers(); + jest.useRealTimers(); + }); + + beforeEach(() => { + mockedFirstProductionWorkflowSuccess.mockClear(); + mockedFirstWorkflowDataLoad.mockClear(); + mockedError.mockClear(); + }); + + afterEach(() => {}); + + describe('workflowExecutionCompleted', () => { + test('should fail with an invalid workflowId', async () => { + const workflow = { + id: 'abcde', + name: '', + active: false, + createdAt: new Date(), + updatedAt: new Date(), + nodes: [], + connections: {}, + }; + const runData = { + finished: true, + data: { resultData: { runData: {} } }, + mode: 'internal' as WorkflowExecuteMode, + startedAt: new Date(), + }; + await workflowExecutionCompleted(workflow, runData); + expect(mockedError).toBeCalledTimes(1); + }); + + test('should create metrics for production successes', async () => { + // Call the function with a production success result, ensure metrics hook gets called + const workflow = { + id: '1', + name: '', + active: false, + createdAt: new Date(), + updatedAt: new Date(), + nodes: [], + connections: {}, + }; + const runData = { + finished: true, + data: { resultData: { runData: {} } }, + mode: 'internal' as WorkflowExecuteMode, + startedAt: new Date(), + }; + await workflowExecutionCompleted(workflow, runData); + expect(mockedFirstProductionWorkflowSuccess).toBeCalledTimes(1); + expect(mockedFirstProductionWorkflowSuccess).toHaveBeenNthCalledWith(1, { + user_id: FAKE_USER_ID, + workflow_id: parseInt(workflow.id, 10), + }); + }); + + test('should only create metrics for production successes', async () => { + // Call the function with a non production success result, ensure metrics hook is never called + const workflow = { + id: '1', + name: '', + active: false, + createdAt: new Date(), + updatedAt: new Date(), + nodes: [], + connections: {}, + }; + const runData = { + finished: false, + data: { resultData: { runData: {} } }, + mode: 'internal' as WorkflowExecuteMode, + startedAt: new Date(), + }; + await workflowExecutionCompleted(workflow, runData); + expect(mockedFirstProductionWorkflowSuccess).toBeCalledTimes(0); + }); + + test('should not send metrics for updated entries', async () => { + // Call the function with the id that causes insert to fail, ensure update is called *and* metrics aren't sent + const mockedError = jest.spyOn(console, 'error'); + const workflow = { + id: '-1', + name: '', + active: false, + createdAt: new Date(), + updatedAt: new Date(), + nodes: [], + connections: {}, + }; + const runData = { + finished: true, + data: { resultData: { runData: {} } }, + mode: 'internal' as WorkflowExecuteMode, + startedAt: new Date(), + }; + mockedError.mockClear(); + await workflowExecutionCompleted(workflow, runData); + expect(mockedError).toBeCalled(); + expect(mockedFirstProductionWorkflowSuccess).toBeCalledTimes(0); + }); + }); + + describe('nodeFetchedData', () => { + test('should fail with an invalid workflowId', async () => { + const workflowId = 'abcde'; + const node = { + id: 'abcde', + name: 'test node', + typeVersion: 1, + type: '', + position: [0, 0] as [number, number], + parameters: {}, + }; + await nodeFetchedData(workflowId, node); + expect(mockedError).toBeCalledTimes(1); + }); + + test('should create metrics when the db is updated', async () => { + // Call the function with a production success result, ensure metrics hook gets called + const workflowId = '1'; + const node = { + id: 'abcde', + name: 'test node', + typeVersion: 1, + type: '', + position: [0, 0] as [number, number], + parameters: {}, + }; + await nodeFetchedData(workflowId, node); + expect(mockedFirstWorkflowDataLoad).toBeCalledTimes(1); + expect(mockedFirstWorkflowDataLoad).toHaveBeenNthCalledWith(1, { + user_id: FAKE_USER_ID, + workflow_id: parseInt(workflowId, 10), + node_type: node.type, + node_id: node.id, + }); + }); + + test('should create metrics with credentials when the db is updated', async () => { + // Call the function with a production success result, ensure metrics hook gets called + const workflowId = '1'; + const node = { + id: 'abcde', + name: 'test node', + typeVersion: 1, + type: '', + position: [0, 0] as [number, number], + parameters: {}, + credentials: { + testCredentials: { + id: '1', + name: 'Test Credentials', + }, + }, + }; + await nodeFetchedData(workflowId, node); + expect(mockedFirstWorkflowDataLoad).toBeCalledTimes(1); + expect(mockedFirstWorkflowDataLoad).toHaveBeenNthCalledWith(1, { + user_id: FAKE_USER_ID, + workflow_id: parseInt(workflowId, 10), + node_type: node.type, + node_id: node.id, + credential_type: 'testCredentials', + credential_id: node.credentials.testCredentials.id, + }); + }); + + test('should not send metrics for entries that already have the flag set', async () => { + // Fetch data for workflow 2 which is set up to not be altered in the mocks + const workflowId = '2'; + const node = { + id: 'abcde', + name: 'test node', + typeVersion: 1, + type: '', + position: [0, 0] as [number, number], + parameters: {}, + }; + await nodeFetchedData(workflowId, node); + expect(mockedFirstWorkflowDataLoad).toBeCalledTimes(0); + }); + }); +}); From d256362be4902b0cccf176011417722a1189f30f Mon Sep 17 00:00:00 2001 From: freyamade Date: Wed, 9 Nov 2022 15:09:22 +0000 Subject: [PATCH 31/49] :hammer: - Fixing config imports --- packages/cli/src/databases/entities/WorkflowStatistics.ts | 6 ++---- .../migrations/mysqldb/1664196174002-WorkflowStatistics.ts | 2 +- .../postgresdb/1664196174001-WorkflowStatistics.ts | 2 +- .../migrations/sqlite/1664196174000-WorkflowStatistics.ts | 2 +- packages/cli/test/unit/Events.test.ts | 2 +- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/cli/src/databases/entities/WorkflowStatistics.ts b/packages/cli/src/databases/entities/WorkflowStatistics.ts index 8aca130b76ded..22800a5fe8e26 100644 --- a/packages/cli/src/databases/entities/WorkflowStatistics.ts +++ b/packages/cli/src/databases/entities/WorkflowStatistics.ts @@ -1,12 +1,10 @@ -/* eslint-disable import/no-cycle */ import { Column, Entity, RelationId, ManyToOne, PrimaryColumn, UpdateDateColumn } from 'typeorm'; -import * as config from '../../../config'; +import config from '@/config'; import { DatabaseType } from '../..'; import { WorkflowEntity } from './WorkflowEntity'; -// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types function getTimestampSyntax() { - const dbType = config.getEnv('database.type'); + const dbType = config.getEnv('database.type') as DatabaseType; const map: { [key in DatabaseType]: string } = { sqlite: "STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')", diff --git a/packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts index 262df435c4d84..a4ef5f9d4bd97 100644 --- a/packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts +++ b/packages/cli/src/databases/migrations/mysqldb/1664196174002-WorkflowStatistics.ts @@ -1,6 +1,6 @@ import { MigrationInterface, QueryRunner } from 'typeorm'; import { logMigrationEnd, logMigrationStart } from '../../utils/migrationHelpers'; -import config from '../../../../config'; +import config from '@/config'; export class WorkflowStatistics1664196174002 implements MigrationInterface { name = 'WorkflowStatistics1664196174002'; diff --git a/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts index 86d09605aa696..1023d8b3c6385 100644 --- a/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts +++ b/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts @@ -1,6 +1,6 @@ import { MigrationInterface, QueryRunner } from 'typeorm'; import { getTablePrefix, logMigrationEnd, logMigrationStart } from '../../utils/migrationHelpers'; -import config from '../../../../config'; +import config from '@/config'; export class WorkflowStatistics1664196174001 implements MigrationInterface { name = 'WorkflowStatistics1664196174001'; diff --git a/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts index 8da8e0a55b8a8..f400350ccd4e4 100644 --- a/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts +++ b/packages/cli/src/databases/migrations/sqlite/1664196174000-WorkflowStatistics.ts @@ -1,6 +1,6 @@ import { MigrationInterface, QueryRunner } from 'typeorm'; import { logMigrationEnd, logMigrationStart } from '../../utils/migrationHelpers'; -import config from '../../../../config'; +import config from '@/config'; export class WorkflowStatistics1664196174000 implements MigrationInterface { name = 'WorkflowStatistics1664196174000'; diff --git a/packages/cli/test/unit/Events.test.ts b/packages/cli/test/unit/Events.test.ts index 3d56e4113e3da..5ac9129cb45c4 100644 --- a/packages/cli/test/unit/Events.test.ts +++ b/packages/cli/test/unit/Events.test.ts @@ -1,4 +1,4 @@ -import config from '../../config'; +import config from '@/config'; import { InternalHooksManager } from '../../src'; import { nodeFetchedData, workflowExecutionCompleted } from '../../src/events/WorkflowStatistics'; import { WorkflowExecuteMode } from 'n8n-workflow'; From 36141f96f968e6593e2f97d111a96c1a76637553 Mon Sep 17 00:00:00 2001 From: freyamade Date: Thu, 10 Nov 2022 18:07:37 +0000 Subject: [PATCH 32/49] :hammer: - fixed update issue for mysql --- packages/cli/src/events/WorkflowStatistics.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/events/WorkflowStatistics.ts b/packages/cli/src/events/WorkflowStatistics.ts index 9fd890e67779d..00f1328a5783b 100644 --- a/packages/cli/src/events/WorkflowStatistics.ts +++ b/packages/cli/src/events/WorkflowStatistics.ts @@ -51,7 +51,7 @@ export async function workflowExecutionCompleted( // Do we just assume it's a conflict error? If there is any other sort of error in the DB it should trigger here too await Db.collections.WorkflowStatistics.update( { workflowId, name }, - { count: () => '"count" + 1' }, + { count: () => 'count + 1' }, ); } } From 25a8dd872dc2acbdc8911ca33ca402dc7665ca2a Mon Sep 17 00:00:00 2001 From: freyamade Date: Fri, 11 Nov 2022 09:10:18 +0000 Subject: [PATCH 33/49] :bug: - Fix first event having null timestamp --- .../src/databases/entities/WorkflowStatistics.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/databases/entities/WorkflowStatistics.ts b/packages/cli/src/databases/entities/WorkflowStatistics.ts index 22800a5fe8e26..e0dc8713a088f 100644 --- a/packages/cli/src/databases/entities/WorkflowStatistics.ts +++ b/packages/cli/src/databases/entities/WorkflowStatistics.ts @@ -1,4 +1,12 @@ -import { Column, Entity, RelationId, ManyToOne, PrimaryColumn, UpdateDateColumn } from 'typeorm'; +import { + CreateDateColumn, + Column, + Entity, + RelationId, + ManyToOne, + PrimaryColumn, + UpdateDateColumn, +} from 'typeorm'; import config from '@/config'; import { DatabaseType } from '../..'; import { WorkflowEntity } from './WorkflowEntity'; @@ -28,6 +36,10 @@ export class WorkflowStatistics { @Column() count: number; + @CreateDateColumn({ + precision: 3, + default: () => getTimestampSyntax(), + }) @UpdateDateColumn({ precision: 3, default: () => getTimestampSyntax(), From d74cba5557987308d0de293d282d6e625f9fd49a Mon Sep 17 00:00:00 2001 From: freyamade Date: Mon, 14 Nov 2022 10:31:10 +0000 Subject: [PATCH 34/49] :bug: - Fix issue where creation timestamps were null --- packages/cli/src/Server.ts | 2 +- .../databases/entities/WorkflowStatistics.ts | 36 ++----------------- .../src/databases/migrations/mysqldb/index.ts | 2 +- .../1664196174001-WorkflowStatistics.ts | 2 +- .../databases/migrations/postgresdb/index.ts | 2 +- .../src/databases/migrations/sqlite/index.ts | 2 +- packages/cli/src/events/WorkflowStatistics.ts | 9 +++-- 7 files changed, 15 insertions(+), 40 deletions(-) diff --git a/packages/cli/src/Server.ts b/packages/cli/src/Server.ts index 4e3f73afe28c1..3b61c62195345 100644 --- a/packages/cli/src/Server.ts +++ b/packages/cli/src/Server.ts @@ -794,7 +794,7 @@ class App { this.app.use(`/${this.restEndpoint}/tags`, tagsController); // Returns parameter values which normally get loaded from an external API or - // get gene// Call the function with the id that causes insert to fail, ensure update is called *and* metrics aren't sentrated dynamically + // get generated dynamically this.app.get( `/${this.restEndpoint}/node-parameter-options`, ResponseHelper.send( diff --git a/packages/cli/src/databases/entities/WorkflowStatistics.ts b/packages/cli/src/databases/entities/WorkflowStatistics.ts index e0dc8713a088f..6ef1e1d6bf79b 100644 --- a/packages/cli/src/databases/entities/WorkflowStatistics.ts +++ b/packages/cli/src/databases/entities/WorkflowStatistics.ts @@ -1,29 +1,7 @@ -import { - CreateDateColumn, - Column, - Entity, - RelationId, - ManyToOne, - PrimaryColumn, - UpdateDateColumn, -} from 'typeorm'; -import config from '@/config'; -import { DatabaseType } from '../..'; +import { Column, Entity, RelationId, ManyToOne, PrimaryColumn } from 'typeorm'; +import { datetimeColumnType } from './AbstractEntity'; import { WorkflowEntity } from './WorkflowEntity'; -function getTimestampSyntax() { - const dbType = config.getEnv('database.type') as DatabaseType; - - const map: { [key in DatabaseType]: string } = { - sqlite: "STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')", - postgresdb: 'CURRENT_TIMESTAMP(3)', - mysqldb: 'CURRENT_TIMESTAMP(3)', - mariadb: 'CURRENT_TIMESTAMP(3)', - }; - - return map[dbType]; -} - export enum StatisticsNames { productionSuccess = 'production_success', productionError = 'production_error', @@ -36,15 +14,7 @@ export class WorkflowStatistics { @Column() count: number; - @CreateDateColumn({ - precision: 3, - default: () => getTimestampSyntax(), - }) - @UpdateDateColumn({ - precision: 3, - default: () => getTimestampSyntax(), - onUpdate: getTimestampSyntax(), - }) + @Column(datetimeColumnType) latestEvent: Date; @PrimaryColumn({ length: 128 }) diff --git a/packages/cli/src/databases/migrations/mysqldb/index.ts b/packages/cli/src/databases/migrations/mysqldb/index.ts index 52c6ad1280e69..20d5eb01a1124 100644 --- a/packages/cli/src/databases/migrations/mysqldb/index.ts +++ b/packages/cli/src/databases/migrations/mysqldb/index.ts @@ -48,8 +48,8 @@ export const mysqlMigrations = [ AddNodeIds1658932910559, AddJsonKeyPinData1659895550980, CreateCredentialsUserRole1660062385367, - WorkflowStatistics1664196174002, CreateWorkflowsEditorRole1663755770894, CreateCredentialUsageTable1665484192213, RemoveCredentialUsageTable1665754637026, + WorkflowStatistics1664196174002, ]; diff --git a/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts b/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts index 1023d8b3c6385..41959520b2574 100644 --- a/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts +++ b/packages/cli/src/databases/migrations/postgresdb/1664196174001-WorkflowStatistics.ts @@ -13,7 +13,7 @@ export class WorkflowStatistics1664196174001 implements MigrationInterface { await queryRunner.query( `CREATE TABLE ${tablePrefix}workflow_statistics ( "count" INTEGER DEFAULT 0, - "latestEvent" DATE, + "latestEvent" TIMESTAMP, "name" VARCHAR(128) NOT NULL, "workflowId" INTEGER, PRIMARY KEY("workflowId", "name"), diff --git a/packages/cli/src/databases/migrations/postgresdb/index.ts b/packages/cli/src/databases/migrations/postgresdb/index.ts index c7722f9cd4c5d..e66c3cd5c15b9 100644 --- a/packages/cli/src/databases/migrations/postgresdb/index.ts +++ b/packages/cli/src/databases/migrations/postgresdb/index.ts @@ -44,8 +44,8 @@ export const postgresMigrations = [ CreateCredentialsUserRole1660062385367, AddNodeIds1658932090381, AddJsonKeyPinData1659902242948, - WorkflowStatistics1664196174001, CreateWorkflowsEditorRole1663755770893, CreateCredentialUsageTable1665484192212, RemoveCredentialUsageTable1665754637025, + WorkflowStatistics1664196174001, ]; diff --git a/packages/cli/src/databases/migrations/sqlite/index.ts b/packages/cli/src/databases/migrations/sqlite/index.ts index fb30002dfad77..288e41af0caa2 100644 --- a/packages/cli/src/databases/migrations/sqlite/index.ts +++ b/packages/cli/src/databases/migrations/sqlite/index.ts @@ -42,10 +42,10 @@ const sqliteMigrations = [ AddNodeIds1658930531669, AddJsonKeyPinData1659888469333, CreateCredentialsUserRole1660062385367, - WorkflowStatistics1664196174000, CreateWorkflowsEditorRole1663755770892, CreateCredentialUsageTable1665484192211, RemoveCredentialUsageTable1665754637024, + WorkflowStatistics1664196174000, ]; export { sqliteMigrations }; diff --git a/packages/cli/src/events/WorkflowStatistics.ts b/packages/cli/src/events/WorkflowStatistics.ts index 00f1328a5783b..02b0f441d8047 100644 --- a/packages/cli/src/events/WorkflowStatistics.ts +++ b/packages/cli/src/events/WorkflowStatistics.ts @@ -32,7 +32,12 @@ export async function workflowExecutionCompleted( // Try insertion and if it fails due to key conflicts then update the existing entry instead try { - await Db.collections.WorkflowStatistics.insert({ count: 1, name, workflowId }); + await Db.collections.WorkflowStatistics.insert({ + count: 1, + name, + workflowId, + latestEvent: new Date(), + }); // If we're here we can check if we're sending the first production success metric if (name !== StatisticsNames.productionSuccess) return; @@ -51,7 +56,7 @@ export async function workflowExecutionCompleted( // Do we just assume it's a conflict error? If there is any other sort of error in the DB it should trigger here too await Db.collections.WorkflowStatistics.update( { workflowId, name }, - { count: () => 'count + 1' }, + { count: () => 'count + 1', latestEvent: new Date() }, ); } } From 6a205e07115670f3e3eb5da9c8425c1d208d7fbb Mon Sep 17 00:00:00 2001 From: freyamade Date: Mon, 14 Nov 2022 14:07:49 +0000 Subject: [PATCH 35/49] :sparkles: - Add permission checking to stats apis --- packages/cli/src/api/workflowStats.api.ts | 34 +++++++++++++++++++---- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/packages/cli/src/api/workflowStats.api.ts b/packages/cli/src/api/workflowStats.api.ts index aedf284a81f24..9e5e2b9e7d281 100644 --- a/packages/cli/src/api/workflowStats.api.ts +++ b/packages/cli/src/api/workflowStats.api.ts @@ -1,3 +1,4 @@ +import { User } from '@/databases/entities/User'; import express from 'express'; import { LoggerProxy } from 'n8n-workflow'; import { @@ -6,6 +7,7 @@ import { IWorkflowStatisticsDataLoaded, IWorkflowStatisticsTimestamps, ResponseHelper, + whereClause, } from '..'; import { WorkflowEntity } from '../databases/entities/WorkflowEntity'; import { StatisticsNames } from '../databases/entities/WorkflowStatistics'; @@ -27,15 +29,37 @@ workflowStatsController.use((req, res, next) => { }); // Helper function that validates the ID, throws an error if not valud -async function checkWorkflowId(workflowId: string): Promise { +async function checkWorkflowId(workflowId: string, user: User): Promise { const workflow = await Db.collections.Workflow.findOne(workflowId); if (!workflow) { throw new ResponseHelper.ResponseError( - `The workflow with the ID "${workflowId}" does not exist.`, + `Workflow with ID "${workflowId}" could not be found..`, 404, 404, ); } + + // Check permissions + const shared = await Db.collections.SharedWorkflow.findOne({ + relations: ['workflow'], + where: whereClause({ + user, + entityType: 'workflow', + entityId: workflowId, + }), + }); + + if (!shared) { + LoggerProxy.info('User attempted to read a workflow without permissions', { + workflowId, + userId: user.id, + }); + throw new ResponseHelper.ResponseError( + `Workflow with ID "${workflowId}" could not be found.`, + undefined, + 400, + ); + } return workflow; } @@ -49,7 +73,7 @@ workflowStatsController.get( const workflowId = req.params.id; // Check that the id is valid - await checkWorkflowId(workflowId); + await checkWorkflowId(workflowId, req.user); // Find the stats for this workflow const stats = await Db.collections.WorkflowStatistics.find({ @@ -100,7 +124,7 @@ workflowStatsController.get( const workflowId = req.params.id; // Check that the id is valid - await checkWorkflowId(workflowId); + await checkWorkflowId(workflowId, req.user); // Find the stats for this workflow const stats = await Db.collections.WorkflowStatistics.find({ @@ -151,7 +175,7 @@ workflowStatsController.get( const workflowId = req.params.id; // Get the corresponding workflow - const workflow = await checkWorkflowId(workflowId); + const workflow = await checkWorkflowId(workflowId, req.user); const data: IWorkflowStatisticsDataLoaded = { dataLoaded: workflow.dataLoaded, From 152169a3baf35a8459c38cab16756e9f327c3252 Mon Sep 17 00:00:00 2001 From: freyamade Date: Mon, 28 Nov 2022 10:00:37 +0000 Subject: [PATCH 36/49] bring files up to date with recent refactors --- .../cli/src/WorkflowExecuteAdditionalData.ts | 1 + packages/cli/src/api/workflowStats.api.ts | 17 ++++------------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/packages/cli/src/WorkflowExecuteAdditionalData.ts b/packages/cli/src/WorkflowExecuteAdditionalData.ts index 986ccc7254d52..5f2ac8b7579ff 100644 --- a/packages/cli/src/WorkflowExecuteAdditionalData.ts +++ b/packages/cli/src/WorkflowExecuteAdditionalData.ts @@ -21,6 +21,7 @@ import { IDataObject, IExecuteData, IExecuteWorkflowInfo, + INode, INodeExecutionData, INodeParameters, IRun, diff --git a/packages/cli/src/api/workflowStats.api.ts b/packages/cli/src/api/workflowStats.api.ts index 9e5e2b9e7d281..69fb4370f55af 100644 --- a/packages/cli/src/api/workflowStats.api.ts +++ b/packages/cli/src/api/workflowStats.api.ts @@ -7,7 +7,6 @@ import { IWorkflowStatisticsDataLoaded, IWorkflowStatisticsTimestamps, ResponseHelper, - whereClause, } from '..'; import { WorkflowEntity } from '../databases/entities/WorkflowEntity'; import { StatisticsNames } from '../databases/entities/WorkflowStatistics'; @@ -32,21 +31,17 @@ workflowStatsController.use((req, res, next) => { async function checkWorkflowId(workflowId: string, user: User): Promise { const workflow = await Db.collections.Workflow.findOne(workflowId); if (!workflow) { - throw new ResponseHelper.ResponseError( - `Workflow with ID "${workflowId}" could not be found..`, - 404, - 404, - ); + throw new ResponseHelper.NotFoundError(`Workflow with ID "${workflowId}" could not be found.`); } // Check permissions const shared = await Db.collections.SharedWorkflow.findOne({ relations: ['workflow'], - where: whereClause({ + where: { user, entityType: 'workflow', entityId: workflowId, - }), + }, }); if (!shared) { @@ -54,11 +49,7 @@ async function checkWorkflowId(workflowId: string, user: User): Promise Date: Tue, 29 Nov 2022 11:16:59 +0000 Subject: [PATCH 37/49] :arrow_up: - Updating posthog-node to latest --- packages/cli/package.json | 2 +- pnpm-lock.yaml | 29 +++++++++++------------------ 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index ae187cb90b16a..394e8d054cf9c 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -165,7 +165,7 @@ "passport-jwt": "^4.0.0", "picocolors": "^1.0.0", "pg": "^8.3.0", - "posthog-node": "^1.3.0", + "posthog-node": "^2.2.2", "prom-client": "^13.1.0", "psl": "^1.8.0", "semver": "^7.3.8", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e4ce259e7b8f0..c4326e076320d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -190,7 +190,7 @@ importers: passport-jwt: ^4.0.0 pg: ^8.3.0 picocolors: ^1.0.0 - posthog-node: ^1.3.0 + posthog-node: ^2.2.2 prom-client: ^13.1.0 psl: ^1.8.0 run-script-os: ^1.0.7 @@ -273,7 +273,7 @@ importers: passport-jwt: 4.0.0 pg: 8.8.0 picocolors: 1.0.0 - posthog-node: 1.3.0 + posthog-node: 2.2.2 prom-client: 13.2.0 psl: 1.9.0 semver: 7.3.8 @@ -7652,10 +7652,11 @@ packages: - debug dev: false - /axios/0.24.0: - resolution: {integrity: sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==} + /axios/0.27.2: + resolution: {integrity: sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==} dependencies: - follow-redirects: 1.15.2_debug@3.2.7 + follow-redirects: 1.15.2 + form-data: 4.0.0 transitivePeerDependencies: - debug dev: false @@ -7672,7 +7673,7 @@ packages: /axios/1.1.3: resolution: {integrity: sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA==} dependencies: - follow-redirects: 1.15.2_debug@3.2.7 + follow-redirects: 1.15.2 form-data: 4.0.0 proxy-from-env: 1.1.0 transitivePeerDependencies: @@ -17421,19 +17422,11 @@ packages: xtend: 4.0.2 dev: false - /posthog-node/1.3.0: - resolution: {integrity: sha512-2+VhqiY/rKIqKIXyvemBFHbeijHE25sP7eKltnqcFqAssUE6+sX6vusN9A4luzToOqHQkUZexiCKxvuGagh7JA==} - engines: {node: '>=4'} - hasBin: true + /posthog-node/2.2.2: + resolution: {integrity: sha512-aXYe/D+28kF63W8Cz53t09ypEORz+ULeDCahdAqhVrRm2scbOXFbtnn0GGhvMpYe45grepLKuwui9KxrZ2ZuMw==} + engines: {node: '>=14.17.0'} dependencies: - axios: 0.24.0 - axios-retry: 3.3.1 - component-type: 1.2.1 - join-component: 1.1.0 - md5: 2.3.0 - ms: 2.1.3 - remove-trailing-slash: 0.1.1 - uuid: 8.3.2 + axios: 0.27.2 transitivePeerDependencies: - debug dev: false From 3d53b55e87c9ee4cfe08de56d27df29be4f322a6 Mon Sep 17 00:00:00 2001 From: freyamade Date: Tue, 29 Nov 2022 11:17:23 +0000 Subject: [PATCH 38/49] :sparkles: - Ensuring backend pushes feat flags to PH --- packages/cli/src/telemetry/index.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/telemetry/index.ts b/packages/cli/src/telemetry/index.ts index b744d53d38e2b..f37261b3b3758 100644 --- a/packages/cli/src/telemetry/index.ts +++ b/packages/cli/src/telemetry/index.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/no-unsafe-call */ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ import RudderStack from '@rudderstack/rudder-sdk-node'; -import PostHog from 'posthog-node'; +import { PostHog } from 'posthog-node'; import { ITelemetryTrackProperties, LoggerProxy } from 'n8n-workflow'; import config from '@/config'; import { IExecutionTrackProperties } from '@/Interfaces'; @@ -191,6 +191,7 @@ export class Telemetry { return Promise.all([ this.postHog.capture({ distinctId: payload.userId, + sendFeatureFlags: true, ...payload, }), this.rudderStack.track(payload), @@ -207,7 +208,7 @@ export class Telemetry { async isFeatureFlagEnabled( featureFlagName: string, { user_id: userId }: ITelemetryTrackProperties = {}, - ): Promise { + ): Promise { if (!this.postHog) return Promise.resolve(false); const fullId = [this.instanceId, userId].join('#'); From c02060bb1e02424ef92e04bb79c9a0bd243c3d5f Mon Sep 17 00:00:00 2001 From: freyamade Date: Wed, 30 Nov 2022 16:11:25 +0000 Subject: [PATCH 39/49] :bug: - use getsharedworkflow function for checking permissions in api --- packages/cli/src/api/workflowStats.api.ts | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/packages/cli/src/api/workflowStats.api.ts b/packages/cli/src/api/workflowStats.api.ts index 69fb4370f55af..d8ddd40fc29e7 100644 --- a/packages/cli/src/api/workflowStats.api.ts +++ b/packages/cli/src/api/workflowStats.api.ts @@ -1,4 +1,5 @@ import { User } from '@/databases/entities/User'; +import { getSharedWorkflowIds } from '@/WorkflowHelpers'; import express from 'express'; import { LoggerProxy } from 'n8n-workflow'; import { @@ -35,16 +36,9 @@ async function checkWorkflowId(workflowId: string, user: User): Promise Date: Wed, 30 Nov 2022 16:39:38 +0000 Subject: [PATCH 40/49] :rotating_light: - fixing slight issue in tests --- packages/cli/test/unit/Telemetry.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/cli/test/unit/Telemetry.test.ts b/packages/cli/test/unit/Telemetry.test.ts index 66c78b217a4cb..d69ac70eef310 100644 --- a/packages/cli/test/unit/Telemetry.test.ts +++ b/packages/cli/test/unit/Telemetry.test.ts @@ -334,7 +334,7 @@ describe('Telemetry', () => { execBuffer = telemetry.getCountsBuffer(); expect(pulseSpy).toBeCalledTimes(1); - expect(spyTrack).toHaveBeenCalledTimes(3); + expect(spyTrack).toHaveBeenCalledTimes(2); expect(spyTrack).toHaveBeenNthCalledWith( 1, 'Workflow execution count', @@ -375,7 +375,6 @@ describe('Telemetry', () => { }, { withPostHog: true }, ); - expect(spyTrack).toHaveBeenNthCalledWith(3, 'pulse'); expect(Object.keys(execBuffer).length).toBe(0); // Adding a second step here because we believe PostHog may use timers for sending data From 4af5c2cef090981d9fb0e1e4064b04185b07811e Mon Sep 17 00:00:00 2001 From: freyamade Date: Wed, 30 Nov 2022 17:12:20 +0000 Subject: [PATCH 41/49] :rotating_light: - more test fixes --- packages/cli/test/unit/Telemetry.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/cli/test/unit/Telemetry.test.ts b/packages/cli/test/unit/Telemetry.test.ts index d69ac70eef310..48ad111b8c574 100644 --- a/packages/cli/test/unit/Telemetry.test.ts +++ b/packages/cli/test/unit/Telemetry.test.ts @@ -334,7 +334,7 @@ describe('Telemetry', () => { execBuffer = telemetry.getCountsBuffer(); expect(pulseSpy).toBeCalledTimes(1); - expect(spyTrack).toHaveBeenCalledTimes(2); + expect(spyTrack).toHaveBeenCalledTimes(3); expect(spyTrack).toHaveBeenNthCalledWith( 1, 'Workflow execution count', @@ -375,11 +375,12 @@ describe('Telemetry', () => { }, { withPostHog: true }, ); + expect(spyTrack).toHaveBeenNthCalledWith(3, 'pulse'); expect(Object.keys(execBuffer).length).toBe(0); // Adding a second step here because we believe PostHog may use timers for sending data // and adding posthog to the above metric was causing the pulseSpy timer to not be ran - jest.advanceTimersToNextTimer(2); + jest.advanceTimersToNextTimer(); execBuffer = telemetry.getCountsBuffer(); expect(Object.keys(execBuffer).length).toBe(0); From d7af982c128065f554e71aea090f107caa82e9cc Mon Sep 17 00:00:00 2001 From: Omar Ajoue Date: Thu, 1 Dec 2022 14:55:53 +0100 Subject: [PATCH 42/49] fix: Remove commented import --- packages/core/src/NodeExecuteFunctions.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/core/src/NodeExecuteFunctions.ts b/packages/core/src/NodeExecuteFunctions.ts index 8ec71485c9912..1722ff56afdf1 100644 --- a/packages/core/src/NodeExecuteFunctions.ts +++ b/packages/core/src/NodeExecuteFunctions.ts @@ -91,7 +91,6 @@ import axios, { import url, { URL, URLSearchParams } from 'url'; import { BinaryDataManager } from './BinaryDataManager'; import { - // eventEmitter, ICredentialTestFunctions, IHookFunctions, ILoadOptionsFunctions, From 407c146beb68154490ad972717d2238f5e78ce2d Mon Sep 17 00:00:00 2001 From: freyamade Date: Tue, 6 Dec 2022 11:15:03 +0000 Subject: [PATCH 43/49] :hammer: - moved changeWorkflowID to middleware --- packages/cli/src/api/workflowStats.api.ts | 38 ++++++++++++----------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/packages/cli/src/api/workflowStats.api.ts b/packages/cli/src/api/workflowStats.api.ts index d8ddd40fc29e7..c7fa26dd618e6 100644 --- a/packages/cli/src/api/workflowStats.api.ts +++ b/packages/cli/src/api/workflowStats.api.ts @@ -1,4 +1,5 @@ import { User } from '@/databases/entities/User'; +import { whereClause } from '@/UserManagement/UserManagementHelper'; import { getSharedWorkflowIds } from '@/WorkflowHelpers'; import express from 'express'; import { LoggerProxy } from 'n8n-workflow'; @@ -19,33 +20,38 @@ export const workflowStatsController = express.Router(); /** * Initialise Logger if needed */ -workflowStatsController.use((req, res, next) => { +workflowStatsController.use(async (req: ExecutionRequest.Get, res, next) => { try { LoggerProxy.getInstance(); } catch (error) { LoggerProxy.init(getLogger()); } + + // Call the checkWorkflowId function here + await checkWorkflowId(req.params.id, req.user); + next(); }); // Helper function that validates the ID, throws an error if not valud -async function checkWorkflowId(workflowId: string, user: User): Promise { - const workflow = await Db.collections.Workflow.findOne(workflowId); - if (!workflow) { - throw new ResponseHelper.NotFoundError(`Workflow with ID "${workflowId}" could not be found.`); - } - +async function checkWorkflowId(workflowId: string, user: User): Promise { // Check permissions - const sharedWorkflowIds = await getSharedWorkflowIds(user); - - if (sharedWorkflowIds.length === 0 || !sharedWorkflowIds.includes(parseInt(workflowId, 10))) { + const shared = await Db.collections.SharedWorkflow.findOne({ + relations: ['workflow'], + where: whereClause({ + user, + entityType: 'workflow', + entityId: workflowId, + }), + }); + + if (!shared) { LoggerProxy.info('User attempted to read a workflow without permissions', { workflowId, userId: user.id, }); throw new ResponseHelper.NotFoundError(`Workflow with ID "${workflowId}" could not be found.`); } - return workflow; } /** @@ -57,9 +63,6 @@ workflowStatsController.get( // Get counts from DB const workflowId = req.params.id; - // Check that the id is valid - await checkWorkflowId(workflowId, req.user); - // Find the stats for this workflow const stats = await Db.collections.WorkflowStatistics.find({ select: ['count', 'name'], @@ -108,9 +111,6 @@ workflowStatsController.get( // Get times from DB const workflowId = req.params.id; - // Check that the id is valid - await checkWorkflowId(workflowId, req.user); - // Find the stats for this workflow const stats = await Db.collections.WorkflowStatistics.find({ select: ['latestEvent', 'name'], @@ -160,7 +160,9 @@ workflowStatsController.get( const workflowId = req.params.id; // Get the corresponding workflow - const workflow = await checkWorkflowId(workflowId, req.user); + const workflow = await Db.collections.Workflow.findOne(workflowId); + // It will be valid if we reach this point, this is just for TS + if (!workflow) return { dataLoaded: false }; const data: IWorkflowStatisticsDataLoaded = { dataLoaded: workflow.dataLoaded, From 7617ed0a9ee4378eb891ea1ad15a748d0c470edb Mon Sep 17 00:00:00 2001 From: freyamade Date: Tue, 6 Dec 2022 11:55:01 +0000 Subject: [PATCH 44/49] :shirt: - clean up lint errors --- packages/cli/src/api/workflowStats.api.ts | 34 +++++++++++------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/packages/cli/src/api/workflowStats.api.ts b/packages/cli/src/api/workflowStats.api.ts index c7fa26dd618e6..c9c76ce959241 100644 --- a/packages/cli/src/api/workflowStats.api.ts +++ b/packages/cli/src/api/workflowStats.api.ts @@ -1,6 +1,5 @@ import { User } from '@/databases/entities/User'; import { whereClause } from '@/UserManagement/UserManagementHelper'; -import { getSharedWorkflowIds } from '@/WorkflowHelpers'; import express from 'express'; import { LoggerProxy } from 'n8n-workflow'; import { @@ -10,29 +9,12 @@ import { IWorkflowStatisticsTimestamps, ResponseHelper, } from '..'; -import { WorkflowEntity } from '../databases/entities/WorkflowEntity'; import { StatisticsNames } from '../databases/entities/WorkflowStatistics'; import { getLogger } from '../Logger'; import { ExecutionRequest } from '../requests'; export const workflowStatsController = express.Router(); -/** - * Initialise Logger if needed - */ -workflowStatsController.use(async (req: ExecutionRequest.Get, res, next) => { - try { - LoggerProxy.getInstance(); - } catch (error) { - LoggerProxy.init(getLogger()); - } - - // Call the checkWorkflowId function here - await checkWorkflowId(req.params.id, req.user); - - next(); -}); - // Helper function that validates the ID, throws an error if not valud async function checkWorkflowId(workflowId: string, user: User): Promise { // Check permissions @@ -54,6 +36,22 @@ async function checkWorkflowId(workflowId: string, user: User): Promise { } } +/** + * Initialise Logger if needed, and check the workflowId is acceptable + */ + workflowStatsController.use(async (req: ExecutionRequest.Get, res, next) => { + try { + LoggerProxy.getInstance(); + } catch (error) { + LoggerProxy.init(getLogger()); + } + + // Call the checkWorkflowId function here + await checkWorkflowId(req.params.id, req.user); + + next(); +}); + /** * GET /workflow-stats/:id/counts/ */ From 7260b01e7aa99f009a00d72d197ff3709529ec95 Mon Sep 17 00:00:00 2001 From: freyamade Date: Tue, 6 Dec 2022 12:41:57 +0000 Subject: [PATCH 45/49] :shirt: - accidental typo --- packages/cli/src/api/workflowStats.api.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/api/workflowStats.api.ts b/packages/cli/src/api/workflowStats.api.ts index c9c76ce959241..6d2f7a877f4e8 100644 --- a/packages/cli/src/api/workflowStats.api.ts +++ b/packages/cli/src/api/workflowStats.api.ts @@ -39,7 +39,7 @@ async function checkWorkflowId(workflowId: string, user: User): Promise { /** * Initialise Logger if needed, and check the workflowId is acceptable */ - workflowStatsController.use(async (req: ExecutionRequest.Get, res, next) => { +workflowStatsController.use(async (req: ExecutionRequest.Get, res, next) => { try { LoggerProxy.getInstance(); } catch (error) { From 8dbd8f8bed52fab94f5cc45f32ed58f6c18a498a Mon Sep 17 00:00:00 2001 From: freyamade Date: Tue, 6 Dec 2022 13:33:02 +0000 Subject: [PATCH 46/49] :hammer: - Made checkWorkflowId work as middleware --- packages/cli/src/api/workflowStats.api.ts | 30 ++++++++++++++++------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/packages/cli/src/api/workflowStats.api.ts b/packages/cli/src/api/workflowStats.api.ts index 6d2f7a877f4e8..7156524c42ff6 100644 --- a/packages/cli/src/api/workflowStats.api.ts +++ b/packages/cli/src/api/workflowStats.api.ts @@ -15,8 +15,8 @@ import { ExecutionRequest } from '../requests'; export const workflowStatsController = express.Router(); -// Helper function that validates the ID, throws an error if not valud -async function checkWorkflowId(workflowId: string, user: User): Promise { +// Helper function that validates the ID, return a flag stating whether the request is allowed +async function checkWorkflowId(workflowId: string, user: User): Promise { // Check permissions const shared = await Db.collections.SharedWorkflow.findOne({ relations: ['workflow'], @@ -32,26 +32,36 @@ async function checkWorkflowId(workflowId: string, user: User): Promise { workflowId, userId: user.id, }); - throw new ResponseHelper.NotFoundError(`Workflow with ID "${workflowId}" could not be found.`); + return false; } + return true; } /** - * Initialise Logger if needed, and check the workflowId is acceptable + * Initialise Logger if needed */ -workflowStatsController.use(async (req: ExecutionRequest.Get, res, next) => { +workflowStatsController.use((req, res, next) => { try { LoggerProxy.getInstance(); } catch (error) { LoggerProxy.init(getLogger()); } - // Call the checkWorkflowId function here - await checkWorkflowId(req.params.id, req.user); - next(); }); +/** + * Check that the workflow ID is valid and allowed to be read by the user + */ +workflowStatsController.use(async (req: ExecutionRequest.Get, res, next) => { + const allowed = await checkWorkflowId(req.params.id, req.user); + if (allowed) next(); + + // Otherwise, make and return an error + const err = new ResponseHelper.NotFoundError(`Workflow ${req.params.id} does not exist.`); + next(err); +}); + /** * GET /workflow-stats/:id/counts/ */ @@ -160,7 +170,9 @@ workflowStatsController.get( // Get the corresponding workflow const workflow = await Db.collections.Workflow.findOne(workflowId); // It will be valid if we reach this point, this is just for TS - if (!workflow) return { dataLoaded: false }; + if (!workflow) { + return { dataLoaded: false }; + } const data: IWorkflowStatisticsDataLoaded = { dataLoaded: workflow.dataLoaded, From 36982187b72357f87cf951e590b5ea03bd83316a Mon Sep 17 00:00:00 2001 From: freyamade Date: Tue, 6 Dec 2022 13:52:53 +0000 Subject: [PATCH 47/49] :bug: - next doesn't halt execution --- packages/cli/src/api/workflowStats.api.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/api/workflowStats.api.ts b/packages/cli/src/api/workflowStats.api.ts index 7156524c42ff6..0fb39c99a552c 100644 --- a/packages/cli/src/api/workflowStats.api.ts +++ b/packages/cli/src/api/workflowStats.api.ts @@ -56,10 +56,11 @@ workflowStatsController.use((req, res, next) => { workflowStatsController.use(async (req: ExecutionRequest.Get, res, next) => { const allowed = await checkWorkflowId(req.params.id, req.user); if (allowed) next(); - - // Otherwise, make and return an error - const err = new ResponseHelper.NotFoundError(`Workflow ${req.params.id} does not exist.`); - next(err); + else { + // Otherwise, make and return an error + const err = new ResponseHelper.NotFoundError(`Workflow ${req.params.id} does not exist.`); + next(err); + } }); /** From c91093355bec44e62d8597d8dfce0b7e33db1631 Mon Sep 17 00:00:00 2001 From: Omar Ajoue Date: Tue, 6 Dec 2022 14:59:26 +0100 Subject: [PATCH 48/49] refactor: add brackets to improve legibility --- packages/cli/src/api/workflowStats.api.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/api/workflowStats.api.ts b/packages/cli/src/api/workflowStats.api.ts index 0fb39c99a552c..a67f911e124ca 100644 --- a/packages/cli/src/api/workflowStats.api.ts +++ b/packages/cli/src/api/workflowStats.api.ts @@ -55,8 +55,9 @@ workflowStatsController.use((req, res, next) => { */ workflowStatsController.use(async (req: ExecutionRequest.Get, res, next) => { const allowed = await checkWorkflowId(req.params.id, req.user); - if (allowed) next(); - else { + if (allowed) { + next(); + } else { // Otherwise, make and return an error const err = new ResponseHelper.NotFoundError(`Workflow ${req.params.id} does not exist.`); next(err); From 850fbb85f31df8813dd3a3c52d355e2e384155e9 Mon Sep 17 00:00:00 2001 From: freyamade Date: Tue, 6 Dec 2022 14:07:14 +0000 Subject: [PATCH 49/49] :shirt: - remove the name `err` --- packages/cli/src/api/workflowStats.api.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/api/workflowStats.api.ts b/packages/cli/src/api/workflowStats.api.ts index 0fb39c99a552c..578f5c8f7ded8 100644 --- a/packages/cli/src/api/workflowStats.api.ts +++ b/packages/cli/src/api/workflowStats.api.ts @@ -58,8 +58,8 @@ workflowStatsController.use(async (req: ExecutionRequest.Get, res, next) => { if (allowed) next(); else { // Otherwise, make and return an error - const err = new ResponseHelper.NotFoundError(`Workflow ${req.params.id} does not exist.`); - next(err); + const response = new ResponseHelper.NotFoundError(`Workflow ${req.params.id} does not exist.`); + next(response); } });