From ba4ea9c024307dcb248c71bf2702371d733d42ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=A4=95=E0=A4=BE=E0=A4=B0=E0=A4=A4=E0=A5=8B=E0=A4=AB?= =?UTF-8?q?=E0=A5=8D=E0=A4=AB=E0=A5=87=E0=A4=B2=E0=A4=B8=E0=A5=8D=E0=A4=95?= =?UTF-8?q?=E0=A5=8D=E0=A4=B0=E0=A4=BF=E0=A4=AA=E0=A5=8D=E0=A4=9F=E2=84=A2?= Date: Thu, 24 Nov 2022 15:52:15 +0100 Subject: [PATCH 01/20] fix: Apply SpecialNodeParameters to the types generated for the frontend (no-changelog) (#4715) fix: apply SpecialNodeParameters to the types generated for the frontend (no-changelog) --- packages/core/bin/generate-ui-types | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/core/bin/generate-ui-types b/packages/core/bin/generate-ui-types index f8377a21b130d..e57dd4885b6cc 100755 --- a/packages/core/bin/generate-ui-types +++ b/packages/core/bin/generate-ui-types @@ -16,7 +16,14 @@ LoggerProxy.init({ const credentialTypes = Object.values(loader.credentialTypes).map((data) => data.type); const nodeTypes = Object.values(loader.nodeTypes) - .map((data) => data.type) + .map((data) => { + const nodeType = NodeHelpers.getVersionedNodeType(data.type); + const applyParameters = NodeHelpers.getSpecialNodeParameters(nodeType); + if (applyParameters.length) { + nodeType.description.properties.unshift(...applyParameters); + } + return data.type; + }) .flatMap((nodeData) => { const allNodeTypes = NodeHelpers.getVersionedNodeTypeAll(nodeData); return allNodeTypes.map((element) => element.description); From 44d6185118f66f804cffe0bbb537ad40611d68ff Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Thu, 24 Nov 2022 15:43:46 +0000 Subject: [PATCH 02/20] =?UTF-8?q?:bookmark:=20Release=C2=A0n8n-workflow@0.?= =?UTF-8?q?126.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/workflow/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/workflow/package.json b/packages/workflow/package.json index f8f84c8ee968f..8e650510eaa3b 100644 --- a/packages/workflow/package.json +++ b/packages/workflow/package.json @@ -1,6 +1,6 @@ { "name": "n8n-workflow", - "version": "0.125.0", + "version": "0.126.0", "description": "Workflow base code of n8n", "license": "SEE LICENSE IN LICENSE.md", "homepage": "https://n8n.io", From 7e70155518f18166deadb49253c9dd95b45c1d70 Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Thu, 24 Nov 2022 15:43:57 +0000 Subject: [PATCH 03/20] :arrow_up: Set n8n-workflow@0.126.0 on n8n-core --- packages/core/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/package.json b/packages/core/package.json index b52dd30f82156..26e3c79ba5dc7 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -53,7 +53,7 @@ "form-data": "^4.0.0", "lodash.get": "^4.4.2", "mime-types": "^2.1.27", - "n8n-workflow": "~0.125.0", + "n8n-workflow": "~0.126.0", "oauth-1.0a": "^2.2.6", "p-cancelable": "^2.0.0", "qs": "^6.10.1", From 78b70499761f54b44e9ead42846cbed4ff2d84e7 Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Thu, 24 Nov 2022 15:43:57 +0000 Subject: [PATCH 04/20] =?UTF-8?q?:bookmark:=20Release=C2=A0n8n-core@0.144.?= =?UTF-8?q?0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/core/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/package.json b/packages/core/package.json index 26e3c79ba5dc7..5a4e6dc1e20a5 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "n8n-core", - "version": "0.143.1", + "version": "0.144.0", "description": "Core functionality of n8n", "license": "SEE LICENSE IN LICENSE.md", "homepage": "https://n8n.io", From 6ba8c9ce8ae2572172d6cef455932e0ba648db2f Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Thu, 24 Nov 2022 15:44:11 +0000 Subject: [PATCH 05/20] :arrow_up: Set n8n-core@0.144.0 and n8n-workflow@0.126.0 on n8n-node-dev --- packages/node-dev/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/node-dev/package.json b/packages/node-dev/package.json index b86f82f6061b2..b0ca1eed241d0 100644 --- a/packages/node-dev/package.json +++ b/packages/node-dev/package.json @@ -59,8 +59,8 @@ "change-case": "^4.1.1", "fast-glob": "^3.2.5", "inquirer": "^7.0.1", - "n8n-core": "~0.143.0", - "n8n-workflow": "~0.125.0", + "n8n-core": "~0.144.0", + "n8n-workflow": "~0.126.0", "oauth-1.0a": "^2.2.6", "replace-in-file": "^6.0.0", "request": "^2.88.2", From b9874b2c4c5479762b8297808e105a6002ab0e7c Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Thu, 24 Nov 2022 15:44:11 +0000 Subject: [PATCH 06/20] =?UTF-8?q?:bookmark:=20Release=C2=A0n8n-node-dev@0.?= =?UTF-8?q?83.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/node-dev/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/node-dev/package.json b/packages/node-dev/package.json index b0ca1eed241d0..39fac76502214 100644 --- a/packages/node-dev/package.json +++ b/packages/node-dev/package.json @@ -1,6 +1,6 @@ { "name": "n8n-node-dev", - "version": "0.82.1", + "version": "0.83.0", "description": "CLI to simplify n8n credentials/node development", "license": "SEE LICENSE IN LICENSE.md", "homepage": "https://n8n.io", From f924c81a517b639a1f73db03eb9508df82cfbd13 Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Thu, 24 Nov 2022 15:44:23 +0000 Subject: [PATCH 07/20] :arrow_up: Set n8n-core@0.144.0 and n8n-workflow@0.126.0 on n8n-nodes-base --- packages/nodes-base/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/nodes-base/package.json b/packages/nodes-base/package.json index 14e3a6ae8ef32..d58552ea28052 100644 --- a/packages/nodes-base/package.json +++ b/packages/nodes-base/package.json @@ -751,7 +751,7 @@ "@types/xml2js": "^0.4.3", "eslint-plugin-n8n-nodes-base": "^1.11.0", "gulp": "^4.0.0", - "n8n-workflow": "~0.125.0" + "n8n-workflow": "~0.126.0" }, "dependencies": { "@kafkajs/confluent-schema-registry": "1.0.6", @@ -790,7 +790,7 @@ "mqtt": "4.2.6", "mssql": "^8.1.2", "mysql2": "~2.3.0", - "n8n-core": "~0.143.0", + "n8n-core": "~0.144.0", "node-html-markdown": "^1.1.3", "node-ssh": "^12.0.0", "nodemailer": "^6.7.1", From bfaf04f463260b55c3f026f394583f12f8b57cf8 Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Thu, 24 Nov 2022 15:44:23 +0000 Subject: [PATCH 08/20] =?UTF-8?q?:bookmark:=20Release=C2=A0n8n-nodes-base@?= =?UTF-8?q?0.202.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/nodes-base/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nodes-base/package.json b/packages/nodes-base/package.json index d58552ea28052..1fc6123315493 100644 --- a/packages/nodes-base/package.json +++ b/packages/nodes-base/package.json @@ -1,6 +1,6 @@ { "name": "n8n-nodes-base", - "version": "0.201.1", + "version": "0.202.0", "description": "Base nodes of n8n", "license": "SEE LICENSE IN LICENSE.md", "homepage": "https://n8n.io", From 90a081d6bfa5fc1a618d38f8fa1b5a18624cc288 Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Thu, 24 Nov 2022 15:45:45 +0000 Subject: [PATCH 09/20] =?UTF-8?q?:bookmark:=20Release=C2=A0n8n-design-syst?= =?UTF-8?q?em@0.44.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/design-system/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/design-system/package.json b/packages/design-system/package.json index 178f33ac57173..5e2af41df2144 100644 --- a/packages/design-system/package.json +++ b/packages/design-system/package.json @@ -1,6 +1,6 @@ { "name": "n8n-design-system", - "version": "0.43.0", + "version": "0.44.0", "license": "SEE LICENSE IN LICENSE.md", "homepage": "https://n8n.io", "author": { From 2f84e5a6075555816bcc16949e6f0aaf37e9e368 Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Thu, 24 Nov 2022 15:46:07 +0000 Subject: [PATCH 10/20] :arrow_up: Set n8n-design-system@0.44.0 and n8n-workflow@0.126.0 on n8n-editor-ui --- packages/editor-ui/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/editor-ui/package.json b/packages/editor-ui/package.json index 351388ff5efaf..517712dd52f66 100644 --- a/packages/editor-ui/package.json +++ b/packages/editor-ui/package.json @@ -56,8 +56,8 @@ "lodash.set": "^4.3.2", "luxon": "^2.3.0", "monaco-editor": "^0.33.0", - "n8n-design-system": "~0.43.0", - "n8n-workflow": "~0.125.0", + "n8n-design-system": "~0.44.0", + "n8n-workflow": "~0.126.0", "normalize-wheel": "^1.0.1", "pinia": "^2.0.22", "prismjs": "^1.17.1", From 203af41c2c11569fa7f28d7d739c98baf89b253e Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Thu, 24 Nov 2022 15:46:08 +0000 Subject: [PATCH 11/20] =?UTF-8?q?:bookmark:=20Release=C2=A0n8n-editor-ui@0?= =?UTF-8?q?.170.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/editor-ui/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/editor-ui/package.json b/packages/editor-ui/package.json index 517712dd52f66..196fb7d43fac0 100644 --- a/packages/editor-ui/package.json +++ b/packages/editor-ui/package.json @@ -1,6 +1,6 @@ { "name": "n8n-editor-ui", - "version": "0.169.0", + "version": "0.170.0", "description": "Workflow Editor UI for n8n", "license": "SEE LICENSE IN LICENSE.md", "homepage": "https://n8n.io", From e1ff7330c45617c32a3c69c408d562c429c3f72d Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Thu, 24 Nov 2022 15:48:12 +0000 Subject: [PATCH 12/20] :arrow_up: Set n8n-core@0.144.0, n8n-editor-ui@0.170.0, n8n-nodes-base@0.202.0 and n8n-workflow@0.126.0 on n8n --- packages/cli/package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index f5f5a11d3e447..db9ac3b14cc21 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -150,10 +150,10 @@ "lodash.split": "^4.4.2", "lodash.unset": "^4.5.2", "mysql2": "~2.3.0", - "n8n-core": "~0.143.0", - "n8n-editor-ui": "~0.169.0", - "n8n-nodes-base": "~0.201.1", - "n8n-workflow": "~0.125.0", + "n8n-core": "~0.144.0", + "n8n-editor-ui": "~0.170.0", + "n8n-nodes-base": "~0.202.0", + "n8n-workflow": "~0.126.0", "nodemailer": "^6.7.1", "oauth-1.0a": "^2.2.6", "open": "^7.0.0", From 63b3cda7603992441aa30b4a011463e91bad9a68 Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Thu, 24 Nov 2022 15:48:12 +0000 Subject: [PATCH 13/20] =?UTF-8?q?:bookmark:=20Release=C2=A0n8n@0.204.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/cli/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index db9ac3b14cc21..ae187cb90b16a 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "n8n", - "version": "0.203.1", + "version": "0.204.0", "description": "n8n Workflow Automation Tool", "license": "SEE LICENSE IN LICENSE.md", "homepage": "https://n8n.io", From 602b1e56d604cd82ccf80e5d8866011632f20390 Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Thu, 24 Nov 2022 16:53:37 +0100 Subject: [PATCH 14/20] :books: Update CHANGELOG.md and main package.json to 0.204.0 --- CHANGELOG.md | 41 ++++++++++++++++++++++++++++++++++++++++- package.json | 2 +- pnpm-lock.yaml | 22 +++++++++++----------- 3 files changed, 52 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d87f7d97e466c..d2ba0a516599b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,43 @@ +# [0.204.0](https://github.com/n8n-io/n8n/compare/n8n@0.203.1...n8n@0.204.0) (2022-11-24) + + +### Bug Fixes + +* **core:** Fix `$items().length` behavior in `executeOnce` mode ([#4694](https://github.com/n8n-io/n8n/issues/4694)) ([b87c122](https://github.com/n8n-io/n8n/commit/b87c12285fa2b339e726f81e271715e258b6a075)) +* **core:** Fix for unused imports ([a6df51b](https://github.com/n8n-io/n8n/commit/a6df51b6e83fa25dad962ab212123edb3c055cac)) +* **core:** Use CredentialsOverwrites when testing credentials ([#4675](https://github.com/n8n-io/n8n/issues/4675)) ([772ec78](https://github.com/n8n-io/n8n/commit/772ec78349b5b6877bf681f3262951e3a4e34fe4)) +* Disable workflow locking due to issues ([#4708](https://github.com/n8n-io/n8n/issues/4708)) ([ee6ac5d](https://github.com/n8n-io/n8n/commit/ee6ac5d3417a2f308a7f4a3cda18c01fcec57faf)) +* **editor:** Fix for missing node connections in dev environment ([#4707](https://github.com/n8n-io/n8n/issues/4707)) ([b18ae18](https://github.com/n8n-io/n8n/commit/b18ae18a6b10fb125f3eed73103e33e3519cd82c)) +* **editor:** Fix missing resource locator component ([#4649](https://github.com/n8n-io/n8n/issues/4649)) ([44182f2](https://github.com/n8n-io/n8n/commit/44182f23a5e62c53209b3fa19edd1727586551ff)) +* **editor:** Prevent node-creator tabs from showing when toggled by CanvasAddButton ([#4661](https://github.com/n8n-io/n8n/issues/4661)) ([60746dc](https://github.com/n8n-io/n8n/commit/60746dc92ee1b6c33015e2a6a0d34bc981aa1dd5)) +* **editor:** Table view column limit tooltip ([#4655](https://github.com/n8n-io/n8n/issues/4655)) ([3ac9ba3](https://github.com/n8n-io/n8n/commit/3ac9ba3491c0dc1de283bc4285a243e02747f971)) +* Fix broken n8n-info-tip slots ([#4665](https://github.com/n8n-io/n8n/issues/4665)) ([6c99223](https://github.com/n8n-io/n8n/commit/6c992233a053db1ea235f785a52a754c1e694555)) +* **IF Node:** Fix "Is Empty" and "Is Not Empty" operation fails for date objects ([#4670](https://github.com/n8n-io/n8n/issues/4670)) ([753f4c9](https://github.com/n8n-io/n8n/commit/753f4c9a7d34c8d3329d4dc024fcf272f6f47ff3)) +* Remove redundant await in node's api request functions without try/catch ([#4639](https://github.com/n8n-io/n8n/issues/4639)) ([67983e8](https://github.com/n8n-io/n8n/commit/67983e8f945397d3fb0be55fdeb47609be92b2cb)) +* **Schedule Trigger Node:** Fixes inconsitent behavior with cron and weekly intervals ([#4558](https://github.com/n8n-io/n8n/issues/4558)) ([2fb8ed8](https://github.com/n8n-io/n8n/commit/2fb8ed825b18118fc0783e95d1551ee2ce8c3a38)) +* Workflow activation should not crash if one of the credential is invalid ([#4671](https://github.com/n8n-io/n8n/issues/4671)) ([c0e13c2](https://github.com/n8n-io/n8n/commit/c0e13c2a8f9374e9c65aae3ce4102e37c993cf74)) + + +### Features + +* Add credentials E2E test suite and page object ([#4596](https://github.com/n8n-io/n8n/issues/4596)) ([b5b44d1](https://github.com/n8n-io/n8n/commit/b5b44d1b598e67ef7e735d7cdfb5233ca72caca6)) +* Add save confirmation modal when leaving sharing modal ([#4683](https://github.com/n8n-io/n8n/issues/4683)) ([173badc](https://github.com/n8n-io/n8n/commit/173badc4e099ebb818686dc5a25c2192c138bcd9)) +* Add share button to workflows list ([#4681](https://github.com/n8n-io/n8n/issues/4681)) ([a356d7b](https://github.com/n8n-io/n8n/commit/a356d7bdbadd5a4c69c61c5a5a30e75e9765e3d2)) +* **core:** Add license support to n8n ([#4566](https://github.com/n8n-io/n8n/issues/4566)) ([30e5d3d](https://github.com/n8n-io/n8n/commit/30e5d3d04c3457780875cc36637c8c1ea14ec783)) +* **core:** Lazy-load nodes and credentials to reduce baseline memory usage ([#4577](https://github.com/n8n-io/n8n/issues/4577)) ([b6c57e1](https://github.com/n8n-io/n8n/commit/b6c57e19fc5683dd7fb9eabb60ec4e89359c59eb)) +* **editor:** Add workflows list status filter ([#4690](https://github.com/n8n-io/n8n/issues/4690)) ([5364e7f](https://github.com/n8n-io/n8n/commit/5364e7fc9250421b799adc28b3e47dc75819ec7d)) +* Show delete button based on workflow permissions ([#4686](https://github.com/n8n-io/n8n/issues/4686)) ([4f64e26](https://github.com/n8n-io/n8n/commit/4f64e26a83c7e62c98d93c38bf3dcb6cdfaadb58)) +* Show toast when saving workflow sharing settings ([#4684](https://github.com/n8n-io/n8n/issues/4684)) ([6f8d0de](https://github.com/n8n-io/n8n/commit/6f8d0de55dc9c3c1cdb17329a8560ee8453c639a)) +* Switch owner subview to all subview if has shared resources ([#4672](https://github.com/n8n-io/n8n/issues/4672)) ([e3e17e5](https://github.com/n8n-io/n8n/commit/e3e17e5dac685b1230e39bbef247312419b71f9b)) +* Use longer stack-traces when error-reporting is enabled ([#4674](https://github.com/n8n-io/n8n/issues/4674)) ([de5b0b0](https://github.com/n8n-io/n8n/commit/de5b0b03fedede680f5a6e0f4dc73770b888bf46)) + + +### Performance Improvements + +* **Code Node:** Improve n8n item key validation performance ([#4669](https://github.com/n8n-io/n8n/issues/4669)) ([740513b](https://github.com/n8n-io/n8n/commit/740513b42440b8760cd488659e92abe9951462b0)) + + + ## [0.203.1](https://github.com/n8n-io/n8n/compare/n8n@0.203.0...n8n@0.203.1) (2022-11-18) ### Bug Fixes @@ -9,7 +49,6 @@ ### Bug Fixes - Add back mapping hint when parameter is focused ([#4634](https://github.com/n8n-io/n8n/issues/4634)) ([b35172e](https://github.com/n8n-io/n8n/commit/b35172e442a131f76c2d902d451356ab937bba48)) -- Change feature flag for starting at empty WF state for PH experiments (no-changelog) ([#4606](https://github.com/n8n-io/n8n/issues/4606)) ([d1ffc58](https://github.com/n8n-io/n8n/commit/d1ffc58aa4947e5b9f956568061a981c8ab37e65)) - **core:** Deduplicate error handling in nodes ([#4319](https://github.com/n8n-io/n8n/issues/4319)) ([c7133ec](https://github.com/n8n-io/n8n/commit/c7133ecd3fe6f022a537b6edb4c006d6786efad2)) - **editor:** Add 'Stop execution' button to execution preview ([#4632](https://github.com/n8n-io/n8n/issues/4632)) ([be7672a](https://github.com/n8n-io/n8n/commit/be7672a177bfcf997ec241af7c628a90312849b1)) - **editor:** Curb direct item access linting ([#4591](https://github.com/n8n-io/n8n/issues/4591)) ([271cd06](https://github.com/n8n-io/n8n/commit/271cd06a6ac6274a83a6a71fe76072281edf3724)) diff --git a/package.json b/package.json index f51d4bcf85d4b..c6865795ff9fd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "n8n", - "version": "0.203.1", + "version": "0.204.0", "private": true, "homepage": "https://n8n.io", "engines": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0bb816cebdc48..9a8fe6527c7d0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -172,10 +172,10 @@ importers: lodash.split: ^4.4.2 lodash.unset: ^4.5.2 mysql2: ~2.3.0 - n8n-core: ~0.143.0 - n8n-editor-ui: ~0.169.0 - n8n-nodes-base: ~0.201.1 - n8n-workflow: ~0.125.0 + n8n-core: ~0.144.0 + n8n-editor-ui: ~0.170.0 + n8n-nodes-base: ~0.202.0 + n8n-workflow: ~0.126.0 nodemailer: ^6.7.1 nodemon: ^2.0.2 oauth-1.0a: ^2.2.6 @@ -346,7 +346,7 @@ importers: form-data: ^4.0.0 lodash.get: ^4.4.2 mime-types: ^2.1.27 - n8n-workflow: ~0.125.0 + n8n-workflow: ~0.126.0 oauth-1.0a: ^2.2.6 p-cancelable: ^2.0.0 qs: ^6.10.1 @@ -517,8 +517,8 @@ importers: lodash.set: ^4.3.2 luxon: ^2.3.0 monaco-editor: ^0.33.0 - n8n-design-system: ~0.43.0 - n8n-workflow: ~0.125.0 + n8n-design-system: ~0.44.0 + n8n-workflow: ~0.126.0 normalize-wheel: ^1.0.1 pinia: ^2.0.22 prismjs: ^1.17.1 @@ -640,8 +640,8 @@ importers: change-case: ^4.1.1 fast-glob: ^3.2.5 inquirer: ^7.0.1 - n8n-core: ~0.143.0 - n8n-workflow: ~0.125.0 + n8n-core: ~0.144.0 + n8n-workflow: ~0.126.0 oauth-1.0a: ^2.2.6 replace-in-file: ^6.0.0 request: ^2.88.2 @@ -731,8 +731,8 @@ importers: mqtt: 4.2.6 mssql: ^8.1.2 mysql2: ~2.3.0 - n8n-core: ~0.143.0 - n8n-workflow: ~0.125.0 + n8n-core: ~0.144.0 + n8n-workflow: ~0.126.0 node-html-markdown: ^1.1.3 node-ssh: ^12.0.0 nodemailer: ^6.7.1 From 07e4743a3e1a93542fac661aabba0a36d95c098a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=A4=95=E0=A4=BE=E0=A4=B0=E0=A4=A4=E0=A5=8B=E0=A4=AB?= =?UTF-8?q?=E0=A5=8D=E0=A4=AB=E0=A5=87=E0=A4=B2=E0=A4=B8=E0=A5=8D=E0=A4=95?= =?UTF-8?q?=E0=A5=8D=E0=A4=B0=E0=A4=BF=E0=A4=AA=E0=A5=8D=E0=A4=9F=E2=84=A2?= Date: Thu, 24 Nov 2022 16:54:43 +0100 Subject: [PATCH 15/20] refactor(core): Reduce memory usage in the Webhook node (#4640) use file streaming to pass webhook binaries around --- packages/cli/src/Server.ts | 22 +++-- packages/core/package.json | 1 + .../core/src/BinaryDataManager/FileSystem.ts | 53 +++++++++-- packages/core/src/BinaryDataManager/index.ts | 88 ++++++++++++++----- packages/core/src/Interfaces.ts | 12 +++ packages/core/src/NodeExecuteFunctions.ts | 82 ++++++++++++++++- packages/editor-ui/src/Interface.ts | 9 +- .../src/components/BinaryDataDisplay.vue | 15 +--- .../src/components/BinaryDataDisplayEmbed.vue | 20 ++--- packages/editor-ui/src/components/RunData.vue | 36 ++++---- packages/editor-ui/src/mixins/restApi.ts | 4 + .../src/plugins/i18n/locales/en.json | 1 + packages/nodes-base/nodes/Wait/Wait.node.ts | 80 ++++++++--------- .../nodes-base/nodes/Webhook/Webhook.node.ts | 82 ++++++++--------- packages/workflow/src/Interfaces.ts | 3 + pnpm-lock.yaml | 3 +- 16 files changed, 332 insertions(+), 179 deletions(-) diff --git a/packages/cli/src/Server.ts b/packages/cli/src/Server.ts index b3346a8b053d3..2558b9bbd22f7 100644 --- a/packages/cli/src/Server.ts +++ b/packages/cli/src/Server.ts @@ -1497,18 +1497,22 @@ class App { // Binary data // ---------------------------------------- - // Returns binary buffer + // Download binary this.app.get( `/${this.restEndpoint}/data/:path`, - ResponseHelper.send(async (req: express.Request, res: express.Response): Promise => { + async (req: express.Request, res: express.Response): Promise => { // TODO UM: check if this needs permission check for UM - const dataPath = req.params.path; - return BinaryDataManager.getInstance() - .retrieveBinaryDataByIdentifier(dataPath) - .then((buffer: Buffer) => { - return buffer.toString('base64'); - }); - }), + const identifier = req.params.path; + const binaryDataManager = BinaryDataManager.getInstance(); + const binaryPath = binaryDataManager.getBinaryPath(identifier); + const { mimeType, fileName, fileSize } = await binaryDataManager.getBinaryMetadata( + identifier, + ); + if (mimeType) res.setHeader('Content-Type', mimeType); + if (fileName) res.setHeader('Content-Disposition', `attachment; filename="${fileName}"`); + res.setHeader('Content-Length', fileSize); + res.sendFile(binaryPath); + }, ); // ---------------------------------------- diff --git a/packages/core/package.json b/packages/core/package.json index 5a4e6dc1e20a5..24803be905418 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -56,6 +56,7 @@ "n8n-workflow": "~0.126.0", "oauth-1.0a": "^2.2.6", "p-cancelable": "^2.0.0", + "pretty-bytes": "^5.6.0", "qs": "^6.10.1", "request": "^2.88.2", "request-promise-native": "^1.0.7", diff --git a/packages/core/src/BinaryDataManager/FileSystem.ts b/packages/core/src/BinaryDataManager/FileSystem.ts index d9a8e4d2a263f..360051ff034ad 100644 --- a/packages/core/src/BinaryDataManager/FileSystem.ts +++ b/packages/core/src/BinaryDataManager/FileSystem.ts @@ -1,8 +1,9 @@ -import { promises as fs } from 'fs'; +import fs from 'fs/promises'; +import { jsonParse } from 'n8n-workflow'; import path from 'path'; import { v4 as uuid } from 'uuid'; -import { IBinaryDataConfig, IBinaryDataManager } from '../Interfaces'; +import { BinaryMetadata, IBinaryDataConfig, IBinaryDataManager } from '../Interfaces'; const PREFIX_METAFILE = 'binarymeta'; const PREFIX_PERSISTED_METAFILE = 'persistedmeta'; @@ -43,17 +44,47 @@ export class BinaryDataFileSystem implements IBinaryDataManager { .then(() => {}); } + async getFileSize(identifier: string): Promise { + const stats = await fs.stat(this.getBinaryPath(identifier)); + return stats.size; + } + + async copyBinaryFile(filePath: string, executionId: string): Promise { + const binaryDataId = this.generateFileName(executionId); + await this.addBinaryIdToPersistMeta(executionId, binaryDataId); + await this.copyFileToLocalStorage(filePath, binaryDataId); + return binaryDataId; + } + + async storeBinaryMetadata(identifier: string, metadata: BinaryMetadata) { + await fs.writeFile(this.getMetadataPath(identifier), JSON.stringify(metadata), { + encoding: 'utf-8', + }); + } + + async getBinaryMetadata(identifier: string): Promise { + return jsonParse(await fs.readFile(this.getMetadataPath(identifier), { encoding: 'utf-8' })); + } + async storeBinaryData(binaryBuffer: Buffer, executionId: string): Promise { const binaryDataId = this.generateFileName(executionId); - return this.addBinaryIdToPersistMeta(executionId, binaryDataId).then(async () => - this.saveToLocalStorage(binaryBuffer, binaryDataId).then(() => binaryDataId), - ); + await this.addBinaryIdToPersistMeta(executionId, binaryDataId); + await this.saveToLocalStorage(binaryBuffer, binaryDataId); + return binaryDataId; } async retrieveBinaryDataByIdentifier(identifier: string): Promise { return this.retrieveFromLocalStorage(identifier); } + getBinaryPath(identifier: string): string { + return path.join(this.storagePath, identifier); + } + + getMetadataPath(identifier: string): string { + return path.join(this.storagePath, `${identifier}.metadata`); + } + async markDataForDeletionByExecutionId(executionId: string): Promise { const tt = new Date(new Date().getTime() + this.binaryDataTTL * 60000); return fs.writeFile( @@ -180,7 +211,7 @@ export class BinaryDataFileSystem implements IBinaryDataManager { } private generateFileName(prefix: string): string { - return `${prefix}_${uuid()}`; + return [prefix, uuid()].join(''); } private getBinaryDataMetaPath() { @@ -196,15 +227,19 @@ export class BinaryDataFileSystem implements IBinaryDataManager { } private async deleteFromLocalStorage(identifier: string) { - return fs.rm(path.join(this.storagePath, identifier)); + return fs.rm(this.getBinaryPath(identifier)); + } + + private async copyFileToLocalStorage(source: string, identifier: string): Promise { + await fs.cp(source, this.getBinaryPath(identifier)); } private async saveToLocalStorage(data: Buffer, identifier: string) { - await fs.writeFile(path.join(this.storagePath, identifier), data); + await fs.writeFile(this.getBinaryPath(identifier), data); } private async retrieveFromLocalStorage(identifier: string): Promise { - const filePath = path.join(this.storagePath, identifier); + const filePath = this.getBinaryPath(identifier); try { return await fs.readFile(filePath); } catch (e) { diff --git a/packages/core/src/BinaryDataManager/index.ts b/packages/core/src/BinaryDataManager/index.ts index 57b9313b533fc..ae65534945dd0 100644 --- a/packages/core/src/BinaryDataManager/index.ts +++ b/packages/core/src/BinaryDataManager/index.ts @@ -1,7 +1,9 @@ -import { IBinaryData, INodeExecutionData } from 'n8n-workflow'; +import prettyBytes from 'pretty-bytes'; +import type { IBinaryData, INodeExecutionData } from 'n8n-workflow'; import { BINARY_ENCODING } from '../Constants'; -import { IBinaryDataConfig, IBinaryDataManager } from '../Interfaces'; +import type { BinaryMetadata, IBinaryDataConfig, IBinaryDataManager } from '../Interfaces'; import { BinaryDataFileSystem } from './FileSystem'; +import { readFile, stat } from 'fs/promises'; export class BinaryDataManager { static instance: BinaryDataManager | undefined; @@ -43,31 +45,59 @@ export class BinaryDataManager { return BinaryDataManager.instance; } + async copyBinaryFile( + binaryData: IBinaryData, + filePath: string, + executionId: string, + ): Promise { + // If a manager handles this binary, copy over the binary file and return its reference id. + const manager = this.managers[this.binaryDataMode]; + if (manager) { + const identifier = await manager.copyBinaryFile(filePath, executionId); + // Add data manager reference id. + binaryData.id = this.generateBinaryId(identifier); + + // Prevent preserving data in memory if handled by a data manager. + binaryData.data = this.binaryDataMode; + + const fileSize = await manager.getFileSize(identifier); + binaryData.fileSize = prettyBytes(fileSize); + + await manager.storeBinaryMetadata(identifier, { + fileName: binaryData.fileName, + mimeType: binaryData.mimeType, + fileSize, + }); + } else { + const { size } = await stat(filePath); + binaryData.fileSize = prettyBytes(size); + binaryData.data = await readFile(filePath, { encoding: BINARY_ENCODING }); + } + + return binaryData; + } + async storeBinaryData( binaryData: IBinaryData, binaryBuffer: Buffer, executionId: string, ): Promise { - const retBinaryData = binaryData; - - // If a manager handles this binary, return the binary data with it's reference id. - if (this.managers[this.binaryDataMode]) { - return this.managers[this.binaryDataMode] - .storeBinaryData(binaryBuffer, executionId) - .then((filename) => { - // Add data manager reference id. - retBinaryData.id = this.generateBinaryId(filename); - - // Prevent preserving data in memory if handled by a data manager. - retBinaryData.data = this.binaryDataMode; - - // Short-circuit return to prevent further actions. - return retBinaryData; - }); + binaryData.fileSize = prettyBytes(binaryBuffer.length); + + // If a manager handles this binary, return the binary data with its reference id. + const manager = this.managers[this.binaryDataMode]; + if (manager) { + const identifier = await manager.storeBinaryData(binaryBuffer, executionId); + // Add data manager reference id. + binaryData.id = this.generateBinaryId(identifier); + + // Prevent preserving data in memory if handled by a data manager. + binaryData.data = this.binaryDataMode; + } else { + // Else fallback to storing this data in memory. + binaryData.data = binaryBuffer.toString(BINARY_ENCODING); } - // Else fallback to storing this data in memory. - retBinaryData.data = binaryBuffer.toString(BINARY_ENCODING); return binaryData; } @@ -88,6 +118,24 @@ export class BinaryDataManager { throw new Error('Storage mode used to store binary data not available'); } + getBinaryPath(identifier: string): string { + const { mode, id } = this.splitBinaryModeFileId(identifier); + if (this.managers[mode]) { + return this.managers[mode].getBinaryPath(id); + } + + throw new Error('Storage mode used to store binary data not available'); + } + + async getBinaryMetadata(identifier: string): Promise { + const { mode, id } = this.splitBinaryModeFileId(identifier); + if (this.managers[mode]) { + return this.managers[mode].getBinaryMetadata(id); + } + + throw new Error('Storage mode used to store binary data not available'); + } + async markDataForDeletionByExecutionId(executionId: string): Promise { if (this.managers[this.binaryDataMode]) { return this.managers[this.binaryDataMode].markDataForDeletionByExecutionId(executionId); diff --git a/packages/core/src/Interfaces.ts b/packages/core/src/Interfaces.ts index da1b3e6f1b9ed..51dec31fbdd63 100644 --- a/packages/core/src/Interfaces.ts +++ b/packages/core/src/Interfaces.ts @@ -260,6 +260,7 @@ export interface IWebhookFunctions extends IWebhookFunctionsBase { filePath?: string, mimeType?: string, ): Promise; + copyBinaryFile(filePath: string, fileName: string, mimeType?: string): Promise; request: (uriOrObject: string | IDataObject | any, options?: IDataObject) => Promise; requestWithAuthentication( this: IAllExecuteFunctions, @@ -306,10 +307,21 @@ export interface IBinaryDataConfig { persistedBinaryDataTTL: number; } +export interface BinaryMetadata { + fileName?: string; + mimeType?: string; + fileSize: number; +} + export interface IBinaryDataManager { init(startPurger: boolean): Promise; + getFileSize(filePath: string): Promise; + copyBinaryFile(filePath: string, executionId: string): Promise; + storeBinaryMetadata(identifier: string, metadata: BinaryMetadata): Promise; + getBinaryMetadata(identifier: string): Promise; storeBinaryData(binaryBuffer: Buffer, executionId: string): Promise; retrieveBinaryDataByIdentifier(identifier: string): Promise; + getBinaryPath(identifier: string): string; markDataForDeletionByExecutionId(executionId: string): Promise; deleteMarkedFiles(): Promise; deleteBinaryDataByIdentifier(identifier: string): Promise; diff --git a/packages/core/src/NodeExecuteFunctions.ts b/packages/core/src/NodeExecuteFunctions.ts index 0e2d10e7b2b70..0bc5d51c7619c 100644 --- a/packages/core/src/NodeExecuteFunctions.ts +++ b/packages/core/src/NodeExecuteFunctions.ts @@ -64,6 +64,7 @@ import { NodeExecutionWithMetadata, IPairedItemData, deepCopy, + BinaryFileType, } from 'n8n-workflow'; import { Agent } from 'https'; @@ -77,8 +78,8 @@ import FormData from 'form-data'; import path from 'path'; import { OptionsWithUri, OptionsWithUrl, RequestCallback, RequiredUriUrl } from 'request'; import requestPromise, { RequestPromiseOptions } from 'request-promise-native'; -import { fromBuffer } from 'file-type'; -import { lookup } from 'mime-types'; +import FileType from 'file-type'; +import { lookup, extension } from 'mime-types'; import { IncomingHttpHeaders } from 'http'; import axios, { AxiosError, @@ -830,6 +831,13 @@ export async function getBinaryDataBuffer( return BinaryDataManager.getInstance().retrieveBinaryData(binaryData); } +function fileTypeFromMimeType(mimeType: string): BinaryFileType | undefined { + if (mimeType.startsWith('image/')) return 'image'; + if (mimeType.startsWith('video/')) return 'video'; + if (mimeType.startsWith('text/') || mimeType.startsWith('application/json')) return 'text'; + return; +} + /** * Store an incoming IBinaryData & related buffer using the configured binary data manager. * @@ -846,10 +854,60 @@ export async function setBinaryDataBuffer( return BinaryDataManager.getInstance().storeBinaryData(data, binaryData, executionId); } +export async function copyBinaryFile( + executionId: string, + filePath: string, + fileName: string, + mimeType?: string, +): Promise { + let fileExtension: string | undefined; + if (!mimeType) { + // If no mime type is given figure it out + + if (filePath) { + // Use file path to guess mime type + const mimeTypeLookup = lookup(filePath); + if (mimeTypeLookup) { + mimeType = mimeTypeLookup; + } + } + + if (!mimeType) { + // read the first bytes of the file to guess mime type + const fileTypeData = await FileType.fromFile(filePath); + if (fileTypeData) { + mimeType = fileTypeData.mime; + fileExtension = fileTypeData.ext; + } + } + + if (!mimeType) { + // Fall back to text + mimeType = 'text/plain'; + } + } else if (!fileExtension) { + fileExtension = extension(mimeType) || undefined; + } + + const returnData: IBinaryData = { + mimeType, + fileType: fileTypeFromMimeType(mimeType), + fileExtension, + data: '', + }; + + if (fileName) { + returnData.fileName = fileName; + } else if (filePath) { + returnData.fileName = path.parse(filePath).base; + } + + return BinaryDataManager.getInstance().copyBinaryFile(returnData, filePath, executionId); +} + /** * Takes a buffer and converts it into the format n8n uses. It encodes the binary data as * base64 and adds metadata. - * */ export async function prepareBinaryData( binaryData: Buffer, @@ -871,7 +929,7 @@ export async function prepareBinaryData( if (!mimeType) { // Use buffer to guess mime type - const fileTypeData = await fromBuffer(binaryData); + const fileTypeData = await FileType.fromBuffer(binaryData); if (fileTypeData) { mimeType = fileTypeData.mime; fileExtension = fileTypeData.ext; @@ -882,10 +940,13 @@ export async function prepareBinaryData( // Fall back to text mimeType = 'text/plain'; } + } else if (!fileExtension) { + fileExtension = extension(mimeType) || undefined; } const returnData: IBinaryData = { mimeType, + fileType: fileTypeFromMimeType(mimeType), fileExtension, data: '', }; @@ -3076,6 +3137,19 @@ export function getExecuteWebhookFunctions( async setBinaryDataBuffer(data: IBinaryData, binaryData: Buffer): Promise { return setBinaryDataBuffer.call(this, data, binaryData, additionalData.executionId!); }, + async copyBinaryFile( + filePath: string, + fileName: string, + mimeType?: string, + ): Promise { + return copyBinaryFile.call( + this, + additionalData.executionId!, + filePath, + fileName, + mimeType, + ); + }, async prepareBinaryData( binaryData: Buffer, filePath?: string, diff --git a/packages/editor-ui/src/Interface.ts b/packages/editor-ui/src/Interface.ts index e623057a92653..0a7fed175e8c3 100644 --- a/packages/editor-ui/src/Interface.ts +++ b/packages/editor-ui/src/Interface.ts @@ -215,6 +215,7 @@ export interface IRestApi { retryExecution(id: string, loadWorkflow?: boolean): Promise; getTimezones(): Promise; getBinaryBufferString(dataPath: string): Promise; + getBinaryUrl(dataPath: string): string; } export interface INodeTranslationHeaders { @@ -226,14 +227,6 @@ export interface INodeTranslationHeaders { }; } -export interface IBinaryDisplayData { - index: number; - key: string; - node: string; - outputIndex: number; - runIndex: number; -} - export interface IStartRunData { workflowData: IWorkflowData; startNodes?: string[]; diff --git a/packages/editor-ui/src/components/BinaryDataDisplay.vue b/packages/editor-ui/src/components/BinaryDataDisplay.vue index 59aac23200691..8ef83d206d253 100644 --- a/packages/editor-ui/src/components/BinaryDataDisplay.vue +++ b/packages/editor-ui/src/components/BinaryDataDisplay.vue @@ -20,10 +20,7 @@ diff --git a/packages/design-system/src/components/N8nLink/Link.stories.js b/packages/design-system/src/components/N8nLink/Link.stories.ts similarity index 100% rename from packages/design-system/src/components/N8nLink/Link.stories.js rename to packages/design-system/src/components/N8nLink/Link.stories.ts diff --git a/packages/design-system/src/components/N8nLoading/Loading.stories.js b/packages/design-system/src/components/N8nLoading/Loading.stories.ts similarity index 100% rename from packages/design-system/src/components/N8nLoading/Loading.stories.js rename to packages/design-system/src/components/N8nLoading/Loading.stories.ts diff --git a/packages/design-system/src/components/N8nLoading/Loading.vue b/packages/design-system/src/components/N8nLoading/Loading.vue index 0821dba20a88a..57625b2f00d51 100644 --- a/packages/design-system/src/components/N8nLoading/Loading.vue +++ b/packages/design-system/src/components/N8nLoading/Loading.vue @@ -36,16 +36,15 @@ diff --git a/packages/design-system/src/components/N8nPopover/Popover.stories.js b/packages/design-system/src/components/N8nPopover/Popover.stories.ts similarity index 100% rename from packages/design-system/src/components/N8nPopover/Popover.stories.js rename to packages/design-system/src/components/N8nPopover/Popover.stories.ts diff --git a/packages/design-system/src/components/N8nPopover/Popover.vue b/packages/design-system/src/components/N8nPopover/Popover.vue index 77f598f2506c1..b85cad06e7e5f 100644 --- a/packages/design-system/src/components/N8nPopover/Popover.vue +++ b/packages/design-system/src/components/N8nPopover/Popover.vue @@ -1,7 +1,4 @@ diff --git a/packages/design-system/src/components/N8nRadioButtons/RadioButtons.stories.js b/packages/design-system/src/components/N8nRadioButtons/RadioButtons.stories.ts similarity index 100% rename from packages/design-system/src/components/N8nRadioButtons/RadioButtons.stories.js rename to packages/design-system/src/components/N8nRadioButtons/RadioButtons.stories.ts diff --git a/packages/design-system/src/components/N8nResizeWrapper/ResizeWrapper.stories.js b/packages/design-system/src/components/N8nResizeWrapper/ResizeWrapper.stories.ts similarity index 100% rename from packages/design-system/src/components/N8nResizeWrapper/ResizeWrapper.stories.js rename to packages/design-system/src/components/N8nResizeWrapper/ResizeWrapper.stories.ts diff --git a/packages/design-system/src/components/N8nSelect/Select.vue b/packages/design-system/src/components/N8nSelect/Select.vue index fd94a816f3a0e..b8fd63924e02d 100644 --- a/packages/design-system/src/components/N8nSelect/Select.vue +++ b/packages/design-system/src/components/N8nSelect/Select.vue @@ -32,7 +32,7 @@