feat: Expression extension framework (#4372)

* ⚡ Introduce a framework for expression extension

* 💡 Add some inline comments

* ⚡ Introduce hash alias for encrypt

* ⚡ Introduce a manual granular level approach to shadowing/overrideing extensions

* 🔥 Cleanup comments

* ⚡ Introduce a basic method of extension for native functions

* ⚡ Add length to StringExtension

* ⚡ Add number type to extension return types

* ⚡ Temporarily introduce DateTime with extension

* ⚡ Cleanup comments

* ⚡ Organize imports

* ♻️ Fix up some typings

* ⚡ Fix typings

* ♻️ Remove unnecessary resolve of expression

* ⚡ Extensions Improvement

* ♻️ Refactor EXPRESSION_EXTENSION_METHODS

* ♻️ Refactor EXPRESSION_EXTENSION_METHODS

* ♻️ Update extraArgs types

* ♻️ Fix tests

* ♻️ Fix bind type issue

* ♻️ Fixing duration type issue

* ♻️ Refactor to allow overrides on native methods

* ♻️ Temporarily remove Date Extensions to pass tests

* feat(dt-functions): introduce date expression extensions (#4045)

* 🎉 Add Date Extensions into the mix

* ✨ Introduce additional date extension methods

* ✅ Add Date Expression Extension tests

* 🔧 Add ability to debug tests

* ♻️ Refactor extension for native types

* 🔥 Move sayHi method to String Extension class

* ♻️ Update scope when binding member methods

* ✅ Add String Extension tests

* feat(dt-functions): introduce array expression extensions (#4044)

* ✨ Introduce Array Extensions

* ✅ Add Array Expression tests

* feat(dt-functions): introduce number expression extensions (#4046)

* 🎉 Introduce Number Extensions

* ⚡ Support more shared extensions

* ⚡ Improve handling of name collision

* ✅ Update tests

* Fixed up tests

* 🔥 Remove remove markdown

* :recylce: Replace remove-markdown dependencies with implementation

* ♻️ Replace remove-markdown dependencies with implementation

* ✅ Update tests

* ♻️ Fix scoping and cleanup

* ♻️ Update comments and errors

* ♻️ Fix linting errors

* ➖ Remove unused dependencies

* fix: expression extension not working with multiple extensions

* refactor: change extension transform to be more efficient

* test: update most test to work with new extend function

* fix: update and fix type error in config

* refactor: replace babel with recast

* feat: add hashing functions to string extension

* fix: removed export

* test: add extension parser and transform tests

* fix: vite tests breaking

* refactor: remove commented out code

* fix: parse dates passed from $json in extend function

* refactor: review feedback changes for date extensions

* refactor: review feedback changes for number extensions

* fix: date extension beginningOf test

* fix: broken build from merge

* fix: another merge issue

* refactor: address review feedback (remove ignores)

* feat: new extension functions and tests

* feat: non-dot notation functions

* test: most of the other tests

* fix: toSentenceCase for node versions below 16.6

* feat: add $if and $not expression extensions

* Fix test to work on every timezone

* lint: fix remaining lint issues

Co-authored-by: Csaba Tuncsik <csaba@n8n.io>
Co-authored-by: Omar Ajoue <krynble@gmail.com>

.vscode/launch.json

@@ -5,6 +5,17 @@
 	"version": "0.2.0",
 	"configurations": [
 		{
+      "type": "node",
+      "request": "launch",
+      "name": "Jest: current file",
+      "program": "${workspaceFolder}/node_modules/.bin/jest",
+      "args": ["${fileBasenameNoExtension}"],
+      "console": "integratedTerminal",
+      "windows": {
+        "program": "${workspaceFolder}/node_modules/jest/bin/jest"
+      }
+    },
+		{
 			"name": "Attach to running n8n",
 			"processId": "${command:PickProcess}",
 			"request": "attach",

packages/editor-ui/tsconfig.json

@@ -10,6 +10,7 @@
 		"importHelpers": true,
 		"incremental": false,
 		"allowSyntheticDefaultImports": true,
+		"resolveJsonModule": true,
 		"baseUrl": ".",
 		"types": ["vitest/globals"],
 		"paths": {

packages/editor-ui/vite.config.ts

@@ -43,6 +43,10 @@ const lodashAliases = ['orderBy', 'camelCase', 'cloneDeep', 'isEqual', 'startCas
 
 export default mergeConfig(
 	defineConfig({
+		define: {
+			// This causes test to fail but is required for actually running it
+			...(process.env.NODE_ENV !== 'test' ? { global: 'globalThis' } : {}),
+		},
 		plugins: [
 			legacy({
 				targets: ['defaults', 'not IE 11'],

packages/workflow/package.json

@@ -39,6 +39,8 @@
     "dist/**/*"
   ],
   "devDependencies": {
+    "@n8n_io/eslint-config": "",
+    "@types/crypto-js": "^4.1.1",
     "@types/express": "^4.17.6",
     "@types/jmespath": "^0.15.0",
     "@types/lodash.get": "^4.4.6",
@@ -50,12 +52,16 @@
   },
   "dependencies": {
     "@n8n_io/riot-tmpl": "^2.0.0",
+    "crypto-js": "^4.1.1",
     "jmespath": "^0.16.0",
+    "js-base64": "^3.7.2",
     "lodash.get": "^4.4.2",
     "lodash.isequal": "^4.5.0",
     "lodash.merge": "^4.6.2",
     "lodash.set": "^4.3.2",
     "luxon": "~2.3.0",
+    "recast": "^0.21.5",
+    "transliteration": "^2.3.5",
     "xml2js": "^0.4.23"
   }
 }

packages/workflow/src/Expression.ts

@@ -14,10 +14,21 @@ import {
 	NodeParameterValueType,
 	WorkflowExecuteMode,
 } from './Interfaces';
-import { ExpressionError } from './ExpressionError';
+import { ExpressionError, ExpressionExtensionError } from './ExpressionError';
 import { WorkflowDataProxy } from './WorkflowDataProxy';
 import type { Workflow } from './Workflow';
 
+// eslint-disable-next-line import/no-cycle
+import { extend, hasExpressionExtension, hasNativeMethod } from './Extensions';
+import {
+	ExpressionChunk,
+	ExpressionCode,
+	joinExpression,
+	splitExpression,
+} from './Extensions/ExpressionParser';
+import { extendTransform } from './Extensions/ExpressionExtension';
+import { extendedFunctions } from './Extensions/ExtendedFunctions';
+
 // Set it to use double curly brackets instead of single ones
 tmpl.brackets.set('{{ }}');
 
@@ -242,6 +253,11 @@ export class Expression {
 		data.Boolean = Boolean;
 		data.Symbol = Symbol;
 
+		// expression extensions
+		data.extend = extend;
+
+		Object.assign(data, extendedFunctions);
+
 		const constructorValidation = new RegExp(/\.\s*constructor/gm);
 		if (parameterValue.match(constructorValidation)) {
 			throw new ExpressionError('Expression contains invalid constructor function call', {
@@ -252,7 +268,8 @@ export class Expression {
 		}
 
 		// Execute the expression
-		const returnValue = this.renderExpression(parameterValue, data);
+		const extendedExpression = this.extendSyntax(parameterValue);
+		const returnValue = this.renderExpression(extendedExpression, data);
 		if (typeof returnValue === 'function') {
 			if (returnValue.name === '$') throw new Error('invalid syntax');
 			throw new Error('This is a function. Please add ()');
@@ -267,7 +284,10 @@ export class Expression {
 		return returnValue;
 	}
 
-	private renderExpression(expression: string, data: IWorkflowDataProxyData): tmpl.ReturnValue {
+	private renderExpression(
+		expression: string,
+		data: IWorkflowDataProxyData,
+	): tmpl.ReturnValue | undefined {
 		try {
 			return tmpl.tmpl(expression, data);
 		} catch (error) {
@@ -279,10 +299,43 @@ export class Expression {
 				}
 			}
 		}
-
 		return null;
 	}
 
+	extendSyntax(bracketedExpression: string): string {
+		if (!hasExpressionExtension(bracketedExpression) || hasNativeMethod(bracketedExpression))
+			return bracketedExpression;
+
+		const chunks = splitExpression(bracketedExpression);
+
+		const extendedChunks = chunks.map((chunk): ExpressionChunk => {
+			if (chunk.type === 'code') {
+				const output = extendTransform(chunk.text);
+
+				// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+				if (!output?.code) {
+					throw new ExpressionExtensionError('Failed to extend syntax');
+				}
+
+				let text = output.code;
+				// We need to cut off any trailing semicolons. These cause issues
+				// with certain types of expression and cause the whole expression
+				// to fail.
+				if (text.trim().endsWith(';')) {
+					text = text.trim().slice(0, -1);
+				}
+
+				return {
+					...chunk,
+					text,
+				} as ExpressionCode;
+			}
+			return chunk;
+		});
+
+		return joinExpression(extendedChunks);
+	}
+
 	/**
 	 * Resolves value of parameter. But does not work for workflow-data.
 	 *
@@ -439,6 +492,7 @@ export class Expression {
 					selfData,
 				);
 			}
+
 			return this.resolveSimpleParameterValue(
 				value as NodeParameterValue,
 				siblingParameters,

packages/workflow/src/ExpressionError.ts

@@ -50,3 +50,10 @@ export class ExpressionError extends ExecutionBaseError {
 		}
 	}
 }
+
+export class ExpressionExtensionError extends ExpressionError {
+	constructor(message: string) {
+		super(message);
+		this.context.failExecution = true;
+	}
+}