Skip to content

Commit

Permalink
feat(cli): Load all nodes and credentials code in isolation - N8N-4362 (
Browse files Browse the repository at this point in the history 
#3906)

[N8N-4362] Load all nodes and credentials code in isolation

Co-authored-by: Omar Ajoue <krynble@gmail.com>
  • Loading branch information
@netroy @krynble
netroy and krynble authored Sep 9, 2022
1 parent 9267e8f commit b450e97
Show file tree
Hide file tree
Showing 3 changed files with 52 additions and 94 deletions.
9 changes: 8 additions & 1 deletion packages/cli/src/CommunityNodes/helpers.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import { promisify } from 'util';
import { exec } from 'child_process';
import { access as fsAccess, mkdir as fsMkdir } from 'fs/promises';

import { createContext, Script } from 'vm';
import axios from 'axios';
import { UserSettings } from 'n8n-core';
import { LoggerProxy, PublicInstalledPackage } from 'n8n-workflow';
Expand Down Expand Up @@ -235,3 +235,10 @@ export const isClientError = (error: Error): boolean => {
export function isNpmError(error: unknown): error is { code: number; stdout: string } {
return typeof error === 'object' && error !== null && 'code' in error && 'stdout' in error;
}

const context = createContext({ require });
export const loadClassInIsolation = (filePath: string, className: string) => {
const script = new Script(`new (require('${filePath}').${className})()`);
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return script.runInContext(context);
};
101 changes: 33 additions & 68 deletions packages/cli/src/LoadNodesAndCredentials.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ import config from '../config';
import { NodeTypes } from '.';
import { InstalledPackages } from './databases/entities/InstalledPackages';
import { InstalledNodes } from './databases/entities/InstalledNodes';
import { executeCommand } from './CommunityNodes/helpers';
import { executeCommand, loadClassInIsolation } from './CommunityNodes/helpers';
import { RESPONSE_ERROR_MESSAGES } from './constants';
import {
persistInstalledPackageData,
Expand All @@ -46,6 +46,14 @@ import {

const CUSTOM_NODES_CATEGORY = 'Custom Nodes';

function toJSON() {
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return {
...this,
authenticate: typeof this.authenticate === 'function' ? {} : this.authenticate,
};
}

class LoadNodesAndCredentialsClass {
nodeTypes: INodeTypeData = {};

Expand Down Expand Up @@ -104,10 +112,8 @@ class LoadNodesAndCredentialsClass {
await fsAccess(checkPath);
// Folder exists, so use it.
return path.dirname(checkPath);
} catch (error) {
} catch (_) {
// Folder does not exist so get next one
// eslint-disable-next-line no-continue
continue;
}
}
throw new Error('Could not find "node_modules" folder!');
Expand Down Expand Up @@ -144,8 +150,7 @@ class LoadNodesAndCredentialsClass {
if (process.env[CUSTOM_EXTENSION_ENV] !== undefined) {
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
const customExtensionFolders = process.env[CUSTOM_EXTENSION_ENV]!.split(';');
// eslint-disable-next-line prefer-spread
customDirectories.push.apply(customDirectories, customExtensionFolders);
customDirectories.push(...customExtensionFolders);
}

for (const directory of customDirectories) {
Expand Down Expand Up @@ -192,26 +197,16 @@ class LoadNodesAndCredentialsClass {
* @param {string} filePath The file to read credentials from
* @returns {Promise<void>}
*/
async loadCredentialsFromFile(credentialName: string, filePath: string): Promise<void> {
// eslint-disable-next-line import/no-dynamic-require, global-require, @typescript-eslint/no-var-requires
const tempModule = require(filePath);

loadCredentialsFromFile(credentialName: string, filePath: string): void {
let tempCredential: ICredentialType;
try {
tempCredential = loadClassInIsolation(filePath, credentialName);

// Add serializer method "toJSON" to the class so that authenticate method (if defined)
// gets mapped to the authenticate attribute before it is sent to the client.
// The authenticate property is used by the client to decide whether or not to
// include the credential type in the predefined credentials (HTTP node)
// eslint-disable-next-line func-names
tempModule[credentialName].prototype.toJSON = function () {
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return {
...this,
authenticate: typeof this.authenticate === 'function' ? {} : this.authenticate,
};
};

tempCredential = new tempModule[credentialName]() as ICredentialType;
Object.assign(tempCredential, { toJSON });

if (tempCredential.icon && tempCredential.icon.startsWith('file:')) {
// If a file icon gets used add the full path
Expand Down Expand Up @@ -353,28 +348,24 @@ class LoadNodesAndCredentialsClass {
* @param {string} filePath The file to read node from
* @returns {Promise<void>}
*/
async loadNodeFromFile(
loadNodeFromFile(
packageName: string,
nodeName: string,
filePath: string,
): Promise<INodeTypeNameVersion | undefined> {
): INodeTypeNameVersion | undefined {
let tempNode: INodeType | INodeVersionedType;
let fullNodeName: string;
let nodeVersion = 1;

try {
// eslint-disable-next-line import/no-dynamic-require, global-require, @typescript-eslint/no-var-requires
const tempModule = require(filePath);
tempNode = new tempModule[nodeName]();
tempNode = loadClassInIsolation(filePath, nodeName);
this.addCodex({ node: tempNode, filePath, isCustom: packageName === 'CUSTOM' });
} catch (error) {
// eslint-disable-next-line no-console, @typescript-eslint/restrict-template-expressions
console.error(`Error loading node "${nodeName}" from: "${filePath}" - ${error.message}`);
throw error;
}

// eslint-disable-next-line prefer-const
fullNodeName = `${packageName}.${tempNode.description.name}`;
const fullNodeName = `${packageName}.${tempNode.description.name}`;
tempNode.description.name = fullNodeName;

if (tempNode.description.icon !== undefined && tempNode.description.icon.startsWith('file:')) {
Expand All @@ -385,13 +376,6 @@ class LoadNodesAndCredentialsClass {
)}`;
}

if (tempNode.hasOwnProperty('executeSingle')) {
this.logger.warn(
`"executeSingle" will get deprecated soon. Please update the code of node "${packageName}.${nodeName}" to use "execute" instead!`,
{ filePath },
);
}

if (tempNode.hasOwnProperty('nodeVersions')) {
const versionedNodeType = (tempNode as INodeVersionedType).getNodeType();
this.addCodex({ node: versionedNodeType, filePath, isCustom: packageName === 'CUSTOM' });
Expand Down Expand Up @@ -491,8 +475,7 @@ class LoadNodesAndCredentialsClass {

node.description.codex = codex;
} catch (_) {
// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
this.logger.debug(`No codex available for: ${filePath.split('/').pop()}`);
this.logger.debug(`No codex available for: ${filePath.split('/').pop() ?? ''}`);

if (isCustom) {
node.description.codex = {
Expand All @@ -512,22 +495,15 @@ class LoadNodesAndCredentialsClass {
async loadDataFromDirectory(setPackageName: string, directory: string): Promise<void> {
const files = await glob(path.join(directory, '**/*.@(node|credentials).js'));

let fileName: string;
let type: string;

// eslint-disable-next-line @typescript-eslint/no-explicit-any
const loadPromises: any[] = [];
for (const filePath of files) {
[fileName, type] = path.parse(filePath).name.split('.');
const [fileName, type] = path.parse(filePath).name.split('.');

if (type === 'node') {
loadPromises.push(this.loadNodeFromFile(setPackageName, fileName, filePath));
this.loadNodeFromFile(setPackageName, fileName, filePath);
} else if (type === 'credentials') {
loadPromises.push(this.loadCredentialsFromFile(fileName, filePath));
this.loadCredentialsFromFile(fileName, filePath);
}
}

await Promise.all(loadPromises);
}

async readPackageJson(packagePath: string): Promise<IN8nNodePackageJson> {
Expand All @@ -545,42 +521,31 @@ class LoadNodesAndCredentialsClass {
async loadDataFromPackage(packagePath: string): Promise<INodeTypeNameVersion[]> {
// Get the absolute path of the package
const packageFile = await this.readPackageJson(packagePath);
// if (!packageFile.hasOwnProperty('n8n')) {
if (!packageFile.n8n) {
return [];
}

const packageName = packageFile.name;

let tempPath: string;
let filePath: string;

const { nodes, credentials } = packageFile.n8n;
const returnData: INodeTypeNameVersion[] = [];

// Read all node types
let fileName: string;
let type: string;
if (packageFile.n8n.hasOwnProperty('nodes') && Array.isArray(packageFile.n8n.nodes)) {
for (filePath of packageFile.n8n.nodes) {
tempPath = path.join(packagePath, filePath);
[fileName, type] = path.parse(filePath).name.split('.');
const loadData = await this.loadNodeFromFile(packageName, fileName, tempPath);
if (Array.isArray(nodes)) {
for (const filePath of nodes) {
const tempPath = path.join(packagePath, filePath);
const [fileName] = path.parse(filePath).name.split('.');
const loadData = this.loadNodeFromFile(packageName, fileName, tempPath);
if (loadData) {
returnData.push(loadData);
}
}
}

// Read all credential types
if (
packageFile.n8n.hasOwnProperty('credentials') &&
Array.isArray(packageFile.n8n.credentials)
) {
for (filePath of packageFile.n8n.credentials) {
tempPath = path.join(packagePath, filePath);
// eslint-disable-next-line @typescript-eslint/no-unused-vars
[fileName, type] = path.parse(filePath).name.split('.');
// eslint-disable-next-line @typescript-eslint/no-floating-promises
if (Array.isArray(credentials)) {
for (const filePath of credentials) {
const tempPath = path.join(packagePath, filePath);
const [fileName] = path.parse(filePath).name.split('.');
this.loadCredentialsFromFile(fileName, tempPath);
}
}
Expand Down
36 changes: 11 additions & 25 deletions packages/cli/src/WorkflowRunnerProcess.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ import { getLogger } from './Logger';
import config from '../config';
import { InternalHooksManager } from './InternalHooksManager';
import { checkPermissionsForExecution } from './UserManagement/UserManagementHelper';
import { loadClassInIsolation } from './CommunityNodes/helpers';

export class WorkflowRunnerProcess {
data: IWorkflowExecutionDataProcessWithExecution | undefined;
Expand Down Expand Up @@ -92,41 +93,30 @@ export class WorkflowRunnerProcess {
workflowId: this.data.workflowData.id,
});

let className: string;
let tempNode: INodeType;
let tempCredential: ICredentialType;
let filePath: string;

this.startedAt = new Date();

// Load the required nodes
const nodeTypesData: INodeTypeData = {};
// eslint-disable-next-line no-restricted-syntax
for (const nodeTypeName of Object.keys(this.data.nodeTypeData)) {
className = this.data.nodeTypeData[nodeTypeName].className;

filePath = this.data.nodeTypeData[nodeTypeName].sourcePath;
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, import/no-dynamic-require, global-require, @typescript-eslint/no-var-requires
const tempModule = require(filePath);
let tempNode: INodeType;
const { className, sourcePath } = this.data.nodeTypeData[nodeTypeName];

try {
// eslint-disable-next-line @typescript-eslint/no-unsafe-call
const nodeObject = new tempModule[className]();
const nodeObject = loadClassInIsolation(sourcePath, className);
if (nodeObject.getNodeType !== undefined) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-call
tempNode = nodeObject.getNodeType();
} else {
tempNode = nodeObject;
}
// eslint-disable-next-line @typescript-eslint/no-unsafe-call
tempNode = new tempModule[className]() as INodeType;
} catch (error) {
throw new Error(`Error loading node "${nodeTypeName}" from: "${filePath}"`);
throw new Error(`Error loading node "${nodeTypeName}" from: "${sourcePath}"`);
}

nodeTypesData[nodeTypeName] = {
type: tempNode,
sourcePath: filePath,
sourcePath,
};
}

Expand All @@ -137,22 +127,18 @@ export class WorkflowRunnerProcess {
const credentialsTypeData: ICredentialTypeData = {};
// eslint-disable-next-line no-restricted-syntax
for (const credentialTypeName of Object.keys(this.data.credentialsTypeData)) {
className = this.data.credentialsTypeData[credentialTypeName].className;

filePath = this.data.credentialsTypeData[credentialTypeName].sourcePath;
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, import/no-dynamic-require, global-require, @typescript-eslint/no-var-requires
const tempModule = require(filePath);
let tempCredential: ICredentialType;
const { className, sourcePath } = this.data.credentialsTypeData[credentialTypeName];

try {
// eslint-disable-next-line @typescript-eslint/no-unsafe-call
tempCredential = new tempModule[className]() as ICredentialType;
tempCredential = loadClassInIsolation(sourcePath, className);
} catch (error) {
throw new Error(`Error loading credential "${credentialTypeName}" from: "${filePath}"`);
throw new Error(`Error loading credential "${credentialTypeName}" from: "${sourcePath}"`);
}

credentialsTypeData[credentialTypeName] = {
type: tempCredential,
sourcePath: filePath,
sourcePath,
};
}

Expand Down

0 comments on commit b450e97

Please sign in to comment.