Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(cli): Load all nodes and credentials code in isolation - N8N-4362 #3906

Merged
merged 3 commits into from
Sep 9, 2022
Merged

feat(cli): Load all nodes and credentials code in isolation - N8N-4362 #3906

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
551d5bd
[N8N-4362] Load all nodes and credentials code in isolation
netroy Aug 19, 2022
d8f8caf
Minor fixes and refactors
krynble Aug 30, 2022
0dc47c8
reuse the same context for loading modules
netroy Aug 31, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion packages/cli/src/CommunityNodes/helpers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import { promisify } from 'util';
import { exec } from 'child_process';
import { access as fsAccess, mkdir as fsMkdir } from 'fs/promises';

import { createContext, Script } from 'vm';
import axios from 'axios';
import { UserSettings } from 'n8n-core';
import { LoggerProxy, PublicInstalledPackage } from 'n8n-workflow';
Expand Down Expand Up @@ -235,3 +235,10 @@ export const isClientError = (error: Error): boolean => {
export function isNpmError(error: unknown): error is { code: number; stdout: string } {
return typeof error === 'object' && error !== null && 'code' in error && 'stdout' in error;
}

const context = createContext({ require });
export const loadClassInIsolation = (filePath: string, className: string) => {
const script = new Script(`new (require('${filePath}').${className})()`);
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return script.runInContext(context);
};
101 changes: 33 additions & 68 deletions packages/cli/src/LoadNodesAndCredentials.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ import config from '../config';
import { NodeTypes } from '.';
import { InstalledPackages } from './databases/entities/InstalledPackages';
import { InstalledNodes } from './databases/entities/InstalledNodes';
import { executeCommand } from './CommunityNodes/helpers';
import { executeCommand, loadClassInIsolation } from './CommunityNodes/helpers';
import { RESPONSE_ERROR_MESSAGES } from './constants';
import {
persistInstalledPackageData,
Expand All @@ -46,6 +46,14 @@ import {

const CUSTOM_NODES_CATEGORY = 'Custom Nodes';

function toJSON() {
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return {
...this,
authenticate: typeof this.authenticate === 'function' ? {} : this.authenticate,
};
}

class LoadNodesAndCredentialsClass {
nodeTypes: INodeTypeData = {};

Expand Down Expand Up @@ -104,10 +112,8 @@ class LoadNodesAndCredentialsClass {
await fsAccess(checkPath);
// Folder exists, so use it.
return path.dirname(checkPath);
} catch (error) {
} catch (_) {
// Folder does not exist so get next one
// eslint-disable-next-line no-continue
continue;
}
}
throw new Error('Could not find "node_modules" folder!');
Expand Down Expand Up @@ -144,8 +150,7 @@ class LoadNodesAndCredentialsClass {
if (process.env[CUSTOM_EXTENSION_ENV] !== undefined) {
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
const customExtensionFolders = process.env[CUSTOM_EXTENSION_ENV]!.split(';');
// eslint-disable-next-line prefer-spread
customDirectories.push.apply(customDirectories, customExtensionFolders);
customDirectories.push(...customExtensionFolders);
}

for (const directory of customDirectories) {
Expand Down Expand Up @@ -192,26 +197,16 @@ class LoadNodesAndCredentialsClass {
* @param {string} filePath The file to read credentials from
* @returns {Promise<void>}
*/
async loadCredentialsFromFile(credentialName: string, filePath: string): Promise<void> {
// eslint-disable-next-line import/no-dynamic-require, global-require, @typescript-eslint/no-var-requires
const tempModule = require(filePath);

loadCredentialsFromFile(credentialName: string, filePath: string): void {
let tempCredential: ICredentialType;
try {
tempCredential = loadClassInIsolation(filePath, credentialName);

// Add serializer method "toJSON" to the class so that authenticate method (if defined)
// gets mapped to the authenticate attribute before it is sent to the client.
// The authenticate property is used by the client to decide whether or not to
// include the credential type in the predefined credentials (HTTP node)
// eslint-disable-next-line func-names
tempModule[credentialName].prototype.toJSON = function () {
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return {
...this,
authenticate: typeof this.authenticate === 'function' ? {} : this.authenticate,
};
};

tempCredential = new tempModule[credentialName]() as ICredentialType;
Object.assign(tempCredential, { toJSON });

if (tempCredential.icon && tempCredential.icon.startsWith('file:')) {
// If a file icon gets used add the full path
Expand Down Expand Up @@ -353,28 +348,24 @@ class LoadNodesAndCredentialsClass {
* @param {string} filePath The file to read node from
* @returns {Promise<void>}
*/
async loadNodeFromFile(
loadNodeFromFile(
packageName: string,
nodeName: string,
filePath: string,
): Promise<INodeTypeNameVersion | undefined> {
): INodeTypeNameVersion | undefined {
let tempNode: INodeType | INodeVersionedType;
let fullNodeName: string;
let nodeVersion = 1;

try {
// eslint-disable-next-line import/no-dynamic-require, global-require, @typescript-eslint/no-var-requires
const tempModule = require(filePath);
tempNode = new tempModule[nodeName]();
tempNode = loadClassInIsolation(filePath, nodeName);
this.addCodex({ node: tempNode, filePath, isCustom: packageName === 'CUSTOM' });
} catch (error) {
// eslint-disable-next-line no-console, @typescript-eslint/restrict-template-expressions
console.error(`Error loading node "${nodeName}" from: "${filePath}" - ${error.message}`);
throw error;
}

// eslint-disable-next-line prefer-const
fullNodeName = `${packageName}.${tempNode.description.name}`;
const fullNodeName = `${packageName}.${tempNode.description.name}`;
tempNode.description.name = fullNodeName;

if (tempNode.description.icon !== undefined && tempNode.description.icon.startsWith('file:')) {
Expand All @@ -385,13 +376,6 @@ class LoadNodesAndCredentialsClass {
)}`;
}

if (tempNode.hasOwnProperty('executeSingle')) {
this.logger.warn(
`"executeSingle" will get deprecated soon. Please update the code of node "${packageName}.${nodeName}" to use "execute" instead!`,
{ filePath },
);
}

if (tempNode.hasOwnProperty('nodeVersions')) {
const versionedNodeType = (tempNode as INodeVersionedType).getNodeType();
this.addCodex({ node: versionedNodeType, filePath, isCustom: packageName === 'CUSTOM' });
Expand Down Expand Up @@ -491,8 +475,7 @@ class LoadNodesAndCredentialsClass {

node.description.codex = codex;
} catch (_) {
// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
this.logger.debug(`No codex available for: ${filePath.split('/').pop()}`);
this.logger.debug(`No codex available for: ${filePath.split('/').pop() ?? ''}`);

if (isCustom) {
node.description.codex = {
Expand All @@ -512,22 +495,15 @@ class LoadNodesAndCredentialsClass {
async loadDataFromDirectory(setPackageName: string, directory: string): Promise<void> {
const files = await glob(path.join(directory, '**/*.@(node|credentials).js'));

let fileName: string;
let type: string;

// eslint-disable-next-line @typescript-eslint/no-explicit-any
const loadPromises: any[] = [];
for (const filePath of files) {
[fileName, type] = path.parse(filePath).name.split('.');
const [fileName, type] = path.parse(filePath).name.split('.');

if (type === 'node') {
loadPromises.push(this.loadNodeFromFile(setPackageName, fileName, filePath));
this.loadNodeFromFile(setPackageName, fileName, filePath);
} else if (type === 'credentials') {
loadPromises.push(this.loadCredentialsFromFile(fileName, filePath));
this.loadCredentialsFromFile(fileName, filePath);
}
}

await Promise.all(loadPromises);
}

async readPackageJson(packagePath: string): Promise<IN8nNodePackageJson> {
Expand All @@ -545,42 +521,31 @@ class LoadNodesAndCredentialsClass {
async loadDataFromPackage(packagePath: string): Promise<INodeTypeNameVersion[]> {
// Get the absolute path of the package
const packageFile = await this.readPackageJson(packagePath);
// if (!packageFile.hasOwnProperty('n8n')) {
if (!packageFile.n8n) {
return [];
}

const packageName = packageFile.name;

let tempPath: string;
let filePath: string;

const { nodes, credentials } = packageFile.n8n;
const returnData: INodeTypeNameVersion[] = [];

// Read all node types
let fileName: string;
let type: string;
if (packageFile.n8n.hasOwnProperty('nodes') && Array.isArray(packageFile.n8n.nodes)) {
for (filePath of packageFile.n8n.nodes) {
tempPath = path.join(packagePath, filePath);
[fileName, type] = path.parse(filePath).name.split('.');
const loadData = await this.loadNodeFromFile(packageName, fileName, tempPath);
if (Array.isArray(nodes)) {
for (const filePath of nodes) {
const tempPath = path.join(packagePath, filePath);
const [fileName] = path.parse(filePath).name.split('.');
const loadData = this.loadNodeFromFile(packageName, fileName, tempPath);
if (loadData) {
returnData.push(loadData);
}
}
}

// Read all credential types
if (
packageFile.n8n.hasOwnProperty('credentials') &&
Array.isArray(packageFile.n8n.credentials)
) {
for (filePath of packageFile.n8n.credentials) {
tempPath = path.join(packagePath, filePath);
// eslint-disable-next-line @typescript-eslint/no-unused-vars
[fileName, type] = path.parse(filePath).name.split('.');
// eslint-disable-next-line @typescript-eslint/no-floating-promises
if (Array.isArray(credentials)) {
for (const filePath of credentials) {
const tempPath = path.join(packagePath, filePath);
const [fileName] = path.parse(filePath).name.split('.');
this.loadCredentialsFromFile(fileName, tempPath);
}
}
Expand Down
36 changes: 11 additions & 25 deletions packages/cli/src/WorkflowRunnerProcess.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ import { getLogger } from './Logger';
import config from '../config';
import { InternalHooksManager } from './InternalHooksManager';
import { checkPermissionsForExecution } from './UserManagement/UserManagementHelper';
import { loadClassInIsolation } from './CommunityNodes/helpers';

export class WorkflowRunnerProcess {
data: IWorkflowExecutionDataProcessWithExecution | undefined;
Expand Down Expand Up @@ -92,41 +93,30 @@ export class WorkflowRunnerProcess {
workflowId: this.data.workflowData.id,
});

let className: string;
let tempNode: INodeType;
let tempCredential: ICredentialType;
let filePath: string;

this.startedAt = new Date();

// Load the required nodes
const nodeTypesData: INodeTypeData = {};
// eslint-disable-next-line no-restricted-syntax
for (const nodeTypeName of Object.keys(this.data.nodeTypeData)) {
className = this.data.nodeTypeData[nodeTypeName].className;

filePath = this.data.nodeTypeData[nodeTypeName].sourcePath;
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, import/no-dynamic-require, global-require, @typescript-eslint/no-var-requires
const tempModule = require(filePath);
let tempNode: INodeType;
const { className, sourcePath } = this.data.nodeTypeData[nodeTypeName];

try {
// eslint-disable-next-line @typescript-eslint/no-unsafe-call
const nodeObject = new tempModule[className]();
const nodeObject = loadClassInIsolation(sourcePath, className);
if (nodeObject.getNodeType !== undefined) {
// eslint-disable-next-line @typescript-eslint/no-unsafe-call
tempNode = nodeObject.getNodeType();
} else {
tempNode = nodeObject;
}
// eslint-disable-next-line @typescript-eslint/no-unsafe-call
tempNode = new tempModule[className]() as INodeType;
} catch (error) {
throw new Error(`Error loading node "${nodeTypeName}" from: "${filePath}"`);
throw new Error(`Error loading node "${nodeTypeName}" from: "${sourcePath}"`);
}

nodeTypesData[nodeTypeName] = {
type: tempNode,
sourcePath: filePath,
sourcePath,
};
}

Expand All @@ -137,22 +127,18 @@ export class WorkflowRunnerProcess {
const credentialsTypeData: ICredentialTypeData = {};
// eslint-disable-next-line no-restricted-syntax
for (const credentialTypeName of Object.keys(this.data.credentialsTypeData)) {
className = this.data.credentialsTypeData[credentialTypeName].className;

filePath = this.data.credentialsTypeData[credentialTypeName].sourcePath;
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, import/no-dynamic-require, global-require, @typescript-eslint/no-var-requires
const tempModule = require(filePath);
let tempCredential: ICredentialType;
const { className, sourcePath } = this.data.credentialsTypeData[credentialTypeName];

try {
// eslint-disable-next-line @typescript-eslint/no-unsafe-call
tempCredential = new tempModule[className]() as ICredentialType;
tempCredential = loadClassInIsolation(sourcePath, className);
} catch (error) {
throw new Error(`Error loading credential "${credentialTypeName}" from: "${filePath}"`);
throw new Error(`Error loading credential "${credentialTypeName}" from: "${sourcePath}"`);
}

credentialsTypeData[credentialTypeName] = {
type: tempCredential,
sourcePath: filePath,
sourcePath,
};
}

Expand Down