fix(HTTP Request Node): Determine binary file name from content-dispo…

…sition headers
n8n-io · Sep 5, 2023 · 32a8aff · 32a8aff
1 parent cc37a50
commit 32a8aff
Show file tree

Hide file tree

Showing 9 changed files with 90 additions and 109 deletions.
diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -71,8 +71,6 @@
     "@types/bcryptjs": "^2.4.2",
     "@types/compression": "1.0.1",
     "@types/connect-history-api-fallback": "^1.3.1",
-    "@types/content-disposition": "^0.5.5",
-    "@types/content-type": "^1.1.5",
     "@types/convict": "^6.1.1",
     "@types/cookie-parser": "^1.4.2",
     "@types/express": "^4.17.6",
@@ -121,8 +119,6 @@
     "class-validator": "^0.14.0",
     "compression": "^1.7.4",
     "connect-history-api-fallback": "^1.6.0",
-    "content-disposition": "^0.5.4",
-    "content-type": "^1.0.4",
     "convict": "^6.2.4",
     "cookie-parser": "^1.4.6",
     "crypto-js": "~4.1.1",

diff --git a/packages/cli/src/middlewares/bodyParser.ts b/packages/cli/src/middlewares/bodyParser.ts
@@ -1,9 +1,8 @@
-import { parse as parseContentDisposition } from 'content-disposition';
-import { parse as parseContentType } from 'content-type';
 import getRawBody from 'raw-body';
 import type { Request, RequestHandler } from 'express';
 import { parse as parseQueryString } from 'querystring';
 import { Parser as XmlParser } from 'xml2js';
+import { parseIncomingMessage } from 'n8n-core';
 import { jsonParse } from 'n8n-workflow';
 import config from '@/config';
 import { UnprocessableRequestError } from '@/ResponseHelper';
@@ -17,26 +16,7 @@ const xmlParser = new XmlParser({
 
 const payloadSizeMax = config.getEnv('endpoints.payloadSizeMax');
 export const rawBodyReader: RequestHandler = async (req, res, next) => {
-	if ('content-type' in req.headers) {
-		const { type: contentType, parameters } = (() => {
-			try {
-				return parseContentType(req);
-			} catch {
-				return { type: undefined, parameters: undefined };
-			}
-		})();
-		req.contentType = contentType;
-		req.encoding = (parameters?.charset ?? 'utf-8').toLowerCase() as BufferEncoding;
-
-		const contentDispositionHeader = req.headers['content-disposition'];
-		if (contentDispositionHeader?.length) {
-			const {
-				type,
-				parameters: { filename },
-			} = parseContentDisposition(contentDispositionHeader);
-			req.contentDisposition = { type, filename };
-		}
-	}
+	parseIncomingMessage(req);
 
 	req.readRawBody = async () => {
 		if (!req.rawBody) {

diff --git a/packages/core/package.json b/packages/core/package.json
@@ -35,6 +35,8 @@
   ],
   "devDependencies": {
     "@types/concat-stream": "^2.0.0",
+    "@types/content-disposition": "^0.5.5",
+    "@types/content-type": "^1.1.5",
     "@types/cron": "~1.7.1",
     "@types/crypto-js": "^4.0.1",
     "@types/express": "^4.17.6",
@@ -47,6 +49,8 @@
     "axios": "^0.21.1",
     "@n8n/client-oauth2": "workspace:*",
     "concat-stream": "^2.0.0",
+    "content-disposition": "^0.5.4",
+    "content-type": "^1.0.4",
     "cron": "~1.7.2",
     "crypto-js": "~4.1.1",
     "fast-glob": "^3.2.5",

diff --git a/packages/core/src/NodeExecuteFunctions.ts b/packages/core/src/NodeExecuteFunctions.ts
@@ -78,10 +78,11 @@ import {
 	validateFieldType,
 	NodeSSLError,
 } from 'n8n-workflow';
-
+import { parse as parseContentDisposition } from 'content-disposition';
+import { parse as parseContentType } from 'content-type';
 import pick from 'lodash/pick';
 import { Agent } from 'https';
-import { IncomingMessage } from 'http';
+import { IncomingMessage, type IncomingHttpHeaders } from 'http';
 import { stringify } from 'qs';
 import type { Token } from 'oauth-1.0a';
 import clientOAuth1 from 'oauth-1.0a';
@@ -100,7 +101,6 @@ import type { OptionsWithUri, OptionsWithUrl } from 'request';
 import type { RequestPromiseOptions } from 'request-promise-native';
 import FileType from 'file-type';
 import { lookup, extension } from 'mime-types';
-import type { IncomingHttpHeaders } from 'http';
 import type {
 	AxiosError,
 	AxiosPromise,
@@ -600,6 +600,29 @@ type ConfigObject = {
 	simple?: boolean;
 };
 
+export function parseIncomingMessage(message: IncomingMessage) {
+	if ('content-type' in message.headers) {
+		const { type: contentType, parameters } = (() => {
+			try {
+				return parseContentType(message);
+			} catch {
+				return { type: undefined, parameters: undefined };
+			}
+		})();
+		message.contentType = contentType;
+		message.encoding = (parameters?.charset ?? 'utf-8').toLowerCase() as BufferEncoding;
+
+		const contentDispositionHeader = message.headers['content-disposition'];
+		if (contentDispositionHeader?.length) {
+			const {
+				type,
+				parameters: { filename },
+			} = parseContentDisposition(contentDispositionHeader);
+			message.contentDisposition = { type, filename };
+		}
+	}
+}
+
 export async function proxyRequestToAxios(
 	workflow: Workflow | undefined,
 	additionalData: IWorkflowExecuteAdditionalData | undefined,
@@ -654,35 +677,22 @@ export async function proxyRequestToAxios(
 
 	try {
 		const response = await requestFn();
-		if (configObject.resolveWithFullResponse === true) {
-			let body = response.data;
-			if (response.data === '') {
-				if (axiosConfig.responseType === 'arraybuffer') {
-					body = Buffer.alloc(0);
-				} else {
-					body = undefined;
-				}
-			}
-			await additionalData?.hooks?.executeHookFunctions('nodeFetchedData', [workflow?.id, node]);
-			return {
-				body,
-				headers: response.headers,
-				statusCode: response.status,
-				statusMessage: response.statusText,
-				request: response.request,
-			};
-		} else {
-			let body = response.data;
-			if (response.data === '') {
-				if (axiosConfig.responseType === 'arraybuffer') {
-					body = Buffer.alloc(0);
-				} else {
-					body = undefined;
-				}
-			}
-			await additionalData?.hooks?.executeHookFunctions('nodeFetchedData', [workflow?.id, node]);
-			return body;
+		let body = response.data;
+		if (body instanceof IncomingMessage && axiosConfig.responseType === 'stream') {
+			parseIncomingMessage(body);
+		} else if (body === '') {
+			body = axiosConfig.responseType === 'arraybuffer' ? Buffer.alloc(0) : undefined;
 		}
+		await additionalData?.hooks?.executeHookFunctions('nodeFetchedData', [workflow?.id, node]);
+		return configObject.resolveWithFullResponse
+			? {
+					body,
+					headers: response.headers,
+					statusCode: response.status,
+					statusMessage: response.statusText,
+					request: response.request,
+			  }
+			: body;
 	} catch (error) {
 		const { config, response } = error;
 
@@ -998,6 +1008,19 @@ async function prepareBinaryData(
 	mimeType?: string,
 ): Promise<IBinaryData> {
 	let fileExtension: string | undefined;
+	if (binaryData instanceof IncomingMessage) {
+		if (!filePath) {
+			try {
+				filePath =
+					binaryData.contentDisposition?.filename ??
+					new URL(binaryData.responseUrl).pathname.slice(1);
+			} catch {}
+		}
+		if (!mimeType) {
+			mimeType = binaryData.contentType;
+		}
+	}
+
 	if (!mimeType) {
 		// If no mime type is given figure it out
 

diff --git a/packages/nodes-base/nodes/HttpRequest/V1/HttpRequestV1.node.ts b/packages/nodes-base/nodes/HttpRequest/V1/HttpRequestV1.node.ts
@@ -1,3 +1,5 @@
+import type { Readable } from 'stream';
+
 import type {
 	IExecuteFunctions,
 	IDataObject,
@@ -991,7 +993,6 @@ export class HttpRequestV1 implements INodeType {
 			response = response.value;
 
 			const options = this.getNodeParameter('options', itemIndex, {});
-			const url = this.getNodeParameter('url', itemIndex) as string;
 
 			const fullResponse = !!options.fullResponse;
 
@@ -1014,8 +1015,7 @@ export class HttpRequestV1 implements INodeType {
 					Object.assign(newItem.binary, items[itemIndex].binary);
 				}
 
-				const fileName = url.split('/').pop();
-
+				let binaryData: Buffer | Readable;
 				if (fullResponse) {
 					const returnItem: IDataObject = {};
 					for (const property of fullResponseProperties) {
@@ -1026,20 +1026,13 @@ export class HttpRequestV1 implements INodeType {
 					}
 
 					newItem.json = returnItem;
-
-					newItem.binary![dataPropertyName] = await this.helpers.prepareBinaryData(
-						response!.body as Buffer,
-						fileName,
-					);
+					binaryData = response!.body;
 				} else {
 					newItem.json = items[itemIndex].json;
-
-					newItem.binary![dataPropertyName] = await this.helpers.prepareBinaryData(
-						response! as Buffer,
-						fileName,
-					);
+					binaryData = response;
 				}
 
+				newItem.binary![dataPropertyName] = await this.helpers.prepareBinaryData(binaryData);
 				returnItems.push(newItem);
 			} else if (responseFormat === 'string') {
 				const dataPropertyName = this.getNodeParameter('dataPropertyName', 0);

diff --git a/packages/nodes-base/nodes/HttpRequest/V2/HttpRequestV2.node.ts b/packages/nodes-base/nodes/HttpRequest/V2/HttpRequestV2.node.ts
@@ -1,3 +1,5 @@
+import type { Readable } from 'stream';
+
 import type {
 	IDataObject,
 	IExecuteFunctions,
@@ -1044,7 +1046,6 @@ export class HttpRequestV2 implements INodeType {
 			response = response.value;
 
 			const options = this.getNodeParameter('options', itemIndex, {});
-			const url = this.getNodeParameter('url', itemIndex) as string;
 
 			const fullResponse = !!options.fullResponse;
 
@@ -1067,8 +1068,7 @@ export class HttpRequestV2 implements INodeType {
 					Object.assign(newItem.binary, items[itemIndex].binary);
 				}
 
-				const fileName = url.split('/').pop();
-
+				let binaryData: Buffer | Readable;
 				if (fullResponse) {
 					const returnItem: IDataObject = {};
 					for (const property of fullResponseProperties) {
@@ -1079,20 +1079,13 @@ export class HttpRequestV2 implements INodeType {
 					}
 
 					newItem.json = returnItem;
-
-					newItem.binary![dataPropertyName] = await this.helpers.prepareBinaryData(
-						response!.body as Buffer,
-						fileName,
-					);
+					binaryData = response!.body;
 				} else {
 					newItem.json = items[itemIndex].json;
-
-					newItem.binary![dataPropertyName] = await this.helpers.prepareBinaryData(
-						response! as Buffer,
-						fileName,
-					);
+					binaryData = response;
 				}
 
+				newItem.binary![dataPropertyName] = await this.helpers.prepareBinaryData(binaryData);
 				returnItems.push(newItem);
 			} else if (responseFormat === 'string') {
 				const dataPropertyName = this.getNodeParameter('dataPropertyName', 0);

diff --git a/packages/nodes-base/nodes/HttpRequest/V3/HttpRequestV3.node.ts b/packages/nodes-base/nodes/HttpRequest/V3/HttpRequestV3.node.ts
@@ -1452,8 +1452,6 @@ export class HttpRequestV3 implements INodeType {
 
 			response = response.value;
 
-			const url = this.getNodeParameter('url', itemIndex) as string;
-
 			let responseFormat = this.getNodeParameter(
 				'options.response.response.responseFormat',
 				0,
@@ -1525,8 +1523,7 @@ export class HttpRequestV3 implements INodeType {
 					Object.assign(newItem.binary as IBinaryKeyData, items[itemIndex].binary);
 				}
 
-				const fileName = url.split('/').pop();
-
+				let binaryData: Buffer | Readable;
 				if (fullResponse) {
 					const returnItem: IDataObject = {};
 					for (const property of fullResponseProperties) {
@@ -1537,19 +1534,12 @@ export class HttpRequestV3 implements INodeType {
 					}
 
 					newItem.json = returnItem;
-
-					newItem.binary![outputPropertyName] = await this.helpers.prepareBinaryData(
-						response!.body as Buffer | Readable,
-						fileName,
-					);
+					binaryData = response!.body;
 				} else {
 					newItem.json = items[itemIndex].json;
-
-					newItem.binary![outputPropertyName] = await this.helpers.prepareBinaryData(
-						response! as Buffer | Readable,
-						fileName,
-					);
+					binaryData = response;
 				}
+				newItem.binary![outputPropertyName] = await this.helpers.prepareBinaryData(binaryData);
 
 				returnItems.push(newItem);
 			} else if (responseFormat === 'text') {

diff --git a/packages/workflow/src/index.ts b/packages/workflow/src/index.ts
@@ -57,5 +57,7 @@ declare module 'http' {
 		rawBody: Buffer;
 		readRawBody(): Promise<void>;
 		_body: boolean;
+		// This gets added by the `follow-redirects` package
+		responseUrl: string;
 	}
 }
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml