From 115a367caeb9cbec8597d328cd969fff5ab6d941 Mon Sep 17 00:00:00 2001 From: Elias Meire Date: Tue, 11 Feb 2025 12:45:55 +0100 Subject: [PATCH] fix(Gmail Trigger Node): Prevent error for empty emails, improve type safety (#13171) --- .../nodes/Google/Gmail/GmailTrigger.node.ts | 150 +++++++++--------- .../Google/Gmail/test/GmailTrigger.test.ts | 59 +++++++ .../__snapshots__/GmailTrigger.test.ts.snap | 96 +++++++++++ .../nodes-base/nodes/Google/Gmail/types.ts | 35 +++- 4 files changed, 267 insertions(+), 73 deletions(-) create mode 100644 packages/nodes-base/nodes/Google/Gmail/test/__snapshots__/GmailTrigger.test.ts.snap diff --git a/packages/nodes-base/nodes/Google/Gmail/GmailTrigger.node.ts b/packages/nodes-base/nodes/Google/Gmail/GmailTrigger.node.ts index b82a578b94ca5..246854f529ee8 100644 --- a/packages/nodes-base/nodes/Google/Gmail/GmailTrigger.node.ts +++ b/packages/nodes-base/nodes/Google/Gmail/GmailTrigger.node.ts @@ -1,12 +1,12 @@ import { DateTime } from 'luxon'; import type { - IPollFunctions, IDataObject, ILoadOptionsFunctions, INodeExecutionData, INodePropertyOptions, INodeType, INodeTypeDescription, + IPollFunctions, } from 'n8n-workflow'; import { NodeConnectionType } from 'n8n-workflow'; @@ -17,6 +17,15 @@ import { prepareQuery, simplifyOutput, } from './GenericFunctions'; +import type { + GmailTriggerFilters, + GmailTriggerOptions, + GmailWorkflowStaticData, + GmailWorkflowStaticDataDictionary, + Label, + Message, + MessageListResponse, +} from './types'; export class GmailTrigger implements INodeType { description: INodeTypeDescription = { @@ -206,12 +215,12 @@ export class GmailTrigger implements INodeType { async getLabels(this: ILoadOptionsFunctions): Promise { const returnData: INodePropertyOptions[] = []; - const labels = await googleApiRequestAllItems.call( + const labels = (await googleApiRequestAllItems.call( this, 'labels', 'GET', '/gmail/v1/users/me/labels', - ); + )) as Label[]; for (const label of labels) { returnData.push({ @@ -234,50 +243,53 @@ export class GmailTrigger implements INodeType { }; async poll(this: IPollFunctions): Promise { - const workflowStaticData = this.getWorkflowStaticData('node'); + const workflowStaticData = this.getWorkflowStaticData('node') as + | GmailWorkflowStaticData + | GmailWorkflowStaticDataDictionary; const node = this.getNode(); - let nodeStaticData = workflowStaticData; + let nodeStaticData = (workflowStaticData ?? {}) as GmailWorkflowStaticData; if (node.typeVersion > 1) { const nodeName = node.name; - if (workflowStaticData[nodeName] === undefined) { - workflowStaticData[nodeName] = {} as IDataObject; - nodeStaticData = workflowStaticData[nodeName] as IDataObject; - } else { - nodeStaticData = workflowStaticData[nodeName] as IDataObject; + const dictionary = workflowStaticData as GmailWorkflowStaticDataDictionary; + if (!(nodeName in workflowStaticData)) { + dictionary[nodeName] = {}; } - } - let responseData; + nodeStaticData = dictionary[nodeName]; + } const now = Math.floor(DateTime.now().toSeconds()).toString(); - const startDate = (nodeStaticData.lastTimeChecked as string) || +now; + const startDate = nodeStaticData.lastTimeChecked ?? +now; const endDate = +now; - const options = this.getNodeParameter('options', {}) as IDataObject; - const filters = this.getNodeParameter('filters', {}) as IDataObject; + const options = this.getNodeParameter('options', {}) as GmailTriggerOptions; + const filters = this.getNodeParameter('filters', {}) as GmailTriggerFilters; + + let responseData: INodeExecutionData[] = []; try { const qs: IDataObject = {}; - filters.receivedAfter = startDate; + const allFilters: GmailTriggerFilters = { ...filters, receivedAfter: startDate }; if (this.getMode() === 'manual') { qs.maxResults = 1; - delete filters.receivedAfter; + delete allFilters.receivedAfter; } - Object.assign(qs, prepareQuery.call(this, filters, 0), options); + Object.assign(qs, prepareQuery.call(this, allFilters, 0), options); - responseData = await googleApiRequest.call( + const messagesResponse: MessageListResponse = await googleApiRequest.call( this, 'GET', '/gmail/v1/users/me/messages', {}, qs, ); - responseData = responseData.messages; - if (!responseData?.length) { + const messages = messagesResponse.messages ?? []; + + if (!messages.length) { nodeStaticData.lastTimeChecked = endDate; return null; } @@ -291,48 +303,47 @@ export class GmailTrigger implements INodeType { qs.format = 'raw'; } - let includeDrafts; + let includeDrafts = false; if (node.typeVersion > 1.1) { - includeDrafts = (qs.includeDrafts as boolean) ?? false; + includeDrafts = filters.includeDrafts ?? false; } else { - includeDrafts = (qs.includeDrafts as boolean) ?? true; + includeDrafts = filters.includeDrafts ?? true; } + delete qs.includeDrafts; - const withoutDrafts = []; - for (let i = 0; i < responseData.length; i++) { - responseData[i] = await googleApiRequest.call( + for (const message of messages) { + const fullMessage = (await googleApiRequest.call( this, 'GET', - `/gmail/v1/users/me/messages/${responseData[i].id}`, + `/gmail/v1/users/me/messages/${message.id}`, {}, qs, - ); + )) as Message; + if (!includeDrafts) { - if (responseData[i].labelIds.includes('DRAFT')) { + if (fullMessage.labelIds?.includes('DRAFT')) { continue; } } - if (!simple && responseData?.length) { + + if (!simple) { const dataPropertyNameDownload = - (options.dataPropertyAttachmentsPrefixName as string) || 'attachment_'; + options.dataPropertyAttachmentsPrefixName || 'attachment_'; - responseData[i] = await parseRawEmail.call( - this, - responseData[i], - dataPropertyNameDownload, - ); + const parsed = await parseRawEmail.call(this, fullMessage, dataPropertyNameDownload); + responseData.push(parsed); + } else { + responseData.push({ json: fullMessage }); } - withoutDrafts.push(responseData[i]); } - if (!includeDrafts) { - responseData = withoutDrafts; - } - - if (simple && responseData?.length) { + if (simple) { responseData = this.helpers.returnJsonArray( - await simplifyOutput.call(this, responseData as IDataObject[]), + await simplifyOutput.call( + this, + responseData.map((item) => item.json), + ), ); } } catch (error) { @@ -349,52 +360,49 @@ export class GmailTrigger implements INodeType { }, ); } - if (!responseData?.length) { + if (!responseData.length) { nodeStaticData.lastTimeChecked = endDate; return null; } const emailsWithInvalidDate = new Set(); - const getEmailDateAsSeconds = (email: IDataObject): number => { + + const getEmailDateAsSeconds = (email: Message): number => { let date; + if (email.internalDate) { - date = +(email.internalDate as string) / 1000; + date = +email.internalDate / 1000; } else if (email.date) { - date = +DateTime.fromJSDate(new Date(email.date as string)).toSeconds(); - } else { - date = +DateTime.fromJSDate( - new Date((email?.headers as IDataObject)?.date as string), - ).toSeconds(); + date = +DateTime.fromJSDate(new Date(email.date)).toSeconds(); + } else if (email.headers?.date) { + date = +DateTime.fromJSDate(new Date(email.headers.date)).toSeconds(); } if (!date || isNaN(date)) { - emailsWithInvalidDate.add(email.id as string); + emailsWithInvalidDate.add(email.id); return +startDate; } return date; }; - const lastEmailDate = (responseData as IDataObject[]).reduce((lastDate, { json }) => { - const emailDate = getEmailDateAsSeconds(json as IDataObject); + const lastEmailDate = responseData.reduce((lastDate, { json }) => { + const emailDate = getEmailDateAsSeconds(json as Message); return emailDate > lastDate ? emailDate : lastDate; }, 0); - const nextPollPossibleDuplicates = (responseData as IDataObject[]).reduce( - (duplicates, { json }) => { - const emailDate = getEmailDateAsSeconds(json as IDataObject); - return emailDate <= lastEmailDate - ? duplicates.concat((json as IDataObject).id as string) - : duplicates; - }, - Array.from(emailsWithInvalidDate), - ); - - const possibleDuplicates = (nodeStaticData.possibleDuplicates as string[]) || []; - if (possibleDuplicates.length) { - responseData = (responseData as IDataObject[]).filter(({ json }) => { - const { id } = json as IDataObject; - return !possibleDuplicates.includes(id as string); + const nextPollPossibleDuplicates = responseData + .filter((item) => item.json) + .reduce((duplicates, { json }) => { + const emailDate = getEmailDateAsSeconds(json as Message); + return emailDate <= lastEmailDate ? duplicates.concat((json as Message).id) : duplicates; + }, Array.from(emailsWithInvalidDate)); + + const possibleDuplicates = new Set(nodeStaticData.possibleDuplicates ?? []); + if (possibleDuplicates.size > 0) { + responseData = responseData.filter(({ json }) => { + if (!json || typeof json.id !== 'string') return false; + return !possibleDuplicates.has(json.id); }); } @@ -402,7 +410,7 @@ export class GmailTrigger implements INodeType { nodeStaticData.lastTimeChecked = lastEmailDate || endDate; if (Array.isArray(responseData) && responseData.length) { - return [responseData as INodeExecutionData[]]; + return [responseData]; } return null; diff --git a/packages/nodes-base/nodes/Google/Gmail/test/GmailTrigger.test.ts b/packages/nodes-base/nodes/Google/Gmail/test/GmailTrigger.test.ts index 91362843514cf..c8aebdf6940ec 100644 --- a/packages/nodes-base/nodes/Google/Gmail/test/GmailTrigger.test.ts +++ b/packages/nodes-base/nodes/Google/Gmail/test/GmailTrigger.test.ts @@ -212,4 +212,63 @@ describe('GmailTrigger', () => { expect(response).toEqual(null); }); + + it('should handle duplicates and different date fields', async () => { + const messageListResponse: MessageListResponse = { + messages: [ + createListMessage({ id: '1' }), + createListMessage({ id: '2' }), + createListMessage({ id: '3' }), + createListMessage({ id: '4' }), + createListMessage({ id: '5' }), + ], + resultSizeEstimate: 123, + }; + + nock(baseUrl) + .get('/gmail/v1/users/me/labels') + .reply(200, { labels: [{ id: 'testLabelId', name: 'Test Label Name' }] }); + nock(baseUrl).get(new RegExp('/gmail/v1/users/me/messages?.*')).reply(200, messageListResponse); + nock(baseUrl) + .get(new RegExp('/gmail/v1/users/me/messages/1?.*')) + .reply(200, createMessage({ id: '1', internalDate: '1727777957863', date: undefined })); + nock(baseUrl) + .get(new RegExp('/gmail/v1/users/me/messages/2?.*')) + .reply(200, createMessage({ id: '2', internalDate: undefined, date: '1727777957863' })); + nock(baseUrl) + .get(new RegExp('/gmail/v1/users/me/messages/3?.*')) + .reply( + 200, + createMessage({ + id: '3', + internalDate: undefined, + date: undefined, + headers: { date: 'Thu, 5 Dec 2024 08:30:00 -0800' }, + }), + ); + nock(baseUrl) + .get(new RegExp('/gmail/v1/users/me/messages/4?.*')) + .reply( + 200, + createMessage({ + id: '4', + internalDate: undefined, + date: undefined, + headers: undefined, + }), + ); + nock(baseUrl).get(new RegExp('/gmail/v1/users/me/messages/5?.*')).reply(200, {}); + + const { response } = await testPollingTriggerNode(GmailTrigger, { + node: { parameters: { simple: true } }, + workflowStaticData: { + 'Gmail Trigger': { + lastTimeChecked: new Date('2024-10-31').getTime() / 1000, + possibleDuplicates: ['1'], + }, + }, + }); + + expect(response).toMatchSnapshot(); + }); }); diff --git a/packages/nodes-base/nodes/Google/Gmail/test/__snapshots__/GmailTrigger.test.ts.snap b/packages/nodes-base/nodes/Google/Gmail/test/__snapshots__/GmailTrigger.test.ts.snap new file mode 100644 index 0000000000000..a94fc6b5c05ee --- /dev/null +++ b/packages/nodes-base/nodes/Google/Gmail/test/__snapshots__/GmailTrigger.test.ts.snap @@ -0,0 +1,96 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`GmailTrigger should handle duplicates and different date fields 1`] = ` +[ + [ + { + "json": { + "date": "1727777957863", + "historyId": "testHistoryId", + "id": "2", + "labels": [ + { + "id": "testLabelId", + "name": "Test Label Name", + }, + ], + "payload": { + "body": { + "attachmentId": "testAttachmentId", + "data": "dGVzdA==", + "size": 4, + }, + "filename": "foo.txt", + "mimeType": "text/plain", + "partId": "testPartId", + "parts": [], + }, + "raw": "dGVzdA==", + "sizeEstimate": 4, + "snippet": "test", + "testHeader": "testHeaderValue", + "threadId": "testThreadId", + }, + }, + { + "json": { + "headers": { + "date": "Thu, 5 Dec 2024 08:30:00 -0800", + }, + "historyId": "testHistoryId", + "id": "3", + "labels": [ + { + "id": "testLabelId", + "name": "Test Label Name", + }, + ], + "payload": { + "body": { + "attachmentId": "testAttachmentId", + "data": "dGVzdA==", + "size": 4, + }, + "filename": "foo.txt", + "mimeType": "text/plain", + "partId": "testPartId", + "parts": [], + }, + "raw": "dGVzdA==", + "sizeEstimate": 4, + "snippet": "test", + "testHeader": "testHeaderValue", + "threadId": "testThreadId", + }, + }, + { + "json": { + "historyId": "testHistoryId", + "id": "4", + "labels": [ + { + "id": "testLabelId", + "name": "Test Label Name", + }, + ], + "payload": { + "body": { + "attachmentId": "testAttachmentId", + "data": "dGVzdA==", + "size": 4, + }, + "filename": "foo.txt", + "mimeType": "text/plain", + "partId": "testPartId", + "parts": [], + }, + "raw": "dGVzdA==", + "sizeEstimate": 4, + "snippet": "test", + "testHeader": "testHeaderValue", + "threadId": "testThreadId", + }, + }, + ], +] +`; diff --git a/packages/nodes-base/nodes/Google/Gmail/types.ts b/packages/nodes-base/nodes/Google/Gmail/types.ts index 5e519a242f919..545de29a75075 100644 --- a/packages/nodes-base/nodes/Google/Gmail/types.ts +++ b/packages/nodes-base/nodes/Google/Gmail/types.ts @@ -4,7 +4,9 @@ export type Message = { labelIds: string[]; snippet: string; historyId: string; - internalDate: string; + date?: string; + headers?: Record; + internalDate?: string; sizeEstimate: number; raw: string; payload: MessagePart; @@ -13,7 +15,7 @@ export type Message = { export type ListMessage = Pick; export type MessageListResponse = { - messages: ListMessage[]; + messages?: ListMessage[]; nextPageToken?: string; resultSizeEstimate: number; }; @@ -37,3 +39,32 @@ type MessagePartBody = { size: number; data: string; }; + +export type Label = { + id: string; + name: string; + messageListVisibility?: 'hide'; + labelListVisibility?: 'labelHide'; + type?: 'system'; +}; + +export type GmailWorkflowStaticData = { + lastTimeChecked?: number; + possibleDuplicates?: string[]; +}; +export type GmailWorkflowStaticDataDictionary = Record; + +export type GmailTriggerOptions = Partial<{ + dataPropertyAttachmentsPrefixName: string; + downloadAttachments: boolean; +}>; + +export type GmailTriggerFilters = Partial<{ + sender: string; + q: string; + includeSpamTrash: boolean; + includeDrafts: boolean; + readStatus: 'read' | 'unread' | 'both'; + labelIds: string[]; + receivedAfter: number; +}>;