From 3ccb0987a150965bcf46ac6171abcbfcad549162 Mon Sep 17 00:00:00 2001 From: bosiraphael Date: Mon, 3 Jun 2024 17:29:39 +0200 Subject: [PATCH 01/13] remove old logic --- ...ssaging-blocklist-reimport-messages.job.ts | 58 ------------------- .../listeners/messaging-blocklist.listener.ts | 24 +------- 2 files changed, 1 insertion(+), 81 deletions(-) delete mode 100644 packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-reimport-messages.job.ts diff --git a/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-reimport-messages.job.ts b/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-reimport-messages.job.ts deleted file mode 100644 index 12952b6ea336..000000000000 --- a/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-reimport-messages.job.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; - -import { MessageQueueJob } from 'src/engine/integrations/message-queue/interfaces/message-queue-job.interface'; - -import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator'; -import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository'; -import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; - -export type BlocklistReimportMessagesJobData = { - workspaceId: string; - workspaceMemberId: string; - handle: string; -}; - -@Injectable() -export class BlocklistReimportMessagesJob - implements MessageQueueJob -{ - private readonly logger = new Logger(BlocklistReimportMessagesJob.name); - - constructor( - @InjectObjectMetadataRepository(ConnectedAccountWorkspaceEntity) - private readonly connectedAccountRepository: ConnectedAccountRepository, - ) {} - - async handle(data: BlocklistReimportMessagesJobData): Promise { - const { workspaceId, workspaceMemberId, handle } = data; - - this.logger.log( - `Reimporting messages from handle ${handle} in workspace ${workspaceId} for workspace member ${workspaceMemberId}`, - ); - - const connectedAccount = - await this.connectedAccountRepository.getAllByWorkspaceMemberId( - workspaceMemberId, - workspaceId, - ); - - if (!connectedAccount || connectedAccount.length === 0) { - this.logger.error( - `No connected account found for workspace member ${workspaceMemberId} in workspace ${workspaceId}`, - ); - - return; - } - - // TODO: reimplement that - // await this.gmailMessageListFetchJob.fetchConnectedAccountThreads( - // workspaceId, - // connectedAccount[0].id, - // [handle], - // ); - - this.logger.log( - `Reimporting messages from ${handle} in workspace ${workspaceId} for workspace member ${workspaceMemberId} done`, - ); - } -} diff --git a/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts b/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts index 8d2ff5e1f940..302dbceb507b 100644 --- a/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts +++ b/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts @@ -6,10 +6,6 @@ import { ObjectRecordDeleteEvent } from 'src/engine/integrations/event-emitter/t import { MessageQueue } from 'src/engine/integrations/message-queue/message-queue.constants'; import { MessageQueueService } from 'src/engine/integrations/message-queue/services/message-queue.service'; import { BlocklistWorkspaceEntity } from 'src/modules/connected-account/standard-objects/blocklist.workspace-entity'; -import { - BlocklistReimportMessagesJob, - BlocklistReimportMessagesJobData, -} from 'src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-reimport-messages.job'; import { BlocklistItemDeleteMessagesJobData, BlocklistItemDeleteMessagesJob, @@ -39,16 +35,7 @@ export class MessagingBlocklistListener { @OnEvent('blocklist.deleted') async handleDeletedEvent( payload: ObjectRecordDeleteEvent, - ) { - await this.messageQueueService.add( - BlocklistReimportMessagesJob.name, - { - workspaceId: payload.workspaceId, - workspaceMemberId: payload.properties.before.workspaceMember.id, - handle: payload.properties.before.handle, - }, - ); - } + ) {} @OnEvent('blocklist.updated') async handleUpdatedEvent( @@ -61,14 +48,5 @@ export class MessagingBlocklistListener { blocklistItemId: payload.recordId, }, ); - - await this.messageQueueService.add( - BlocklistReimportMessagesJob.name, - { - workspaceId: payload.workspaceId, - workspaceMemberId: payload.properties.after.workspaceMember.id, - handle: payload.properties.before.handle, - }, - ); } } From f8176be53c6157a4f79773f262d829a1892874a7 Mon Sep 17 00:00:00 2001 From: bosiraphael Date: Tue, 4 Jun 2024 16:25:29 +0200 Subject: [PATCH 02/13] reset FULL_MESSAGES_LIST_FETCH_PENDING --- .../listeners/messaging-blocklist.listener.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts b/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts index 302dbceb507b..fb8c8950023e 100644 --- a/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts +++ b/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts @@ -11,12 +11,15 @@ import { BlocklistItemDeleteMessagesJob, } from 'src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job'; import { ObjectRecordUpdateEvent } from 'src/engine/integrations/event-emitter/types/object-record-update.event'; +import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; +import { MessagingChannelSyncStatusService } from 'src/modules/messaging/common/services/messaging-channel-sync-status.service'; @Injectable() export class MessagingBlocklistListener { constructor( @Inject(MessageQueue.messagingQueue) private readonly messageQueueService: MessageQueueService, + private readonly messagingChannelSyncStatusService: MessagingChannelSyncStatusService, ) {} @OnEvent('blocklist.created') @@ -35,7 +38,12 @@ export class MessagingBlocklistListener { @OnEvent('blocklist.deleted') async handleDeletedEvent( payload: ObjectRecordDeleteEvent, - ) {} + ) { + await this.messagingChannelSyncStatusService.resetAndScheduleFullMessageListFetch( + MessageChannelWorkspaceEntity.name, + payload.workspaceId, + ); + } @OnEvent('blocklist.updated') async handleUpdatedEvent( From 80aa19ce8202effc4b1dd56e009eddf9d19d5208 Mon Sep 17 00:00:00 2001 From: bosiraphael Date: Wed, 5 Jun 2024 16:38:10 +0200 Subject: [PATCH 03/13] blocklist is working --- .../listeners/messaging-blocklist.listener.ts | 33 +++++++++++++++++-- .../messaging-blocklist-manager.module.ts | 15 +++++++-- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts b/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts index fb8c8950023e..32033474d442 100644 --- a/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts +++ b/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts @@ -11,15 +11,23 @@ import { BlocklistItemDeleteMessagesJob, } from 'src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job'; import { ObjectRecordUpdateEvent } from 'src/engine/integrations/event-emitter/types/object-record-update.event'; -import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; +import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator'; +import { ConnectedAccountRepository } from 'src/modules/connected-account/repositories/connected-account.repository'; +import { ConnectedAccountWorkspaceEntity } from 'src/modules/connected-account/standard-objects/connected-account.workspace-entity'; +import { MessageChannelRepository } from 'src/modules/messaging/common/repositories/message-channel.repository'; import { MessagingChannelSyncStatusService } from 'src/modules/messaging/common/services/messaging-channel-sync-status.service'; +import { MessageChannelWorkspaceEntity } from 'src/modules/messaging/common/standard-objects/message-channel.workspace-entity'; @Injectable() export class MessagingBlocklistListener { constructor( @Inject(MessageQueue.messagingQueue) private readonly messageQueueService: MessageQueueService, + @InjectObjectMetadataRepository(ConnectedAccountWorkspaceEntity) + private readonly connectedAccountRepository: ConnectedAccountRepository, private readonly messagingChannelSyncStatusService: MessagingChannelSyncStatusService, + @InjectObjectMetadataRepository(MessageChannelWorkspaceEntity) + private readonly messageChannelRepository: MessageChannelRepository, ) {} @OnEvent('blocklist.created') @@ -39,9 +47,28 @@ export class MessagingBlocklistListener { async handleDeletedEvent( payload: ObjectRecordDeleteEvent, ) { + const workspaceMemberId = payload.properties.before.workspaceMember.id; + const workspaceId = payload.workspaceId; + + const connectedAccount = + await this.connectedAccountRepository.getAllByWorkspaceMemberId( + workspaceMemberId, + workspaceId, + ); + + if (!connectedAccount || connectedAccount.length === 0) { + return; + } + + const messageChannel = + await this.messageChannelRepository.getByConnectedAccountId( + connectedAccount[0].id, + workspaceId, + ); + await this.messagingChannelSyncStatusService.resetAndScheduleFullMessageListFetch( - MessageChannelWorkspaceEntity.name, - payload.workspaceId, + messageChannel[0].id, + workspaceId, ); } diff --git a/packages/twenty-server/src/modules/messaging/blocklist-manager/messaging-blocklist-manager.module.ts b/packages/twenty-server/src/modules/messaging/blocklist-manager/messaging-blocklist-manager.module.ts index 0d59ad486348..0ed9ad1fe66a 100644 --- a/packages/twenty-server/src/modules/messaging/blocklist-manager/messaging-blocklist-manager.module.ts +++ b/packages/twenty-server/src/modules/messaging/blocklist-manager/messaging-blocklist-manager.module.ts @@ -1,8 +1,19 @@ import { Module } from '@nestjs/common'; +import { BlocklistItemDeleteMessagesJob } from 'src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job'; +import { MessagingBlocklistListener } from 'src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener'; +import { MessagingCommonModule } from 'src/modules/messaging/common/messaging-common.module'; +import { MessagingMessageCleanerModule } from 'src/modules/messaging/message-cleaner/messaging-message-cleaner.module'; + @Module({ - imports: [], - providers: [], + imports: [MessagingCommonModule, MessagingMessageCleanerModule], + providers: [ + MessagingBlocklistListener, + { + provide: BlocklistItemDeleteMessagesJob.name, + useClass: BlocklistItemDeleteMessagesJob, + }, + ], exports: [], }) export class MessagingBlocklistManagerModule {} From 5fc783a08f69c38b850adfde746c2e9dd3ded273 Mon Sep 17 00:00:00 2001 From: bosiraphael Date: Wed, 5 Jun 2024 17:32:42 +0200 Subject: [PATCH 04/13] add subdomainsdeletion for blocklist --- .../message-channel-message-association.repository.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/twenty-server/src/modules/messaging/common/repositories/message-channel-message-association.repository.ts b/packages/twenty-server/src/modules/messaging/common/repositories/message-channel-message-association.repository.ts index fddbf24c6a38..82573a2ede2b 100644 --- a/packages/twenty-server/src/modules/messaging/common/repositories/message-channel-message-association.repository.ts +++ b/packages/twenty-server/src/modules/messaging/common/repositories/message-channel-message-association.repository.ts @@ -86,11 +86,12 @@ export class MessageChannelMessageAssociationRepository { JOIN ${dataSourceSchema}."message" ON "messageChannelMessageAssociation"."messageId" = ${dataSourceSchema}."message"."id" JOIN ${dataSourceSchema}."messageParticipant" "messageParticipant" ON ${dataSourceSchema}."message"."id" = "messageParticipant"."messageId" WHERE "messageParticipant"."handle" ${ - isHandleDomain ? 'ILIKE' : '=' + isHandleDomain ? '~*' : '=' } $1 AND "messageParticipant"."role" = ANY($2) AND "messageChannelMessageAssociation"."messageChannelId" = ANY($3)`, [ isHandleDomain - ? `%${messageParticipantHandle}` + ? // eslint-disable-next-line no-useless-escape + `.+@(.+\.)?${messageParticipantHandle.slice(1)}` : messageParticipantHandle, rolesToDelete, messageChannelIds, From fa301b997270b9e45aefe7f3169af1504bf3ca57 Mon Sep 17 00:00:00 2001 From: bosiraphael Date: Wed, 5 Jun 2024 17:34:16 +0200 Subject: [PATCH 05/13] update deleted event --- .../listeners/messaging-blocklist.listener.ts | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts b/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts index 32033474d442..c852bb9edb41 100644 --- a/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts +++ b/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts @@ -76,12 +76,36 @@ export class MessagingBlocklistListener { async handleUpdatedEvent( payload: ObjectRecordUpdateEvent, ) { + const workspaceMemberId = payload.properties.before.workspaceMember.id; + const workspaceId = payload.workspaceId; + await this.messageQueueService.add( BlocklistItemDeleteMessagesJob.name, { - workspaceId: payload.workspaceId, + workspaceId, blocklistItemId: payload.recordId, }, ); + + const connectedAccount = + await this.connectedAccountRepository.getAllByWorkspaceMemberId( + workspaceMemberId, + workspaceId, + ); + + if (!connectedAccount || connectedAccount.length === 0) { + return; + } + + const messageChannel = + await this.messageChannelRepository.getByConnectedAccountId( + connectedAccount[0].id, + workspaceId, + ); + + await this.messagingChannelSyncStatusService.resetAndScheduleFullMessageListFetch( + messageChannel[0].id, + workspaceId, + ); } } From 3114e6d46533ee71e8a747cedf071a4099c44b7d Mon Sep 17 00:00:00 2001 From: bosiraphael Date: Wed, 5 Jun 2024 17:34:16 +0200 Subject: [PATCH 06/13] update updated listener --- .../listeners/messaging-blocklist.listener.ts | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts b/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts index 32033474d442..c852bb9edb41 100644 --- a/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts +++ b/packages/twenty-server/src/modules/messaging/blocklist-manager/listeners/messaging-blocklist.listener.ts @@ -76,12 +76,36 @@ export class MessagingBlocklistListener { async handleUpdatedEvent( payload: ObjectRecordUpdateEvent, ) { + const workspaceMemberId = payload.properties.before.workspaceMember.id; + const workspaceId = payload.workspaceId; + await this.messageQueueService.add( BlocklistItemDeleteMessagesJob.name, { - workspaceId: payload.workspaceId, + workspaceId, blocklistItemId: payload.recordId, }, ); + + const connectedAccount = + await this.connectedAccountRepository.getAllByWorkspaceMemberId( + workspaceMemberId, + workspaceId, + ); + + if (!connectedAccount || connectedAccount.length === 0) { + return; + } + + const messageChannel = + await this.messageChannelRepository.getByConnectedAccountId( + connectedAccount[0].id, + workspaceId, + ); + + await this.messagingChannelSyncStatusService.resetAndScheduleFullMessageListFetch( + messageChannel[0].id, + workspaceId, + ); } } From 554987ac720fc5243d866e19ef72947b707faaed Mon Sep 17 00:00:00 2001 From: bosiraphael Date: Wed, 5 Jun 2024 17:42:39 +0200 Subject: [PATCH 07/13] update isEmailBlocklisted.util --- .../utils/is-email-blocklisted.util.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/twenty-server/src/modules/calendar-messaging-participant/utils/is-email-blocklisted.util.ts b/packages/twenty-server/src/modules/calendar-messaging-participant/utils/is-email-blocklisted.util.ts index 4231863bfcbe..8b150844b729 100644 --- a/packages/twenty-server/src/modules/calendar-messaging-participant/utils/is-email-blocklisted.util.ts +++ b/packages/twenty-server/src/modules/calendar-messaging-participant/utils/is-email-blocklisted.util.ts @@ -8,7 +8,9 @@ export const isEmailBlocklisted = ( return blocklist.some((item) => { if (item.startsWith('@')) { - return email.endsWith(item); + const domain = email.split('@')[1]; + + return domain === item.slice(1) || domain.endsWith(`.${item.slice(1)}`); } return email === item; From 0ab8d95f9dd3935d571ae96da1d718f97d28b311 Mon Sep 17 00:00:00 2001 From: bosiraphael Date: Wed, 5 Jun 2024 17:46:01 +0200 Subject: [PATCH 08/13] update tests --- .../__tests__/is-email-blocklisted.util.spec.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/twenty-server/src/modules/calendar-messaging-participant/utils/__tests__/is-email-blocklisted.util.spec.ts b/packages/twenty-server/src/modules/calendar-messaging-participant/utils/__tests__/is-email-blocklisted.util.spec.ts index 0ea2eacb34d0..a0a5ce84fa48 100644 --- a/packages/twenty-server/src/modules/calendar-messaging-participant/utils/__tests__/is-email-blocklisted.util.spec.ts +++ b/packages/twenty-server/src/modules/calendar-messaging-participant/utils/__tests__/is-email-blocklisted.util.spec.ts @@ -22,6 +22,20 @@ describe('isEmailBlocklisted', () => { expect(result).toBe(false); }); + it('should return true for subdomains', () => { + const email = 'hello@twenty.twenty.com'; + const blocklist = ['@twenty.com']; + const result = isEmailBlocklisted(email, blocklist); + + expect(result).toBe(true); + }); + it('should return false for domains which end with blocklisted domain but are not subdomains', () => { + const email = 'hello@twentytwenty.com'; + const blocklist = ['@twenty.com']; + const result = isEmailBlocklisted(email, blocklist); + + expect(result).toBe(false); + }); it('should return false if email is undefined', () => { const email = undefined; const blocklist = ['@example.com']; From 5542ca65d7b238571a3b7d925b342745fc72df2c Mon Sep 17 00:00:00 2001 From: bosiraphael Date: Tue, 11 Jun 2024 11:09:21 +0200 Subject: [PATCH 09/13] import emails from channel even if domain is blocklisted --- .../utils/is-email-blocklisted.util.ts | 3 ++- .../google-calendar-sync.service.ts | 11 +++++------ .../utils/filter-out-blocklisted-events.util.ts | 4 +++- .../messaging-gmail-messages-import.service.ts | 1 + .../utils/filter-emails.util.ts | 15 +++++++++++++-- 5 files changed, 24 insertions(+), 10 deletions(-) diff --git a/packages/twenty-server/src/modules/calendar-messaging-participant/utils/is-email-blocklisted.util.ts b/packages/twenty-server/src/modules/calendar-messaging-participant/utils/is-email-blocklisted.util.ts index 8b150844b729..af21bbe1ed96 100644 --- a/packages/twenty-server/src/modules/calendar-messaging-participant/utils/is-email-blocklisted.util.ts +++ b/packages/twenty-server/src/modules/calendar-messaging-participant/utils/is-email-blocklisted.util.ts @@ -1,8 +1,9 @@ export const isEmailBlocklisted = ( + channelHandle: string, email: string | null | undefined, blocklist: string[], ): boolean => { - if (!email) { + if (!email || email === channelHandle) { return false; } diff --git a/packages/twenty-server/src/modules/calendar/services/google-calendar-sync/google-calendar-sync.service.ts b/packages/twenty-server/src/modules/calendar/services/google-calendar-sync/google-calendar-sync.service.ts index 762bf0609ba0..ab64146e618a 100644 --- a/packages/twenty-server/src/modules/calendar/services/google-calendar-sync/google-calendar-sync.service.ts +++ b/packages/twenty-server/src/modules/calendar/services/google-calendar-sync/google-calendar-sync.service.ts @@ -121,14 +121,13 @@ export class GoogleCalendarSyncService { const blocklist = await this.getBlocklist(workspaceMemberId, workspaceId); - let filteredEvents = filterOutBlocklistedEvents(events, blocklist).filter( - (event) => event.status !== 'cancelled', - ); + let filteredEvents = filterOutBlocklistedEvents( + calendarChannel.handle, + events, + blocklist, + ).filter((event) => event.status !== 'cancelled'); if (emailOrDomainToReimport) { - // We still need to filter the events to only keep the ones that have the email or domain we want to reimport - // because the q parameter in the list method also filters the events that have the email or domain in their summary, description ... - // The q parameter allows us to narrow down the events filteredEvents = filteredEvents.filter( (event) => event.attendees?.some( diff --git a/packages/twenty-server/src/modules/calendar/utils/filter-out-blocklisted-events.util.ts b/packages/twenty-server/src/modules/calendar/utils/filter-out-blocklisted-events.util.ts index eafc669b1ae6..067e49ea5020 100644 --- a/packages/twenty-server/src/modules/calendar/utils/filter-out-blocklisted-events.util.ts +++ b/packages/twenty-server/src/modules/calendar/utils/filter-out-blocklisted-events.util.ts @@ -3,6 +3,7 @@ import { calendar_v3 as calendarV3 } from 'googleapis'; import { isEmailBlocklisted } from 'src/modules/calendar-messaging-participant/utils/is-email-blocklisted.util'; export const filterOutBlocklistedEvents = ( + calendarChannelHandle: string, events: calendarV3.Schema$Event[], blocklist: string[], ) => { @@ -12,7 +13,8 @@ export const filterOutBlocklistedEvents = ( } return event.attendees.every( - (attendee) => !isEmailBlocklisted(attendee.email, blocklist), + (attendee) => + !isEmailBlocklisted(calendarChannelHandle, attendee.email, blocklist), ); }); }; diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-messages-import.service.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-messages-import.service.ts index 73a9d485dabb..4b721754c8ef 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-messages-import.service.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/gmail/services/messaging-gmail-messages-import.service.ts @@ -112,6 +112,7 @@ export class MessagingGmailMessagesImportService { ); const messagesToSave = filterEmails( + messageChannel.handle, allMessages, blocklist.map((blocklistItem) => blocklistItem.handle), ); diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/filter-emails.util.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/filter-emails.util.ts index 9e1ebecfe95b..3bd9819f27d4 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/utils/filter-emails.util.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/utils/filter-emails.util.ts @@ -2,14 +2,20 @@ import { isEmailBlocklisted } from 'src/modules/calendar-messaging-participant/u import { GmailMessage } from 'src/modules/messaging/message-import-manager/drivers/gmail/types/gmail-message'; // Todo: refactor this into several utils -export const filterEmails = (messages: GmailMessage[], blocklist: string[]) => { +export const filterEmails = ( + messageChannelHandle: string, + messages: GmailMessage[], + blocklist: string[], +) => { return filterOutBlocklistedMessages( + messageChannelHandle, filterOutIcsAttachments(filterOutNonPersonalEmails(messages)), blocklist, ); }; const filterOutBlocklistedMessages = ( + messageChannelHandle: string, messages: GmailMessage[], blocklist: string[], ) => { @@ -19,7 +25,12 @@ const filterOutBlocklistedMessages = ( } return message.participants.every( - (participant) => !isEmailBlocklisted(participant.handle, blocklist), + (participant) => + !isEmailBlocklisted( + messageChannelHandle, + participant.handle, + blocklist, + ), ); }); }; From 142d958d54d440cb7854373d7a60bd6b80fda91a Mon Sep 17 00:00:00 2001 From: bosiraphael Date: Tue, 11 Jun 2024 11:20:53 +0200 Subject: [PATCH 10/13] add test --- .../is-email-blocklisted.util.spec.ts | 40 +++++++++++++------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/packages/twenty-server/src/modules/calendar-messaging-participant/utils/__tests__/is-email-blocklisted.util.spec.ts b/packages/twenty-server/src/modules/calendar-messaging-participant/utils/__tests__/is-email-blocklisted.util.spec.ts index a0a5ce84fa48..c7c6af34c30f 100644 --- a/packages/twenty-server/src/modules/calendar-messaging-participant/utils/__tests__/is-email-blocklisted.util.spec.ts +++ b/packages/twenty-server/src/modules/calendar-messaging-participant/utils/__tests__/is-email-blocklisted.util.spec.ts @@ -2,52 +2,68 @@ import { isEmailBlocklisted } from 'src/modules/calendar-messaging-participant/u describe('isEmailBlocklisted', () => { it('should return true if email is blocklisted', () => { - const email = 'hello@example.com'; - const blocklist = ['hello@example.com', 'hey@example.com']; - const result = isEmailBlocklisted(email, blocklist); + const channelHandle = 'abc@example.com'; + const email = 'hello@twenty.com'; + const blocklist = ['hello@twenty.com', 'hey@twenty.com']; + const result = isEmailBlocklisted(channelHandle, email, blocklist); expect(result).toBe(true); }); it('should return false if email is not blocklisted', () => { + const channelHandle = 'abc@example.com'; const email = 'hello@twenty.com'; const blocklist = ['hey@example.com']; - const result = isEmailBlocklisted(email, blocklist); + const result = isEmailBlocklisted(channelHandle, email, blocklist); expect(result).toBe(false); }); it('should return false if email is null', () => { + const channelHandle = 'abc@twenty.com'; const email = null; const blocklist = ['@example.com']; - const result = isEmailBlocklisted(email, blocklist); + const result = isEmailBlocklisted(channelHandle, email, blocklist); expect(result).toBe(false); }); it('should return true for subdomains', () => { + const channelHandle = 'abc@example.com'; const email = 'hello@twenty.twenty.com'; const blocklist = ['@twenty.com']; - const result = isEmailBlocklisted(email, blocklist); + const result = isEmailBlocklisted(channelHandle, email, blocklist); expect(result).toBe(true); }); it('should return false for domains which end with blocklisted domain but are not subdomains', () => { + const channelHandle = 'abc@example.com'; const email = 'hello@twentytwenty.com'; const blocklist = ['@twenty.com']; - const result = isEmailBlocklisted(email, blocklist); + const result = isEmailBlocklisted(channelHandle, email, blocklist); expect(result).toBe(false); }); it('should return false if email is undefined', () => { + const channelHandle = 'abc@example.com'; const email = undefined; - const blocklist = ['@example.com']; - const result = isEmailBlocklisted(email, blocklist); + const blocklist = ['@twenty.com']; + const result = isEmailBlocklisted(channelHandle, email, blocklist); expect(result).toBe(false); }); it('should return true if email ends with blocklisted domain', () => { - const email = 'hello@example.com'; - const blocklist = ['@example.com']; - const result = isEmailBlocklisted(email, blocklist); + const channelHandle = 'abc@example.com'; + const email = 'hello@twenty.com'; + const blocklist = ['@twenty.com']; + const result = isEmailBlocklisted(channelHandle, email, blocklist); expect(result).toBe(true); }); + + it('should return false if email is same as channel handle', () => { + const channelHandle = 'hello@twenty.com'; + const email = 'hello@twenty.com'; + const blocklist = ['@twenty.com']; + const result = isEmailBlocklisted(channelHandle, email, blocklist); + + expect(result).toBe(false); + }); }); From bd14add792078f23e12410830bc5a7d9d195b120 Mon Sep 17 00:00:00 2001 From: bosiraphael Date: Tue, 11 Jun 2024 12:40:59 +0200 Subject: [PATCH 11/13] modify deleteByMessageParticipantHandleAndMessageChannelIdsAndRoles --- ...-channel-message-association.repository.ts | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/packages/twenty-server/src/modules/messaging/common/repositories/message-channel-message-association.repository.ts b/packages/twenty-server/src/modules/messaging/common/repositories/message-channel-message-association.repository.ts index 82573a2ede2b..4cfbfde8237a 100644 --- a/packages/twenty-server/src/modules/messaging/common/repositories/message-channel-message-association.repository.ts +++ b/packages/twenty-server/src/modules/messaging/common/repositories/message-channel-message-association.repository.ts @@ -79,16 +79,31 @@ export class MessageChannelMessageAssociationRepository { const isHandleDomain = messageParticipantHandle.startsWith('@'); + const messageChannels = + await this.workspaceDataSourceService.executeRawQuery( + `SELECT * FROM ${dataSourceSchema}."messageChannel" + WHERE "id" = ANY($1)`, + [messageChannelIds], + workspaceId, + transactionManager, + ); + + const messageChannelHandles = messageChannels.map( + (messageChannel: { handle: string }) => messageChannel.handle, + ); + const messageChannelMessageAssociationIdsToDelete = await this.workspaceDataSourceService.executeRawQuery( `SELECT "messageChannelMessageAssociation".id FROM ${dataSourceSchema}."messageChannelMessageAssociation" "messageChannelMessageAssociation" JOIN ${dataSourceSchema}."message" ON "messageChannelMessageAssociation"."messageId" = ${dataSourceSchema}."message"."id" JOIN ${dataSourceSchema}."messageParticipant" "messageParticipant" ON ${dataSourceSchema}."message"."id" = "messageParticipant"."messageId" - WHERE "messageParticipant"."handle" ${ - isHandleDomain ? '~*' : '=' - } $1 AND "messageParticipant"."role" = ANY($2) AND "messageChannelMessageAssociation"."messageChannelId" = ANY($3)`, + WHERE "messageParticipant"."handle" != ALL($1) + AND "messageParticipant"."handle" ${isHandleDomain ? '~*' : '='} $2 + AND "messageParticipant"."role" = ANY($3) + AND "messageChannelMessageAssociation"."messageChannelId" = ANY($4)`, [ + messageChannelHandles, isHandleDomain ? // eslint-disable-next-line no-useless-escape `.+@(.+\.)?${messageParticipantHandle.slice(1)}` From b450aea831c6b8db7dc82503728d6f793d891a8f Mon Sep 17 00:00:00 2001 From: bosiraphael Date: Tue, 11 Jun 2024 15:20:16 +0200 Subject: [PATCH 12/13] line break --- .../message-channel-message-association.repository.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/twenty-server/src/modules/messaging/common/repositories/message-channel-message-association.repository.ts b/packages/twenty-server/src/modules/messaging/common/repositories/message-channel-message-association.repository.ts index 4cfbfde8237a..1d630748e899 100644 --- a/packages/twenty-server/src/modules/messaging/common/repositories/message-channel-message-association.repository.ts +++ b/packages/twenty-server/src/modules/messaging/common/repositories/message-channel-message-association.repository.ts @@ -82,7 +82,7 @@ export class MessageChannelMessageAssociationRepository { const messageChannels = await this.workspaceDataSourceService.executeRawQuery( `SELECT * FROM ${dataSourceSchema}."messageChannel" - WHERE "id" = ANY($1)`, + WHERE "id" = ANY($1)`, [messageChannelIds], workspaceId, transactionManager, From c64cd80668b463805e834ccfed0da0c5bfd4a387 Mon Sep 17 00:00:00 2001 From: bosiraphael Date: Tue, 11 Jun 2024 17:06:29 +0200 Subject: [PATCH 13/13] seems to work --- ...ging-blocklist-item-delete-messages.job.ts | 14 +++++++----- ...-channel-message-association.repository.ts | 22 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job.ts b/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job.ts index ac2f6defaa97..3a52e42deb17 100644 --- a/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job.ts +++ b/packages/twenty-server/src/modules/messaging/blocklist-manager/jobs/messaging-blocklist-item-delete-messages.job.ts @@ -66,12 +66,14 @@ export class BlocklistItemDeleteMessagesJob const rolesToDelete: ('from' | 'to')[] = ['from', 'to']; - await this.messageChannelMessageAssociationRepository.deleteByMessageParticipantHandleAndMessageChannelIdsAndRoles( - handle, - messageChannelIds, - rolesToDelete, - workspaceId, - ); + for (const messageChannelId of messageChannelIds) { + await this.messageChannelMessageAssociationRepository.deleteByMessageParticipantHandleAndMessageChannelIdAndRoles( + handle, + messageChannelId, + rolesToDelete, + workspaceId, + ); + } await this.threadCleanerService.cleanWorkspaceThreads(workspaceId); diff --git a/packages/twenty-server/src/modules/messaging/common/repositories/message-channel-message-association.repository.ts b/packages/twenty-server/src/modules/messaging/common/repositories/message-channel-message-association.repository.ts index 1d630748e899..8699a7670f3a 100644 --- a/packages/twenty-server/src/modules/messaging/common/repositories/message-channel-message-association.repository.ts +++ b/packages/twenty-server/src/modules/messaging/common/repositories/message-channel-message-association.repository.ts @@ -67,9 +67,9 @@ export class MessageChannelMessageAssociationRepository { ); } - public async deleteByMessageParticipantHandleAndMessageChannelIdsAndRoles( + public async deleteByMessageParticipantHandleAndMessageChannelIdAndRoles( messageParticipantHandle: string, - messageChannelIds: string[], + messageChannelId: string, rolesToDelete: ('from' | 'to' | 'cc' | 'bcc')[], workspaceId: string, transactionManager?: EntityManager, @@ -79,18 +79,16 @@ export class MessageChannelMessageAssociationRepository { const isHandleDomain = messageParticipantHandle.startsWith('@'); - const messageChannels = + const messageChannel = await this.workspaceDataSourceService.executeRawQuery( `SELECT * FROM ${dataSourceSchema}."messageChannel" - WHERE "id" = ANY($1)`, - [messageChannelIds], + WHERE "id" = $1`, + [messageChannelId], workspaceId, transactionManager, ); - const messageChannelHandles = messageChannels.map( - (messageChannel: { handle: string }) => messageChannel.handle, - ); + const messageChannelHandle = messageChannel[0].handle; const messageChannelMessageAssociationIdsToDelete = await this.workspaceDataSourceService.executeRawQuery( @@ -98,18 +96,18 @@ export class MessageChannelMessageAssociationRepository { FROM ${dataSourceSchema}."messageChannelMessageAssociation" "messageChannelMessageAssociation" JOIN ${dataSourceSchema}."message" ON "messageChannelMessageAssociation"."messageId" = ${dataSourceSchema}."message"."id" JOIN ${dataSourceSchema}."messageParticipant" "messageParticipant" ON ${dataSourceSchema}."message"."id" = "messageParticipant"."messageId" - WHERE "messageParticipant"."handle" != ALL($1) + WHERE "messageParticipant"."handle" != $1 AND "messageParticipant"."handle" ${isHandleDomain ? '~*' : '='} $2 AND "messageParticipant"."role" = ANY($3) - AND "messageChannelMessageAssociation"."messageChannelId" = ANY($4)`, + AND "messageChannelMessageAssociation"."messageChannelId" = $4`, [ - messageChannelHandles, + messageChannelHandle, isHandleDomain ? // eslint-disable-next-line no-useless-escape `.+@(.+\.)?${messageParticipantHandle.slice(1)}` : messageParticipantHandle, rolesToDelete, - messageChannelIds, + messageChannelId, ], workspaceId, transactionManager,