Merge pull request #327 from kmcwebdev/development

Development

package.json

@@ -55,7 +55,8 @@
     "pusher": "^5.1.3",
     "reflect-metadata": "^0.1.13",
     "rxjs": "^7.8.1",
-    "zod": "^3.22.2"
+    "stream-buffers": "^3.0.2",
+    "zod": "^3.22.3"
   },
   "devDependencies": {
     "@nestjs/cli": "^10.1.18",
@@ -69,6 +70,7 @@
     "@types/multer": "^1.4.8",
     "@types/node": "^20.3.1",
     "@types/pg": "^8.10.2",
+    "@types/stream-buffers": "^3.0.5",
     "@types/supertest": "^2.0.12",
     "@typescript-eslint/eslint-plugin": "^6.5.0",
     "@typescript-eslint/parser": "^6.5.0",

src/common/interface/filestack-response.interface.ts

@@ -0,0 +1,16 @@
+export interface FilestackResponse {
+  _file: {
+    name: string;
+    size: number;
+    type: string;
+    slice: () => Blob;
+  };
+  _sanitizeOptions?: unknown;
+  status: string;
+  handle: string;
+  url: string;
+  container?: string;
+  key?: string;
+  uploadTags?: unknown;
+  workflows?: unknown;
+}

src/common/schema/config.schema.ts

@@ -222,6 +222,22 @@ export const configSchema = z.object({
     invalid_type_error:
       'REIMBURSEMENT_ATTACHMENT_UPLOAD_ACCESS must be one of "public" or "private"',
   }),
+  LEXISNEXIS_ATTACHMENT_UPLOAD_CONTAINER: z
+    .string({
+      description: 'Filestack upload container',
+      required_error: 'LEXISNEXIS_ATTACHMENT_UPLOAD_CONTAINER is required',
+      invalid_type_error:
+        'LEXISNEXIS_ATTACHMENT_UPLOAD_CONTAINER must be a string',
+    })
+    .nonempty({
+      message: 'UPLOAD_CONTAINER must not be empty',
+    }),
+  LEXISNEXIS_ATTACHMENT_UPLOAD_ACCESS: z.enum(['public', 'private'], {
+    description: 'Filestack upload access',
+    required_error: 'LEXISNEXIS_ATTACHMENT_UPLOAD_ACCESS is required',
+    invalid_type_error:
+      'LEXISNEXIS_ATTACHMENT_UPLOAD_ACCESS must be one of "public" or "private"',
+  }),
   COMMON_UPLOAD_CONTAINER: z
     .string({
       description: 'Filestack upload container',

src/legal-and-compliance/common/interface/lexisnexis-download-metadata.inteface.ts

@@ -0,0 +1,6 @@
+export interface LexisnexisDownloadMetadata {
+  lexisnexis_search_id: string;
+  search_query: string;
+  download_id: string;
+  category: string;
+}

src/legal-and-compliance/legal-and-compliance.controller.ts

@@ -1,6 +1,7 @@
 import { Controller, Get, Query } from '@nestjs/common';
 import { LegalAndComplianceService } from './services/legal-and-compliance.service';
 import { LexisnexisSearchDto } from './common/dto/lexisnexis-search.dto';
+import { Public } from 'src/auth/common/decorator/public.decorator';
 
 @Controller('legal-and-compliance')
 export class LegalAndComplianceController {
@@ -12,4 +13,12 @@ export class LegalAndComplianceController {
   lexisnexisSearch(@Query() query: LexisnexisSearchDto) {
     return this.legalAndComplianceService.lexisnexisSearch(query);
   }
+
+  @Public()
+  @Get('test')
+  test() {
+    this.legalAndComplianceService.test();
+
+    return 'OK';
+  }
 }

src/legal-and-compliance/legal-and-compliance.module.ts

@@ -5,10 +5,15 @@ import { MemphisDevModule } from 'src/memphis-dev/memphis-dev.module';
 import { LegalAndComplianceController } from './legal-and-compliance.controller';
 import { LegalAndComplianceService } from './services/legal-and-compliance.service';
 import { LacLexisnexisSearchService } from './services/memphis/lexisnexis-search.memphis.service';
+import { LacLexisnexisDownloadService } from './services/memphis/lexisnexis-download.memphis.service';
 
 @Module({
   imports: [ScheduleModule.forRoot(), HttpModule, MemphisDevModule],
   controllers: [LegalAndComplianceController],
-  providers: [LegalAndComplianceService, LacLexisnexisSearchService],
+  providers: [
+    LegalAndComplianceService,
+    LacLexisnexisSearchService,
+    LacLexisnexisDownloadService,
+  ],
 })
 export class LegalAndComplianceModule {}

src/legal-and-compliance/services/legal-and-compliance.service.ts

@@ -51,4 +51,13 @@ export class LegalAndComplianceService {
 
     return insertJob;
   }
+
+  test() {
+    return this.eventEmitter.emit('lac-lexisnexis-download', {
+      lexisnexis_search_id: '6cd2673a-b4d0-4d35-855b-9129b2987ba6',
+      search_query: 'Oriental game limited',
+      category: 'negativeNews',
+      download_id: '8IL5csh0nyKXHbiDhdS98ZF3khrhHI-Z0QYXf4XV8og',
+    });
+  }
 }

src/legal-and-compliance/services/memphis/lexisnexis-download.memphis.service.ts

@@ -0,0 +1,155 @@
+import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { Consumer, MemphisService, Message, Producer } from 'memphis-dev';
+import { filestackClient } from 'src/common/lib/filestack';
+import { WritableStreamBuffer } from 'stream-buffers';
+import { OnEvent } from '@nestjs/event-emitter';
+import { HttpService } from '@nestjs/axios';
+import { catchError, firstValueFrom } from 'rxjs';
+import { AxiosError } from 'axios';
+import { InjectKysely } from 'nestjs-kysely';
+import { DB } from 'src/common/types';
+import { LexisnexisDownloadMetadata } from 'src/legal-and-compliance/common/interface/lexisnexis-download-metadata.inteface';
+import { FilestackResponse } from 'src/common/interface/filestack-response.interface';
+
+@Injectable()
+export class LacLexisnexisDownloadService implements OnModuleInit {
+  private readonly logger = new Logger(LacLexisnexisDownloadService.name);
+
+  consumer: Consumer;
+  producer: Producer;
+
+  uploadLocation = '';
+  uploadContainer = '';
+  uploadAccess = '';
+
+  constructor(
+    @InjectKysely() private readonly pgsql: DB,
+    private readonly configService: ConfigService,
+    private readonly httpService: HttpService,
+    private readonly memphisService: MemphisService,
+  ) {
+    this.uploadLocation = this.configService.get('UPLOAD_LOCATION');
+    this.uploadContainer = this.configService.get(
+      'LEXISNEXIS_ATTACHMENT_UPLOAD_CONTAINER',
+    );
+    this.uploadAccess = this.configService.get(
+      'LEXISNEXIS_ATTACHMENT_UPLOAD_ACCESS',
+    );
+  }
+
+  @OnEvent('lac-lexisnexis-download')
+  async triggerMemphisEvent(data: LexisnexisDownloadMetadata) {
+    return await this.producer.produce({
+      message: Buffer.from(JSON.stringify(data)),
+    });
+  }
+
+  async onModuleInit() {
+    try {
+      this.consumer = await this.memphisService.consumer({
+        stationName: 'erp.lac.lexisnexis-download',
+        consumerName: 'erp.lac.lexisnexis-download.consumer-name',
+        consumerGroup: 'erp.lac.lexisnexis-download.consumer-group',
+        pullIntervalMs: 10000,
+      });
+
+      this.consumer.on('message', async (message: Message) => {
+        const data: LexisnexisDownloadMetadata = JSON.parse(
+          message.getData().toString(),
+        );
+
+        const base64Pdf = await firstValueFrom(
+          this.httpService
+            .get<{ fileBase64: string; status: string }>(
+              '/api/v1/diligence/download',
+              {
+                baseURL: 'https://metabase.moreover.com',
+                headers: {
+                  'Content-Type': 'application/json',
+                },
+                params: {
+                  downloadId: data.download_id,
+                },
+              },
+            )
+            .pipe(
+              catchError(async (error: AxiosError) => {
+                const response = error.response.data as { status: string };
+
+                this.logger.error(JSON.stringify(error?.response?.data));
+
+                await this.pgsql
+                  .updateTable('lexisnexis_search')
+                  .set({
+                    finished_at: new Date(),
+                    report_generation_status: 'Failed',
+                    report_generation_desc: `Download - ${response.status}`,
+                  })
+                  .where('lexisnexis_search_id', '=', data.lexisnexis_search_id)
+                  .executeTakeFirstOrThrow();
+
+                throw Error('Failed to perform lexisnexis search');
+              }),
+            ),
+        );
+
+        if (base64Pdf.status === 200 && base64Pdf.data.status === 'COMPLETE') {
+          const pdf = Buffer.from(base64Pdf.data.fileBase64, 'base64');
+          const streamBuffer = new WritableStreamBuffer();
+
+          streamBuffer.write(pdf);
+          streamBuffer.end();
+
+          const contents = streamBuffer.getContents();
+
+          const dateNumber = Date.now();
+
+          const filename = `${data.search_query
+            .replace(/\s/g, '-')
+            .toLowerCase()}_${data.category.toLowerCase()}`;
+
+          const uniqueFilename = `${filename}_${dateNumber}.pdf`.toLowerCase();
+
+          if (!contents) throw new Error('PDF content is empty');
+
+          const fileHandle: FilestackResponse = await filestackClient.upload(
+            contents,
+            {
+              tags: {
+                search_query: filename,
+              },
+            },
+            {
+              location: this.uploadLocation,
+              filename: uniqueFilename,
+              container: this.uploadContainer,
+              access: this.uploadAccess,
+            },
+          );
+
+          await this.pgsql
+            .updateTable('lexisnexis_search')
+            .set({
+              pdf_report_url: fileHandle.url,
+            })
+            .where('lexisnexis_search_id', '=', data.lexisnexis_search_id)
+            .execute();
+
+          message.ack();
+        }
+      });
+
+      this.producer = await this.memphisService.producer({
+        stationName: 'erp.lac.lexisnexis-download',
+        producerName: 'erp.lac.lexisnexis-download.producer-name',
+      });
+
+      this.logger.log(
+        'Memphis legal and compliance lexisnexis download station is ready ⬇️  🚀',
+      );
+    } catch (error: any) {
+      this.logger.error(error.message);
+    }
+  }
+}