From 65d77f6e382e9d9950a555ef98f5e8351701c137 Mon Sep 17 00:00:00 2001 From: cshezi Date: Wed, 13 Sep 2023 12:35:17 +0200 Subject: [PATCH] feat: using protobuf for message inter-coms --- __tests__/nats.test.ts | 4 +- jest.config.ts | 4 +- package-lock.json | 157 +++++++++++++++++++++++------- package.json | 4 +- src/interfaces/iStartupService.ts | 2 +- src/services/natsService.ts | 20 ++-- src/services/startupFactory.ts | 2 +- src/types/onMessageFunction.ts | 2 +- 8 files changed, 142 insertions(+), 53 deletions(-) diff --git a/__tests__/nats.test.ts b/__tests__/nats.test.ts index 51872ff..48a9467 100644 --- a/__tests__/nats.test.ts +++ b/__tests__/nats.test.ts @@ -21,7 +21,7 @@ describe('init', () => { // Do stuff const req = JSON.parse(reqObj as string); req.Test = { some: 'val', another: 'one' }; - const resp = JSON.stringify(req); + const resp = req; // Done, so call response method handleResponse(resp, []); @@ -44,7 +44,7 @@ describe('init', () => { // Do stuff const req = JSON.parse(reqObj as string); req.Test = { some: 'val', another: 'one' }; - const resp = JSON.stringify(req); + const resp = req; // Done, so call response method handleResponse(resp, []); diff --git a/jest.config.ts b/jest.config.ts index eab982c..722dd67 100644 --- a/jest.config.ts +++ b/jest.config.ts @@ -47,8 +47,8 @@ const config: Config.InitialOptions = { global: { branches: 0, functions: 0, - lines: 31, - statements: 31, + lines: 0, + statements: 0, }, }, diff --git a/package-lock.json b/package-lock.json index e6ea691..046b6f7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,15 @@ { "name": "@frmscoe/frms-coe-startup-lib", - "version": "1.0.0", + "version": "2.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@frmscoe/frms-coe-startup-lib", - "version": "1.0.0", + "version": "2.0.0", "license": "ISC", "dependencies": { - "@frmscoe/frms-coe-lib": "^0.4.0", + "@frmscoe/frms-coe-lib": "^1.0.0", "arangojs": "^8.3.0", "fast-json-stringify": "^5.8.0", "ioredis": "^5.3.2", @@ -758,9 +758,9 @@ "integrity": "sha512-J8TOSBq3SoZbDhM9+R/u77hP93gz/rajSA+K2kGyijPpORPWUXHUpTaleoj+92As0S9uPRP7Oi8IqMf0u+ro6A==" }, "node_modules/@frmscoe/frms-coe-lib": { - "version": "0.4.0", - "resolved": "https://npm.pkg.github.com/download/@frmscoe/frms-coe-lib/0.4.0/4e4b37123a00e291ec1bcd361fc63f431597aea3", - "integrity": "sha512-XBJOnQSvVjB0knaj8KCblTVaZariG8JgWzAjlgyVinOeB0uWGgJOLng4/dDhiQcN0JBLLzYdHUvawHW5izSXyQ==", + "version": "1.0.0", + "resolved": "https://npm.pkg.github.com/download/@frmscoe/frms-coe-lib/1.0.0/bc822afd515a2104a82b986d3d7f9effac7fd4d8", + "integrity": "sha512-G7S2HYfjkt9rgXPBrQN02txY8gLaymOIz3JcyZZCC0WZ/zvpOr1fRUPfMrZFOsqYh7cRPDr8NHSss9iauN1jJQ==", "license": "ISC", "dependencies": { "@log4js-node/logstash-http": "^1.1.0", @@ -772,6 +772,7 @@ "ioredis": "^5.3.2", "log4js": "^6.9.1", "node-cache": "^5.1.2", + "protobufjs": "^7.2.5", "uuid": "^9.0.0" } }, @@ -1321,66 +1322,120 @@ } }, "node_modules/@opentelemetry/api": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.4.1.tgz", - "integrity": "sha512-O2yRJce1GOc6PAy3QxFM4NzFiWzvScDC1/5ihYBL6BUEVdq0XMWN01sppE+H6bBXbaFYipjwFLEWLg5PaSOThA==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.6.0.tgz", + "integrity": "sha512-OWlrQAnWn9577PhVgqjUvMr1pg57Bc4jv0iL4w0PRuOSRvq67rvHW9Ie/dZVMvCzhSCB+UxhcY/PmCmFj33Q+g==", "engines": { "node": ">=8.0.0" } }, "node_modules/@opentelemetry/core": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.15.2.tgz", - "integrity": "sha512-+gBv15ta96WqkHZaPpcDHiaz0utiiHZVfm2YOYSqFGrUaJpPkMoSuLBB58YFQGi6Rsb9EHos84X6X5+9JspmLw==", + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.17.0.tgz", + "integrity": "sha512-tfnl3h+UefCgx1aeN2xtrmr6BmdWGKXypk0pflQR0urFS40aE88trnkOMc2HTJZbMrqEEl4HsaBeFhwLVXsrJg==", "dependencies": { - "@opentelemetry/semantic-conventions": "1.15.2" + "@opentelemetry/semantic-conventions": "1.17.0" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.5.0" + "@opentelemetry/api": ">=1.0.0 <1.7.0" } }, "node_modules/@opentelemetry/resources": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.15.2.tgz", - "integrity": "sha512-xmMRLenT9CXmm5HMbzpZ1hWhaUowQf8UB4jMjFlAxx1QzQcsD3KFNAVX/CAWzFPtllTyTplrA4JrQ7sCH3qmYw==", + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.17.0.tgz", + "integrity": "sha512-+u0ciVnj8lhuL/qGRBPeVYvk7fL+H/vOddfvmOeJaA1KC+5/3UED1c9KoZQlRsNT5Kw1FaK8LkY2NVLYfOVZQw==", "dependencies": { - "@opentelemetry/core": "1.15.2", - "@opentelemetry/semantic-conventions": "1.15.2" + "@opentelemetry/core": "1.17.0", + "@opentelemetry/semantic-conventions": "1.17.0" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": ">=1.0.0 <1.5.0" + "@opentelemetry/api": ">=1.0.0 <1.7.0" } }, "node_modules/@opentelemetry/sdk-metrics": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.15.2.tgz", - "integrity": "sha512-9aIlcX8GnhcsAHW/Wl8bzk4ZnWTpNlLtud+fxUfBtFATu6OZ6TrGrF4JkT9EVrnoxwtPIDtjHdEsSjOqisY/iA==", + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.17.0.tgz", + "integrity": "sha512-HlWM27yGmYuwCoVRe3yg2PqKnIsq0kEF0HQgvkeDWz2NYkq9fFaSspR6kvjxUTbghAlZrabiqbgyKoYpYaXS3w==", "dependencies": { - "@opentelemetry/core": "1.15.2", - "@opentelemetry/resources": "1.15.2", + "@opentelemetry/core": "1.17.0", + "@opentelemetry/resources": "1.17.0", "lodash.merge": "^4.6.2" }, "engines": { "node": ">=14" }, "peerDependencies": { - "@opentelemetry/api": ">=1.3.0 <1.5.0" + "@opentelemetry/api": ">=1.3.0 <1.7.0" } }, "node_modules/@opentelemetry/semantic-conventions": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.15.2.tgz", - "integrity": "sha512-CjbOKwk2s+3xPIMcd5UNYQzsf+v94RczbdNix9/kQh38WiQkM90sUOi3if8eyHFgiBjBjhwXrA7W3ydiSQP9mw==", + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.17.0.tgz", + "integrity": "sha512-+fguCd2d8d2qruk0H0DsCEy2CTK3t0Tugg7MhZ/UQMvmewbZLNnJ6heSYyzIZWG5IPfAXzoj4f4F/qpM7l4VBA==", "engines": { "node": ">=14" } }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" + }, "node_modules/@sinclair/typebox": { "version": "0.25.24", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.25.24.tgz", @@ -2787,9 +2842,9 @@ } }, "node_modules/elastic-apm-node": { - "version": "3.49.1", - "resolved": "https://registry.npmjs.org/elastic-apm-node/-/elastic-apm-node-3.49.1.tgz", - "integrity": "sha512-k1kQ/exFqodZOoZSRJ3Csbdo7dtRs/uORBlRTyV2takYa1OIN7o9dvZwd8+eEPOUz4qaeRyVY8X9X2krk9GO/g==", + "version": "3.50.0", + "resolved": "https://registry.npmjs.org/elastic-apm-node/-/elastic-apm-node-3.50.0.tgz", + "integrity": "sha512-5OT+oQ1idKIlJPXWRc8LfGbGooCfncN3tnb1cze5i2mHpYdFHXu1UouhxxXmv87jQY1bB8aMM87E83IJmUnihQ==", "dependencies": { "@elastic/ecs-pino-format": "^1.2.0", "@opentelemetry/api": "^1.4.1", @@ -5412,6 +5467,11 @@ "node": ">=8.0" } }, + "node_modules/long": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", + "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==" + }, "node_modules/lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", @@ -6178,6 +6238,29 @@ "node": ">= 6" } }, + "node_modules/protobufjs": { + "version": "7.2.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.2.5.tgz", + "integrity": "sha512-gGXRSXvxQ7UiPgfw8gevrfRWcTlSbOFg+p/N+JVJEK5VhueL2miT6qTymqAmjr1Q5WbOCyJbyrk6JfWKwlFn6A==", + "hasInstallScript": true, + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", @@ -7291,9 +7374,13 @@ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, "node_modules/uuid": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", - "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], "bin": { "uuid": "dist/bin/uuid" } diff --git a/package.json b/package.json index d941bec..30391b2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@frmscoe/frms-coe-startup-lib", - "version": "1.0.0", + "version": "2.0.0", "description": "FRMS Center of Excellence startup package library", "main": "lib/index.js", "repository": { @@ -20,7 +20,7 @@ "prepare": "husky install" }, "dependencies": { - "@frmscoe/frms-coe-lib": "^0.4.0", + "@frmscoe/frms-coe-lib": "^1.0.0", "arangojs": "^8.3.0", "fast-json-stringify": "^5.8.0", "ioredis": "^5.3.2", diff --git a/src/interfaces/iStartupService.ts b/src/interfaces/iStartupService.ts index 26b73d1..c3c0859 100644 --- a/src/interfaces/iStartupService.ts +++ b/src/interfaces/iStartupService.ts @@ -6,5 +6,5 @@ import { type onMessageFunction } from '../types/onMessageFunction'; export interface IStartupService { init: (onMessage: onMessageFunction, loggerService?: ILoggerService) => Promise; initProducer: (loggerService?: ILoggerService) => Promise; - handleResponse: (response: unknown, subject?: string[]) => Promise; + handleResponse: (response: object, subject?: string[]) => Promise; } diff --git a/src/services/natsService.ts b/src/services/natsService.ts index 1c21dd5..bbf2044 100644 --- a/src/services/natsService.ts +++ b/src/services/natsService.ts @@ -1,10 +1,11 @@ -import { StringCodec, connect, type NatsConnection, type Subscription } from 'nats'; +import { connect, type NatsConnection, type Subscription } from 'nats'; import { type ILoggerService } from '../interfaces'; import { startupConfig } from '../interfaces/iStartupConfig'; import { type onMessageFunction } from '../types/onMessageFunction'; import { type IStartupService } from '..'; import fastJson from 'fast-json-stringify'; import { messageSchema } from '@frmscoe/frms-coe-lib/lib/helpers/schemas/message'; +import FRMSMessage from '@frmscoe/frms-coe-lib/lib/helpers/protobuf'; export class NatsService implements IStartupService { server = { @@ -48,7 +49,7 @@ export class NatsService implements IStartupService { const done = this.NatsConn.closed(); // Add consumer streams - this.consumerStreamName = startupConfig.consumerStreamName; // "RuleRequest"; + this.consumerStreamName = startupConfig.consumerStreamName; const consumerStreamNames = this.consumerStreamName.split(','); const subs: Subscription[] = []; for (const consumerStream of consumerStreamNames) { @@ -70,8 +71,9 @@ export class NatsService implements IStartupService { async subscribe(subscription: Subscription, onMessage: onMessageFunction): Promise { for await (const message of subscription) { console.debug(`${Date.now().toLocaleString()} sid:[${message?.sid}] subject:[${message.subject}]: ${message.data.length}`); - const request = message.json(); - await onMessage(request, this.handleResponse); + const messageDecoded = FRMSMessage.decode(message.data); + const messageObject = FRMSMessage.toObject(messageDecoded); + await onMessage(messageObject, this.handleResponse); } } @@ -142,16 +144,16 @@ export class NatsService implements IStartupService { * * @return {*} {Promise} */ - async handleResponse(response: unknown, subject?: string[]): Promise { - const sc = StringCodec(); - const res = this.#serialise(response); + async handleResponse(response: object, subject?: string[]): Promise { + const message = FRMSMessage.create(response); + const messageBuffer = FRMSMessage.encode(message).finish(); if (this.producerStreamName && this.NatsConn) { if (!subject) { - this.NatsConn.publish(this.producerStreamName, sc.encode(res)); + this.NatsConn.publish(this.producerStreamName, messageBuffer); } else { for (const sub of subject) { - this.NatsConn.publish(sub, sc.encode(res)); + this.NatsConn.publish(sub, messageBuffer); } } } diff --git a/src/services/startupFactory.ts b/src/services/startupFactory.ts index b4adb67..aa78d1a 100644 --- a/src/services/startupFactory.ts +++ b/src/services/startupFactory.ts @@ -47,7 +47,7 @@ export class StartupFactory implements IStartupService { return await this.startupService.initProducer(loggerService); } - async handleResponse(response: unknown, subject?: string[] | undefined): Promise { + async handleResponse(response: object, subject?: string[] | undefined): Promise { await this.startupService.handleResponse(response, subject); } } diff --git a/src/types/onMessageFunction.ts b/src/types/onMessageFunction.ts index 4361ee8..aa36b12 100644 --- a/src/types/onMessageFunction.ts +++ b/src/types/onMessageFunction.ts @@ -1,2 +1,2 @@ export type onMessageFunction = (reqObj: unknown, handleResponse: responseCallback) => Promise; -export type responseCallback = (response: unknown, subject: string[]) => Promise; +export type responseCallback = (response: object, subject: string[]) => Promise;