From d9fd7e7c440602cda70d0c12584c74498d4181b5 Mon Sep 17 00:00:00 2001 From: Audrius Vaitonis Date: Mon, 17 Jul 2023 16:53:03 +0100 Subject: [PATCH 1/8] feat(APIM-344): add log redacting for bootstrap, PinoLogger and TypeOrm --- .env.sample | 1 + .github/workflows/deployment.yml | 1 + docker-compose.yml | 1 + package-lock.json | 49 ++++--- package.json | 1 + src/config/app.config.test.ts | 135 ++++++++++++++++++ src/config/app.config.ts | 3 +- src/constants/index.ts | 4 + .../redact-strings-paths.constant.ts | 10 ++ src/constants/redact-strings.constant.ts | 9 ++ src/helpers/redact-errors.helper.test.ts | 88 ++++++++++++ src/helpers/redact-errors.helper.ts | 27 ++++ src/logging/build-key-to-redact.test.ts | 34 +++++ src/logging/build-key-to-redact.ts | 10 ++ src/logging/console-logger-with-redact.ts | 21 +++ src/logging/log-keys-to-redact.test.ts | 80 +++++++++++ src/logging/log-keys-to-redact.ts | 77 ++++++++++ src/logging/logging-interceptor.helper.ts | 20 +++ src/main.module.ts | 32 +++++ src/main.ts | 7 +- src/modules/constants/constants.service.ts | 14 +- src/modules/currencies/currencies.service.ts | 14 +- .../exposure-period.service.ts | 12 +- src/modules/http/http.constants.ts | 2 + .../interest-rates/interest-rates.service.ts | 12 +- src/modules/markets/markets.service.ts | 12 +- src/modules/numbers/numbers.service.test.ts | 11 +- src/modules/numbers/numbers.service.ts | 13 +- .../premium-schedules.service.ts | 14 +- .../sector-industries.service.ts | 14 +- .../yield-rates/yield-rates.service.ts | 14 +- .../get-docs-yaml.api-test.ts.snap | 2 +- 32 files changed, 673 insertions(+), 71 deletions(-) create mode 100644 src/config/app.config.test.ts create mode 100644 src/constants/redact-strings-paths.constant.ts create mode 100644 src/constants/redact-strings.constant.ts create mode 100644 src/helpers/redact-errors.helper.test.ts create mode 100644 src/helpers/redact-errors.helper.ts create mode 100644 src/logging/build-key-to-redact.test.ts create mode 100644 src/logging/build-key-to-redact.ts create mode 100644 src/logging/console-logger-with-redact.ts create mode 100644 src/logging/log-keys-to-redact.test.ts create mode 100644 src/logging/log-keys-to-redact.ts create mode 100644 src/logging/logging-interceptor.helper.ts diff --git a/.env.sample b/.env.sample index 5d6b3eba..b3f44894 100644 --- a/.env.sample +++ b/.env.sample @@ -2,6 +2,7 @@ PORT= LOG_LEVEL=debug +REDACT_LOGS=false # Swagger SWAGGER_USER= diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index 1fcd9c59..b60309ab 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -123,6 +123,7 @@ jobs: "PORT=${{ secrets.PORT }}" \ "NODE_ENV=${{ secrets.NODE_ENV }}" \ "LOG_LEVEL=${{ secrets.LOG_LEVEL }}" \ + "REDACT_LOGS=${{ secrets.REDACT_LOGS }}" \ "TZ=${{ secrets.TZ }}" \ "SWAGGER_USER=${{ secrets.SWAGGER_USER }}" \ "SWAGGER_PASSWORD=${{ secrets.SWAGGER_PASSWORD }}" \ diff --git a/docker-compose.yml b/docker-compose.yml index 5b8fd768..58087a05 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,6 +18,7 @@ services: PORT: NODE_ENV: LOG_LEVEL: + REDACT_LOGS: TZ: SWAGGER_USER: SWAGGER_PASSWORD: diff --git a/package-lock.json b/package-lock.json index 889dfdff..749a2dac 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42,17 +42,18 @@ "devDependencies": { "@commitlint/cli": "^17.6.6", "@commitlint/config-conventional": "^17.6.6", - "@nestjs/cli": "^10.1.8", + "@nestjs/cli": "^10.1.7", "@nestjs/schematics": "^10.0.1", "@nestjs/testing": "^10.0.5", "@tsconfig/node20": "^1.0.2", "@types/chance": "^1.1.3", "@types/compression": "^1.7.2", "@types/express": "^4.17.17", - "@types/jest": "^29.5.2", + "@types/jest": "^29.5.3", + "@types/lodash": "^4.14.195", "@types/node": "^20.4.1", "@types/supertest": "^2.0.12", - "@typescript-eslint/eslint-plugin": "^5.61.0", + "@typescript-eslint/eslint-plugin": "^6.0.0", "@typescript-eslint/parser": "^5.61.0", "chance": "^1.1.11", "cspell": "^6.31.1", @@ -1759,7 +1760,7 @@ "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", - "devOptional": true, + "dev": true, "dependencies": { "@jridgewell/trace-mapping": "0.3.9" }, @@ -1771,7 +1772,7 @@ "version": "0.3.9", "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", - "devOptional": true, + "dev": true, "dependencies": { "@jridgewell/resolve-uri": "^3.0.3", "@jridgewell/sourcemap-codec": "^1.4.10" @@ -2304,7 +2305,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", "integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==", - "devOptional": true, + "dev": true, "engines": { "node": ">=6.0.0" } @@ -2332,7 +2333,7 @@ "version": "1.4.15", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", - "devOptional": true + "dev": true }, "node_modules/@jridgewell/trace-mapping": { "version": "0.3.18", @@ -2912,25 +2913,25 @@ "version": "1.0.9", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz", "integrity": "sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA==", - "devOptional": true + "dev": true }, "node_modules/@tsconfig/node12": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", - "devOptional": true + "dev": true }, "node_modules/@tsconfig/node14": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", - "devOptional": true + "dev": true }, "node_modules/@tsconfig/node16": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", - "devOptional": true + "dev": true }, "node_modules/@tsconfig/node20": { "version": "1.0.2", @@ -3130,6 +3131,12 @@ "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", "dev": true }, + "node_modules/@types/lodash": { + "version": "4.14.195", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.195.tgz", + "integrity": "sha512-Hwx9EUgdwf2GLarOjQp5ZH8ZmblzcbTBC2wtQWNKARBSxM9ezRIAUpeDTgoQRAFB0+8CNWXVA9+MaSOzOF3nPg==", + "dev": true + }, "node_modules/@types/mime": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", @@ -3146,7 +3153,7 @@ "version": "20.4.1", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.4.1.tgz", "integrity": "sha512-JIzsAvJeA/5iY6Y/OxZbv1lUcc8dNSE77lb2gnBH+/PJ3lFR1Ccvgwl5JWnHAkNHcRsT0TbpVOsiMKZ1F/yyJg==", - "devOptional": true + "dev": true }, "node_modules/@types/normalize-package-data": { "version": "2.4.1", @@ -3826,7 +3833,7 @@ "version": "8.10.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.10.0.tgz", "integrity": "sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw==", - "devOptional": true, + "dev": true, "bin": { "acorn": "bin/acorn" }, @@ -3856,7 +3863,7 @@ "version": "8.2.0", "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.2.0.tgz", "integrity": "sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==", - "devOptional": true, + "dev": true, "engines": { "node": ">=0.4.0" } @@ -4050,7 +4057,7 @@ "version": "4.1.3", "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", - "devOptional": true + "dev": true }, "node_modules/argparse": { "version": "2.0.1", @@ -5401,7 +5408,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", - "devOptional": true + "dev": true }, "node_modules/cross-spawn": { "version": "7.0.3", @@ -5966,7 +5973,7 @@ "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", - "devOptional": true, + "dev": true, "engines": { "node": ">=0.3.1" } @@ -10487,7 +10494,7 @@ "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", - "devOptional": true + "dev": true }, "node_modules/makeerror": { "version": "1.0.12", @@ -13705,7 +13712,7 @@ "version": "10.9.1", "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==", - "devOptional": true, + "dev": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -14217,7 +14224,7 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", - "devOptional": true + "dev": true }, "node_modules/v8-to-istanbul": { "version": "9.1.0", @@ -14611,7 +14618,7 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", - "devOptional": true, + "dev": true, "engines": { "node": ">=6" } diff --git a/package.json b/package.json index 7630693d..d67dc12a 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ "@types/compression": "^1.7.2", "@types/express": "^4.17.17", "@types/jest": "^29.5.3", + "@types/lodash": "^4.14.195", "@types/node": "^20.4.1", "@types/supertest": "^2.0.12", "@typescript-eslint/eslint-plugin": "^6.0.0", diff --git a/src/config/app.config.test.ts b/src/config/app.config.test.ts new file mode 100644 index 00000000..4197317d --- /dev/null +++ b/src/config/app.config.test.ts @@ -0,0 +1,135 @@ +import { RandomValueGenerator } from '@ukef-test/support/generator/random-value-generator'; + +import appConfig from './app.config'; +import { InvalidConfigException } from './invalid-config.exception'; + +describe('appConfig', () => { + const valueGenerator = new RandomValueGenerator(); + + let originalProcessEnv: NodeJS.ProcessEnv; + + beforeEach(() => { + originalProcessEnv = process.env; + }); + + afterEach(() => { + process.env = originalProcessEnv; + }); + + describe('parsing LOG_LEVEL', () => { + it('throws an InvalidConfigException if LOG_LEVEL is specified but is not a valid log level', () => { + replaceEnvironmentVariables({ + LOG_LEVEL: 'not-a-real-log-level', + }); + + const gettingTheAppConfig = () => appConfig(); + + expect(gettingTheAppConfig).toThrow(InvalidConfigException); + expect(gettingTheAppConfig).toThrow(`LOG_LEVEL must be one of fatal,error,warn,info,debug,trace,silent or not specified.`); + }); + + it('uses info as the logLevel if LOG_LEVEL is not specified', () => { + replaceEnvironmentVariables({}); + + const config = appConfig(); + + expect(config.logLevel).toBe('info'); + }); + + it('uses info as the logLevel if LOG_LEVEL is empty', () => { + replaceEnvironmentVariables({ + LOG_LEVEL: '', + }); + + const config = appConfig(); + + expect(config.logLevel).toBe('info'); + }); + + it.each([ + { + LOG_LEVEL: 'fatal', + }, + { + LOG_LEVEL: 'error', + }, + { + LOG_LEVEL: 'warn', + }, + { + LOG_LEVEL: 'info', + }, + { + LOG_LEVEL: 'debug', + }, + { + LOG_LEVEL: 'trace', + }, + { + LOG_LEVEL: 'silent', + }, + ])('uses LOG_LEVEL as the logLevel if LOG_LEVEL is valid ($LOG_LEVEL)', ({ LOG_LEVEL }) => { + replaceEnvironmentVariables({ + LOG_LEVEL, + }); + + const config = appConfig(); + + expect(config.logLevel).toBe(LOG_LEVEL); + }); + }); + + describe('parsing REDACT_LOGS', () => { + it('sets redactLogs to true if REDACT_LOGS is true', () => { + replaceEnvironmentVariables({ + REDACT_LOGS: 'true', + }); + + const config = appConfig(); + + expect(config.redactLogs).toBe(true); + }); + + it('sets redactLogs to false if REDACT_LOGS is false', () => { + replaceEnvironmentVariables({ + REDACT_LOGS: 'false', + }); + + const config = appConfig(); + + expect(config.redactLogs).toBe(false); + }); + + it('sets redactLogs to true if REDACT_LOGS is not specified', () => { + replaceEnvironmentVariables({}); + + const config = appConfig(); + + expect(config.redactLogs).toBe(true); + }); + + it('sets redactLogs to true if REDACT_LOGS is the empty string', () => { + replaceEnvironmentVariables({ + REDACT_LOGS: '', + }); + + const config = appConfig(); + + expect(config.redactLogs).toBe(true); + }); + + it('sets redactLogs to true if REDACT_LOGS is any string other than true or false', () => { + replaceEnvironmentVariables({ + REDACT_LOGS: valueGenerator.string(), + }); + + const config = appConfig(); + + expect(config.redactLogs).toBe(true); + }); + }); + + const replaceEnvironmentVariables = (newEnvVariables: Record): void => { + process.env = newEnvVariables; + }; +}); diff --git a/src/config/app.config.ts b/src/config/app.config.ts index 003f4824..f38e82c5 100644 --- a/src/config/app.config.ts +++ b/src/config/app.config.ts @@ -14,7 +14,7 @@ export default registerAs('app', (): Record => { env: process.env.APP_ENV || 'development', versioning: { - enable: process.env.HTTP_VERSIONING_ENABLE === 'true' || false, + enable: process.env.HTTP_VERSIONING_ENABLE === 'true', prefix: 'v', version: process.env.HTTP_VERSION || '1', }, @@ -23,5 +23,6 @@ export default registerAs('app', (): Record => { port: process.env.HTTP_PORT ? Number.parseInt(process.env.HTTP_PORT, 10) : 3003, apiKey: process.env.API_KEY, logLevel: process.env.LOG_LEVEL || 'info', + redactLogs: process.env.REDACT_LOGS !== 'false', }; }); diff --git a/src/constants/index.ts b/src/constants/index.ts index 3ea8323d..b0c176ab 100644 --- a/src/constants/index.ts +++ b/src/constants/index.ts @@ -7,6 +7,8 @@ * 3. Auth strategy * 4. Products * 5. Customers + * 6. Strings to redact + * 7. Strings locations to redact */ export * from './auth.constant'; @@ -16,3 +18,5 @@ export * from './date.constant'; export * from './enums'; export * from './products.constant'; export * from './ukef-id.constant'; +export * from './redact-strings.constant'; +export * from './redact-strings-paths.constant'; diff --git a/src/constants/redact-strings-paths.constant.ts b/src/constants/redact-strings-paths.constant.ts new file mode 100644 index 00000000..1e2fc3bb --- /dev/null +++ b/src/constants/redact-strings-paths.constant.ts @@ -0,0 +1,10 @@ +export const REDACT_STRING_PATHS = [ + 'driverError.originalError.message', + 'driverError.originalError.stack', + 'driverError.message', + 'driverError.stack', + 'message', + 'stack', + 'originalError.message', + 'originalError.stack', +]; diff --git a/src/constants/redact-strings.constant.ts b/src/constants/redact-strings.constant.ts new file mode 100644 index 00000000..d43044dd --- /dev/null +++ b/src/constants/redact-strings.constant.ts @@ -0,0 +1,9 @@ +export const REDACT_STRINGS = [ + {searchValue: /(Login failed for user ').*(')/g, replaceValue: '$1[Redacted]$2'}, + {searchValue: process.env.DATABASE_USERNAME, replaceValue: '[RedactedDomain]'}, + {searchValue: process.env.DATABASE_PASSWORD, replaceValue: '[RedactedDomain]'}, + {searchValue: process.env.DATABASE_MDM_HOST, replaceValue: '[RedactedDomain]'}, + {searchValue: process.env.DATABASE_CEDAR_HOST, replaceValue: '[RedactedDomain]'}, + {searchValue: process.env.DATABASE_NUMBER_GENERATOR_HOST, replaceValue: '[RedactedDomain]'}, + {searchValue: process.env.DATABASE_CIS_HOST, replaceValue: '[RedactedDomain]'}, +]; diff --git a/src/helpers/redact-errors.helper.test.ts b/src/helpers/redact-errors.helper.test.ts new file mode 100644 index 00000000..96c6a2f1 --- /dev/null +++ b/src/helpers/redact-errors.helper.test.ts @@ -0,0 +1,88 @@ +import { RandomValueGenerator } from '@ukef-test/support/generator/random-value-generator'; +import { redactError } from './redact-errors.helper'; +import { regexToString } from './regex.helper'; +import { REDACT_STRING_PATHS } from '@ukef/constants'; + +describe('Redact errors helper', () => { + const valueGenerator = new RandomValueGenerator(); + + describe('redactError', () => { + const domain = valueGenerator.httpsUrl(); + const otherSensitivefield = valueGenerator.word(); + const message = `ConnectionError: Failed to connect to ${domain}, ${otherSensitivefield}`; + const redactedMessage = `ConnectionError: Failed to connect to [RedactedDomain], [Redacted]`; + const redactStrings = [ + {searchValue: domain, replaceValue: '[RedactedDomain]'}, + {searchValue: otherSensitivefield, replaceValue: '[Redacted]'}, + ]; + const error = { + message: message, + stack: message, + originalError: { + message: message, + stack: message, + safe: 'Nothing sensitive', + }, + driverError: { + message: message, + stack: message, + originalError: { + message: message, + stack: message, + safe: 'Nothing sensitive', + } + }, + }; + const expectedError = { + message: redactedMessage, + stack: redactedMessage, + originalError: { + message: redactedMessage, + stack: redactedMessage, + safe: 'Nothing sensitive', + }, + driverError: { + message: redactedMessage, + stack: redactedMessage, + originalError: { + message: redactedMessage, + stack: redactedMessage, + safe: 'Nothing sensitive', + } + }, + }; + + it('replaces sensitive data in input object', () => { + const redacted = redactError(true, REDACT_STRING_PATHS, redactStrings, error); + expect(redacted).toStrictEqual(expectedError); + }); + + it('returns original input if redactLogs is set to false', () => { + const redacted = redactError(false, REDACT_STRING_PATHS, redactStrings, error); + expect(redacted).toStrictEqual(error); + }); + + it('replaces sensitive data in different input object', () => { + const error = { + field1: message, + field2: { + field3: message, + safe: 'Nothing sensitive', + }, + }; + const expectedError = { + field1: redactedMessage, + field2: { + field3: redactedMessage, + safe: 'Nothing sensitive', + }, + }; + const redactPaths = [ + 'field1', + 'field2.field3', + ]; + const redacted = redactError(true, redactPaths, redactStrings, error); + expect(redacted).toStrictEqual(expectedError); + }); + }); +}); diff --git a/src/helpers/redact-errors.helper.ts b/src/helpers/redact-errors.helper.ts new file mode 100644 index 00000000..f645c968 --- /dev/null +++ b/src/helpers/redact-errors.helper.ts @@ -0,0 +1,27 @@ +import { REDACT_STRINGS, REDACT_STRING_PATHS } from "@ukef/constants"; + +import {get, set} from "lodash"; + +// This helper function is used to redact sensitive data in Error object strings. +export const redactError = (redactLogs: boolean, redactPaths: string[], redactStrings: {searchValue: string | RegExp, replaceValue: string}[], err: any): any => { + if (!redactLogs) { + return err; + } + redactPaths.forEach((path) => { + //REDACT_STRING_PATHS.forEach((path) => { + let value: string = get(err, path); + if (value) { + const safeValue = redactString(redactStrings, value); + set(err, path, safeValue); + } + }) + return err; +}; + +const redactString = (redactStrings: {searchValue: string | RegExp, replaceValue: string}[], string: string): string => { + let safeSTring: string = string; + redactStrings.forEach( (redact) => { + safeSTring = safeSTring.replaceAll(redact.searchValue, redact.replaceValue); + }); + return safeSTring; +} diff --git a/src/logging/build-key-to-redact.test.ts b/src/logging/build-key-to-redact.test.ts new file mode 100644 index 00000000..49df4053 --- /dev/null +++ b/src/logging/build-key-to-redact.test.ts @@ -0,0 +1,34 @@ +import { RandomValueGenerator } from '@ukef-test/support/generator/random-value-generator'; + +import { buildKeyToRedact } from './build-key-to-redact'; + +describe('buildKeyToRedact', () => { + const valueGenerator = new RandomValueGenerator(); + const part1 = valueGenerator.string(); + const part2 = valueGenerator.string(); + const part3 = valueGenerator.string(); + + it('returns an empty string if there are no parts to build', () => { + const keyToRedact = buildKeyToRedact([]); + + expect(keyToRedact).toBe(''); + }); + + it('returns the string in the format [""] if there is only 1 part to build', () => { + const keyToRedact = buildKeyToRedact([part1]); + + expect(keyToRedact).toBe(`["${part1}"]`); + }); + + it('returns the parts joined in the format [""][""] if there are 2 parts', () => { + const keyToRedact = buildKeyToRedact([part1, part2]); + + expect(keyToRedact).toBe(`["${part1}"]["${part2}"]`); + }); + + it('returns the parts joined in the format [""][""] if there are more than 2 parts', () => { + const keyToRedact = buildKeyToRedact([part1, part2, part3]); + + expect(keyToRedact).toBe(`["${part1}"]["${part2}"]["${part3}"]`); + }); +}); diff --git a/src/logging/build-key-to-redact.ts b/src/logging/build-key-to-redact.ts new file mode 100644 index 00000000..ca17309f --- /dev/null +++ b/src/logging/build-key-to-redact.ts @@ -0,0 +1,10 @@ +const prefix = `["`; +const suffix = `"]`; +const joinSeparator = `${suffix}${prefix}`; + +export const buildKeyToRedact = (parts: string[]): string => { + if (!parts.length) { + return ''; + } + return `${prefix}${parts.join(joinSeparator)}${suffix}`; +}; diff --git a/src/logging/console-logger-with-redact.ts b/src/logging/console-logger-with-redact.ts new file mode 100644 index 00000000..b63d32d8 --- /dev/null +++ b/src/logging/console-logger-with-redact.ts @@ -0,0 +1,21 @@ +import { ConsoleLogger } from '@nestjs/common'; + +export class ConsoleLoggerWithRedact extends ConsoleLogger { + + private stringPatternsToRedact: [{searchValue: string | RegExp, replaceValue, string}]; + + constructor(stringPatternsToRedact) { + super(); + this.stringPatternsToRedact = stringPatternsToRedact; + } + + error(message: any, stack?: string, context?: string) { + let cleanStack = stack; + if (typeof stack == 'string') { + this.stringPatternsToRedact.forEach( (redact) => { + cleanStack = stack.replace(redact.searchValue, redact.replaceValue); + }) + } + super.error(message, cleanStack, context); + } +} \ No newline at end of file diff --git a/src/logging/log-keys-to-redact.test.ts b/src/logging/log-keys-to-redact.test.ts new file mode 100644 index 00000000..45cf2495 --- /dev/null +++ b/src/logging/log-keys-to-redact.test.ts @@ -0,0 +1,80 @@ +import { RandomValueGenerator } from '@ukef-test/support/generator/random-value-generator'; + +import { buildKeyToRedact } from './build-key-to-redact'; +import { logKeysToRedact, LogKeysToRedactOptions } from './log-keys-to-redact'; + +describe('logKeysToRedact', () => { + const valueGenerator = new RandomValueGenerator(); + const options: Omit = { + clientRequest: { + logKey: valueGenerator.string(), + headersLogKey: valueGenerator.string(), + }, + outgoingRequest: { + logKey: valueGenerator.string(), + headersLogKey: valueGenerator.string(), + }, + error: { + logKey: valueGenerator.string(), + sensitiveChildKeys: [valueGenerator.string(), valueGenerator.string()], + }, + dbError: { + logKey: valueGenerator.string(), + sensitiveChildKeys: [valueGenerator.string(), valueGenerator.string()], + }, + }; + + describe('when redactLogs is false', () => { + it('returns an empty array', () => { + expect(logKeysToRedact({ redactLogs: false, ...options })).toStrictEqual([]); + }); + }); + + describe('when redactLogs is true', () => { + let result: string[]; + + beforeEach(() => { + result = logKeysToRedact({ redactLogs: true, ...options }); + }); + + it('includes the headers of a client request', () => { + const { logKey, headersLogKey } = options.clientRequest; + + expect(result).toContain(buildKeyToRedact([logKey, headersLogKey])); + }); + + it('includes the headers of an outgoing request', () => { + const { logKey, headersLogKey } = options.outgoingRequest; + + expect(result).toContain(buildKeyToRedact([logKey, headersLogKey])); + }); + + it('includes all sensitive child keys of an error', () => { + const { logKey, sensitiveChildKeys } = options.error; + + expect(result).toContain(buildKeyToRedact([logKey, sensitiveChildKeys[0]])); + expect(result).toContain(buildKeyToRedact([logKey, sensitiveChildKeys[1]])); + }); + + it('includes all sensitive child keys of an inner error', () => { + const { logKey, sensitiveChildKeys } = options.error; + + expect(result).toContain(buildKeyToRedact([logKey, 'innerError', sensitiveChildKeys[0]])); + expect(result).toContain(buildKeyToRedact([logKey, 'innerError', sensitiveChildKeys[1]])); + }); + + it('includes all sensitive child keys of an error cause', () => { + const { logKey, sensitiveChildKeys } = options.error; + + expect(result).toContain(buildKeyToRedact([logKey, 'options', 'cause', sensitiveChildKeys[0]])); + expect(result).toContain(buildKeyToRedact([logKey, 'options', 'cause', sensitiveChildKeys[1]])); + }); + + it(`includes all sensitive child keys of an error cause's inner error`, () => { + const { logKey, sensitiveChildKeys } = options.error; + + expect(result).toContain(buildKeyToRedact([logKey, 'options', 'cause', 'innerError', sensitiveChildKeys[0]])); + expect(result).toContain(buildKeyToRedact([logKey, 'options', 'cause', 'innerError', sensitiveChildKeys[1]])); + }); + }); +}); diff --git a/src/logging/log-keys-to-redact.ts b/src/logging/log-keys-to-redact.ts new file mode 100644 index 00000000..299ab50e --- /dev/null +++ b/src/logging/log-keys-to-redact.ts @@ -0,0 +1,77 @@ +import { buildKeyToRedact } from './build-key-to-redact'; + +export interface LogKeysToRedactOptions { + redactLogs: boolean; + clientRequest: { + logKey: string; + headersLogKey: string; + }; + outgoingRequest: { + logKey: string; + headersLogKey: string; + }; + error: { + logKey: string; + sensitiveChildKeys: string[]; + }; + dbError: { + logKey: string; + sensitiveChildKeys: string[]; + }; +} + +export const logKeysToRedact = ({ redactLogs, clientRequest, outgoingRequest, error, dbError }: LogKeysToRedactOptions): string[] => { + if (!redactLogs) { + return []; + } + const keys = [ + ...getClientRequestLogKeysToRedact(clientRequest), + ...getOutgoingRequestLogKeysToRedact(outgoingRequest), + ...getErrorLogKeysToRedact(error), + ...getDbErrorLogKeysToRedact(dbError), + ]; + + return keys; +}; + +const getClientRequestLogKeysToRedact = ({ logKey, headersLogKey }: LogKeysToRedactOptions['clientRequest']): string[] => [ + // We redact the client request headers as they contain the secret API key that the client uses to authenticate with our API. + buildKeyToRedact([logKey, headersLogKey]), +]; + +const getOutgoingRequestLogKeysToRedact = ({ logKey, headersLogKey }: LogKeysToRedactOptions['outgoingRequest']): string[] => { + return [ + // We redact the outgoing request headers as they contain: + // - our Basic auth details for Informatica + buildKeyToRedact([logKey, headersLogKey]), + ]; +}; + +const getErrorLogKeysToRedact = ({ logKey, sensitiveChildKeys }: LogKeysToRedactOptions['error']): string[] => { + const innerErrorKey = 'innerError'; + const causeNestedErrorKey = ['options', 'cause']; + return sensitiveChildKeys.flatMap((childKey) => [ + buildKeyToRedact([logKey, childKey]), + // Some errors are wrapped in a new error and logged as the `innerError` field on the new error. + // Some errors also contain a `cause` field containing a wrapped error. + // We need to make sure the sensitive child keys are still redacted in these cases. + buildKeyToRedact([logKey, innerErrorKey, childKey]), + buildKeyToRedact([logKey, ...causeNestedErrorKey, childKey]), + buildKeyToRedact([logKey, ...causeNestedErrorKey, innerErrorKey, childKey]), + ]); +}; + +const getDbErrorLogKeysToRedact = ({ logKey, sensitiveChildKeys }: LogKeysToRedactOptions['dbError']): string[] => { + const innerErrorKey = ['originalError', 'info']; + const driverNestedErrorKey = ['driverError']; + return sensitiveChildKeys.flatMap((childKey) => [ + // Some errors are wrapped in a new error and logged as the `originalError` field on the new error. + // Some errors also contain a `driverError` field containing a wrapped error. + // We need to make sure the sensitive child keys are still redacted in these cases. + buildKeyToRedact([logKey, childKey]), + buildKeyToRedact([logKey, ...driverNestedErrorKey, childKey]), + buildKeyToRedact([logKey, ...innerErrorKey, childKey]), + buildKeyToRedact([logKey, ...driverNestedErrorKey, ...innerErrorKey, childKey]), + ]); +}; + diff --git a/src/logging/logging-interceptor.helper.ts b/src/logging/logging-interceptor.helper.ts new file mode 100644 index 00000000..c2655c63 --- /dev/null +++ b/src/logging/logging-interceptor.helper.ts @@ -0,0 +1,20 @@ +import { CallHandler, ExecutionContext, Injectable, NestInterceptor } from '@nestjs/common'; +import { PinoLogger } from 'nestjs-pino'; +import { Observable } from 'rxjs'; +import { tap } from 'rxjs/operators'; + +@Injectable() +export class LoggingInterceptor implements NestInterceptor { + constructor(private readonly logger: PinoLogger) {} + + intercept(context: ExecutionContext, next: CallHandler): Observable { + const requestBody = context.switchToHttp().getRequest().body; + this.logger.debug({ requestBody }, 'Handling the following request from the client.'); + + return next.handle().pipe( + tap((responseBody) => { + this.logger.debug({ responseBody }, 'Returning the following response to the client.'); + }), + ); + } +} diff --git a/src/main.module.ts b/src/main.module.ts index fbc6c02f..6b981ba8 100644 --- a/src/main.module.ts +++ b/src/main.module.ts @@ -1,9 +1,14 @@ import { Module } from '@nestjs/common'; import { ConfigModule, ConfigService } from '@nestjs/config'; import config from '@ukef/config'; +import { APP_INTERCEPTOR } from '@nestjs/core'; import { MdmModule } from '@ukef/modules/mdm.module'; import { LoggerModule } from 'nestjs-pino'; +import { logKeysToRedact } from './logging/log-keys-to-redact'; +import { LoggingInterceptor } from './logging/logging-interceptor.helper'; +import { HEADERS_LOG_KEY, OUTGOING_REQUEST_LOG_KEY } from '@ukef/modules/http/http.constants'; + @Module({ imports: [ ConfigModule.forRoot({ @@ -25,10 +30,37 @@ import { LoggerModule } from 'nestjs-pino'; singleLine: true, }, }, + redact: logKeysToRedact({ + redactLogs: config.get('app.redactLogs'), + clientRequest: { + logKey: 'req', + headersLogKey: 'headers', + }, + outgoingRequest: { + logKey: OUTGOING_REQUEST_LOG_KEY, + headersLogKey: HEADERS_LOG_KEY, + }, + error: { + logKey: 'err', + sensitiveChildKeys: [ + // The `config` has basic authentication details for Informatica. + 'config', + ], + }, + dbError: { + logKey: 'err', + sensitiveChildKeys: [ + // The `serverName` has db server address. + 'serverName', + ], + }, + }), }, }), }), MdmModule, ], + controllers: [], + providers: [{ provide: APP_INTERCEPTOR, useClass: LoggingInterceptor }], }) export class MainModule {} diff --git a/src/main.ts b/src/main.ts index ac024a74..11edb1f8 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,14 +1,15 @@ -import { Logger as NestLogger } from '@nestjs/common'; import { NestApplication, NestFactory } from '@nestjs/core'; import { MainModule } from '@ukef/main.module'; import { App } from './app'; +import { ConsoleLoggerWithRedact } from './logging/console-logger-with-redact'; +import { REDACT_STRINGS } from './constants'; const main = async () => { - const nestApp: NestApplication = await NestFactory.create(MainModule, { bufferLogs: true }); + const logger = new ConsoleLoggerWithRedact( REDACT_STRINGS ); + const nestApp: NestApplication = await NestFactory.create(MainModule, { logger, bufferLogs: false }); const app = new App(nestApp); - const logger = new NestLogger(); logger.log(`==========================================================`); logger.log(`Main app will serve on PORT ${app.port}`, 'MainApplication'); logger.log(`==========================================================`); diff --git a/src/modules/constants/constants.service.ts b/src/modules/constants/constants.service.ts index 038b0243..692e8581 100644 --- a/src/modules/constants/constants.service.ts +++ b/src/modules/constants/constants.service.ts @@ -1,18 +1,22 @@ -import { Injectable, InternalServerErrorException, Logger, NotFoundException } from '@nestjs/common'; +import { Injectable, InternalServerErrorException, NotFoundException } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import { DATABASE } from '@ukef/constants'; +import { DATABASE, REDACT_STRINGS, REDACT_STRING_PATHS } from '@ukef/constants'; import { DbResponseHelper } from '@ukef/helpers/db-response.helper'; import { Repository } from 'typeorm'; import { ConstantSpiEntity } from './entities/constants-spi.entity'; +import { PinoLogger } from 'nestjs-pino'; +import { redactError } from '@ukef/helpers/redact-errors.helper'; +import { ConfigService } from '@nestjs/config'; @Injectable() export class ConstantsService { - private readonly logger = new Logger(); constructor( @InjectRepository(ConstantSpiEntity, DATABASE.CIS) private readonly constantsCisRepository: Repository, + private readonly logger: PinoLogger, + private readonly config: ConfigService, ) {} async find(oecdRiskCategory: number, category: string): Promise { @@ -35,10 +39,10 @@ export class ConstantsService { return transformedResults; } catch (err) { if (err instanceof NotFoundException) { - this.logger.warn(err); + this.logger.warn(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); throw err; } - this.logger.error(err); + this.logger.error(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); throw new InternalServerErrorException(); } } diff --git a/src/modules/currencies/currencies.service.ts b/src/modules/currencies/currencies.service.ts index 151c6d18..92518ef5 100644 --- a/src/modules/currencies/currencies.service.ts +++ b/src/modules/currencies/currencies.service.ts @@ -1,15 +1,17 @@ -import { Injectable, InternalServerErrorException, Logger, NotFoundException } from '@nestjs/common'; +import { Injectable, InternalServerErrorException, NotFoundException } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import { DATABASE, DATE } from '@ukef/constants'; +import { DATABASE, DATE, REDACT_STRINGS, REDACT_STRING_PATHS } from '@ukef/constants'; import { DbResponseHelper } from '@ukef/helpers/db-response.helper'; import { DataSource, Equal, Repository } from 'typeorm'; import { CurrencyEntity } from './entities/currency.entity'; import { CurrencyExchangeEntity } from './entities/currency-exchange.entity'; +import { PinoLogger } from 'nestjs-pino'; +import { redactError } from '@ukef/helpers/redact-errors.helper'; +import { ConfigService } from '@nestjs/config'; @Injectable() export class CurrenciesService { - private readonly logger = new Logger(); constructor( @InjectRepository(CurrencyEntity, DATABASE.MDM) private readonly currency: Repository, @@ -17,6 +19,8 @@ export class CurrenciesService { private readonly currencyExchange: DataSource, @InjectRepository(CurrencyExchangeEntity, DATABASE.CEDAR) private readonly currencyExchangeRepository: Repository, + private readonly logger: PinoLogger, + private readonly config: ConfigService, ) {} async findAll(): Promise { @@ -65,10 +69,10 @@ export class CurrenciesService { return renamedResults; } catch (err) { if (err instanceof NotFoundException) { - this.logger.warn(err); + this.logger.warn(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); throw err; } else { - this.logger.error(err); + this.logger.error(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); throw new InternalServerErrorException(); } } diff --git a/src/modules/exposure-period/exposure-period.service.ts b/src/modules/exposure-period/exposure-period.service.ts index 0a71556a..9566d1a0 100644 --- a/src/modules/exposure-period/exposure-period.service.ts +++ b/src/modules/exposure-period/exposure-period.service.ts @@ -1,17 +1,21 @@ import { Injectable, InternalServerErrorException, Logger, NotFoundException } from '@nestjs/common'; import { InjectDataSource } from '@nestjs/typeorm'; -import { DATABASE } from '@ukef/constants'; +import { DATABASE, REDACT_STRINGS, REDACT_STRING_PATHS } from '@ukef/constants'; import { DataSource } from 'typeorm'; import { ExposurePeriodDto } from './dto/exposure-period.dto'; +import { PinoLogger } from 'nestjs-pino'; +import { redactError } from '@ukef/helpers/redact-errors.helper'; +import { ConfigService } from '@nestjs/config'; @Injectable() export class ExposurePeriodService { - private readonly logger = new Logger(); constructor( @InjectDataSource(DATABASE.MDM) private readonly mdmDataSource: DataSource, + private readonly logger: PinoLogger, + private readonly config: ConfigService, ) {} async calculate(startDate: string, endDate: string, productGroup: string): Promise { @@ -26,10 +30,10 @@ export class ExposurePeriodService { return { exposurePeriod: results[0].EXPOSURE_PERIOD }; } catch (err) { if (err instanceof NotFoundException) { - this.logger.warn(err); + this.logger.warn(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); throw err; } else { - this.logger.error(err); + this.logger.error(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); throw new InternalServerErrorException(); } } diff --git a/src/modules/http/http.constants.ts b/src/modules/http/http.constants.ts index 88119275..e0cf5df5 100644 --- a/src/modules/http/http.constants.ts +++ b/src/modules/http/http.constants.ts @@ -2,3 +2,5 @@ export const OUTGOING_REQUEST_LOG_KEY = 'outgoingRequest'; export const INCOMING_RESPONSE_LOG_KEY = 'incomingResponse'; export const BODY_LOG_KEY = 'data'; export const HEADERS_LOG_KEY = 'headers'; +export const SENSITIVE_REQUEST_HEADER_NAMES = 'headers'; +export const SENSITIVE_RESPONSE_HEADER_NAMES = 'headers'; diff --git a/src/modules/interest-rates/interest-rates.service.ts b/src/modules/interest-rates/interest-rates.service.ts index f2f5cf38..751138ae 100644 --- a/src/modules/interest-rates/interest-rates.service.ts +++ b/src/modules/interest-rates/interest-rates.service.ts @@ -1,23 +1,27 @@ -import { Injectable, InternalServerErrorException, Logger } from '@nestjs/common'; +import { Injectable, InternalServerErrorException } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import { DATABASE, DATE } from '@ukef/constants'; +import { DATABASE, DATE, REDACT_STRINGS, REDACT_STRING_PATHS } from '@ukef/constants'; import { Equal, Repository } from 'typeorm'; import { InterestRatesEntity } from './entities/interest-rate.entity'; +import { PinoLogger } from 'nestjs-pino'; +import { redactError } from '@ukef/helpers/redact-errors.helper'; +import { ConfigService } from '@nestjs/config'; @Injectable() export class InterestRatesService { - private readonly logger = new Logger(); constructor( @InjectRepository(InterestRatesEntity, DATABASE.CEDAR) private readonly interestRates: Repository, + private readonly logger: PinoLogger, + private readonly config: ConfigService, ) {} findAll(): Promise { try { return this.interestRates.find({ where: { effectiveTo: Equal(new Date(DATE.MAXIMUM_LIMIT)) } }); } catch (err) { - this.logger.error(err); + this.logger.error(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); throw new InternalServerErrorException(); } } diff --git a/src/modules/markets/markets.service.ts b/src/modules/markets/markets.service.ts index 04739520..32b43ab3 100644 --- a/src/modules/markets/markets.service.ts +++ b/src/modules/markets/markets.service.ts @@ -1,18 +1,22 @@ import { Injectable, InternalServerErrorException, Logger } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import { DATABASE } from '@ukef/constants'; +import { DATABASE, REDACT_STRINGS, REDACT_STRING_PATHS } from '@ukef/constants'; import { DbResponseHelper } from '@ukef/helpers/db-response.helper'; import { Repository } from 'typeorm'; import { MarketEntity } from './entities/market.entity'; +import { PinoLogger } from 'nestjs-pino'; +import { redactError } from '@ukef/helpers/redact-errors.helper'; +import { ConfigService } from '@nestjs/config'; @Injectable() export class MarketsService { - private readonly logger = new Logger(); constructor( @InjectRepository(MarketEntity, DATABASE.CIS) private readonly marketsRepository: Repository, + private readonly logger: PinoLogger, + private readonly config: ConfigService, ) {} async find(active?: string, search?: string): Promise { @@ -44,8 +48,8 @@ export class MarketsService { })); return mappedResults; - } catch (err) { - this.logger.error(err); + } catch (err: any) { + this.logger.error(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); throw new InternalServerErrorException(); } } diff --git a/src/modules/numbers/numbers.service.test.ts b/src/modules/numbers/numbers.service.test.ts index e37b549e..69b463e1 100644 --- a/src/modules/numbers/numbers.service.test.ts +++ b/src/modules/numbers/numbers.service.test.ts @@ -2,10 +2,16 @@ import { Test } from '@nestjs/testing'; import { Repository } from 'typeorm'; import { NumbersService } from './numbers.service'; +import { PinoLogger } from 'nestjs-pino'; +import { ConfigModule, ConfigService } from '@nestjs/config'; +import { NumbersModule } from './numbers.module'; describe('NumbersService', () => { let service: NumbersService; + + const logger = new PinoLogger({}); + const unSortedUkefIds = [ { id: 201, @@ -128,10 +134,9 @@ describe('NumbersService', () => { })); beforeAll(async () => { - const module = await Test.createTestingModule({ - providers: [NumbersService, { provide: 'mssql-number-generator_UkefIdRepository', useFactory: repositoryMockFactory }], + const module = await Test.createTestingModule({ + providers: [NumbersService, PinoLogger, { provide: 'pino-params', useValue: {}}, ConfigService, { provide: 'mssql-number-generator_UkefIdRepository', useFactory: repositoryMockFactory }], }).compile(); - service = module.get(NumbersService); }); diff --git a/src/modules/numbers/numbers.service.ts b/src/modules/numbers/numbers.service.ts index 97f9ad51..98df67c0 100644 --- a/src/modules/numbers/numbers.service.ts +++ b/src/modules/numbers/numbers.service.ts @@ -1,22 +1,25 @@ import { BadRequestException, Injectable, InternalServerErrorException, Logger, NotFoundException } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import { DATABASE } from '@ukef/constants'; +import { DATABASE, REDACT_STRINGS, REDACT_STRING_PATHS } from '@ukef/constants'; import { Repository } from 'typeorm'; import { CreateUkefIdDto } from './dto/create-ukef-id.dto'; import { UkefId } from './entities/ukef-id.entity'; +import { PinoLogger } from 'nestjs-pino'; +import { redactError } from '@ukef/helpers/redact-errors.helper'; +import { ConfigService } from '@nestjs/config'; @Injectable() export class NumbersService { - private readonly logger = new Logger(); constructor( @InjectRepository(UkefId, DATABASE.NUMBER_GENERATOR) private readonly numberRepository: Repository, + private readonly logger: PinoLogger, + private readonly config: ConfigService, ) {} async create(createUkefIdDto: CreateUkefIdDto[]): Promise { - // TODO: DB calls are async and will generate IDs that are not in order. Extra code to order ids is required, or calls need to be made in async order. // TODO: new IDs of type 1 and 2 could be checked if they are used in ACBS. ACBS might be down, but generation still should work. const activeRequests = createUkefIdDto.map((createNumber) => { return this.numberRepository @@ -44,10 +47,10 @@ export class NumbersService { return this.mapFieldsFromDbToApi(dbNumber[0]); } catch (err) { if (err instanceof NotFoundException || err instanceof BadRequestException) { - this.logger.warn(err); + this.logger.warn(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); throw err; } else { - this.logger.error(err); + this.logger.error(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); throw new InternalServerErrorException(); } } diff --git a/src/modules/premium-schedules/premium-schedules.service.ts b/src/modules/premium-schedules/premium-schedules.service.ts index a069cb5e..05df69f4 100644 --- a/src/modules/premium-schedules/premium-schedules.service.ts +++ b/src/modules/premium-schedules/premium-schedules.service.ts @@ -1,19 +1,23 @@ import { Injectable, InternalServerErrorException, Logger, NotFoundException } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import { DATABASE } from '@ukef/constants'; +import { DATABASE, REDACT_STRINGS, REDACT_STRING_PATHS } from '@ukef/constants'; import { Response } from 'express'; import { Equal, Repository } from 'typeorm'; import { DbResponseHelper } from '../../helpers/db-response.helper'; import { CreatePremiumScheduleDto } from './dto/create-premium-schedule.dto'; import { PremiumScheduleEntity } from './entities/premium-schedule.entity'; +import { PinoLogger } from 'nestjs-pino'; +import { redactError } from '@ukef/helpers/redact-errors.helper'; +import { ConfigService } from '@nestjs/config'; @Injectable() export class PremiumSchedulesService { - private readonly logger = new Logger(); constructor( @InjectRepository(PremiumScheduleEntity, DATABASE.MDM) private readonly premiumSchedulesRepository: Repository, + private readonly logger: PinoLogger, + private readonly config: ConfigService, ) {} async find(facilityId: string): Promise { @@ -28,10 +32,10 @@ export class PremiumSchedulesService { return results; } catch (err) { if (err instanceof NotFoundException) { - this.logger.warn(err); + this.logger.warn(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); throw err; } else { - this.logger.error(err); + this.logger.error(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); throw new InternalServerErrorException(); } } @@ -88,7 +92,7 @@ export class PremiumSchedulesService { return transformedResults; } catch (err) { - this.logger.error(err); + this.logger.error(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); throw new InternalServerErrorException(); } } diff --git a/src/modules/sector-industries/sector-industries.service.ts b/src/modules/sector-industries/sector-industries.service.ts index a1437db6..1ea26db3 100644 --- a/src/modules/sector-industries/sector-industries.service.ts +++ b/src/modules/sector-industries/sector-industries.service.ts @@ -1,16 +1,20 @@ -import { Injectable, InternalServerErrorException, Logger, NotFoundException } from '@nestjs/common'; +import { Injectable, InternalServerErrorException, NotFoundException } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import { DATABASE, DATE } from '@ukef/constants'; +import { DATABASE, DATE, REDACT_STRINGS, REDACT_STRING_PATHS } from '@ukef/constants'; import { Equal, Repository } from 'typeorm'; import { SectorIndustryEntity } from './entities/sector-industry.entity'; +import { PinoLogger } from 'nestjs-pino'; +import { redactError } from '@ukef/helpers/redact-errors.helper'; +import { ConfigService } from '@nestjs/config'; @Injectable() export class SectorIndustriesService { - private readonly logger = new Logger(); constructor( @InjectRepository(SectorIndustryEntity, DATABASE.MDM) private readonly sectorIndustries: Repository, + private readonly logger: PinoLogger, + private readonly config: ConfigService, ) {} async find(ukefSectorId: string, ukefIndustryId: string): Promise { @@ -33,10 +37,10 @@ export class SectorIndustriesService { return results; } catch (err) { if (err instanceof NotFoundException) { - this.logger.warn(err); + this.logger.warn(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); throw err; } else { - this.logger.error(err); + this.logger.error(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); throw new InternalServerErrorException(); } } diff --git a/src/modules/yield-rates/yield-rates.service.ts b/src/modules/yield-rates/yield-rates.service.ts index 4e2d56b8..a5a6a0b6 100644 --- a/src/modules/yield-rates/yield-rates.service.ts +++ b/src/modules/yield-rates/yield-rates.service.ts @@ -1,16 +1,20 @@ -import { Injectable, InternalServerErrorException, Logger, NotFoundException } from '@nestjs/common'; +import { Injectable, InternalServerErrorException, NotFoundException } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import { DATABASE, DATE } from '@ukef/constants'; +import { DATABASE, DATE, REDACT_STRINGS, REDACT_STRING_PATHS } from '@ukef/constants'; import { Equal, LessThanOrEqual, MoreThan, Repository } from 'typeorm'; import { YieldRateEntity } from './entities/yield-rate.entity'; +import { PinoLogger } from 'nestjs-pino'; +import { redactError } from '@ukef/helpers/redact-errors.helper'; +import { ConfigService } from '@nestjs/config'; @Injectable() export class YieldRatesService { - private readonly logger = new Logger(); constructor( @InjectRepository(YieldRateEntity, DATABASE.CEDAR) private readonly yieldRateRepository: Repository, + private readonly logger: PinoLogger, + private readonly config: ConfigService, ) {} async find(searchDate: string): Promise { @@ -29,10 +33,10 @@ export class YieldRatesService { return results; } catch (err) { if (err instanceof NotFoundException) { - this.logger.warn(err); + this.logger.warn(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); throw err; } else { - this.logger.error(err); + this.logger.error(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); throw new InternalServerErrorException(); } } diff --git a/test/docs/__snapshots__/get-docs-yaml.api-test.ts.snap b/test/docs/__snapshots__/get-docs-yaml.api-test.ts.snap index 0e5b0b8c..3307631b 100644 --- a/test/docs/__snapshots__/get-docs-yaml.api-test.ts.snap +++ b/test/docs/__snapshots__/get-docs-yaml.api-test.ts.snap @@ -135,7 +135,7 @@ paths: example: '00302069' description: The unique UKEF id of the customer to search for. schema: - pattern: /^\\d{8}$/ + pattern: ^\\d{8}$ type: string - name: name required: false From 8b84a142a0d729edc6995d8e35e03edaeb0c767f Mon Sep 17 00:00:00 2001 From: Audrius Vaitonis Date: Tue, 18 Jul 2023 10:12:29 +0100 Subject: [PATCH 2/8] feat(APIM-344): fix lint errors, some new issues and some are in the old code --- package-lock.json | 1 + package.json | 1 + src/constants/index.ts | 2 +- src/constants/redact-strings.constant.ts | 14 ++++++------ src/helpers/redact-errors.helper.test.ts | 20 ++++++++--------- src/helpers/redact-errors.helper.ts | 22 ++++++++++--------- src/logging/console-logger-with-redact.ts | 9 ++++---- src/logging/log-keys-to-redact.ts | 1 - src/main.module.ts | 4 ++-- src/main.ts | 4 ++-- src/modules/auth/strategy/api-key.strategy.ts | 5 ++++- src/modules/constants/constants.service.ts | 9 ++++---- src/modules/currencies/currencies.service.ts | 8 +++---- .../exposure-period.service.ts | 11 +++++----- .../exception/informatica.exception.ts | 5 ++++- src/modules/informatica/informatica.module.ts | 1 + .../interest-rates/interest-rates.service.ts | 8 +++---- src/modules/markets/markets.service.ts | 11 +++++----- src/modules/numbers/numbers.service.test.ts | 19 ++++++++-------- src/modules/numbers/numbers.service.ts | 11 +++++----- .../premium-schedules.service.ts | 12 +++++----- .../sector-industries.service.ts | 8 +++---- .../yield-rates/yield-rates.service.ts | 8 +++---- 23 files changed, 100 insertions(+), 94 deletions(-) diff --git a/package-lock.json b/package-lock.json index f6b328d1..bd283508 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,6 +25,7 @@ "date-fns": "^2.30.0", "dotenv": "^16.3.1", "express-basic-auth": "^1.2.1", + "lodash": "^4.17.21", "mssql": "^9.1.1", "nestjs-pino": "^3.3.0", "passport": "^0.6.0", diff --git a/package.json b/package.json index 97d9a8a3..df1f6920 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "date-fns": "^2.30.0", "dotenv": "^16.3.1", "express-basic-auth": "^1.2.1", + "lodash": "^4.17.21", "mssql": "^9.1.1", "nestjs-pino": "^3.3.0", "passport": "^0.6.0", diff --git a/src/constants/index.ts b/src/constants/index.ts index b0c176ab..7bd70773 100644 --- a/src/constants/index.ts +++ b/src/constants/index.ts @@ -17,6 +17,6 @@ export * from './database-name.constant'; export * from './date.constant'; export * from './enums'; export * from './products.constant'; -export * from './ukef-id.constant'; export * from './redact-strings.constant'; export * from './redact-strings-paths.constant'; +export * from './ukef-id.constant'; diff --git a/src/constants/redact-strings.constant.ts b/src/constants/redact-strings.constant.ts index d43044dd..7d4e1c97 100644 --- a/src/constants/redact-strings.constant.ts +++ b/src/constants/redact-strings.constant.ts @@ -1,9 +1,9 @@ export const REDACT_STRINGS = [ - {searchValue: /(Login failed for user ').*(')/g, replaceValue: '$1[Redacted]$2'}, - {searchValue: process.env.DATABASE_USERNAME, replaceValue: '[RedactedDomain]'}, - {searchValue: process.env.DATABASE_PASSWORD, replaceValue: '[RedactedDomain]'}, - {searchValue: process.env.DATABASE_MDM_HOST, replaceValue: '[RedactedDomain]'}, - {searchValue: process.env.DATABASE_CEDAR_HOST, replaceValue: '[RedactedDomain]'}, - {searchValue: process.env.DATABASE_NUMBER_GENERATOR_HOST, replaceValue: '[RedactedDomain]'}, - {searchValue: process.env.DATABASE_CIS_HOST, replaceValue: '[RedactedDomain]'}, + { searchValue: /(Login failed for user ').*(')/g, replaceValue: '$1[Redacted]$2' }, + { searchValue: process.env.DATABASE_USERNAME, replaceValue: '[RedactedDomain]' }, + { searchValue: process.env.DATABASE_PASSWORD, replaceValue: '[RedactedDomain]' }, + { searchValue: process.env.DATABASE_MDM_HOST, replaceValue: '[RedactedDomain]' }, + { searchValue: process.env.DATABASE_CEDAR_HOST, replaceValue: '[RedactedDomain]' }, + { searchValue: process.env.DATABASE_NUMBER_GENERATOR_HOST, replaceValue: '[RedactedDomain]' }, + { searchValue: process.env.DATABASE_CIS_HOST, replaceValue: '[RedactedDomain]' }, ]; diff --git a/src/helpers/redact-errors.helper.test.ts b/src/helpers/redact-errors.helper.test.ts index 96c6a2f1..2944bc79 100644 --- a/src/helpers/redact-errors.helper.test.ts +++ b/src/helpers/redact-errors.helper.test.ts @@ -1,7 +1,7 @@ +import { REDACT_STRING_PATHS } from '@ukef/constants'; import { RandomValueGenerator } from '@ukef-test/support/generator/random-value-generator'; + import { redactError } from './redact-errors.helper'; -import { regexToString } from './regex.helper'; -import { REDACT_STRING_PATHS } from '@ukef/constants'; describe('Redact errors helper', () => { const valueGenerator = new RandomValueGenerator(); @@ -12,8 +12,8 @@ describe('Redact errors helper', () => { const message = `ConnectionError: Failed to connect to ${domain}, ${otherSensitivefield}`; const redactedMessage = `ConnectionError: Failed to connect to [RedactedDomain], [Redacted]`; const redactStrings = [ - {searchValue: domain, replaceValue: '[RedactedDomain]'}, - {searchValue: otherSensitivefield, replaceValue: '[Redacted]'}, + { searchValue: domain, replaceValue: '[RedactedDomain]' }, + { searchValue: otherSensitivefield, replaceValue: '[Redacted]' }, ]; const error = { message: message, @@ -30,7 +30,7 @@ describe('Redact errors helper', () => { message: message, stack: message, safe: 'Nothing sensitive', - } + }, }, }; const expectedError = { @@ -48,17 +48,19 @@ describe('Redact errors helper', () => { message: redactedMessage, stack: redactedMessage, safe: 'Nothing sensitive', - } + }, }, }; it('replaces sensitive data in input object', () => { const redacted = redactError(true, REDACT_STRING_PATHS, redactStrings, error); + expect(redacted).toStrictEqual(expectedError); }); it('returns original input if redactLogs is set to false', () => { const redacted = redactError(false, REDACT_STRING_PATHS, redactStrings, error); + expect(redacted).toStrictEqual(error); }); @@ -77,11 +79,9 @@ describe('Redact errors helper', () => { safe: 'Nothing sensitive', }, }; - const redactPaths = [ - 'field1', - 'field2.field3', - ]; + const redactPaths = ['field1', 'field2.field3']; const redacted = redactError(true, redactPaths, redactStrings, error); + expect(redacted).toStrictEqual(expectedError); }); }); diff --git a/src/helpers/redact-errors.helper.ts b/src/helpers/redact-errors.helper.ts index f645c968..d4b05159 100644 --- a/src/helpers/redact-errors.helper.ts +++ b/src/helpers/redact-errors.helper.ts @@ -1,27 +1,29 @@ -import { REDACT_STRINGS, REDACT_STRING_PATHS } from "@ukef/constants"; - -import {get, set} from "lodash"; +import { get, set } from 'lodash'; // This helper function is used to redact sensitive data in Error object strings. -export const redactError = (redactLogs: boolean, redactPaths: string[], redactStrings: {searchValue: string | RegExp, replaceValue: string}[], err: any): any => { +export const redactError = ( + redactLogs: boolean, + redactPaths: string[], + redactStrings: { searchValue: string | RegExp; replaceValue: string }[], + err: any, +): any => { if (!redactLogs) { return err; } redactPaths.forEach((path) => { - //REDACT_STRING_PATHS.forEach((path) => { - let value: string = get(err, path); + const value: string = get(err, path); if (value) { const safeValue = redactString(redactStrings, value); set(err, path, safeValue); } - }) + }); return err; }; -const redactString = (redactStrings: {searchValue: string | RegExp, replaceValue: string}[], string: string): string => { +const redactString = (redactStrings: { searchValue: string | RegExp; replaceValue: string }[], string: string): string => { let safeSTring: string = string; - redactStrings.forEach( (redact) => { + redactStrings.forEach((redact) => { safeSTring = safeSTring.replaceAll(redact.searchValue, redact.replaceValue); }); return safeSTring; -} +}; diff --git a/src/logging/console-logger-with-redact.ts b/src/logging/console-logger-with-redact.ts index b63d32d8..5523d9d7 100644 --- a/src/logging/console-logger-with-redact.ts +++ b/src/logging/console-logger-with-redact.ts @@ -1,8 +1,7 @@ import { ConsoleLogger } from '@nestjs/common'; export class ConsoleLoggerWithRedact extends ConsoleLogger { - - private stringPatternsToRedact: [{searchValue: string | RegExp, replaceValue, string}]; + private stringPatternsToRedact: [{ searchValue: string | RegExp; replaceValue; string }]; constructor(stringPatternsToRedact) { super(); @@ -12,10 +11,10 @@ export class ConsoleLoggerWithRedact extends ConsoleLogger { error(message: any, stack?: string, context?: string) { let cleanStack = stack; if (typeof stack == 'string') { - this.stringPatternsToRedact.forEach( (redact) => { + this.stringPatternsToRedact.forEach((redact) => { cleanStack = stack.replace(redact.searchValue, redact.replaceValue); - }) + }); } super.error(message, cleanStack, context); } -} \ No newline at end of file +} diff --git a/src/logging/log-keys-to-redact.ts b/src/logging/log-keys-to-redact.ts index 299ab50e..cf7d3c42 100644 --- a/src/logging/log-keys-to-redact.ts +++ b/src/logging/log-keys-to-redact.ts @@ -74,4 +74,3 @@ const getDbErrorLogKeysToRedact = ({ logKey, sensitiveChildKeys }: LogKeysToReda buildKeyToRedact([logKey, ...driverNestedErrorKey, ...innerErrorKey, childKey]), ]); }; - diff --git a/src/main.module.ts b/src/main.module.ts index 6b981ba8..d93a5299 100644 --- a/src/main.module.ts +++ b/src/main.module.ts @@ -1,13 +1,13 @@ import { Module } from '@nestjs/common'; import { ConfigModule, ConfigService } from '@nestjs/config'; -import config from '@ukef/config'; import { APP_INTERCEPTOR } from '@nestjs/core'; +import config from '@ukef/config'; +import { HEADERS_LOG_KEY, OUTGOING_REQUEST_LOG_KEY } from '@ukef/modules/http/http.constants'; import { MdmModule } from '@ukef/modules/mdm.module'; import { LoggerModule } from 'nestjs-pino'; import { logKeysToRedact } from './logging/log-keys-to-redact'; import { LoggingInterceptor } from './logging/logging-interceptor.helper'; -import { HEADERS_LOG_KEY, OUTGOING_REQUEST_LOG_KEY } from '@ukef/modules/http/http.constants'; @Module({ imports: [ diff --git a/src/main.ts b/src/main.ts index 11edb1f8..851011bc 100644 --- a/src/main.ts +++ b/src/main.ts @@ -2,11 +2,11 @@ import { NestApplication, NestFactory } from '@nestjs/core'; import { MainModule } from '@ukef/main.module'; import { App } from './app'; -import { ConsoleLoggerWithRedact } from './logging/console-logger-with-redact'; import { REDACT_STRINGS } from './constants'; +import { ConsoleLoggerWithRedact } from './logging/console-logger-with-redact'; const main = async () => { - const logger = new ConsoleLoggerWithRedact( REDACT_STRINGS ); + const logger = new ConsoleLoggerWithRedact(REDACT_STRINGS); const nestApp: NestApplication = await NestFactory.create(MainModule, { logger, bufferLogs: false }); const app = new App(nestApp); diff --git a/src/modules/auth/strategy/api-key.strategy.ts b/src/modules/auth/strategy/api-key.strategy.ts index 07753717..b1d8ab88 100644 --- a/src/modules/auth/strategy/api-key.strategy.ts +++ b/src/modules/auth/strategy/api-key.strategy.ts @@ -8,7 +8,10 @@ import { AuthService } from '../auth.service'; @Injectable() export class ApiKeyStrategy extends PassportStrategy(HeaderAPIKeyStrategy, AUTH.STRATEGY) { - constructor(private readonly authService: AuthService, private readonly configService: ConfigService) { + constructor( + private readonly authService: AuthService, + private readonly configService: ConfigService, + ) { const headerKeyApiKey: string = configService.get('app.apiKeyStrategy'); super({ header: headerKeyApiKey, prefix: '' }, true, (apiKey: string, done: (arg0: UnauthorizedException, arg1: boolean) => void) => { const hasValidKey = this.authService.validateApiKey(apiKey); diff --git a/src/modules/constants/constants.service.ts b/src/modules/constants/constants.service.ts index 692e8581..c3832ead 100644 --- a/src/modules/constants/constants.service.ts +++ b/src/modules/constants/constants.service.ts @@ -1,17 +1,16 @@ import { Injectable, InternalServerErrorException, NotFoundException } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; import { InjectRepository } from '@nestjs/typeorm'; -import { DATABASE, REDACT_STRINGS, REDACT_STRING_PATHS } from '@ukef/constants'; +import { DATABASE, REDACT_STRING_PATHS, REDACT_STRINGS } from '@ukef/constants'; import { DbResponseHelper } from '@ukef/helpers/db-response.helper'; +import { redactError } from '@ukef/helpers/redact-errors.helper'; +import { PinoLogger } from 'nestjs-pino'; import { Repository } from 'typeorm'; import { ConstantSpiEntity } from './entities/constants-spi.entity'; -import { PinoLogger } from 'nestjs-pino'; -import { redactError } from '@ukef/helpers/redact-errors.helper'; -import { ConfigService } from '@nestjs/config'; @Injectable() export class ConstantsService { - constructor( @InjectRepository(ConstantSpiEntity, DATABASE.CIS) private readonly constantsCisRepository: Repository, diff --git a/src/modules/currencies/currencies.service.ts b/src/modules/currencies/currencies.service.ts index 92518ef5..cecbc927 100644 --- a/src/modules/currencies/currencies.service.ts +++ b/src/modules/currencies/currencies.service.ts @@ -1,14 +1,14 @@ import { Injectable, InternalServerErrorException, NotFoundException } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; import { InjectRepository } from '@nestjs/typeorm'; -import { DATABASE, DATE, REDACT_STRINGS, REDACT_STRING_PATHS } from '@ukef/constants'; +import { DATABASE, DATE, REDACT_STRING_PATHS, REDACT_STRINGS } from '@ukef/constants'; import { DbResponseHelper } from '@ukef/helpers/db-response.helper'; +import { redactError } from '@ukef/helpers/redact-errors.helper'; +import { PinoLogger } from 'nestjs-pino'; import { DataSource, Equal, Repository } from 'typeorm'; import { CurrencyEntity } from './entities/currency.entity'; import { CurrencyExchangeEntity } from './entities/currency-exchange.entity'; -import { PinoLogger } from 'nestjs-pino'; -import { redactError } from '@ukef/helpers/redact-errors.helper'; -import { ConfigService } from '@nestjs/config'; @Injectable() export class CurrenciesService { diff --git a/src/modules/exposure-period/exposure-period.service.ts b/src/modules/exposure-period/exposure-period.service.ts index 9566d1a0..cc34785c 100644 --- a/src/modules/exposure-period/exposure-period.service.ts +++ b/src/modules/exposure-period/exposure-period.service.ts @@ -1,16 +1,15 @@ -import { Injectable, InternalServerErrorException, Logger, NotFoundException } from '@nestjs/common'; +import { Injectable, InternalServerErrorException, NotFoundException } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; import { InjectDataSource } from '@nestjs/typeorm'; -import { DATABASE, REDACT_STRINGS, REDACT_STRING_PATHS } from '@ukef/constants'; +import { DATABASE, REDACT_STRING_PATHS, REDACT_STRINGS } from '@ukef/constants'; +import { redactError } from '@ukef/helpers/redact-errors.helper'; +import { PinoLogger } from 'nestjs-pino'; import { DataSource } from 'typeorm'; import { ExposurePeriodDto } from './dto/exposure-period.dto'; -import { PinoLogger } from 'nestjs-pino'; -import { redactError } from '@ukef/helpers/redact-errors.helper'; -import { ConfigService } from '@nestjs/config'; @Injectable() export class ExposurePeriodService { - constructor( @InjectDataSource(DATABASE.MDM) private readonly mdmDataSource: DataSource, diff --git a/src/modules/informatica/exception/informatica.exception.ts b/src/modules/informatica/exception/informatica.exception.ts index 47e3b67e..668edf31 100644 --- a/src/modules/informatica/exception/informatica.exception.ts +++ b/src/modules/informatica/exception/informatica.exception.ts @@ -1,5 +1,8 @@ export class InformaticaException extends Error { - constructor(message: string, public readonly innerError?: Error) { + constructor( + message: string, + public readonly innerError?: Error, + ) { super(message); this.name = this.constructor.name; } diff --git a/src/modules/informatica/informatica.module.ts b/src/modules/informatica/informatica.module.ts index 64eb4890..b51ca005 100644 --- a/src/modules/informatica/informatica.module.ts +++ b/src/modules/informatica/informatica.module.ts @@ -4,6 +4,7 @@ import { InformaticaConfig, KEY as INFORMATICA_CONFIG_KEY } from '@ukef/config/i import { HttpModule } from '@ukef/modules/http/http.module'; import { InformaticaService } from './informatica.service'; + @Module({ imports: [ HttpModule.registerAsync({ diff --git a/src/modules/interest-rates/interest-rates.service.ts b/src/modules/interest-rates/interest-rates.service.ts index 751138ae..de55bcfc 100644 --- a/src/modules/interest-rates/interest-rates.service.ts +++ b/src/modules/interest-rates/interest-rates.service.ts @@ -1,12 +1,12 @@ import { Injectable, InternalServerErrorException } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; import { InjectRepository } from '@nestjs/typeorm'; -import { DATABASE, DATE, REDACT_STRINGS, REDACT_STRING_PATHS } from '@ukef/constants'; +import { DATABASE, DATE, REDACT_STRING_PATHS, REDACT_STRINGS } from '@ukef/constants'; +import { redactError } from '@ukef/helpers/redact-errors.helper'; +import { PinoLogger } from 'nestjs-pino'; import { Equal, Repository } from 'typeorm'; import { InterestRatesEntity } from './entities/interest-rate.entity'; -import { PinoLogger } from 'nestjs-pino'; -import { redactError } from '@ukef/helpers/redact-errors.helper'; -import { ConfigService } from '@nestjs/config'; @Injectable() export class InterestRatesService { diff --git a/src/modules/markets/markets.service.ts b/src/modules/markets/markets.service.ts index 32b43ab3..882347b4 100644 --- a/src/modules/markets/markets.service.ts +++ b/src/modules/markets/markets.service.ts @@ -1,17 +1,16 @@ -import { Injectable, InternalServerErrorException, Logger } from '@nestjs/common'; +import { Injectable, InternalServerErrorException } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; import { InjectRepository } from '@nestjs/typeorm'; -import { DATABASE, REDACT_STRINGS, REDACT_STRING_PATHS } from '@ukef/constants'; +import { DATABASE, REDACT_STRING_PATHS, REDACT_STRINGS } from '@ukef/constants'; import { DbResponseHelper } from '@ukef/helpers/db-response.helper'; +import { redactError } from '@ukef/helpers/redact-errors.helper'; +import { PinoLogger } from 'nestjs-pino'; import { Repository } from 'typeorm'; import { MarketEntity } from './entities/market.entity'; -import { PinoLogger } from 'nestjs-pino'; -import { redactError } from '@ukef/helpers/redact-errors.helper'; -import { ConfigService } from '@nestjs/config'; @Injectable() export class MarketsService { - constructor( @InjectRepository(MarketEntity, DATABASE.CIS) private readonly marketsRepository: Repository, diff --git a/src/modules/numbers/numbers.service.test.ts b/src/modules/numbers/numbers.service.test.ts index 69b463e1..2479c62a 100644 --- a/src/modules/numbers/numbers.service.test.ts +++ b/src/modules/numbers/numbers.service.test.ts @@ -1,17 +1,12 @@ +import { ConfigService } from '@nestjs/config'; import { Test } from '@nestjs/testing'; +import { PinoLogger } from 'nestjs-pino'; import { Repository } from 'typeorm'; import { NumbersService } from './numbers.service'; -import { PinoLogger } from 'nestjs-pino'; -import { ConfigModule, ConfigService } from '@nestjs/config'; -import { NumbersModule } from './numbers.module'; describe('NumbersService', () => { let service: NumbersService; - - - const logger = new PinoLogger({}); - const unSortedUkefIds = [ { id: 201, @@ -134,8 +129,14 @@ describe('NumbersService', () => { })); beforeAll(async () => { - const module = await Test.createTestingModule({ - providers: [NumbersService, PinoLogger, { provide: 'pino-params', useValue: {}}, ConfigService, { provide: 'mssql-number-generator_UkefIdRepository', useFactory: repositoryMockFactory }], + const module = await Test.createTestingModule({ + providers: [ + NumbersService, + PinoLogger, + { provide: 'pino-params', useValue: {} }, + ConfigService, + { provide: 'mssql-number-generator_UkefIdRepository', useFactory: repositoryMockFactory }, + ], }).compile(); service = module.get(NumbersService); }); diff --git a/src/modules/numbers/numbers.service.ts b/src/modules/numbers/numbers.service.ts index 98df67c0..6afd1639 100644 --- a/src/modules/numbers/numbers.service.ts +++ b/src/modules/numbers/numbers.service.ts @@ -1,17 +1,16 @@ -import { BadRequestException, Injectable, InternalServerErrorException, Logger, NotFoundException } from '@nestjs/common'; +import { BadRequestException, Injectable, InternalServerErrorException, NotFoundException } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; import { InjectRepository } from '@nestjs/typeorm'; -import { DATABASE, REDACT_STRINGS, REDACT_STRING_PATHS } from '@ukef/constants'; +import { DATABASE, REDACT_STRING_PATHS, REDACT_STRINGS } from '@ukef/constants'; +import { redactError } from '@ukef/helpers/redact-errors.helper'; +import { PinoLogger } from 'nestjs-pino'; import { Repository } from 'typeorm'; import { CreateUkefIdDto } from './dto/create-ukef-id.dto'; import { UkefId } from './entities/ukef-id.entity'; -import { PinoLogger } from 'nestjs-pino'; -import { redactError } from '@ukef/helpers/redact-errors.helper'; -import { ConfigService } from '@nestjs/config'; @Injectable() export class NumbersService { - constructor( @InjectRepository(UkefId, DATABASE.NUMBER_GENERATOR) private readonly numberRepository: Repository, diff --git a/src/modules/premium-schedules/premium-schedules.service.ts b/src/modules/premium-schedules/premium-schedules.service.ts index 05df69f4..f669e2d1 100644 --- a/src/modules/premium-schedules/premium-schedules.service.ts +++ b/src/modules/premium-schedules/premium-schedules.service.ts @@ -1,15 +1,15 @@ -import { Injectable, InternalServerErrorException, Logger, NotFoundException } from '@nestjs/common'; +import { Injectable, InternalServerErrorException, NotFoundException } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; import { InjectRepository } from '@nestjs/typeorm'; -import { DATABASE, REDACT_STRINGS, REDACT_STRING_PATHS } from '@ukef/constants'; +import { DATABASE, REDACT_STRING_PATHS, REDACT_STRINGS } from '@ukef/constants'; +import { DbResponseHelper } from '@ukef/helpers/db-response.helper'; +import { redactError } from '@ukef/helpers/redact-errors.helper'; import { Response } from 'express'; +import { PinoLogger } from 'nestjs-pino'; import { Equal, Repository } from 'typeorm'; -import { DbResponseHelper } from '../../helpers/db-response.helper'; import { CreatePremiumScheduleDto } from './dto/create-premium-schedule.dto'; import { PremiumScheduleEntity } from './entities/premium-schedule.entity'; -import { PinoLogger } from 'nestjs-pino'; -import { redactError } from '@ukef/helpers/redact-errors.helper'; -import { ConfigService } from '@nestjs/config'; @Injectable() export class PremiumSchedulesService { diff --git a/src/modules/sector-industries/sector-industries.service.ts b/src/modules/sector-industries/sector-industries.service.ts index 1ea26db3..9aa36a25 100644 --- a/src/modules/sector-industries/sector-industries.service.ts +++ b/src/modules/sector-industries/sector-industries.service.ts @@ -1,12 +1,12 @@ import { Injectable, InternalServerErrorException, NotFoundException } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; import { InjectRepository } from '@nestjs/typeorm'; -import { DATABASE, DATE, REDACT_STRINGS, REDACT_STRING_PATHS } from '@ukef/constants'; +import { DATABASE, DATE, REDACT_STRING_PATHS, REDACT_STRINGS } from '@ukef/constants'; +import { redactError } from '@ukef/helpers/redact-errors.helper'; +import { PinoLogger } from 'nestjs-pino'; import { Equal, Repository } from 'typeorm'; import { SectorIndustryEntity } from './entities/sector-industry.entity'; -import { PinoLogger } from 'nestjs-pino'; -import { redactError } from '@ukef/helpers/redact-errors.helper'; -import { ConfigService } from '@nestjs/config'; @Injectable() export class SectorIndustriesService { diff --git a/src/modules/yield-rates/yield-rates.service.ts b/src/modules/yield-rates/yield-rates.service.ts index a5a6a0b6..ea56c5de 100644 --- a/src/modules/yield-rates/yield-rates.service.ts +++ b/src/modules/yield-rates/yield-rates.service.ts @@ -1,12 +1,12 @@ import { Injectable, InternalServerErrorException, NotFoundException } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; import { InjectRepository } from '@nestjs/typeorm'; -import { DATABASE, DATE, REDACT_STRINGS, REDACT_STRING_PATHS } from '@ukef/constants'; +import { DATABASE, DATE, REDACT_STRING_PATHS, REDACT_STRINGS } from '@ukef/constants'; +import { redactError } from '@ukef/helpers/redact-errors.helper'; +import { PinoLogger } from 'nestjs-pino'; import { Equal, LessThanOrEqual, MoreThan, Repository } from 'typeorm'; import { YieldRateEntity } from './entities/yield-rate.entity'; -import { PinoLogger } from 'nestjs-pino'; -import { redactError } from '@ukef/helpers/redact-errors.helper'; -import { ConfigService } from '@nestjs/config'; @Injectable() export class YieldRatesService { From 9f3442a9ab7b96d346622feddbe04969148e9f85 Mon Sep 17 00:00:00 2001 From: Audrius Vaitonis Date: Tue, 18 Jul 2023 16:19:08 +0100 Subject: [PATCH 3/8] feat(APIM-344): change how string redaction helper is called --- src/constants/redact-strings.constant.ts | 4 +- src/helpers/redact-errors.helper.test.ts | 88 ------------------- src/helpers/redact-errors.helper.ts | 29 ------ src/logging/log-keys-to-redact.test.ts | 28 ++++++ src/main.module.ts | 8 +- src/modules/constants/constants.service.ts | 9 +- src/modules/currencies/currencies.service.ts | 9 +- .../exposure-period.service.ts | 9 +- .../interest-rates/interest-rates.service.ts | 7 +- src/modules/markets/markets.service.ts | 7 +- src/modules/numbers/numbers.service.ts | 9 +- .../premium-schedules.service.ts | 11 +-- .../sector-industries.service.ts | 9 +- .../yield-rates/yield-rates.service.ts | 9 +- 14 files changed, 63 insertions(+), 173 deletions(-) delete mode 100644 src/helpers/redact-errors.helper.test.ts delete mode 100644 src/helpers/redact-errors.helper.ts diff --git a/src/constants/redact-strings.constant.ts b/src/constants/redact-strings.constant.ts index 7d4e1c97..a58a6b33 100644 --- a/src/constants/redact-strings.constant.ts +++ b/src/constants/redact-strings.constant.ts @@ -1,7 +1,7 @@ export const REDACT_STRINGS = [ { searchValue: /(Login failed for user ').*(')/g, replaceValue: '$1[Redacted]$2' }, - { searchValue: process.env.DATABASE_USERNAME, replaceValue: '[RedactedDomain]' }, - { searchValue: process.env.DATABASE_PASSWORD, replaceValue: '[RedactedDomain]' }, + { searchValue: process.env.DATABASE_USERNAME, replaceValue: '[Redacted]' }, + { searchValue: process.env.DATABASE_PASSWORD, replaceValue: '[Redacted]' }, { searchValue: process.env.DATABASE_MDM_HOST, replaceValue: '[RedactedDomain]' }, { searchValue: process.env.DATABASE_CEDAR_HOST, replaceValue: '[RedactedDomain]' }, { searchValue: process.env.DATABASE_NUMBER_GENERATOR_HOST, replaceValue: '[RedactedDomain]' }, diff --git a/src/helpers/redact-errors.helper.test.ts b/src/helpers/redact-errors.helper.test.ts deleted file mode 100644 index 2944bc79..00000000 --- a/src/helpers/redact-errors.helper.test.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { REDACT_STRING_PATHS } from '@ukef/constants'; -import { RandomValueGenerator } from '@ukef-test/support/generator/random-value-generator'; - -import { redactError } from './redact-errors.helper'; - -describe('Redact errors helper', () => { - const valueGenerator = new RandomValueGenerator(); - - describe('redactError', () => { - const domain = valueGenerator.httpsUrl(); - const otherSensitivefield = valueGenerator.word(); - const message = `ConnectionError: Failed to connect to ${domain}, ${otherSensitivefield}`; - const redactedMessage = `ConnectionError: Failed to connect to [RedactedDomain], [Redacted]`; - const redactStrings = [ - { searchValue: domain, replaceValue: '[RedactedDomain]' }, - { searchValue: otherSensitivefield, replaceValue: '[Redacted]' }, - ]; - const error = { - message: message, - stack: message, - originalError: { - message: message, - stack: message, - safe: 'Nothing sensitive', - }, - driverError: { - message: message, - stack: message, - originalError: { - message: message, - stack: message, - safe: 'Nothing sensitive', - }, - }, - }; - const expectedError = { - message: redactedMessage, - stack: redactedMessage, - originalError: { - message: redactedMessage, - stack: redactedMessage, - safe: 'Nothing sensitive', - }, - driverError: { - message: redactedMessage, - stack: redactedMessage, - originalError: { - message: redactedMessage, - stack: redactedMessage, - safe: 'Nothing sensitive', - }, - }, - }; - - it('replaces sensitive data in input object', () => { - const redacted = redactError(true, REDACT_STRING_PATHS, redactStrings, error); - - expect(redacted).toStrictEqual(expectedError); - }); - - it('returns original input if redactLogs is set to false', () => { - const redacted = redactError(false, REDACT_STRING_PATHS, redactStrings, error); - - expect(redacted).toStrictEqual(error); - }); - - it('replaces sensitive data in different input object', () => { - const error = { - field1: message, - field2: { - field3: message, - safe: 'Nothing sensitive', - }, - }; - const expectedError = { - field1: redactedMessage, - field2: { - field3: redactedMessage, - safe: 'Nothing sensitive', - }, - }; - const redactPaths = ['field1', 'field2.field3']; - const redacted = redactError(true, redactPaths, redactStrings, error); - - expect(redacted).toStrictEqual(expectedError); - }); - }); -}); diff --git a/src/helpers/redact-errors.helper.ts b/src/helpers/redact-errors.helper.ts deleted file mode 100644 index d4b05159..00000000 --- a/src/helpers/redact-errors.helper.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { get, set } from 'lodash'; - -// This helper function is used to redact sensitive data in Error object strings. -export const redactError = ( - redactLogs: boolean, - redactPaths: string[], - redactStrings: { searchValue: string | RegExp; replaceValue: string }[], - err: any, -): any => { - if (!redactLogs) { - return err; - } - redactPaths.forEach((path) => { - const value: string = get(err, path); - if (value) { - const safeValue = redactString(redactStrings, value); - set(err, path, safeValue); - } - }); - return err; -}; - -const redactString = (redactStrings: { searchValue: string | RegExp; replaceValue: string }[], string: string): string => { - let safeSTring: string = string; - redactStrings.forEach((redact) => { - safeSTring = safeSTring.replaceAll(redact.searchValue, redact.replaceValue); - }); - return safeSTring; -}; diff --git a/src/logging/log-keys-to-redact.test.ts b/src/logging/log-keys-to-redact.test.ts index 45cf2495..155171f8 100644 --- a/src/logging/log-keys-to-redact.test.ts +++ b/src/logging/log-keys-to-redact.test.ts @@ -76,5 +76,33 @@ describe('logKeysToRedact', () => { expect(result).toContain(buildKeyToRedact([logKey, 'options', 'cause', 'innerError', sensitiveChildKeys[0]])); expect(result).toContain(buildKeyToRedact([logKey, 'options', 'cause', 'innerError', sensitiveChildKeys[1]])); }); + + it('includes all sensitive child keys of an dbError', () => { + const { logKey, sensitiveChildKeys } = options.dbError; + + expect(result).toContain(buildKeyToRedact([logKey, sensitiveChildKeys[0]])); + expect(result).toContain(buildKeyToRedact([logKey, sensitiveChildKeys[1]])); + }); + + it('includes all sensitive child keys of an original dbError', () => { + const { logKey, sensitiveChildKeys } = options.dbError; + + expect(result).toContain(buildKeyToRedact([logKey, 'originalError', 'info', sensitiveChildKeys[0]])); + expect(result).toContain(buildKeyToRedact([logKey, 'originalError', 'info', sensitiveChildKeys[1]])); + }); + + it('includes all sensitive child keys of an dbError driverError', () => { + const { logKey, sensitiveChildKeys } = options.dbError; + + expect(result).toContain(buildKeyToRedact([logKey, 'driverError', sensitiveChildKeys[0]])); + expect(result).toContain(buildKeyToRedact([logKey, 'driverError', sensitiveChildKeys[1]])); + }); + + it(`includes all sensitive child keys of an dbError driverError's original error`, () => { + const { logKey, sensitiveChildKeys } = options.dbError; + + expect(result).toContain(buildKeyToRedact([logKey, 'driverError', 'originalError', 'info', sensitiveChildKeys[0]])); + expect(result).toContain(buildKeyToRedact([logKey, 'driverError', 'originalError', 'info', sensitiveChildKeys[1]])); + }); }); }); diff --git a/src/main.module.ts b/src/main.module.ts index d93a5299..b0583583 100644 --- a/src/main.module.ts +++ b/src/main.module.ts @@ -6,6 +6,8 @@ import { HEADERS_LOG_KEY, OUTGOING_REQUEST_LOG_KEY } from '@ukef/modules/http/ht import { MdmModule } from '@ukef/modules/mdm.module'; import { LoggerModule } from 'nestjs-pino'; +import { REDACT_STRING_PATHS, REDACT_STRINGS } from './constants'; +import { redactStringsInLogArgs } from './helpers/redact-strings-in-log-args.helper'; import { logKeysToRedact } from './logging/log-keys-to-redact'; import { LoggingInterceptor } from './logging/logging-interceptor.helper'; @@ -30,6 +32,11 @@ import { LoggingInterceptor } from './logging/logging-interceptor.helper'; singleLine: true, }, }, + hooks: { + logMethod(inputArgs, method) { + return method.apply(this, redactStringsInLogArgs(config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, inputArgs)); + }, + }, redact: logKeysToRedact({ redactLogs: config.get('app.redactLogs'), clientRequest: { @@ -60,7 +67,6 @@ import { LoggingInterceptor } from './logging/logging-interceptor.helper'; }), MdmModule, ], - controllers: [], providers: [{ provide: APP_INTERCEPTOR, useClass: LoggingInterceptor }], }) export class MainModule {} diff --git a/src/modules/constants/constants.service.ts b/src/modules/constants/constants.service.ts index c3832ead..1c81761a 100644 --- a/src/modules/constants/constants.service.ts +++ b/src/modules/constants/constants.service.ts @@ -1,9 +1,7 @@ import { Injectable, InternalServerErrorException, NotFoundException } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; import { InjectRepository } from '@nestjs/typeorm'; -import { DATABASE, REDACT_STRING_PATHS, REDACT_STRINGS } from '@ukef/constants'; +import { DATABASE } from '@ukef/constants'; import { DbResponseHelper } from '@ukef/helpers/db-response.helper'; -import { redactError } from '@ukef/helpers/redact-errors.helper'; import { PinoLogger } from 'nestjs-pino'; import { Repository } from 'typeorm'; @@ -15,7 +13,6 @@ export class ConstantsService { @InjectRepository(ConstantSpiEntity, DATABASE.CIS) private readonly constantsCisRepository: Repository, private readonly logger: PinoLogger, - private readonly config: ConfigService, ) {} async find(oecdRiskCategory: number, category: string): Promise { @@ -38,10 +35,10 @@ export class ConstantsService { return transformedResults; } catch (err) { if (err instanceof NotFoundException) { - this.logger.warn(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); + this.logger.warn(err); throw err; } - this.logger.error(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); + this.logger.error(err); throw new InternalServerErrorException(); } } diff --git a/src/modules/currencies/currencies.service.ts b/src/modules/currencies/currencies.service.ts index cecbc927..2b60217d 100644 --- a/src/modules/currencies/currencies.service.ts +++ b/src/modules/currencies/currencies.service.ts @@ -1,9 +1,7 @@ import { Injectable, InternalServerErrorException, NotFoundException } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; import { InjectRepository } from '@nestjs/typeorm'; -import { DATABASE, DATE, REDACT_STRING_PATHS, REDACT_STRINGS } from '@ukef/constants'; +import { DATABASE, DATE } from '@ukef/constants'; import { DbResponseHelper } from '@ukef/helpers/db-response.helper'; -import { redactError } from '@ukef/helpers/redact-errors.helper'; import { PinoLogger } from 'nestjs-pino'; import { DataSource, Equal, Repository } from 'typeorm'; @@ -20,7 +18,6 @@ export class CurrenciesService { @InjectRepository(CurrencyExchangeEntity, DATABASE.CEDAR) private readonly currencyExchangeRepository: Repository, private readonly logger: PinoLogger, - private readonly config: ConfigService, ) {} async findAll(): Promise { @@ -69,10 +66,10 @@ export class CurrenciesService { return renamedResults; } catch (err) { if (err instanceof NotFoundException) { - this.logger.warn(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); + this.logger.warn(err); throw err; } else { - this.logger.error(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); + this.logger.error(err); throw new InternalServerErrorException(); } } diff --git a/src/modules/exposure-period/exposure-period.service.ts b/src/modules/exposure-period/exposure-period.service.ts index cc34785c..2519b45e 100644 --- a/src/modules/exposure-period/exposure-period.service.ts +++ b/src/modules/exposure-period/exposure-period.service.ts @@ -1,8 +1,6 @@ import { Injectable, InternalServerErrorException, NotFoundException } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; import { InjectDataSource } from '@nestjs/typeorm'; -import { DATABASE, REDACT_STRING_PATHS, REDACT_STRINGS } from '@ukef/constants'; -import { redactError } from '@ukef/helpers/redact-errors.helper'; +import { DATABASE } from '@ukef/constants'; import { PinoLogger } from 'nestjs-pino'; import { DataSource } from 'typeorm'; @@ -14,7 +12,6 @@ export class ExposurePeriodService { @InjectDataSource(DATABASE.MDM) private readonly mdmDataSource: DataSource, private readonly logger: PinoLogger, - private readonly config: ConfigService, ) {} async calculate(startDate: string, endDate: string, productGroup: string): Promise { @@ -29,10 +26,10 @@ export class ExposurePeriodService { return { exposurePeriod: results[0].EXPOSURE_PERIOD }; } catch (err) { if (err instanceof NotFoundException) { - this.logger.warn(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); + this.logger.warn(err); throw err; } else { - this.logger.error(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); + this.logger.error(err); throw new InternalServerErrorException(); } } diff --git a/src/modules/interest-rates/interest-rates.service.ts b/src/modules/interest-rates/interest-rates.service.ts index de55bcfc..124b9bfa 100644 --- a/src/modules/interest-rates/interest-rates.service.ts +++ b/src/modules/interest-rates/interest-rates.service.ts @@ -1,8 +1,6 @@ import { Injectable, InternalServerErrorException } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; import { InjectRepository } from '@nestjs/typeorm'; -import { DATABASE, DATE, REDACT_STRING_PATHS, REDACT_STRINGS } from '@ukef/constants'; -import { redactError } from '@ukef/helpers/redact-errors.helper'; +import { DATABASE, DATE } from '@ukef/constants'; import { PinoLogger } from 'nestjs-pino'; import { Equal, Repository } from 'typeorm'; @@ -14,14 +12,13 @@ export class InterestRatesService { @InjectRepository(InterestRatesEntity, DATABASE.CEDAR) private readonly interestRates: Repository, private readonly logger: PinoLogger, - private readonly config: ConfigService, ) {} findAll(): Promise { try { return this.interestRates.find({ where: { effectiveTo: Equal(new Date(DATE.MAXIMUM_LIMIT)) } }); } catch (err) { - this.logger.error(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); + this.logger.error(err); throw new InternalServerErrorException(); } } diff --git a/src/modules/markets/markets.service.ts b/src/modules/markets/markets.service.ts index 882347b4..c7520f6a 100644 --- a/src/modules/markets/markets.service.ts +++ b/src/modules/markets/markets.service.ts @@ -1,9 +1,7 @@ import { Injectable, InternalServerErrorException } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; import { InjectRepository } from '@nestjs/typeorm'; -import { DATABASE, REDACT_STRING_PATHS, REDACT_STRINGS } from '@ukef/constants'; +import { DATABASE } from '@ukef/constants'; import { DbResponseHelper } from '@ukef/helpers/db-response.helper'; -import { redactError } from '@ukef/helpers/redact-errors.helper'; import { PinoLogger } from 'nestjs-pino'; import { Repository } from 'typeorm'; @@ -15,7 +13,6 @@ export class MarketsService { @InjectRepository(MarketEntity, DATABASE.CIS) private readonly marketsRepository: Repository, private readonly logger: PinoLogger, - private readonly config: ConfigService, ) {} async find(active?: string, search?: string): Promise { @@ -48,7 +45,7 @@ export class MarketsService { return mappedResults; } catch (err: any) { - this.logger.error(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); + this.logger.error(err); throw new InternalServerErrorException(); } } diff --git a/src/modules/numbers/numbers.service.ts b/src/modules/numbers/numbers.service.ts index 6afd1639..66d38d32 100644 --- a/src/modules/numbers/numbers.service.ts +++ b/src/modules/numbers/numbers.service.ts @@ -1,8 +1,6 @@ import { BadRequestException, Injectable, InternalServerErrorException, NotFoundException } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; import { InjectRepository } from '@nestjs/typeorm'; -import { DATABASE, REDACT_STRING_PATHS, REDACT_STRINGS } from '@ukef/constants'; -import { redactError } from '@ukef/helpers/redact-errors.helper'; +import { DATABASE } from '@ukef/constants'; import { PinoLogger } from 'nestjs-pino'; import { Repository } from 'typeorm'; @@ -15,7 +13,6 @@ export class NumbersService { @InjectRepository(UkefId, DATABASE.NUMBER_GENERATOR) private readonly numberRepository: Repository, private readonly logger: PinoLogger, - private readonly config: ConfigService, ) {} async create(createUkefIdDto: CreateUkefIdDto[]): Promise { @@ -46,10 +43,10 @@ export class NumbersService { return this.mapFieldsFromDbToApi(dbNumber[0]); } catch (err) { if (err instanceof NotFoundException || err instanceof BadRequestException) { - this.logger.warn(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); + this.logger.warn(err); throw err; } else { - this.logger.error(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); + this.logger.error(err); throw new InternalServerErrorException(); } } diff --git a/src/modules/premium-schedules/premium-schedules.service.ts b/src/modules/premium-schedules/premium-schedules.service.ts index f669e2d1..eb13247a 100644 --- a/src/modules/premium-schedules/premium-schedules.service.ts +++ b/src/modules/premium-schedules/premium-schedules.service.ts @@ -1,9 +1,7 @@ import { Injectable, InternalServerErrorException, NotFoundException } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; import { InjectRepository } from '@nestjs/typeorm'; -import { DATABASE, REDACT_STRING_PATHS, REDACT_STRINGS } from '@ukef/constants'; +import { DATABASE } from '@ukef/constants'; import { DbResponseHelper } from '@ukef/helpers/db-response.helper'; -import { redactError } from '@ukef/helpers/redact-errors.helper'; import { Response } from 'express'; import { PinoLogger } from 'nestjs-pino'; import { Equal, Repository } from 'typeorm'; @@ -17,7 +15,6 @@ export class PremiumSchedulesService { @InjectRepository(PremiumScheduleEntity, DATABASE.MDM) private readonly premiumSchedulesRepository: Repository, private readonly logger: PinoLogger, - private readonly config: ConfigService, ) {} async find(facilityId: string): Promise { @@ -32,10 +29,10 @@ export class PremiumSchedulesService { return results; } catch (err) { if (err instanceof NotFoundException) { - this.logger.warn(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); + this.logger.warn(err); throw err; } else { - this.logger.error(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); + this.logger.error(err); throw new InternalServerErrorException(); } } @@ -92,7 +89,7 @@ export class PremiumSchedulesService { return transformedResults; } catch (err) { - this.logger.error(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); + this.logger.error(err); throw new InternalServerErrorException(); } } diff --git a/src/modules/sector-industries/sector-industries.service.ts b/src/modules/sector-industries/sector-industries.service.ts index 9aa36a25..df3aa89a 100644 --- a/src/modules/sector-industries/sector-industries.service.ts +++ b/src/modules/sector-industries/sector-industries.service.ts @@ -1,8 +1,6 @@ import { Injectable, InternalServerErrorException, NotFoundException } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; import { InjectRepository } from '@nestjs/typeorm'; -import { DATABASE, DATE, REDACT_STRING_PATHS, REDACT_STRINGS } from '@ukef/constants'; -import { redactError } from '@ukef/helpers/redact-errors.helper'; +import { DATABASE, DATE } from '@ukef/constants'; import { PinoLogger } from 'nestjs-pino'; import { Equal, Repository } from 'typeorm'; @@ -14,7 +12,6 @@ export class SectorIndustriesService { @InjectRepository(SectorIndustryEntity, DATABASE.MDM) private readonly sectorIndustries: Repository, private readonly logger: PinoLogger, - private readonly config: ConfigService, ) {} async find(ukefSectorId: string, ukefIndustryId: string): Promise { @@ -37,10 +34,10 @@ export class SectorIndustriesService { return results; } catch (err) { if (err instanceof NotFoundException) { - this.logger.warn(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); + this.logger.warn(err); throw err; } else { - this.logger.error(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); + this.logger.error(err); throw new InternalServerErrorException(); } } diff --git a/src/modules/yield-rates/yield-rates.service.ts b/src/modules/yield-rates/yield-rates.service.ts index ea56c5de..a2c4a2ff 100644 --- a/src/modules/yield-rates/yield-rates.service.ts +++ b/src/modules/yield-rates/yield-rates.service.ts @@ -1,8 +1,6 @@ import { Injectable, InternalServerErrorException, NotFoundException } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; import { InjectRepository } from '@nestjs/typeorm'; -import { DATABASE, DATE, REDACT_STRING_PATHS, REDACT_STRINGS } from '@ukef/constants'; -import { redactError } from '@ukef/helpers/redact-errors.helper'; +import { DATABASE, DATE } from '@ukef/constants'; import { PinoLogger } from 'nestjs-pino'; import { Equal, LessThanOrEqual, MoreThan, Repository } from 'typeorm'; @@ -14,7 +12,6 @@ export class YieldRatesService { @InjectRepository(YieldRateEntity, DATABASE.CEDAR) private readonly yieldRateRepository: Repository, private readonly logger: PinoLogger, - private readonly config: ConfigService, ) {} async find(searchDate: string): Promise { @@ -33,10 +30,10 @@ export class YieldRatesService { return results; } catch (err) { if (err instanceof NotFoundException) { - this.logger.warn(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); + this.logger.warn(err); throw err; } else { - this.logger.error(redactError(this.config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, err)); + this.logger.error(err); throw new InternalServerErrorException(); } } From 3a3b64687187fc3f4c8406da5399ef8135b6a1d1 Mon Sep 17 00:00:00 2001 From: Audrius Vaitonis Date: Wed, 19 Jul 2023 11:01:40 +0100 Subject: [PATCH 4/8] feat(APIM-344): changed redaction in crashed bootstrap and changed redaction in recovered bootstrap --- .../redact-strings-paths.constant.ts | 2 + .../redact-strings-in-log-args.helper.test.ts | 96 +++++++++++++++++++ .../redact-strings-in-log-args.helper.ts | 31 ++++++ src/logging/console-logger-with-redact.ts | 10 +- src/main.module.ts | 2 +- src/main.ts | 4 +- src/modules/markets/markets.service.ts | 2 +- 7 files changed, 142 insertions(+), 5 deletions(-) create mode 100644 src/helpers/redact-strings-in-log-args.helper.test.ts create mode 100644 src/helpers/redact-strings-in-log-args.helper.ts diff --git a/src/constants/redact-strings-paths.constant.ts b/src/constants/redact-strings-paths.constant.ts index 1e2fc3bb..648fd4e5 100644 --- a/src/constants/redact-strings-paths.constant.ts +++ b/src/constants/redact-strings-paths.constant.ts @@ -7,4 +7,6 @@ export const REDACT_STRING_PATHS = [ 'stack', 'originalError.message', 'originalError.stack', + 'err.message', + 'err.stack', ]; diff --git a/src/helpers/redact-strings-in-log-args.helper.test.ts b/src/helpers/redact-strings-in-log-args.helper.test.ts new file mode 100644 index 00000000..7e8386e9 --- /dev/null +++ b/src/helpers/redact-strings-in-log-args.helper.test.ts @@ -0,0 +1,96 @@ +import { REDACT_STRING_PATHS } from '@ukef/constants'; +import { RandomValueGenerator } from '@ukef-test/support/generator/random-value-generator'; + +import { redactStringsInLogArgs } from './redact-strings-in-log-args.helper'; + +describe('Redact errors helper', () => { + const valueGenerator = new RandomValueGenerator(); + + describe('redactStringsInLogArgs', () => { + const domain = valueGenerator.httpsUrl(); + const otherSensitivefield = valueGenerator.word(); + const message = `ConnectionError: Failed to connect to ${domain}, ${otherSensitivefield}`; + const redactedMessage = `ConnectionError: Failed to connect to [RedactedDomain], [Redacted]`; + const redactStrings = [ + { searchValue: domain, replaceValue: '[RedactedDomain]' }, + { searchValue: otherSensitivefield, replaceValue: '[Redacted]' }, + ]; + const args = [ + { + message: message, + stack: message, + originalError: { + message: message, + stack: message, + safe: 'Nothing sensitive', + }, + driverError: { + message: message, + stack: message, + originalError: { + message: message, + stack: message, + safe: 'Nothing sensitive', + }, + }, + }, + ]; + const expectedResult = [ + { + message: redactedMessage, + stack: redactedMessage, + originalError: { + message: redactedMessage, + stack: redactedMessage, + safe: 'Nothing sensitive', + }, + driverError: { + message: redactedMessage, + stack: redactedMessage, + originalError: { + message: redactedMessage, + stack: redactedMessage, + safe: 'Nothing sensitive', + }, + }, + }, + ]; + + it('replaces sensitive data in input object', () => { + const redacted = redactStringsInLogArgs(true, REDACT_STRING_PATHS, redactStrings, args); + + expect(redacted).toStrictEqual(expectedResult); + }); + + it('returns original input if redactLogs is set to false', () => { + const redacted = redactStringsInLogArgs(false, REDACT_STRING_PATHS, redactStrings, args); + + expect(redacted).toStrictEqual(args); + }); + + it('replaces sensitive data in different input object', () => { + const args = [ + { + field1: message, + field2: { + field3: message, + safe: 'Nothing sensitive', + }, + }, + ]; + const expectedResult = [ + { + field1: redactedMessage, + field2: { + field3: redactedMessage, + safe: 'Nothing sensitive', + }, + }, + ]; + const redactPaths = ['field1', 'field2.field3']; + const redacted = redactStringsInLogArgs(true, redactPaths, redactStrings, args); + + expect(redacted).toStrictEqual(expectedResult); + }); + }); +}); diff --git a/src/helpers/redact-strings-in-log-args.helper.ts b/src/helpers/redact-strings-in-log-args.helper.ts new file mode 100644 index 00000000..345a08c2 --- /dev/null +++ b/src/helpers/redact-strings-in-log-args.helper.ts @@ -0,0 +1,31 @@ +import { get, set } from 'lodash'; + +// This helper function is used to redact sensitive data in Error object strings. +export const redactStringsInLogArgs = ( + redactLogs: boolean, + redactPaths: string[], + redactStrings: { searchValue: string | RegExp; replaceValue: string }[], + args: any, +): any => { + if (!redactLogs) { + return args; + } + args.forEach((arg, index) => { + redactPaths.forEach((path) => { + const value: string = get(arg, path); + if (value) { + const safeValue = redactString(redactStrings, value); + set(args[index], path, safeValue); + } + }); + }); + return args; +}; + +const redactString = (redactStrings: { searchValue: string | RegExp; replaceValue: string }[], string: string): string => { + let safeString: string = string; + redactStrings.forEach((redact) => { + safeString = safeString.replaceAll(redact.searchValue, redact.replaceValue); + }); + return safeString; +}; diff --git a/src/logging/console-logger-with-redact.ts b/src/logging/console-logger-with-redact.ts index 5523d9d7..b76a9973 100644 --- a/src/logging/console-logger-with-redact.ts +++ b/src/logging/console-logger-with-redact.ts @@ -9,12 +9,18 @@ export class ConsoleLoggerWithRedact extends ConsoleLogger { } error(message: any, stack?: string, context?: string) { + let cleanMessage = message; let cleanStack = stack; + if (typeof message == 'string') { + this.stringPatternsToRedact.forEach((redact) => { + cleanMessage = cleanMessage.replace(redact.searchValue, redact.replaceValue); + }); + } if (typeof stack == 'string') { this.stringPatternsToRedact.forEach((redact) => { - cleanStack = stack.replace(redact.searchValue, redact.replaceValue); + cleanStack = cleanStack.replace(redact.searchValue, redact.replaceValue); }); } - super.error(message, cleanStack, context); + super.error(cleanMessage, cleanStack, context); } } diff --git a/src/main.module.ts b/src/main.module.ts index b0583583..a86aae59 100644 --- a/src/main.module.ts +++ b/src/main.module.ts @@ -33,7 +33,7 @@ import { LoggingInterceptor } from './logging/logging-interceptor.helper'; }, }, hooks: { - logMethod(inputArgs, method) { + logMethod(inputArgs: any, method) { return method.apply(this, redactStringsInLogArgs(config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, inputArgs)); }, }, diff --git a/src/main.ts b/src/main.ts index 851011bc..e835eed2 100644 --- a/src/main.ts +++ b/src/main.ts @@ -6,8 +6,10 @@ import { REDACT_STRINGS } from './constants'; import { ConsoleLoggerWithRedact } from './logging/console-logger-with-redact'; const main = async () => { + // ConsoleLoggerWithRedact is used just if NestFactory.create is fails completely. + // Buffered logs are passed to PinoLogger if NestFactory.create is successfull, ConsoleLoggerWithRedact is not used. const logger = new ConsoleLoggerWithRedact(REDACT_STRINGS); - const nestApp: NestApplication = await NestFactory.create(MainModule, { logger, bufferLogs: false }); + const nestApp: NestApplication = await NestFactory.create(MainModule, { logger, bufferLogs: true }); const app = new App(nestApp); logger.log(`==========================================================`); diff --git a/src/modules/markets/markets.service.ts b/src/modules/markets/markets.service.ts index c7520f6a..68fb6406 100644 --- a/src/modules/markets/markets.service.ts +++ b/src/modules/markets/markets.service.ts @@ -44,7 +44,7 @@ export class MarketsService { })); return mappedResults; - } catch (err: any) { + } catch (err) { this.logger.error(err); throw new InternalServerErrorException(); } From 4950a0487933cd1797639df1c27526b0cf1de2ec Mon Sep 17 00:00:00 2001 From: Audrius Vaitonis Date: Wed, 19 Jul 2023 14:19:35 +0100 Subject: [PATCH 5/8] feat(APIM-344): unit test for logger class ConsoleLoggerWithRedact --- .../redact-strings-in-log-args.helper.test.ts | 6 +-- .../console-logger-with-redact.test.ts | 42 +++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 src/logging/console-logger-with-redact.test.ts diff --git a/src/helpers/redact-strings-in-log-args.helper.test.ts b/src/helpers/redact-strings-in-log-args.helper.test.ts index 7e8386e9..bfb76ad0 100644 --- a/src/helpers/redact-strings-in-log-args.helper.test.ts +++ b/src/helpers/redact-strings-in-log-args.helper.test.ts @@ -8,12 +8,12 @@ describe('Redact errors helper', () => { describe('redactStringsInLogArgs', () => { const domain = valueGenerator.httpsUrl(); - const otherSensitivefield = valueGenerator.word(); - const message = `ConnectionError: Failed to connect to ${domain}, ${otherSensitivefield}`; + const otherSensitiveField = valueGenerator.word(); + const message = `ConnectionError: Failed to connect to ${domain}, ${otherSensitiveField}`; const redactedMessage = `ConnectionError: Failed to connect to [RedactedDomain], [Redacted]`; const redactStrings = [ { searchValue: domain, replaceValue: '[RedactedDomain]' }, - { searchValue: otherSensitivefield, replaceValue: '[Redacted]' }, + { searchValue: otherSensitiveField, replaceValue: '[Redacted]' }, ]; const args = [ { diff --git a/src/logging/console-logger-with-redact.test.ts b/src/logging/console-logger-with-redact.test.ts new file mode 100644 index 00000000..e3ff05a7 --- /dev/null +++ b/src/logging/console-logger-with-redact.test.ts @@ -0,0 +1,42 @@ +import { ConsoleLogger } from '@nestjs/common'; +import { RandomValueGenerator } from '@ukef-test/support/generator/random-value-generator'; + +import { ConsoleLoggerWithRedact } from './console-logger-with-redact'; + +const mockedSuperError = jest.spyOn(ConsoleLogger.prototype, 'error'); + +describe('ConsoleLoggerWithRedact', () => { + const valueGenerator = new RandomValueGenerator(); + const domain = valueGenerator.httpsUrl(); + const otherSensitiveField = valueGenerator.word(); + const message = `ConnectionError: Failed to connect to ${domain}, ${otherSensitiveField}`; + const redactedMessage = `ConnectionError: Failed to connect to [RedactedDomain], [Redacted]`; + const redactStrings = [ + { searchValue: domain, replaceValue: '[RedactedDomain]' }, + { searchValue: otherSensitiveField, replaceValue: '[Redacted]' }, + ]; + const logger = new ConsoleLoggerWithRedact(redactStrings); + + beforeEach(() => { + mockedSuperError.mockClear(); + }); + + it('redacts sensitive data in `message`', () => { + logger.error(message); + + expect(mockedSuperError).toHaveBeenCalledWith(redactedMessage, undefined, undefined); + }); + + it('redacts sensitive data in `message` and second parameter `stack`', () => { + logger.error(message, message); + + expect(mockedSuperError).toHaveBeenCalledWith(redactedMessage, redactedMessage, undefined); + }); + + it('redacts sensitive data in `message` and second parameter `stack`, also passes context', () => { + const context = valueGenerator.word(); + logger.error(message, message, context); + + expect(mockedSuperError).toHaveBeenCalledWith(redactedMessage, redactedMessage, context); + }); +}); From f1c101a8afc3789c5e131be93a139511b26a333d Mon Sep 17 00:00:00 2001 From: Audrius Vaitonis Date: Wed, 19 Jul 2023 15:03:44 +0100 Subject: [PATCH 6/8] feat(APIM-344): use process variable to enable/disable redacting at bootstrap --- src/logging/console-logger-with-redact.ts | 1 + src/main.ts | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/logging/console-logger-with-redact.ts b/src/logging/console-logger-with-redact.ts index b76a9973..9637cc27 100644 --- a/src/logging/console-logger-with-redact.ts +++ b/src/logging/console-logger-with-redact.ts @@ -8,6 +8,7 @@ export class ConsoleLoggerWithRedact extends ConsoleLogger { this.stringPatternsToRedact = stringPatternsToRedact; } + // Simplified, because has just single signature, function from ConsoleLogger. error(message: any, stack?: string, context?: string) { let cleanMessage = message; let cleanStack = stack; diff --git a/src/main.ts b/src/main.ts index e835eed2..bf63d572 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,3 +1,4 @@ +import { Logger as NestLogger } from '@nestjs/common'; import { NestApplication, NestFactory } from '@nestjs/core'; import { MainModule } from '@ukef/main.module'; @@ -6,9 +7,9 @@ import { REDACT_STRINGS } from './constants'; import { ConsoleLoggerWithRedact } from './logging/console-logger-with-redact'; const main = async () => { - // ConsoleLoggerWithRedact is used just if NestFactory.create is fails completely. - // Buffered logs are passed to PinoLogger if NestFactory.create is successfull, ConsoleLoggerWithRedact is not used. - const logger = new ConsoleLoggerWithRedact(REDACT_STRINGS); + // If REDACT_LOGS is true use ConsoleLoggerWithRedact. ConsoleLoggerWithRedact is used just if `NestFactory.create` is fails completely. + // If `NestFactory.create` doesn't fail completely, then buffered logs are passed to PinoLogger. NestLogger and ConsoleLoggerWithRedact are not used. + const logger = process.env.REDACT_LOGS !== 'false' ? new ConsoleLoggerWithRedact(REDACT_STRINGS) : new NestLogger(); const nestApp: NestApplication = await NestFactory.create(MainModule, { logger, bufferLogs: true }); const app = new App(nestApp); From 709477d1ce6a8a8638be08522af57e3cf2cf94d1 Mon Sep 17 00:00:00 2001 From: Audrius Vaitonis Date: Wed, 19 Jul 2023 15:56:47 +0100 Subject: [PATCH 7/8] feat(APIM-344): added test to redact using regex in 'redactStringsInLogArgs' --- .../redact-strings-in-log-args.helper.test.ts | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/helpers/redact-strings-in-log-args.helper.test.ts b/src/helpers/redact-strings-in-log-args.helper.test.ts index bfb76ad0..375f8529 100644 --- a/src/helpers/redact-strings-in-log-args.helper.test.ts +++ b/src/helpers/redact-strings-in-log-args.helper.test.ts @@ -68,6 +68,35 @@ describe('Redact errors helper', () => { expect(redacted).toStrictEqual(args); }); + it('replaces sensitive data in input object using regex', () => { + const redactStrings = [{ searchValue: /(Login failed for user ').*(')/g, replaceValue: '$1[Redacted]$2' }]; + const otherSensitiveValue = valueGenerator.word(); + const messageforRegex = `Connection error: Login failed for user '${otherSensitiveValue}'`; + const redactedMessage = `Connection error: Login failed for user '[Redacted]'`; + const args = [ + { + message: messageforRegex, + stack: messageforRegex, + originalError: { + message: messageforRegex, + }, + }, + ]; + const expectedResult = [ + { + message: redactedMessage, + stack: redactedMessage, + originalError: { + message: redactedMessage, + }, + }, + ]; + + const redacted = redactStringsInLogArgs(true, REDACT_STRING_PATHS, redactStrings, args); + + expect(redacted).toStrictEqual(expectedResult); + }); + it('replaces sensitive data in different input object', () => { const args = [ { From da02c071096e7c1ba07bc90b9e1b06a77829357e Mon Sep 17 00:00:00 2001 From: Audrius Vaitonis Date: Tue, 25 Jul 2023 13:05:45 +0100 Subject: [PATCH 8/8] feat(APIM-344): fix 2 PR feedback items, change comment and var type --- src/helpers/redact-strings-in-log-args.helper.ts | 2 +- src/main.module.ts | 2 +- src/main.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/helpers/redact-strings-in-log-args.helper.ts b/src/helpers/redact-strings-in-log-args.helper.ts index 345a08c2..d50e76c8 100644 --- a/src/helpers/redact-strings-in-log-args.helper.ts +++ b/src/helpers/redact-strings-in-log-args.helper.ts @@ -5,7 +5,7 @@ export const redactStringsInLogArgs = ( redactLogs: boolean, redactPaths: string[], redactStrings: { searchValue: string | RegExp; replaceValue: string }[], - args: any, + args: any[], ): any => { if (!redactLogs) { return args; diff --git a/src/main.module.ts b/src/main.module.ts index a86aae59..6fb41e02 100644 --- a/src/main.module.ts +++ b/src/main.module.ts @@ -33,7 +33,7 @@ import { LoggingInterceptor } from './logging/logging-interceptor.helper'; }, }, hooks: { - logMethod(inputArgs: any, method) { + logMethod(inputArgs: any[], method) { return method.apply(this, redactStringsInLogArgs(config.get('app.redactLogs'), REDACT_STRING_PATHS, REDACT_STRINGS, inputArgs)); }, }, diff --git a/src/main.ts b/src/main.ts index bf63d572..bbace709 100644 --- a/src/main.ts +++ b/src/main.ts @@ -7,7 +7,7 @@ import { REDACT_STRINGS } from './constants'; import { ConsoleLoggerWithRedact } from './logging/console-logger-with-redact'; const main = async () => { - // If REDACT_LOGS is true use ConsoleLoggerWithRedact. ConsoleLoggerWithRedact is used just if `NestFactory.create` is fails completely. + // If REDACT_LOGS is true use ConsoleLoggerWithRedact. ConsoleLoggerWithRedact is used just if `NestFactory.create` fails completely. // If `NestFactory.create` doesn't fail completely, then buffered logs are passed to PinoLogger. NestLogger and ConsoleLoggerWithRedact are not used. const logger = process.env.REDACT_LOGS !== 'false' ? new ConsoleLoggerWithRedact(REDACT_STRINGS) : new NestLogger(); const nestApp: NestApplication = await NestFactory.create(MainModule, { logger, bufferLogs: true });