From fc4ab6207e1c8c2544afc4323613fa64911270af Mon Sep 17 00:00:00 2001 From: Etienne Hardy Date: Fri, 11 Nov 2022 15:13:07 -0500 Subject: [PATCH 1/8] Addressing the following Redis related issues. --- README.md | 47 ++++++++++++++++++++-------------------- docker-compose.base.yml | 1 + src/graphql/resolvers.ts | 2 +- src/index.ts | 8 +++++++ src/logger.ts | 9 +++++--- 5 files changed, 40 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 850635aa..34576919 100644 --- a/README.md +++ b/README.md @@ -181,26 +181,27 @@ You can: The following are the different environment variables that are looked up that allow configuring the schema registry in different ways. -| Variable Name | Description | Default | -| --------------------- | ----------------------------------------------------------------------------- | ------------------------- | -| DB_HOST | Host name of the MySQL server | gql-schema-registry-db | -| DB_USERNAME | Username to connect to MySQL | root | -| DB_SECRET | Password used to connect to MySQL | root | -| DB_PORT | Port used when connecting to MySQL | 3306 | -| DB_NAME | Name of the MySQL database to connect to | schema-registry | -| DB_EXECUTE_MIGRATIONS | Controls whether DB migrations are executed upon registry startup or not | true | -| REDIS_HOST | Host name of the Redis server | gql-schema-registry-redis | -| REDIS_PORT | Port used when connecting to Redis | 6379 | -| REDIS_SECRET | Password used to connect to Redis | Empty | -| ASSETS_URL | Controls the url that web assets are served from | localhost:6001 | -| NODE_ENV | Specifies the environment. Use _production_ to load js/css from `dist/assets` | Empty | -| ASYNC_SCHEMA_UPDATES | Specifies if async achema updates is enabled | false | -| KAFKA_BROKER_HOST | Host name of the Kafka broker, used if ASYNC_SCHEMA_UPDATES = true | gql-schema-registry-kafka | -| KAFKA_BROKER_PORT | Port used when connecting to Kafka, used if ASYNC_SCHEMA_UPDATES = true | 9092 | -| KAFKA_SCHEMA_TOPIC | Topic with new schema | graphql-schema-updates | -| KAFKA_QUERIES_TOPIC | Topic with new schema | graphql-queries | -| LOG_LEVEL | Minimum level of logs to output | info | -| LOG_TYPE | Output log type, supports pretty or json. | pretty | +| Variable Name | Description | Default | +|------------------------|-------------------------------------------------------------------------------|---------------------------| +| DB_HOST | Host name of the MySQL server | gql-schema-registry-db | +| DB_USERNAME | Username to connect to MySQL | root | +| DB_SECRET | Password used to connect to MySQL | root | +| DB_PORT | Port used when connecting to MySQL | 3306 | +| DB_NAME | Name of the MySQL database to connect to | schema-registry | +| DB_EXECUTE_MIGRATIONS | Controls whether DB migrations are executed upon registry startup or not | true | +| REDIS_HOST | Host name of the Redis server | gql-schema-registry-redis | +| REDIS_PORT | Port used when connecting to Redis | 6379 | +| REDIS_SECRET | Password used to connect to Redis | Empty | +| ASSETS_URL | Controls the url that web assets are served from | localhost:6001 | +| NODE_ENV | Specifies the environment. Use _production_ to load js/css from `dist/assets` | Empty | +| ASYNC_SCHEMA_UPDATES | Specifies if async achema updates is enabled | false | +| KAFKA_BROKER_HOST | Host name of the Kafka broker, used if ASYNC_SCHEMA_UPDATES = true | gql-schema-registry-kafka | +| KAFKA_BROKER_PORT | Port used when connecting to Kafka, used if ASYNC_SCHEMA_UPDATES = true | 9092 | +| KAFKA_SCHEMA_TOPIC | Topic with new schema | graphql-schema-updates | +| KAFKA_QUERIES_TOPIC | Topic with new schema | graphql-queries | +| LOG_LEVEL | Minimum level of logs to output | info | +| LOG_TYPE | Output log type, supports pretty or json. | pretty | +| LOG_STREAMING_ENABLED | Controls whether logs are streamed over Redis to be presented in UI | false | For development we rely on docker network and use hostnames from `docker-compose.yml`. Node service uses to connect to mysql & redis and change it if you install it with own setup. @@ -390,13 +391,13 @@ See main [blog post](https://medium.com/pipedrive-engineering/journey-to-federat

🟢 GET /health

 - + returns "ok" when service is up

🟢 GET /schema/latest

 - + Simplified version of /schema/compose where latest versions from different services are composed. Some services prefer this to use this natural schema composition, as its natural and time-based. @@ -404,7 +405,7 @@ Some services prefer this to use this natural schema composition, as its natural

🟡 POST /schema/compose

 - + Advanced version of schema composition, where you need to provide services & their versions. Used by graphql gateway to fetch schema based on currently running containers. diff --git a/docker-compose.base.yml b/docker-compose.base.yml index bf11a31d..675c4885 100644 --- a/docker-compose.base.yml +++ b/docker-compose.base.yml @@ -25,6 +25,7 @@ services: image: redis:6-alpine ports: - 6004:6379 + command: redis-server --requirepass password environment: SERVICE_NAME: 'gql-schema-registry-redis' networks: diff --git a/src/graphql/resolvers.ts b/src/graphql/resolvers.ts index 5a82db0e..bfda4266 100644 --- a/src/graphql/resolvers.ts +++ b/src/graphql/resolvers.ts @@ -79,7 +79,7 @@ export default { ); // @ts-ignore - return logs?.redis; + return process.env.LOG_STREAMING_ENABLED === 'true' ? logs?.redis : ''; }, search: async (_, { filter }) => { diff --git a/src/index.ts b/src/index.ts index bc28c30f..3d352c7e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,6 +5,8 @@ import * as kafka from './kafka'; import config from './config'; import router from './router'; import { logger } from './logger'; +import diplomat from './diplomat'; +import redis from './redis'; const app = express(); @@ -100,6 +102,12 @@ export default async function init() { return server; } + const redisConfig = diplomat.getServiceInstance('gql-schema-registry-redis'); + + if (redisConfig.host) { + redis.initRedis(); + } + if (config.asyncSchemaUpdates) { kafka.init(); } diff --git a/src/logger.ts b/src/logger.ts index 5983a1ce..677f0829 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -3,11 +3,14 @@ import RedisTransport from 'winston-redis-stream'; const logTransports = [new transports.Console()]; -if (process.env.REDIS_HOST) { +if (process.env.LOG_STREAMING_ENABLED === 'true' && process.env.REDIS_HOST) { try { const redis = new RedisTransport({ - host: process.env.REDIS_HOST, - port: process.env.REDIS_PORT, + redis: { + host: process.env.REDIS_HOST, + port: process.env.REDIS_PORT, + password: process.env.REDIS_SECRET, + }, channel: 'logs', }); logTransports.push(redis); From 70046648b221c996775b3f31ed97091abaf3c6e2 Mon Sep 17 00:00:00 2001 From: Etienne Hardy Date: Fri, 25 Nov 2022 09:59:19 -0500 Subject: [PATCH 2/8] Fixing the Redis integration tests. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f03851e4..93207353 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "test:unit:coverage": "jest --config test/unit/jest.coverage.cjs --color --detectOpenHandles --forceExit", "test:unit:frontend:coverage": "jest --config test/unit-frontend/jest.config.ts --color --detectOpenHandles --forceExit", "test:watch": "jest --config jest.config.ts --watch", - "test:integration": "DB_HOST=localhost DB_PORT=6000 REDIS_HOST=localhost REDIS_PORT=6004 PORT=6001 jest --config test/integration/jest.config.cjs --color --detectOpenHandles --runInBand --forceExit", + "test:integration": "DB_HOST=localhost DB_PORT=6000 REDIS_HOST=localhost REDIS_PORT=6004 REDIS_SECRET=password PORT=6001 jest --config test/integration/jest.config.cjs --color --detectOpenHandles --runInBand --forceExit", "test:integration:coverage": "DB_HOST=localhost DB_PORT=6000 REDIS_HOST=localhost REDIS_PORT=6004 PORT=6001 jest --config test/integration/jest.coverage.cjs --color --detectOpenHandles --runInBand --forceExit", "test:functional": "./node_modules/.bin/jest --config test/functional/jest.config.cjs", "test:performance": "TARGET_URL=localhost:6001 k6 run test/perf/minimum.test.js", From 687aed2f99d4c90694bbb7fc89239c6c03d8646d Mon Sep 17 00:00:00 2001 From: Etienne Hardy Date: Fri, 25 Nov 2022 10:12:37 -0500 Subject: [PATCH 3/8] Addressing lint errors. --- src/graphql/resolvers.ts | 8 ++++++-- src/index.ts | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/graphql/resolvers.ts b/src/graphql/resolvers.ts index bfda4266..57f507d4 100644 --- a/src/graphql/resolvers.ts +++ b/src/graphql/resolvers.ts @@ -78,8 +78,12 @@ export default { ) ); - // @ts-ignore - return process.env.LOG_STREAMING_ENABLED === 'true' ? logs?.redis : ''; + if (process.env.LOG_STREAMING_ENABLED === 'true') { + // @ts-ignore + return logs?.redis; + } + + return ''; }, search: async (_, { filter }) => { diff --git a/src/index.ts b/src/index.ts index 3d352c7e..cece4631 100644 --- a/src/index.ts +++ b/src/index.ts @@ -102,7 +102,9 @@ export default async function init() { return server; } - const redisConfig = diplomat.getServiceInstance('gql-schema-registry-redis'); + const redisConfig = diplomat.getServiceInstance( + 'gql-schema-registry-redis' + ); if (redisConfig.host) { redis.initRedis(); From d3ea110184a1279a6c0911fb993f7a6e03303b7e Mon Sep 17 00:00:00 2001 From: Etienne Hardy Date: Fri, 25 Nov 2022 10:39:01 -0500 Subject: [PATCH 4/8] Setting the Redis password for the integration tests running on Github. --- docker-compose.functional.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.functional.yml b/docker-compose.functional.yml index a2371dbd..23c986b7 100644 --- a/docker-compose.functional.yml +++ b/docker-compose.functional.yml @@ -13,6 +13,7 @@ services: - WITH_WEBPACK=${WITH_WEBPACK-1} - DB_SCHEMA_REGISTRY=gql-schema-registry-db - NODE_ENV=production + - REDIS_PASSWORD=password volumes: - .:/app/ restart: always From 3e0ef94086f4514432503959c74afc90003b7971 Mon Sep 17 00:00:00 2001 From: Etienne Hardy Date: Sat, 26 Nov 2022 13:20:50 -0500 Subject: [PATCH 5/8] Hopefully fixing the integration tests when run on Github. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 93207353..2679f2fd 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "test:unit:frontend:coverage": "jest --config test/unit-frontend/jest.config.ts --color --detectOpenHandles --forceExit", "test:watch": "jest --config jest.config.ts --watch", "test:integration": "DB_HOST=localhost DB_PORT=6000 REDIS_HOST=localhost REDIS_PORT=6004 REDIS_SECRET=password PORT=6001 jest --config test/integration/jest.config.cjs --color --detectOpenHandles --runInBand --forceExit", - "test:integration:coverage": "DB_HOST=localhost DB_PORT=6000 REDIS_HOST=localhost REDIS_PORT=6004 PORT=6001 jest --config test/integration/jest.coverage.cjs --color --detectOpenHandles --runInBand --forceExit", + "test:integration:coverage": "DB_HOST=localhost DB_PORT=6000 REDIS_HOST=localhost REDIS_PORT=6004 REDIS_SECRET=password PORT=6001 jest --config test/integration/jest.coverage.cjs --color --detectOpenHandles --runInBand --forceExit", "test:functional": "./node_modules/.bin/jest --config test/functional/jest.config.cjs", "test:performance": "TARGET_URL=localhost:6001 k6 run test/perf/minimum.test.js", "test:performance:dockerized": "docker-compose -f docker-compose.perf-tests.yml run --rm k6 run /scripts/minimum.test.js", From 4b794512e3f47cf8022cac4c50aa2b2c66db7150 Mon Sep 17 00:00:00 2001 From: Etienne Hardy Date: Mon, 28 Nov 2022 08:02:27 -0500 Subject: [PATCH 6/8] Making the LOG_STREAMING_ENABLED environment variable default to true. --- README.md | 2 +- docker-compose.functional.yml | 1 + src/config.ts | 7 +++++-- src/graphql/resolvers.ts | 2 +- src/logger.ts | 3 ++- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 34576919..eb667cfd 100644 --- a/README.md +++ b/README.md @@ -201,7 +201,7 @@ The following are the different environment variables that are looked up that al | KAFKA_QUERIES_TOPIC | Topic with new schema | graphql-queries | | LOG_LEVEL | Minimum level of logs to output | info | | LOG_TYPE | Output log type, supports pretty or json. | pretty | -| LOG_STREAMING_ENABLED | Controls whether logs are streamed over Redis to be presented in UI | false | +| LOG_STREAMING_ENABLED | Controls whether logs are streamed over Redis to be presented in UI | true | For development we rely on docker network and use hostnames from `docker-compose.yml`. Node service uses to connect to mysql & redis and change it if you install it with own setup. diff --git a/docker-compose.functional.yml b/docker-compose.functional.yml index 23c986b7..50b80549 100644 --- a/docker-compose.functional.yml +++ b/docker-compose.functional.yml @@ -13,6 +13,7 @@ services: - WITH_WEBPACK=${WITH_WEBPACK-1} - DB_SCHEMA_REGISTRY=gql-schema-registry-db - NODE_ENV=production + - REDIS_HOST=gql-schema-registry-redis - REDIS_PASSWORD=password volumes: - .:/app/ diff --git a/src/config.ts b/src/config.ts index 8109c89a..d07adf9f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -25,8 +25,11 @@ export default { }, }, asyncSchemaUpdates: booleanFor(process.env.ASYNC_SCHEMA_UPDATES), + logStreamingEnabled: + booleanFor(process.env.LOG_STREAMING_ENABLED, 'true') && + process.env.REDIS_HOST !== undefined, }; -function booleanFor(variable) { - return (variable || 'false').toLowerCase() === 'true'; +function booleanFor(variable, defaultValue = 'false') { + return (variable || defaultValue).toLowerCase() === 'true'; } diff --git a/src/graphql/resolvers.ts b/src/graphql/resolvers.ts index 57f507d4..1bb227fc 100644 --- a/src/graphql/resolvers.ts +++ b/src/graphql/resolvers.ts @@ -78,7 +78,7 @@ export default { ) ); - if (process.env.LOG_STREAMING_ENABLED === 'true') { + if (config.logStreamingEnabled) { // @ts-ignore return logs?.redis; } diff --git a/src/logger.ts b/src/logger.ts index 677f0829..26f1a3bd 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -1,9 +1,10 @@ import { createLogger, transports, format } from 'winston'; import RedisTransport from 'winston-redis-stream'; +import config from './config'; const logTransports = [new transports.Console()]; -if (process.env.LOG_STREAMING_ENABLED === 'true' && process.env.REDIS_HOST) { +if (config.logStreamingEnabled) { try { const redis = new RedisTransport({ redis: { From 831710d20ae91ff2cf6bee470151c437c36f9226 Mon Sep 17 00:00:00 2001 From: Etienne Hardy Date: Mon, 28 Nov 2022 11:09:40 -0500 Subject: [PATCH 7/8] Properly specifying the Redis password for functional tests. --- docker-compose.functional.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.functional.yml b/docker-compose.functional.yml index 50b80549..54ab26be 100644 --- a/docker-compose.functional.yml +++ b/docker-compose.functional.yml @@ -14,7 +14,7 @@ services: - DB_SCHEMA_REGISTRY=gql-schema-registry-db - NODE_ENV=production - REDIS_HOST=gql-schema-registry-redis - - REDIS_PASSWORD=password + - REDIS_SECRET=password volumes: - .:/app/ restart: always From 4e401442aa48110a480afb8bcfbf70aeaa58b181 Mon Sep 17 00:00:00 2001 From: Etienne Hardy Date: Fri, 2 Dec 2022 07:53:07 -0500 Subject: [PATCH 8/8] Fixing the Redis configurations in the docker-compose.dev.yml file. --- docker-compose.dev.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index e3bcdba8..100d5de5 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -17,6 +17,8 @@ services: - KAFKA_BROKER_HOST=gql-schema-registry-kafka - KAFKA_BROKER_PORT=9092 - KAFKA_SCHEMA_TOPIC=graphql-schema-updates + - REDIS_HOST=gql-schema-registry-redis + - REDIS_SECRET=password volumes: - .:/app/ restart: always