From fa776bd824c4fc21c247d4e625a7ec231c7cb552 Mon Sep 17 00:00:00 2001 From: Stefano Azzalini Date: Mon, 1 Apr 2024 07:49:42 +0200 Subject: [PATCH 1/3] Basic implementation --- docs/modules/redpanda.md | 24 +++++ mkdocs.yml | 1 + packages/modules/redpanda/jest.config.ts | 11 +++ packages/modules/redpanda/package.json | 39 ++++++++ packages/modules/redpanda/src/bootstrap.yaml | 7 ++ .../modules/redpanda/src/entrypoint-tc.sh | 8 ++ packages/modules/redpanda/src/index.ts | 1 + .../redpanda/src/redpanda-container.test.ts | 67 ++++++++++++++ .../redpanda/src/redpanda-container.ts | 91 +++++++++++++++++++ .../modules/redpanda/src/redpanda.yaml.hbs | 61 +++++++++++++ packages/modules/redpanda/tsconfig.build.json | 13 +++ packages/modules/redpanda/tsconfig.json | 21 +++++ 12 files changed, 344 insertions(+) create mode 100644 docs/modules/redpanda.md create mode 100644 packages/modules/redpanda/jest.config.ts create mode 100644 packages/modules/redpanda/package.json create mode 100644 packages/modules/redpanda/src/bootstrap.yaml create mode 100644 packages/modules/redpanda/src/entrypoint-tc.sh create mode 100644 packages/modules/redpanda/src/index.ts create mode 100644 packages/modules/redpanda/src/redpanda-container.test.ts create mode 100644 packages/modules/redpanda/src/redpanda-container.ts create mode 100644 packages/modules/redpanda/src/redpanda.yaml.hbs create mode 100644 packages/modules/redpanda/tsconfig.build.json create mode 100644 packages/modules/redpanda/tsconfig.json diff --git a/docs/modules/redpanda.md b/docs/modules/redpanda.md new file mode 100644 index 000000000..3db36485c --- /dev/null +++ b/docs/modules/redpanda.md @@ -0,0 +1,24 @@ +# Redpanda + +Testcontainers can be used to automatically instantiate and manage [Redpanda](https://redpanda.com/) containers. +More precisely Testcontainers uses the official Docker images for [Redpanda](https://hub.docker.com/r/redpandadata/redpanda) + +!!! note + This module uses features provided in `docker.redpanda.com/redpandadata/redpanda`. + +## Install + + +```bash +npm install @testcontainers/redpanda --save-dev +``` + +## Example + + +[Connect:](../../packages/modules/redpanda/src/redpanda-container.test.ts) inside_block:connectToKafka + + + +[Schema registry:](../../packages/modules/redpanda/src/redpanda-container.test.ts) inside_block:connectToSchemaRegistry + diff --git a/mkdocs.yml b/mkdocs.yml index 0d1fee84b..68cf8bc4c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -57,6 +57,7 @@ nav: - PostgreSQL: modules/postgresql.md - Qdrant: modules/qdrant.md - Redis: modules/redis.md + - Redpanda: modules/redpanda.md - Selenium: modules/selenium.md - Weaviate: modules/weaviate.md - Configuration: configuration.md diff --git a/packages/modules/redpanda/jest.config.ts b/packages/modules/redpanda/jest.config.ts new file mode 100644 index 000000000..1f677baaf --- /dev/null +++ b/packages/modules/redpanda/jest.config.ts @@ -0,0 +1,11 @@ +import type { Config } from "jest"; +import * as path from "path"; + +const config: Config = { + preset: "ts-jest", + moduleNameMapper: { + "^testcontainers$": path.resolve(__dirname, "../../testcontainers/src"), + }, +}; + +export default config; diff --git a/packages/modules/redpanda/package.json b/packages/modules/redpanda/package.json new file mode 100644 index 000000000..2aa1ff907 --- /dev/null +++ b/packages/modules/redpanda/package.json @@ -0,0 +1,39 @@ +{ + "name": "@testcontainers/redpanda", + "version": "10.8.0", + "license": "MIT", + "keywords": [ + "redpanda", + "testing", + "docker", + "testcontainers" + ], + "description": "Redpanda module for Testcontainers", + "homepage": "https://github.com/testcontainers/testcontainers-node#readme", + "repository": { + "type": "git", + "url": "https://github.com/testcontainers/testcontainers-node" + }, + "bugs": { + "url": "https://github.com/testcontainers/testcontainers-node/issues" + }, + "main": "build/index.js", + "files": [ + "build" + ], + "publishConfig": { + "access": "public" + }, + "scripts": { + "prepack": "shx cp ../../../README.md . && shx cp ../../../LICENSE .", + "build": "tsc --project tsconfig.build.json" + }, + "dependencies": { + "handlebars": "^4.7.8", + "testcontainers": "^10.8.0" + }, + "devDependencies": { + "kafkajs": "^2.2.4", + "node-fetch": "^3.3.2" + } +} diff --git a/packages/modules/redpanda/src/bootstrap.yaml b/packages/modules/redpanda/src/bootstrap.yaml new file mode 100644 index 000000000..f723e61b1 --- /dev/null +++ b/packages/modules/redpanda/src/bootstrap.yaml @@ -0,0 +1,7 @@ +# Injected by testcontainers +# This file contains cluster properties which will only be considered when +# starting the cluster for the first time. Afterwards, you can configure cluster +# properties via the Redpanda Admi n API. + +kafka_enable_authorization: false +auto_create_topics_enabled: true diff --git a/packages/modules/redpanda/src/entrypoint-tc.sh b/packages/modules/redpanda/src/entrypoint-tc.sh new file mode 100644 index 000000000..c2f67939b --- /dev/null +++ b/packages/modules/redpanda/src/entrypoint-tc.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# Wait for testcontainer's injected redpanda config with the port only known after docker start +until grep -q "# Injected by testcontainers" "/etc/redpanda/redpanda.yaml" +do + sleep 0.1 +done +exec /entrypoint.sh $@ diff --git a/packages/modules/redpanda/src/index.ts b/packages/modules/redpanda/src/index.ts new file mode 100644 index 000000000..606f65951 --- /dev/null +++ b/packages/modules/redpanda/src/index.ts @@ -0,0 +1 @@ +export { RedpandaContainer, StartedRedpandaContainer } from "./redpanda-container"; diff --git a/packages/modules/redpanda/src/redpanda-container.test.ts b/packages/modules/redpanda/src/redpanda-container.test.ts new file mode 100644 index 000000000..f6272c20c --- /dev/null +++ b/packages/modules/redpanda/src/redpanda-container.test.ts @@ -0,0 +1,67 @@ +import { Kafka, KafkaConfig, logLevel } from "kafkajs"; +import { RedpandaContainer, StartedRedpandaContainer } from "./redpanda-container"; + +describe("RedpandaContainer", () => { + jest.setTimeout(240_000); + + // connectToKafka { + it("should connect", async () => { + const redpandaContainer = await new RedpandaContainer().start(); + await testPubSub(redpandaContainer); + await redpandaContainer.stop(); + }); + // } + + // connectToSchemaRegistry { + it("should connect to schema registry", async () => { + const redpandaContainer = await new RedpandaContainer().start(); + const schemaRegistryUrl = redpandaContainer.getSchemaRegistryAddress(); + + const response = await fetch(`${schemaRegistryUrl}/subjects`, { + method: "GET", + headers: { + "Content-Type": "application/vnd.schemaregistry.v1+json", + }, + }); + + expect(response.status).toBe(200); + + await redpandaContainer.stop(); + }); + // } + + const testPubSub = async ( + redpandaContainer: StartedRedpandaContainer, + additionalConfig: Partial = {} + ) => { + const kafka = new Kafka({ + logLevel: logLevel.NOTHING, + brokers: [redpandaContainer.getBootstrapServers()], + ...additionalConfig, + }); + + const producer = kafka.producer(); + await producer.connect(); + + const consumer = kafka.consumer({ groupId: "test-group" }); + await consumer.connect(); + + await producer.send({ + topic: "test-topic", + messages: [{ value: "test message" }], + }); + + await consumer.subscribe({ topic: "test-topic", fromBeginning: true }); + + const consumedMessage = await new Promise((resolve) => { + consumer.run({ + eachMessage: async ({ message }) => resolve(message.value?.toString()), + }); + }); + + expect(consumedMessage).toBe("test message"); + + await consumer.disconnect(); + await producer.disconnect(); + }; +}); diff --git a/packages/modules/redpanda/src/redpanda-container.ts b/packages/modules/redpanda/src/redpanda-container.ts new file mode 100644 index 000000000..f24d49e39 --- /dev/null +++ b/packages/modules/redpanda/src/redpanda-container.ts @@ -0,0 +1,91 @@ +import os from "os"; +import fs from "fs"; +import path from "path"; +import { compile } from "handlebars"; +import archiver from "archiver"; +import { + AbstractStartedContainer, + GenericContainer, + InspectResult, + StartedTestContainer, + Wait, + getContainerRuntimeClient, +} from "testcontainers"; + +const REDPANDA_PORT = 9092; +const REDPANDA_ADMIN_PORT = 9644; +const SCHEMA_REGISTRY_PORT = 8081; +const REST_PROXY_PORT = 8082; + +export class RedpandaContainer extends GenericContainer { + constructor(image = "docker.redpanda.com/redpandadata/redpanda:v23.3.10") { + super(image); + this.withExposedPorts(REDPANDA_PORT, REDPANDA_ADMIN_PORT, SCHEMA_REGISTRY_PORT, REST_PROXY_PORT) + .withEntrypoint(["/entrypoint-tc.sh"]) + .withUser("root:root") + .withWaitStrategy(Wait.forLogMessage("Successfully started Redpanda!")) + .withCopyFilesToContainer([ + { + source: path.join(__dirname, "entrypoint-tc.sh"), + target: "/entrypoint-tc.sh", + mode: 0o777, + }, + { + source: path.join(__dirname, "bootstrap.yaml"), + target: "/etc/redpanda/.bootstrap.yaml", + }, + ]) + .withCommand(["redpanda", "start", "--mode=dev-container", "--smp=1", "--memory=1G"]); + } + + public override async start(): Promise { + return new StartedRedpandaContainer(await super.start()); + } + + protected override async containerStarting(inspectResult: InspectResult) { + const client = await getContainerRuntimeClient(); + const container = client.container.getById(inspectResult.name.substring(1)); + const renderedRedpandaFile = path.join(os.tmpdir(), `redpanda-${container.id}.yaml`); + fs.writeFileSync( + renderedRedpandaFile, + this.renderRedpandaFile(client.info.containerRuntime.host, inspectResult.ports[REDPANDA_PORT][0].hostPort) + ); + const tar = archiver("tar"); + tar.file(renderedRedpandaFile, { name: "/etc/redpanda/redpanda.yaml" }); + tar.finalize(); + await client.container.putArchive(container, tar, "/"); + fs.unlinkSync(renderedRedpandaFile); + } + + private renderRedpandaFile(host: string, port: number): string { + const template = compile(fs.readFileSync(path.join(__dirname, "redpanda.yaml.hbs"), "utf-8")); + return template({ + kafkaApi: { + advertisedHost: host, + advertisedPort: port, + }, + }); + } +} + +export class StartedRedpandaContainer extends AbstractStartedContainer { + constructor(startedTestContainer: StartedTestContainer) { + super(startedTestContainer); + } + + public getBootstrapServers(): string { + return `${this.getHost()}:${this.getMappedPort(REDPANDA_PORT)}`; + } + + public getSchemaRegistryAddress(): string { + return `http://${this.getHost()}:${this.getMappedPort(SCHEMA_REGISTRY_PORT)}`; + } + + public getAdminAddress(): string { + return `http://${this.getHost()}:${this.getMappedPort(REDPANDA_ADMIN_PORT)}`; + } + + public getRestProxyAddress(): string { + return `http://${this.getHost()}:${this.getMappedPort(REST_PROXY_PORT)}`; + } +} diff --git a/packages/modules/redpanda/src/redpanda.yaml.hbs b/packages/modules/redpanda/src/redpanda.yaml.hbs new file mode 100644 index 000000000..67cb26236 --- /dev/null +++ b/packages/modules/redpanda/src/redpanda.yaml.hbs @@ -0,0 +1,61 @@ +# Injected by testcontainers + +redpanda: + admin: + address: 0.0.0.0 + port: 9644 + + kafka_api: + - address: 0.0.0.0 + name: external + port: 9092 + authentication_method: none + + # This listener is required for the schema registry client. The schema + # registry client connects via an advertised listener like a normal Kafka + # client would do. It can't use the other listener because the mapped + # port is not accessible from within the Redpanda container. + - address: 0.0.0.0 + name: internal + port: 9093 + authentication_method: none + + advertised_kafka_api: + - address: {{ kafkaApi.advertisedHost }} + name: external + port: {{ kafkaApi.advertisedPort }} + - address: 127.0.0.1 + name: internal + port: 9093 + +schema_registry: + schema_registry_api: + - address: "0.0.0.0" + name: main + port: 8081 + authentication_method: none + +schema_registry_client: + brokers: + - address: localhost + port: 9093 + +pandaproxy: + pandaproxy_api: + - address: 0.0.0.0 + port: 8082 + name: proxy-internal + advertised_pandaproxy_api: + - address: 127.0.0.1 + port: 8082 + name: proxy-internal + +pandaproxy_client: + brokers: + - address: localhost + port: 9093 + +rpk: + kafka_api: + brokers: + - localhost:9093 diff --git a/packages/modules/redpanda/tsconfig.build.json b/packages/modules/redpanda/tsconfig.build.json new file mode 100644 index 000000000..0222f6ff1 --- /dev/null +++ b/packages/modules/redpanda/tsconfig.build.json @@ -0,0 +1,13 @@ +{ + "extends": "./tsconfig.json", + "exclude": [ + "build", + "jest.config.ts", + "src/**/*.test.ts" + ], + "references": [ + { + "path": "../../testcontainers" + } + ] +} \ No newline at end of file diff --git a/packages/modules/redpanda/tsconfig.json b/packages/modules/redpanda/tsconfig.json new file mode 100644 index 000000000..39b165817 --- /dev/null +++ b/packages/modules/redpanda/tsconfig.json @@ -0,0 +1,21 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "build", + "paths": { + "testcontainers": [ + "../../testcontainers/src" + ] + } + }, + "exclude": [ + "build", + "jest.config.ts" + ], + "references": [ + { + "path": "../../testcontainers" + } + ] +} \ No newline at end of file From f82dd00f45abbcb6cc87e0e126616a339ec4fce4 Mon Sep 17 00:00:00 2001 From: Stefano Azzalini Date: Mon, 1 Apr 2024 16:59:41 +0200 Subject: [PATCH 2/3] Fix package-lock.json --- package-lock.json | 150 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 148 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index cdefa460b..a1ad9ec00 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4773,6 +4773,10 @@ "resolved": "packages/modules/redis", "link": true }, + "node_modules/@testcontainers/redpanda": { + "resolved": "packages/modules/redpanda", + "link": true + }, "node_modules/@testcontainers/selenium": { "resolved": "packages/modules/selenium", "link": true @@ -7332,6 +7336,15 @@ "node": ">=4" } }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "dev": true, + "engines": { + "node": ">= 12" + } + }, "node_modules/debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", @@ -8368,6 +8381,29 @@ "bser": "2.1.1" } }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, "node_modules/file-entry-cache": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", @@ -8559,6 +8595,18 @@ "node": ">= 14.17" } }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "dev": true, + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/fp-and-or": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/fp-and-or/-/fp-and-or-0.1.4.tgz", @@ -9957,6 +10005,26 @@ "node": ">=14.0.0" } }, + "node_modules/handlebars": { + "version": "4.7.8", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz", + "integrity": "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==", + "dependencies": { + "minimist": "^1.2.5", + "neo-async": "^2.6.2", + "source-map": "^0.6.1", + "wordwrap": "^1.0.0" + }, + "bin": { + "handlebars": "bin/handlebars" + }, + "engines": { + "node": ">=0.4.7" + }, + "optionalDependencies": { + "uglify-js": "^3.1.4" + } + }, "node_modules/has-bigints": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz", @@ -12745,7 +12813,6 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -13312,6 +13379,11 @@ "node": ">= 0.6" } }, + "node_modules/neo-async": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==" + }, "node_modules/neo4j-driver": { "version": "5.17.0", "resolved": "https://registry.npmjs.org/neo4j-driver/-/neo4j-driver-5.17.0.tgz", @@ -13394,6 +13466,25 @@ "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==", "dev": true }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } + }, "node_modules/node-fetch": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", @@ -16283,7 +16374,6 @@ "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -17489,6 +17579,18 @@ "node": ">=4.2.0" } }, + "node_modules/uglify-js": { + "version": "3.17.4", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.17.4.tgz", + "integrity": "sha512-T9q82TJI9e/C1TAxYvfb16xO120tMVFZrGA3f9/P4424DNu6ypK103y0GPFVa17yotwSyZW5iYXgjYHkGrJW/g==", + "optional": true, + "bin": { + "uglifyjs": "bin/uglifyjs" + }, + "engines": { + "node": ">=0.8.0" + } + }, "node_modules/unbox-primitive": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", @@ -17745,6 +17847,15 @@ "uuid": "dist/bin/uuid" } }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, "node_modules/webidl-conversions": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", @@ -17893,6 +18004,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==" + }, "node_modules/wrap-ansi": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", @@ -18380,6 +18496,36 @@ "redis": "^4.6.13" } }, + "packages/modules/redpanda": { + "version": "10.8.0", + "license": "MIT", + "dependencies": { + "handlebars": "^4.7.8", + "testcontainers": "^10.8.0" + }, + "devDependencies": { + "kafkajs": "^2.2.4", + "node-fetch": "^3.3.2" + } + }, + "packages/modules/redpanda/node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "dev": true, + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, "packages/modules/selenium": { "name": "@testcontainers/selenium", "version": "10.8.0", From 469f147edf8a9560a5af78088b4bc247fa528905 Mon Sep 17 00:00:00 2001 From: Stefano Azzalini Date: Sun, 7 Apr 2024 12:14:31 +0200 Subject: [PATCH 3/3] Add missing tests and make use of 'copyContentToContainer' to copy files to container --- docs/modules/redpanda.md | 8 +++ package-lock.json | 1 + .../modules/redpanda/src/entrypoint-tc.sh | 8 --- .../redpanda/src/redpanda-container.test.ts | 26 ++++++++ .../redpanda/src/redpanda-container.ts | 59 ++++++++++++------- 5 files changed, 73 insertions(+), 29 deletions(-) delete mode 100644 packages/modules/redpanda/src/entrypoint-tc.sh diff --git a/docs/modules/redpanda.md b/docs/modules/redpanda.md index 3db36485c..ae452a264 100644 --- a/docs/modules/redpanda.md +++ b/docs/modules/redpanda.md @@ -22,3 +22,11 @@ npm install @testcontainers/redpanda --save-dev [Schema registry:](../../packages/modules/redpanda/src/redpanda-container.test.ts) inside_block:connectToSchemaRegistry + + +[Admin APIs:](../../packages/modules/redpanda/src/redpanda-container.test.ts) inside_block:connectToAdmin + + + +[Rest Proxy:](../../packages/modules/redpanda/src/redpanda-container.test.ts) inside_block:connectToRestProxy + diff --git a/package-lock.json b/package-lock.json index a0c0e2892..a8c7ca4ef 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18639,6 +18639,7 @@ } }, "packages/modules/redpanda": { + "name": "@testcontainers/redpanda", "version": "10.8.0", "license": "MIT", "dependencies": { diff --git a/packages/modules/redpanda/src/entrypoint-tc.sh b/packages/modules/redpanda/src/entrypoint-tc.sh deleted file mode 100644 index c2f67939b..000000000 --- a/packages/modules/redpanda/src/entrypoint-tc.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -# Wait for testcontainer's injected redpanda config with the port only known after docker start -until grep -q "# Injected by testcontainers" "/etc/redpanda/redpanda.yaml" -do - sleep 0.1 -done -exec /entrypoint.sh $@ diff --git a/packages/modules/redpanda/src/redpanda-container.test.ts b/packages/modules/redpanda/src/redpanda-container.test.ts index f6272c20c..c11106053 100644 --- a/packages/modules/redpanda/src/redpanda-container.test.ts +++ b/packages/modules/redpanda/src/redpanda-container.test.ts @@ -30,6 +30,32 @@ describe("RedpandaContainer", () => { }); // } + // connectToAdmin { + it("should connect to admin", async () => { + const redpandaContainer = await new RedpandaContainer().start(); + const adminUrl = `${redpandaContainer.getAdminAddress()}/v1`; + + const response = await fetch(adminUrl); + + expect(response.status).toBe(200); + + await redpandaContainer.stop(); + }); + // } + + // connectToRestProxy { + it("should connect to rest proxy", async () => { + const redpandaContainer = await new RedpandaContainer().start(); + const restProxyUrl = `${redpandaContainer.getRestProxyAddress()}/topics`; + + const response = await fetch(restProxyUrl); + + expect(response.status).toBe(200); + + await redpandaContainer.stop(); + }); + // } + const testPubSub = async ( redpandaContainer: StartedRedpandaContainer, additionalConfig: Partial = {} diff --git a/packages/modules/redpanda/src/redpanda-container.ts b/packages/modules/redpanda/src/redpanda-container.ts index f24d49e39..222a1882c 100644 --- a/packages/modules/redpanda/src/redpanda-container.ts +++ b/packages/modules/redpanda/src/redpanda-container.ts @@ -1,60 +1,77 @@ -import os from "os"; import fs from "fs"; import path from "path"; import { compile } from "handlebars"; -import archiver from "archiver"; import { AbstractStartedContainer, + BoundPorts, GenericContainer, InspectResult, StartedTestContainer, Wait, + WaitStrategy, getContainerRuntimeClient, + waitForContainer, } from "testcontainers"; const REDPANDA_PORT = 9092; const REDPANDA_ADMIN_PORT = 9644; const SCHEMA_REGISTRY_PORT = 8081; const REST_PROXY_PORT = 8082; +const STARTER_SCRIPT = "/testcontainers_start.sh"; +const WAIT_FOR_SCRIPT_MESSAGE = "Waiting for script..."; export class RedpandaContainer extends GenericContainer { + private originalWaitinStrategy: WaitStrategy; + constructor(image = "docker.redpanda.com/redpandadata/redpanda:v23.3.10") { super(image); this.withExposedPorts(REDPANDA_PORT, REDPANDA_ADMIN_PORT, SCHEMA_REGISTRY_PORT, REST_PROXY_PORT) - .withEntrypoint(["/entrypoint-tc.sh"]) .withUser("root:root") .withWaitStrategy(Wait.forLogMessage("Successfully started Redpanda!")) .withCopyFilesToContainer([ - { - source: path.join(__dirname, "entrypoint-tc.sh"), - target: "/entrypoint-tc.sh", - mode: 0o777, - }, { source: path.join(__dirname, "bootstrap.yaml"), target: "/etc/redpanda/.bootstrap.yaml", }, - ]) - .withCommand(["redpanda", "start", "--mode=dev-container", "--smp=1", "--memory=1G"]); + ]); + this.originalWaitinStrategy = this.waitStrategy; } public override async start(): Promise { return new StartedRedpandaContainer(await super.start()); } - protected override async containerStarting(inspectResult: InspectResult) { + protected override async beforeContainerCreated(): Promise { + // Change the wait strategy to wait for a log message from a fake starter script + // so that we can put a real starter script in place at that moment + this.originalWaitinStrategy = this.waitStrategy; + this.waitStrategy = Wait.forLogMessage(WAIT_FOR_SCRIPT_MESSAGE); + this.withEntrypoint(["sh"]); + this.withCommand([ + "-c", + `echo '${WAIT_FOR_SCRIPT_MESSAGE}'; while [ ! -f ${STARTER_SCRIPT} ]; do sleep 0.1; done; ${STARTER_SCRIPT}`, + ]); + } + + protected override async containerStarted( + container: StartedTestContainer, + inspectResult: InspectResult + ): Promise { + const command = "#!/bin/bash\nrpk redpanda start --mode dev-container --smp=1 --memory=1G"; + await container.copyContentToContainer([{ content: command, target: STARTER_SCRIPT, mode: 0o777 }]); + await container.copyContentToContainer([ + { + content: this.renderRedpandaFile(container.getHost(), container.getMappedPort(REDPANDA_PORT)), + target: "/etc/redpanda/redpanda.yaml", + }, + ]); + const client = await getContainerRuntimeClient(); - const container = client.container.getById(inspectResult.name.substring(1)); - const renderedRedpandaFile = path.join(os.tmpdir(), `redpanda-${container.id}.yaml`); - fs.writeFileSync( - renderedRedpandaFile, - this.renderRedpandaFile(client.info.containerRuntime.host, inspectResult.ports[REDPANDA_PORT][0].hostPort) + const dockerContainer = client.container.getById(container.getId()); + const boundPorts = BoundPorts.fromInspectResult(client.info.containerRuntime.hostIps, inspectResult).filter( + this.exposedPorts ); - const tar = archiver("tar"); - tar.file(renderedRedpandaFile, { name: "/etc/redpanda/redpanda.yaml" }); - tar.finalize(); - await client.container.putArchive(container, tar, "/"); - fs.unlinkSync(renderedRedpandaFile); + await waitForContainer(client, dockerContainer, this.originalWaitinStrategy, boundPorts); } private renderRedpandaFile(host: string, port: number): string {