From 44178e0fc04f77dfbd69f4f943eec892a5261c46 Mon Sep 17 00:00:00 2001 From: Dominic Evans Date: Tue, 12 Apr 2022 01:26:17 +0100 Subject: [PATCH 1/2] chore: switch to multi-arch compatible docker images - use "official" zookeeper image from library - build a minimal kafka image on-the-fly for the current arch --- .github/workflows/ci.yml | 2 +- Dockerfile.kafka | 26 ++++++++++ docker-compose.yml | 106 ++++++++++++++++++++++----------------- entrypoint.sh | 26 ++++++++++ 4 files changed, 112 insertions(+), 48 deletions(-) create mode 100644 Dockerfile.kafka create mode 100755 entrypoint.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44792a079..bb4e6852d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: fail-fast: false matrix: go-version: [1.17.x, 1.18.x] - kafka-version: [2.8.1, 3.0.0, 3.1.0] + kafka-version: [2.8.1, 3.0.1, 3.1.0] platform: [ubuntu-latest] env: diff --git a/Dockerfile.kafka b/Dockerfile.kafka new file mode 100644 index 000000000..f501f007a --- /dev/null +++ b/Dockerfile.kafka @@ -0,0 +1,26 @@ +FROM registry.access.redhat.com/ubi8/ubi-minimal:latest + +USER root + +RUN microdnf update \ + && microdnf install curl gzip java-11-openjdk-headless tar \ + && microdnf clean all + +ENV JAVA_HOME=/usr/lib/jvm/jre-11 + +# https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html +# Ensure Java doesn't cache any dns results +RUN cd /etc/java/java-11-openjdk/*/conf/security \ + && sed -e '/networkaddress.cache.ttl/d' -e '/networkaddress.cache.negative.ttl/d' -i java.security \ + && echo 'networkaddress.cache.ttl=0' >> java.security \ + && echo 'networkaddress.cache.negative.ttl=0' >> java.security + +# https://github.com/apache/kafka/blob/0d518aaed158896ee9ee6949b8f38128d1d73634/tests/docker/Dockerfile#L65-L67 +ARG KAFKA_MIRROR="https://s3-us-west-2.amazonaws.com/kafka-packages" +RUN mkdir -p "/opt/kafka-2.8.1" && chmod a+rw /opt/kafka-2.8.1 && curl -s "$KAFKA_MIRROR/kafka_2.12-2.8.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-2.8.1" +RUN mkdir -p "/opt/kafka-3.0.1" && chmod a+rw /opt/kafka-3.0.1 && curl -s "$KAFKA_MIRROR/kafka_2.12-3.0.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-3.0.1" +RUN mkdir -p "/opt/kafka-3.1.0" && chmod a+rw /opt/kafka-3.1.0 && curl -s "$KAFKA_MIRROR/kafka_2.12-3.1.0.tgz" | tar xz --strip-components=1 -C "/opt/kafka-3.1.0" + +COPY entrypoint.sh / + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker-compose.yml b/docker-compose.yml index 0c1481c92..f77a303f1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,45 +1,46 @@ version: '3.7' services: zookeeper-1: - image: 'confluentinc/cp-zookeeper:7.0.1' + image: 'docker.io/library/zookeeper:3.6.3' restart: always environment: - ZOOKEEPER_SERVER_ID: '1' - ZOOKEEPER_SERVERS: 'zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888' - ZOOKEEPER_CLIENT_PORT: '2181' - ZOOKEEPER_PEER_PORT: '2888' - ZOOKEEPER_LEADER_PORT: '3888' - ZOOKEEPER_INIT_LIMIT: '10' - ZOOKEEPER_SYNC_LIMIT: '5' - ZOOKEEPER_MAX_CLIENT_CONNS: '0' + ZOO_MY_ID: '1' + ZOO_SERVERS: 'server.1=zookeeper-1:2888:3888 server.2=zookeeper-2:2888:3888 server.3=zookeeper-3:2888:3888' + ZOO_CFG_EXTRA: 'clientPort=2181 peerPort=2888 leaderPort=3888' + ZOO_INIT_LIMIT: '10' + ZOO_SYNC_LIMIT: '5' + ZOO_MAX_CLIENT_CNXNS: '0' + ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok' zookeeper-2: - image: 'confluentinc/cp-zookeeper:7.0.1' + image: 'docker.io/library/zookeeper:3.6.3' restart: always environment: - ZOOKEEPER_SERVER_ID: '2' - ZOOKEEPER_SERVERS: 'zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888' - ZOOKEEPER_CLIENT_PORT: '2181' - ZOOKEEPER_PEER_PORT: '2888' - ZOOKEEPER_LEADER_PORT: '3888' - ZOOKEEPER_INIT_LIMIT: '10' - ZOOKEEPER_SYNC_LIMIT: '5' - ZOOKEEPER_MAX_CLIENT_CONNS: '0' + ZOO_MY_ID: '2' + ZOO_SERVERS: 'server.1=zookeeper-1:2888:3888 server.2=zookeeper-2:2888:3888 server.3=zookeeper-3:2888:3888' + ZOO_CFG_EXTRA: 'clientPort=2181 peerPort=2888 leaderPort=3888' + ZOO_INIT_LIMIT: '10' + ZOO_SYNC_LIMIT: '5' + ZOO_MAX_CLIENT_CNXNS: '0' + ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok' zookeeper-3: - image: 'confluentinc/cp-zookeeper:7.0.1' + image: 'docker.io/library/zookeeper:3.6.3' restart: always environment: - ZOOKEEPER_SERVER_ID: '3' - ZOOKEEPER_SERVERS: 'zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888' - ZOOKEEPER_CLIENT_PORT: '2181' - ZOOKEEPER_PEER_PORT: '2888' - ZOOKEEPER_LEADER_PORT: '3888' - ZOOKEEPER_INIT_LIMIT: '10' - ZOOKEEPER_SYNC_LIMIT: '5' - ZOOKEEPER_MAX_CLIENT_CONNS: '0' + ZOO_MY_ID: '3' + ZOO_SERVERS: 'server.1=zookeeper-1:2888:3888 server.2=zookeeper-2:2888:3888 server.3=zookeeper-3:2888:3888' + ZOO_CFG_EXTRA: 'clientPort=2181 peerPort=2888 leaderPort=3888' + ZOO_INIT_LIMIT: '10' + ZOO_SYNC_LIMIT: '5' + ZOO_MAX_CLIENT_CNXNS: '0' + ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok' kafka-1: - image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}' + image: 'sarama/fv-kafka' + build: + context: . + dockerfile: Dockerfile.kafka restart: always environment: + KAFKA_VERSION: ${KAFKA_VERSION:-3.1.0} KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29091' KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-1:9091,LISTENER_LOCAL://localhost:29091' @@ -48,16 +49,19 @@ services: KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2' KAFKA_CFG_BROKER_ID: '1' KAFKA_CFG_BROKER_RACK: '1' - KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000' - KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000' + KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '6000' + KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '6000' KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector' KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false' - ALLOW_PLAINTEXT_LISTENER: 'true' kafka-2: - image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}' + image: 'sarama/fv-kafka' + build: + context: . + dockerfile: Dockerfile.kafka restart: always environment: + KAFKA_VERSION: ${KAFKA_VERSION:-3.1.0} KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29092' KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-2:9091,LISTENER_LOCAL://localhost:29092' @@ -66,16 +70,19 @@ services: KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2' KAFKA_CFG_BROKER_ID: '2' KAFKA_CFG_BROKER_RACK: '2' - KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000' - KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000' + KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '6000' + KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '6000' KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector' KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false' - ALLOW_PLAINTEXT_LISTENER: 'true' kafka-3: - image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}' + image: 'sarama/fv-kafka' + build: + context: . + dockerfile: Dockerfile.kafka restart: always environment: + KAFKA_VERSION: ${KAFKA_VERSION:-3.1.0} KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29093' KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-3:9091,LISTENER_LOCAL://localhost:29093' @@ -84,16 +91,19 @@ services: KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2' KAFKA_CFG_BROKER_ID: '3' KAFKA_CFG_BROKER_RACK: '3' - KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000' - KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000' + KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '6000' + KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '6000' KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector' KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false' - ALLOW_PLAINTEXT_LISTENER: 'true' kafka-4: - image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}' + image: 'sarama/fv-kafka' + build: + context: . + dockerfile: Dockerfile.kafka restart: always environment: + KAFKA_VERSION: ${KAFKA_VERSION:-3.1.0} KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29094' KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-4:9091,LISTENER_LOCAL://localhost:29094' @@ -102,16 +112,19 @@ services: KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2' KAFKA_CFG_BROKER_ID: '4' KAFKA_CFG_BROKER_RACK: '4' - KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000' - KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000' + KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '6000' + KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '6000' KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector' KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false' - ALLOW_PLAINTEXT_LISTENER: 'true' kafka-5: - image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}' + image: 'sarama/fv-kafka' + build: + context: . + dockerfile: Dockerfile.kafka restart: always environment: + KAFKA_VERSION: ${KAFKA_VERSION:-3.1.0} KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29095' KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-5:9091,LISTENER_LOCAL://localhost:29095' @@ -120,12 +133,11 @@ services: KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2' KAFKA_CFG_BROKER_ID: '5' KAFKA_CFG_BROKER_RACK: '5' - KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000' - KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000' + KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '6000' + KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '6000' KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector' KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false' - ALLOW_PLAINTEXT_LISTENER: 'true' toxiproxy: image: 'ghcr.io/shopify/toxiproxy:2.3.0' ports: diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 000000000..4aaa4a5e1 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +KAFKA_VERSION="${KAFKA_VERSION:-3.1.0}" +KAFKA_HOME="/opt/kafka-${KAFKA_VERSION}" + +if [ ! -d "${KAFKA_HOME}" ]; then + echo 'Error: KAFKA_VERSION '$KAFKA_VERSION' not available in this image at '$KAFKA_HOME + exit 1 +fi + +cd "${KAFKA_HOME}" || exit 1 + +# discard all empty/commented lines +sed -e '/^#/d' -e '/^$/d' -i"" config/server.properties + +# emulate kafka_configure_from_environment_variables from bitnami/bitnami-docker-kafka +for var in "${!KAFKA_CFG_@}"; do + key="$(echo "$var" | sed -e 's/^KAFKA_CFG_//g' -e 's/_/\./g' -e 's/.*/\L&/')" + sed -e '/^'$key'/d' -i"" config/server.properties + value="${!var}" + echo "$key=$value" >>config/server.properties +done + +sort config/server.properties + +exec bin/kafka-server-start.sh config/server.properties From c97021f924460da1ea5041376d6c5463b484c608 Mon Sep 17 00:00:00 2001 From: Dominic Evans Date: Wed, 13 Apr 2022 14:12:45 +0100 Subject: [PATCH 2/2] chore(test): log containers if they fail to start --- functional_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/functional_test.go b/functional_test.go index 5fbf5b9d4..acb3849b2 100644 --- a/functional_test.go +++ b/functional_test.go @@ -195,6 +195,10 @@ func prepareDockerTestEnvironment(ctx context.Context, env *testEnvironment) err } } if !allBrokersUp { + c := exec.Command("docker-compose", "logs", "-t") + c.Stdout = os.Stdout + c.Stderr = os.Stderr + _ = c.Run() return fmt.Errorf("timed out waiting for broker to come up") }