Merge pull request #2210 from Shopify/dnwe/multi-arch

chore: switch to multi-arch compatible docker images

.github/workflows/ci.yml

@@ -14,7 +14,7 @@ jobs:
       fail-fast: false
       matrix:
         go-version: [1.17.x, 1.18.x]
-        kafka-version: [2.8.1, 3.0.0, 3.1.0]
+        kafka-version: [2.8.1, 3.0.1, 3.1.0]
         platform: [ubuntu-latest]
 
     env:

Dockerfile.kafka

@@ -0,0 +1,26 @@
+FROM registry.access.redhat.com/ubi8/ubi-minimal:latest
+
+USER root
+
+RUN microdnf update \
+    && microdnf install curl gzip java-11-openjdk-headless tar \
+    && microdnf clean all
+
+ENV JAVA_HOME=/usr/lib/jvm/jre-11
+
+# https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html
+# Ensure Java doesn't cache any dns results
+RUN cd /etc/java/java-11-openjdk/*/conf/security \
+  && sed -e '/networkaddress.cache.ttl/d' -e '/networkaddress.cache.negative.ttl/d' -i java.security \
+  && echo 'networkaddress.cache.ttl=0' >> java.security \
+  && echo 'networkaddress.cache.negative.ttl=0' >> java.security
+
+# https://github.com/apache/kafka/blob/0d518aaed158896ee9ee6949b8f38128d1d73634/tests/docker/Dockerfile#L65-L67
+ARG KAFKA_MIRROR="https://s3-us-west-2.amazonaws.com/kafka-packages"
+RUN mkdir -p "/opt/kafka-2.8.1" && chmod a+rw /opt/kafka-2.8.1 && curl -s "$KAFKA_MIRROR/kafka_2.12-2.8.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-2.8.1"
+RUN mkdir -p "/opt/kafka-3.0.1" && chmod a+rw /opt/kafka-3.0.1 && curl -s "$KAFKA_MIRROR/kafka_2.12-3.0.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-3.0.1"
+RUN mkdir -p "/opt/kafka-3.1.0" && chmod a+rw /opt/kafka-3.1.0 && curl -s "$KAFKA_MIRROR/kafka_2.12-3.1.0.tgz" | tar xz --strip-components=1 -C "/opt/kafka-3.1.0"
+
+COPY entrypoint.sh /
+
+ENTRYPOINT ["/entrypoint.sh"]

docker-compose.yml

@@ -1,45 +1,46 @@
 version: '3.7'
 services:
   zookeeper-1:
-    image: 'confluentinc/cp-zookeeper:7.0.1'
+    image: 'docker.io/library/zookeeper:3.6.3'
     restart: always
     environment:
-      ZOOKEEPER_SERVER_ID: '1'
-      ZOOKEEPER_SERVERS: 'zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888'
-      ZOOKEEPER_CLIENT_PORT: '2181'
-      ZOOKEEPER_PEER_PORT: '2888'
-      ZOOKEEPER_LEADER_PORT: '3888'
-      ZOOKEEPER_INIT_LIMIT: '10'
-      ZOOKEEPER_SYNC_LIMIT: '5'
-      ZOOKEEPER_MAX_CLIENT_CONNS: '0'
+      ZOO_MY_ID: '1'
+      ZOO_SERVERS: 'server.1=zookeeper-1:2888:3888 server.2=zookeeper-2:2888:3888 server.3=zookeeper-3:2888:3888'
+      ZOO_CFG_EXTRA: 'clientPort=2181 peerPort=2888 leaderPort=3888'
+      ZOO_INIT_LIMIT: '10'
+      ZOO_SYNC_LIMIT: '5'
+      ZOO_MAX_CLIENT_CNXNS: '0'
+      ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok'
   zookeeper-2:
-    image: 'confluentinc/cp-zookeeper:7.0.1'
+    image: 'docker.io/library/zookeeper:3.6.3'
     restart: always
     environment:
-      ZOOKEEPER_SERVER_ID: '2'
-      ZOOKEEPER_SERVERS: 'zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888'
-      ZOOKEEPER_CLIENT_PORT: '2181'
-      ZOOKEEPER_PEER_PORT: '2888'
-      ZOOKEEPER_LEADER_PORT: '3888'
-      ZOOKEEPER_INIT_LIMIT: '10'
-      ZOOKEEPER_SYNC_LIMIT: '5'
-      ZOOKEEPER_MAX_CLIENT_CONNS: '0'
+      ZOO_MY_ID: '2'
+      ZOO_SERVERS: 'server.1=zookeeper-1:2888:3888 server.2=zookeeper-2:2888:3888 server.3=zookeeper-3:2888:3888'
+      ZOO_CFG_EXTRA: 'clientPort=2181 peerPort=2888 leaderPort=3888'
+      ZOO_INIT_LIMIT: '10'
+      ZOO_SYNC_LIMIT: '5'
+      ZOO_MAX_CLIENT_CNXNS: '0'
+      ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok'
   zookeeper-3:
-    image: 'confluentinc/cp-zookeeper:7.0.1'
+    image: 'docker.io/library/zookeeper:3.6.3'
     restart: always
     environment:
-      ZOOKEEPER_SERVER_ID: '3'
-      ZOOKEEPER_SERVERS: 'zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888'
-      ZOOKEEPER_CLIENT_PORT: '2181'
-      ZOOKEEPER_PEER_PORT: '2888'
-      ZOOKEEPER_LEADER_PORT: '3888'
-      ZOOKEEPER_INIT_LIMIT: '10'
-      ZOOKEEPER_SYNC_LIMIT: '5'
-      ZOOKEEPER_MAX_CLIENT_CONNS: '0'
+      ZOO_MY_ID: '3'
+      ZOO_SERVERS: 'server.1=zookeeper-1:2888:3888 server.2=zookeeper-2:2888:3888 server.3=zookeeper-3:2888:3888'
+      ZOO_CFG_EXTRA: 'clientPort=2181 peerPort=2888 leaderPort=3888'
+      ZOO_INIT_LIMIT: '10'
+      ZOO_SYNC_LIMIT: '5'
+      ZOO_MAX_CLIENT_CNXNS: '0'
+      ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok'
   kafka-1:
-    image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}'
+    image: 'sarama/fv-kafka'
+    build:
+      context: .
+      dockerfile: Dockerfile.kafka
     restart: always
     environment:
+      KAFKA_VERSION: ${KAFKA_VERSION:-3.1.0}
       KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
       KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29091'
       KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-1:9091,LISTENER_LOCAL://localhost:29091'
@@ -48,16 +49,19 @@ services:
       KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
       KAFKA_CFG_BROKER_ID: '1'
       KAFKA_CFG_BROKER_RACK: '1'
-      KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
-      KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
+      KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '6000'
+      KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '6000'
       KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
       KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
       KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
-      ALLOW_PLAINTEXT_LISTENER: 'true'
   kafka-2:
-    image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}'
+    image: 'sarama/fv-kafka'
+    build:
+      context: .
+      dockerfile: Dockerfile.kafka
     restart: always
     environment:
+      KAFKA_VERSION: ${KAFKA_VERSION:-3.1.0}
       KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
       KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29092'
       KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-2:9091,LISTENER_LOCAL://localhost:29092'
@@ -66,16 +70,19 @@ services:
       KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
       KAFKA_CFG_BROKER_ID: '2'
       KAFKA_CFG_BROKER_RACK: '2'
-      KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
-      KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
+      KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '6000'
+      KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '6000'
       KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
       KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
       KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
-      ALLOW_PLAINTEXT_LISTENER: 'true'
   kafka-3:
-    image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}'
+    image: 'sarama/fv-kafka'
+    build:
+      context: .
+      dockerfile: Dockerfile.kafka
     restart: always
     environment:
+      KAFKA_VERSION: ${KAFKA_VERSION:-3.1.0}
       KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
       KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29093'
       KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-3:9091,LISTENER_LOCAL://localhost:29093'
@@ -84,16 +91,19 @@ services:
       KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
       KAFKA_CFG_BROKER_ID: '3'
       KAFKA_CFG_BROKER_RACK: '3'
-      KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
-      KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
+      KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '6000'
+      KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '6000'
       KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
       KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
       KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
-      ALLOW_PLAINTEXT_LISTENER: 'true'
   kafka-4:
-    image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}'
+    image: 'sarama/fv-kafka'
+    build:
+      context: .
+      dockerfile: Dockerfile.kafka
     restart: always
     environment:
+      KAFKA_VERSION: ${KAFKA_VERSION:-3.1.0}
       KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
       KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29094'
       KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-4:9091,LISTENER_LOCAL://localhost:29094'
@@ -102,16 +112,19 @@ services:
       KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
       KAFKA_CFG_BROKER_ID: '4'
       KAFKA_CFG_BROKER_RACK: '4'
-      KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
-      KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
+      KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '6000'
+      KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '6000'
       KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
       KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
       KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
-      ALLOW_PLAINTEXT_LISTENER: 'true'
   kafka-5:
-    image: 'bitnami/kafka:${KAFKA_VERSION:-3.1.0}'
+    image: 'sarama/fv-kafka'
+    build:
+      context: .
+      dockerfile: Dockerfile.kafka
     restart: always
     environment:
+      KAFKA_VERSION: ${KAFKA_VERSION:-3.1.0}
       KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
       KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29095'
       KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-5:9091,LISTENER_LOCAL://localhost:29095'
@@ -120,12 +133,11 @@ services:
       KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
       KAFKA_CFG_BROKER_ID: '5'
       KAFKA_CFG_BROKER_RACK: '5'
-      KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000'
-      KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000'
+      KAFKA_CFG_ZOOKEEPER_SESSION_TIMEOUT_MS: '6000'
+      KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '6000'
       KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
       KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
       KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
-      ALLOW_PLAINTEXT_LISTENER: 'true'
   toxiproxy:
     image: 'ghcr.io/shopify/toxiproxy:2.3.0'
     ports:

entrypoint.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+KAFKA_VERSION="${KAFKA_VERSION:-3.1.0}"
+KAFKA_HOME="/opt/kafka-${KAFKA_VERSION}"
+
+if [ ! -d "${KAFKA_HOME}" ]; then
+    echo 'Error: KAFKA_VERSION '$KAFKA_VERSION' not available in this image at '$KAFKA_HOME
+    exit 1
+fi
+
+cd "${KAFKA_HOME}" || exit 1
+
+# discard all empty/commented lines
+sed -e '/^#/d' -e '/^$/d' -i"" config/server.properties
+
+# emulate kafka_configure_from_environment_variables from bitnami/bitnami-docker-kafka
+for var in "${!KAFKA_CFG_@}"; do
+    key="$(echo "$var" | sed -e 's/^KAFKA_CFG_//g' -e 's/_/\./g' -e 's/.*/\L&/')"
+    sed -e '/^'$key'/d' -i"" config/server.properties
+    value="${!var}"
+    echo "$key=$value" >>config/server.properties
+done
+
+sort config/server.properties
+
+exec bin/kafka-server-start.sh config/server.properties

functional_test.go

@@ -195,6 +195,10 @@ func prepareDockerTestEnvironment(ctx context.Context, env *testEnvironment) err
 		}
 	}
 	if !allBrokersUp {
+		c := exec.Command("docker-compose", "logs", "-t")
+		c.Stdout = os.Stdout
+		c.Stderr = os.Stderr
+		_ = c.Run()
 		return fmt.Errorf("timed out waiting for broker to come up")
 	}