diff --git a/Dockerfile.kafka b/Dockerfile.kafka index fdd5327f8..2e02efb29 100644 --- a/Dockerfile.kafka +++ b/Dockerfile.kafka @@ -16,11 +16,23 @@ RUN cd /etc/java/java-11-openjdk/*/conf/security \ && echo 'networkaddress.cache.ttl=0' >> java.security \ && echo 'networkaddress.cache.negative.ttl=0' >> java.security +ARG SCALA_VERSION="2.13" +ARG KAFKA_VERSION="3.5.1" + # https://github.com/apache/kafka/blob/53eeaad946cd053e9eb1a762972d4efeacb8e4fc/tests/docker/Dockerfile#L65-L69 ARG KAFKA_MIRROR="https://s3-us-west-2.amazonaws.com/kafka-packages" -RUN mkdir -p "/opt/kafka-3.3.2" && chmod a+rw /opt/kafka-3.3.2 && curl -s "$KAFKA_MIRROR/kafka_2.13-3.3.2.tgz" | tar xz --strip-components=1 -C "/opt/kafka-3.3.2" -RUN mkdir -p "/opt/kafka-3.4.1" && chmod a+rw /opt/kafka-3.4.1 && curl -s "$KAFKA_MIRROR/kafka_2.13-3.4.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-3.4.1" -RUN mkdir -p "/opt/kafka-3.5.1" && chmod a+rw /opt/kafka-3.5.1 && curl -s "$KAFKA_MIRROR/kafka_2.13-3.5.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-3.5.1" +RUN mkdir -p "/opt/kafka-${KAFKA_VERSION}" \ + && chmod a+rw "/opt/kafka-${KAFKA_VERSION}" \ + && curl -s "$KAFKA_MIRROR/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" | tar xz --strip-components=1 -C "/opt/kafka-${KAFKA_VERSION}" + +# older kafka versions depend upon jaxb-api being bundled with the JDK, but it +# was removed from Java 11 so work around that by including it in the kafka +# libs dir regardless +RUN cd /tmp \ + && curl -sLO "https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api/2.3.0/jaxb-api-2.3.0.jar" \ + && for DIR in /opt/kafka-*; do cp -v jaxb-api-2.3.0.jar $DIR/libs/ ; done \ + && rm -f jaxb-api-2.3.0.jar + COPY entrypoint.sh / diff --git a/docker-compose.yml b/docker-compose.yml index 22ee21bf9..4133cd8cd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3.7' +version: '3.9' services: zookeeper-1: image: 'docker.io/library/zookeeper:3.6.3' @@ -34,13 +34,16 @@ services: ZOO_MAX_CLIENT_CNXNS: '0' ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok' kafka-1: - image: 'sarama/fv-kafka' + image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}' build: context: . dockerfile: Dockerfile.kafka + args: + KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1} + SCALA_VERSION: ${SCALA_VERSION:-2.13} restart: always environment: - KAFKA_VERSION: ${KAFKA_VERSION:-3.3.2} + KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1} KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29091' KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-1:9091,LISTENER_LOCAL://localhost:29091' @@ -56,13 +59,16 @@ services: KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false' kafka-2: - image: 'sarama/fv-kafka' + image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}' build: context: . dockerfile: Dockerfile.kafka + args: + KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1} + SCALA_VERSION: ${SCALA_VERSION:-2.13} restart: always environment: - KAFKA_VERSION: ${KAFKA_VERSION:-3.3.2} + KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1} KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29092' KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-2:9091,LISTENER_LOCAL://localhost:29092' @@ -78,13 +84,16 @@ services: KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false' kafka-3: - image: 'sarama/fv-kafka' + image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}' build: context: . dockerfile: Dockerfile.kafka + args: + KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1} + SCALA_VERSION: ${SCALA_VERSION:-2.13} restart: always environment: - KAFKA_VERSION: ${KAFKA_VERSION:-3.3.2} + KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1} KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29093' KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-3:9091,LISTENER_LOCAL://localhost:29093' @@ -100,13 +109,16 @@ services: KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false' kafka-4: - image: 'sarama/fv-kafka' + image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}' build: context: . dockerfile: Dockerfile.kafka + args: + KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1} + SCALA_VERSION: ${SCALA_VERSION:-2.13} restart: always environment: - KAFKA_VERSION: ${KAFKA_VERSION:-3.3.2} + KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1} KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29094' KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-4:9091,LISTENER_LOCAL://localhost:29094' @@ -122,13 +134,16 @@ services: KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false' kafka-5: - image: 'sarama/fv-kafka' + image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}' build: context: . dockerfile: Dockerfile.kafka + args: + KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1} + SCALA_VERSION: ${SCALA_VERSION:-2.13} restart: always environment: - KAFKA_VERSION: ${KAFKA_VERSION:-3.3.2} + KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1} KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29095' KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-5:9091,LISTENER_LOCAL://localhost:29095' @@ -146,11 +161,12 @@ services: toxiproxy: image: 'ghcr.io/shopify/toxiproxy:2.4.0' ports: - # The tests themselves actually start the proxies on these ports - - '29091:29091' - - '29092:29092' - - '29093:29093' - - '29094:29094' - - '29095:29095' - # This is the toxiproxy API port - - '8474:8474' + # The tests themselves actually start the proxies on these ports + - '29091:29091' + - '29092:29092' + - '29093:29093' + - '29094:29094' + - '29095:29095' + + # This is the toxiproxy API port + - '8474:8474' diff --git a/entrypoint.sh b/entrypoint.sh index 7b344fae8..d6381c907 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/bash -KAFKA_VERSION="${KAFKA_VERSION:-3.3.2}" +KAFKA_VERSION="${KAFKA_VERSION:-3.5.1}" KAFKA_HOME="/opt/kafka-${KAFKA_VERSION}" if [ ! -d "${KAFKA_HOME}" ]; then diff --git a/functional_test.go b/functional_test.go index 4883dbfe2..4f22cd393 100644 --- a/functional_test.go +++ b/functional_test.go @@ -154,7 +154,7 @@ func prepareDockerTestEnvironment(ctx context.Context, env *testEnvironment) err if version, ok := os.LookupEnv("KAFKA_VERSION"); ok { env.KafkaVersion = version } else { - env.KafkaVersion = "3.3.2" + env.KafkaVersion = "3.5.1" } c := exec.Command("docker-compose", "up", "-d") @@ -179,9 +179,6 @@ func prepareDockerTestEnvironment(ctx context.Context, env *testEnvironment) err } config := NewFunctionalTestConfig() - if err != nil { - return err - } config.Net.DialTimeout = 1 * time.Second config.Net.ReadTimeout = 1 * time.Second config.Net.WriteTimeout = 1 * time.Second @@ -190,9 +187,14 @@ func prepareDockerTestEnvironment(ctx context.Context, env *testEnvironment) err // wait for the kafka brokers to come up allBrokersUp := false + Logger.Printf("waiting for kafka %s brokers to come up...\n", env.KafkaVersion) + time.Sleep(10 * time.Second) + mainLoop: for i := 0; i < 30 && !allBrokersUp; i++ { - Logger.Println("waiting for kafka brokers to come up") + if i > 0 { + Logger.Printf("still waiting for kafka %s brokers to come up...\n", env.KafkaVersion) + } time.Sleep(3 * time.Second) brokersOk := make([]bool, len(env.KafkaBrokerAddrs)) @@ -213,22 +215,28 @@ mainLoop: } err = client.RefreshMetadata() if err != nil { + client.Close() continue } brokers := client.Brokers() if len(brokers) < expectedBrokers { + client.Close() continue } for _, broker := range brokers { err := broker.Open(client.Config()) if err != nil { + client.Close() continue retryLoop } connected, err := broker.Connected() if err != nil || !connected { + broker.Close() + client.Close() continue retryLoop } } + client.Close() brokersOk[j] = true }