From 3d7b37f6dc60658cb1e26f6803c40179b3051ae2 Mon Sep 17 00:00:00 2001 From: Dominic Evans Date: Fri, 11 Aug 2023 09:46:02 +0100 Subject: [PATCH] feat(fvt): experiment with per-kafka-version image Switch to templating the Dockerfile and docker-compose so that each image is uniquely named and contains just the kafka version specified. This will make it easier to have a large matrix test for the main branch whilst keeping the PR checks at 2 or 3 kafka versions Signed-off-by: Dominic Evans --- Dockerfile.kafka | 18 +++++++++++++--- docker-compose.yml | 54 ++++++++++++++++++++++++++++++---------------- entrypoint.sh | 2 +- 3 files changed, 51 insertions(+), 23 deletions(-) diff --git a/Dockerfile.kafka b/Dockerfile.kafka index fdd5327f8..2e02efb29 100644 --- a/Dockerfile.kafka +++ b/Dockerfile.kafka @@ -16,11 +16,23 @@ RUN cd /etc/java/java-11-openjdk/*/conf/security \ && echo 'networkaddress.cache.ttl=0' >> java.security \ && echo 'networkaddress.cache.negative.ttl=0' >> java.security +ARG SCALA_VERSION="2.13" +ARG KAFKA_VERSION="3.5.1" + # https://github.com/apache/kafka/blob/53eeaad946cd053e9eb1a762972d4efeacb8e4fc/tests/docker/Dockerfile#L65-L69 ARG KAFKA_MIRROR="https://s3-us-west-2.amazonaws.com/kafka-packages" -RUN mkdir -p "/opt/kafka-3.3.2" && chmod a+rw /opt/kafka-3.3.2 && curl -s "$KAFKA_MIRROR/kafka_2.13-3.3.2.tgz" | tar xz --strip-components=1 -C "/opt/kafka-3.3.2" -RUN mkdir -p "/opt/kafka-3.4.1" && chmod a+rw /opt/kafka-3.4.1 && curl -s "$KAFKA_MIRROR/kafka_2.13-3.4.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-3.4.1" -RUN mkdir -p "/opt/kafka-3.5.1" && chmod a+rw /opt/kafka-3.5.1 && curl -s "$KAFKA_MIRROR/kafka_2.13-3.5.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-3.5.1" +RUN mkdir -p "/opt/kafka-${KAFKA_VERSION}" \ + && chmod a+rw "/opt/kafka-${KAFKA_VERSION}" \ + && curl -s "$KAFKA_MIRROR/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" | tar xz --strip-components=1 -C "/opt/kafka-${KAFKA_VERSION}" + +# older kafka versions depend upon jaxb-api being bundled with the JDK, but it +# was removed from Java 11 so work around that by including it in the kafka +# libs dir regardless +RUN cd /tmp \ + && curl -sLO "https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api/2.3.0/jaxb-api-2.3.0.jar" \ + && for DIR in /opt/kafka-*; do cp -v jaxb-api-2.3.0.jar $DIR/libs/ ; done \ + && rm -f jaxb-api-2.3.0.jar + COPY entrypoint.sh / diff --git a/docker-compose.yml b/docker-compose.yml index 22ee21bf9..4133cd8cd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3.7' +version: '3.9' services: zookeeper-1: image: 'docker.io/library/zookeeper:3.6.3' @@ -34,13 +34,16 @@ services: ZOO_MAX_CLIENT_CNXNS: '0' ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok' kafka-1: - image: 'sarama/fv-kafka' + image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}' build: context: . dockerfile: Dockerfile.kafka + args: + KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1} + SCALA_VERSION: ${SCALA_VERSION:-2.13} restart: always environment: - KAFKA_VERSION: ${KAFKA_VERSION:-3.3.2} + KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1} KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29091' KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-1:9091,LISTENER_LOCAL://localhost:29091' @@ -56,13 +59,16 @@ services: KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false' kafka-2: - image: 'sarama/fv-kafka' + image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}' build: context: . dockerfile: Dockerfile.kafka + args: + KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1} + SCALA_VERSION: ${SCALA_VERSION:-2.13} restart: always environment: - KAFKA_VERSION: ${KAFKA_VERSION:-3.3.2} + KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1} KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29092' KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-2:9091,LISTENER_LOCAL://localhost:29092' @@ -78,13 +84,16 @@ services: KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false' kafka-3: - image: 'sarama/fv-kafka' + image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}' build: context: . dockerfile: Dockerfile.kafka + args: + KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1} + SCALA_VERSION: ${SCALA_VERSION:-2.13} restart: always environment: - KAFKA_VERSION: ${KAFKA_VERSION:-3.3.2} + KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1} KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29093' KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-3:9091,LISTENER_LOCAL://localhost:29093' @@ -100,13 +109,16 @@ services: KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false' kafka-4: - image: 'sarama/fv-kafka' + image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}' build: context: . dockerfile: Dockerfile.kafka + args: + KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1} + SCALA_VERSION: ${SCALA_VERSION:-2.13} restart: always environment: - KAFKA_VERSION: ${KAFKA_VERSION:-3.3.2} + KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1} KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29094' KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-4:9091,LISTENER_LOCAL://localhost:29094' @@ -122,13 +134,16 @@ services: KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false' kafka-5: - image: 'sarama/fv-kafka' + image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}' build: context: . dockerfile: Dockerfile.kafka + args: + KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1} + SCALA_VERSION: ${SCALA_VERSION:-2.13} restart: always environment: - KAFKA_VERSION: ${KAFKA_VERSION:-3.3.2} + KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1} KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29095' KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-5:9091,LISTENER_LOCAL://localhost:29095' @@ -146,11 +161,12 @@ services: toxiproxy: image: 'ghcr.io/shopify/toxiproxy:2.4.0' ports: - # The tests themselves actually start the proxies on these ports - - '29091:29091' - - '29092:29092' - - '29093:29093' - - '29094:29094' - - '29095:29095' - # This is the toxiproxy API port - - '8474:8474' + # The tests themselves actually start the proxies on these ports + - '29091:29091' + - '29092:29092' + - '29093:29093' + - '29094:29094' + - '29095:29095' + + # This is the toxiproxy API port + - '8474:8474' diff --git a/entrypoint.sh b/entrypoint.sh index 7b344fae8..d6381c907 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/bash -KAFKA_VERSION="${KAFKA_VERSION:-3.3.2}" +KAFKA_VERSION="${KAFKA_VERSION:-3.5.1}" KAFKA_HOME="/opt/kafka-${KAFKA_VERSION}" if [ ! -d "${KAFKA_HOME}" ]; then