feat(fvt): experiment with per-kafka-version image

Switch to templating the Dockerfile and docker-compose so that each image is uniquely named and contains just the kafka version specified. This will make it easier to have a large matrix test for the main branch whilst keeping the PR checks at 2 or 3 kafka versions Signed-off-by: Dominic Evans <dominic.evans@uk.ibm.com>
IBM · Aug 11, 2023 · 3d7b37f · 3d7b37f
1 parent bd81a11
commit 3d7b37f
Show file tree

Hide file tree

Showing 3 changed files with 51 additions and 23 deletions.
diff --git a/Dockerfile.kafka b/Dockerfile.kafka
@@ -16,11 +16,23 @@ RUN cd /etc/java/java-11-openjdk/*/conf/security \
   && echo 'networkaddress.cache.ttl=0' >> java.security \
   && echo 'networkaddress.cache.negative.ttl=0' >> java.security
 
+ARG SCALA_VERSION="2.13"
+ARG KAFKA_VERSION="3.5.1"
+
 # https://github.com/apache/kafka/blob/53eeaad946cd053e9eb1a762972d4efeacb8e4fc/tests/docker/Dockerfile#L65-L69
 ARG KAFKA_MIRROR="https://s3-us-west-2.amazonaws.com/kafka-packages"
-RUN mkdir -p "/opt/kafka-3.3.2" && chmod a+rw /opt/kafka-3.3.2 && curl -s "$KAFKA_MIRROR/kafka_2.13-3.3.2.tgz" | tar xz --strip-components=1 -C "/opt/kafka-3.3.2"
-RUN mkdir -p "/opt/kafka-3.4.1" && chmod a+rw /opt/kafka-3.4.1 && curl -s "$KAFKA_MIRROR/kafka_2.13-3.4.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-3.4.1"
-RUN mkdir -p "/opt/kafka-3.5.1" && chmod a+rw /opt/kafka-3.5.1 && curl -s "$KAFKA_MIRROR/kafka_2.13-3.5.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-3.5.1"
+RUN mkdir -p "/opt/kafka-${KAFKA_VERSION}" \
+ && chmod a+rw "/opt/kafka-${KAFKA_VERSION}" \
+ && curl -s "$KAFKA_MIRROR/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz" | tar xz --strip-components=1 -C "/opt/kafka-${KAFKA_VERSION}"
+
+# older kafka versions depend upon jaxb-api being bundled with the JDK, but it
+# was removed from Java 11 so work around that by including it in the kafka
+# libs dir regardless
+RUN cd /tmp \
+ && curl -sLO "https://repo1.maven.org/maven2/javax/xml/bind/jaxb-api/2.3.0/jaxb-api-2.3.0.jar" \
+ && for DIR in /opt/kafka-*; do cp -v jaxb-api-2.3.0.jar $DIR/libs/ ; done \
+ && rm -f jaxb-api-2.3.0.jar
+
 
 COPY entrypoint.sh /
 

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,4 +1,4 @@
-version: '3.7'
+version: '3.9'
 services:
   zookeeper-1:
     image: 'docker.io/library/zookeeper:3.6.3'
@@ -34,13 +34,16 @@ services:
       ZOO_MAX_CLIENT_CNXNS: '0'
       ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok'
   kafka-1:
-    image: 'sarama/fv-kafka'
+    image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
     build:
       context: .
       dockerfile: Dockerfile.kafka
+      args:
+        KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
+        SCALA_VERSION: ${SCALA_VERSION:-2.13}
     restart: always
     environment:
-      KAFKA_VERSION: ${KAFKA_VERSION:-3.3.2}
+      KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
       KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
       KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29091'
       KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-1:9091,LISTENER_LOCAL://localhost:29091'
@@ -56,13 +59,16 @@ services:
       KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
       KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
   kafka-2:
-    image: 'sarama/fv-kafka'
+    image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
     build:
       context: .
       dockerfile: Dockerfile.kafka
+      args:
+        KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
+        SCALA_VERSION: ${SCALA_VERSION:-2.13}
     restart: always
     environment:
-      KAFKA_VERSION: ${KAFKA_VERSION:-3.3.2}
+      KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
       KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
       KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29092'
       KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-2:9091,LISTENER_LOCAL://localhost:29092'
@@ -78,13 +84,16 @@ services:
       KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
       KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
   kafka-3:
-    image: 'sarama/fv-kafka'
+    image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
     build:
       context: .
       dockerfile: Dockerfile.kafka
+      args:
+        KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
+        SCALA_VERSION: ${SCALA_VERSION:-2.13}
     restart: always
     environment:
-      KAFKA_VERSION: ${KAFKA_VERSION:-3.3.2}
+      KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
       KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
       KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29093'
       KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-3:9091,LISTENER_LOCAL://localhost:29093'
@@ -100,13 +109,16 @@ services:
       KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
       KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
   kafka-4:
-    image: 'sarama/fv-kafka'
+    image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
     build:
       context: .
       dockerfile: Dockerfile.kafka
+      args:
+        KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
+        SCALA_VERSION: ${SCALA_VERSION:-2.13}
     restart: always
     environment:
-      KAFKA_VERSION: ${KAFKA_VERSION:-3.3.2}
+      KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
       KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
       KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29094'
       KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-4:9091,LISTENER_LOCAL://localhost:29094'
@@ -122,13 +134,16 @@ services:
       KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
       KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
   kafka-5:
-    image: 'sarama/fv-kafka'
+    image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
     build:
       context: .
       dockerfile: Dockerfile.kafka
+      args:
+        KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
+        SCALA_VERSION: ${SCALA_VERSION:-2.13}
     restart: always
     environment:
-      KAFKA_VERSION: ${KAFKA_VERSION:-3.3.2}
+      KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
       KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
       KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29095'
       KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-5:9091,LISTENER_LOCAL://localhost:29095'
@@ -146,11 +161,12 @@ services:
   toxiproxy:
     image: 'ghcr.io/shopify/toxiproxy:2.4.0'
     ports:
-      # The tests themselves actually start the proxies on these ports
-      - '29091:29091'
-      - '29092:29092'
-      - '29093:29093'
-      - '29094:29094'
-      - '29095:29095'
-      # This is the toxiproxy API port
-      - '8474:8474'
+    # The tests themselves actually start the proxies on these ports
+    - '29091:29091'
+    - '29092:29092'
+    - '29093:29093'
+    - '29094:29094'
+    - '29095:29095'
+
+    # This is the toxiproxy API port
+    - '8474:8474'
diff --git a/entrypoint.sh b/entrypoint.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-KAFKA_VERSION="${KAFKA_VERSION:-3.3.2}"
+KAFKA_VERSION="${KAFKA_VERSION:-3.5.1}"
 KAFKA_HOME="/opt/kafka-${KAFKA_VERSION}"
 
 if [ ! -d "${KAFKA_HOME}" ]; then