Merge pull request #1528 from fluent/backport-compression

Backport compression

Makefile

@@ -26,8 +26,8 @@ X86_IMAGES := \
 	v1.16/debian-graylog:v1.16.6-debian-graylog-amd64-1.0,v1.16-debian-graylog-amd64-2 \
 	v1.16/debian-papertrail:v1.16.6-debian-papertrail-amd64-1.0,v1.16-debian-papertrail-amd64-2 \
 	v1.16/debian-logzio:v1.16.6-debian-logzio-amd64-1.0,v1.16-debian-logzio-amd64-2 \
-	v1.16/debian-kafka:v1.16.6-debian-kafka-amd64-1.0,v1.16-debian-kafka-amd64-2 \
-	v1.16/debian-kafka2:v1.16.6-debian-kafka2-amd64-1.0,v1.16-debian-kafka2-amd64-2 \
+	v1.16/debian-kafka:v1.16.6-debian-kafka-amd64-1.1,v1.16-debian-kafka-amd64-2 \
+	v1.16/debian-kafka2:v1.16.6-debian-kafka2-amd64-1.1,v1.16-debian-kafka2-amd64-2 \
 	v1.16/debian-kinesis:v1.16.6-debian-kinesis-amd64-1.0,v1.16-debian-kinesis-amd64-2
 
 #	<Dockerfile>:<version>,<tag1>,<tag2>,...
@@ -47,8 +47,8 @@ ARM64_IMAGES := \
 	v1.16/arm64/debian-graylog:v1.16.6-debian-graylog-arm64-1.0,v1.16-debian-graylog-arm64-2 \
 	v1.16/arm64/debian-papertrail:v1.16.6-debian-papertrail-arm64-1.0,v1.16-debian-papertrail-arm64-2 \
 	v1.16/arm64/debian-logzio:v1.16.6-debian-logzio-arm64-1.0,v1.16-debian-logzio-arm64-2 \
-	v1.16/arm64/debian-kafka:v1.16.6-debian-kafka-arm64-1.0,v1.16-debian-kafka-arm64-2 \
-	v1.16/arm64/debian-kafka2:v1.16.6-debian-kafka2-arm64-1.0,v1.16-debian-kafka2-arm64-2 \
+	v1.16/arm64/debian-kafka:v1.16.6-debian-kafka-arm64-1.1,v1.16-debian-kafka-arm64-2 \
+	v1.16/arm64/debian-kafka2:v1.16.6-debian-kafka2-arm64-1.1,v1.16-debian-kafka2-arm64-2 \
 	v1.16/arm64/debian-kinesis:v1.16.6-debian-kinesis-arm64-1.0,v1.16-debian-kinesis-arm64-2
 
 ALL_IMAGES := $(X86_IMAGES) $(ARM64_IMAGES)

README.md

@@ -50,10 +50,10 @@ If you want to use above non published images, build it by yourself. Dockerfile
   - `docker pull fluent/fluentd-kubernetes-daemonset:v1.16.6-debian-graylog-1.0`
   - `docker pull fluent/fluentd-kubernetes-daemonset:v1.16-debian-graylog-2`
 - `Kafka`
-  - `docker pull fluent/fluentd-kubernetes-daemonset:v1.16.6-debian-kafka-1.0`
+  - `docker pull fluent/fluentd-kubernetes-daemonset:v1.16.6-debian-kafka-1.1`
   - `docker pull fluent/fluentd-kubernetes-daemonset:v1.16-debian-kafka-2`
 - `Kafka2`
-  - `docker pull fluent/fluentd-kubernetes-daemonset:v1.16.6-debian-kafka2-1.0`
+  - `docker pull fluent/fluentd-kubernetes-daemonset:v1.16.6-debian-kafka2-1.1`
   - `docker pull fluent/fluentd-kubernetes-daemonset:v1.16-debian-kafka2-2`
 - `Kinesis`
   - `docker pull fluent/fluentd-kubernetes-daemonset:v1.16.6-debian-kinesis-1.0`
@@ -100,10 +100,10 @@ If you want to use above non published images, build it by yourself. Dockerfile
   - `docker pull fluent/fluentd-kubernetes-daemonset:v1.16.6-debian-logzio-amd64-1.0`
   - `docker pull fluent/fluentd-kubernetes-daemonset:v1.16-debian-logzio-amd64-2`
 - `Kafka` [Dockerfile](docker-image/v1.16/debian-kafka/Dockerfile)
-  - `docker pull fluent/fluentd-kubernetes-daemonset:v1.16.6-debian-kafka-amd64-1.0`
+  - `docker pull fluent/fluentd-kubernetes-daemonset:v1.16.6-debian-kafka-amd64-1.1`
   - `docker pull fluent/fluentd-kubernetes-daemonset:v1.16-debian-kafka-amd64-2`
 - `Kafka2` [Dockerfile](docker-image/v1.16/debian-kafka2/Dockerfile)
-  - `docker pull fluent/fluentd-kubernetes-daemonset:v1.16.6-debian-kafka2-amd64-1.0`
+  - `docker pull fluent/fluentd-kubernetes-daemonset:v1.16.6-debian-kafka2-amd64-1.1`
   - `docker pull fluent/fluentd-kubernetes-daemonset:v1.16-debian-kafka2-amd64-2`
 - `Kinesis` [Dockerfile](docker-image/v1.16/debian-kinesis/Dockerfile)
   - `docker pull fluent/fluentd-kubernetes-daemonset:v1.16.6-debian-kinesis-amd64-1.0`
@@ -142,10 +142,10 @@ If you want to use above non published images, build it by yourself. Dockerfile
 - `Papertrail` [Dockerfile](docker-image/v1.16/arm64/debian-papertrail/Dockerfile)
 - `Logzio` [Dockerfile](docker-image/v1.16/arm64/debian-logzio/Dockerfile)
 - `Kafka` [Dockerfile](docker-image/v1.16/arm64/debian-kafka/Dockerfile)
-  - `docker pull fluent/fluentd-kubernetes-daemonset:v1.16.6-debian-kafka-arm64-1.0`
+  - `docker pull fluent/fluentd-kubernetes-daemonset:v1.16.6-debian-kafka-arm64-1.1`
   - `docker pull fluent/fluentd-kubernetes-daemonset:v1.16-debian-kafka-arm64-2`
 - `Kafka2` [Dockerfile](docker-image/v1.16/arm64/debian-kafka2/Dockerfile)
-  - `docker pull fluent/fluentd-kubernetes-daemonset:v1.16.6-debian-kafka2-arm64-1.0`
+  - `docker pull fluent/fluentd-kubernetes-daemonset:v1.16.6-debian-kafka2-arm64-1.1`
   - `docker pull fluent/fluentd-kubernetes-daemonset:v1.16-debian-kafka2-arm64-2`
 - `Kinesis` [Dockerfile](docker-image/v1.16/arm64/debian-kinesis/Dockerfile)
   - `docker pull fluent/fluentd-kubernetes-daemonset:v1.16.6-debian-kinesis-arm64-1.0`

docker-image/v1.16/arm64/debian-kafka/Dockerfile

@@ -21,7 +21,7 @@ ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0
 ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1
 
 COPY Gemfile* /fluentd/
-RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev build-essential autoconf automake libtool pkg-config libsasl2-dev libssl-dev" \
+RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev build-essential autoconf automake libtool pkg-config libsasl2-dev libssl-dev zlib1g-dev libzstd-dev libsnappy-dev" \
   runtimeDeps="krb5-kdc libsasl2-modules-gssapi-mit libsasl2-2" \
       && export DEBIAN_FRONTEND=noninteractive  && apt-get update \
      && apt-get upgrade -y \

docker-image/v1.16/arm64/debian-kafka/Gemfile.lock

@@ -102,7 +102,7 @@ GEM
     ltsv (0.1.2)
     mime-types (3.5.2)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2024.0806)
+    mime-types-data (3.2024.0820)
     mini_portile2 (2.8.7)
     msgpack (1.7.2)
     multi_json (1.15.0)

docker-image/v1.16/arm64/debian-kafka/hooks/post_push

@@ -13,7 +13,7 @@ curl -Lo manifest-tool https://github.com/estesp/manifest-tool/releases/download
 chmod +x manifest-tool
 
 # Tag and push image and manifest list for each additional tag
-for tag in {v1.16.6-debian-kafka-arm64-1.0,v1.16-debian-kafka-arm64-2}; do
+for tag in {v1.16.6-debian-kafka-arm64-1.1,v1.16-debian-kafka-arm64-2}; do
   docker tag $IMAGE_NAME ${repoName}:${tag}
   docker push ${repoName}:${tag}
   # Note: this will fail until both the amd64 and arm64 images have been pushed

docker-image/v1.16/arm64/debian-kafka2/Dockerfile

@@ -21,7 +21,7 @@ ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0
 ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1
 
 COPY Gemfile* /fluentd/
-RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev build-essential autoconf automake libtool pkg-config libsasl2-dev libssl-dev" \
+RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev build-essential autoconf automake libtool pkg-config libsasl2-dev libssl-dev zlib1g-dev libzstd-dev libsnappy-dev" \
   runtimeDeps="krb5-kdc libsasl2-modules-gssapi-mit libsasl2-2" \
       && export DEBIAN_FRONTEND=noninteractive  && apt-get update \
      && apt-get upgrade -y \

docker-image/v1.16/arm64/debian-kafka2/Gemfile.lock

@@ -102,7 +102,7 @@ GEM
     ltsv (0.1.2)
     mime-types (3.5.2)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2024.0806)
+    mime-types-data (3.2024.0820)
     mini_portile2 (2.8.7)
     msgpack (1.7.2)
     multi_json (1.15.0)

docker-image/v1.16/arm64/debian-kafka2/hooks/post_push

@@ -13,7 +13,7 @@ curl -Lo manifest-tool https://github.com/estesp/manifest-tool/releases/download
 chmod +x manifest-tool
 
 # Tag and push image and manifest list for each additional tag
-for tag in {v1.16.6-debian-kafka2-arm64-1.0,v1.16-debian-kafka2-arm64-2}; do
+for tag in {v1.16.6-debian-kafka2-arm64-1.1,v1.16-debian-kafka2-arm64-2}; do
   docker tag $IMAGE_NAME ${repoName}:${tag}
   docker push ${repoName}:${tag}
   # Note: this will fail until both the amd64 and arm64 images have been pushed

docker-image/v1.16/debian-kafka/Dockerfile

@@ -13,7 +13,7 @@ ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0
 ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1
 
 COPY Gemfile* /fluentd/
-RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev build-essential autoconf automake libtool pkg-config libsasl2-dev libssl-dev" \
+RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev build-essential autoconf automake libtool pkg-config libsasl2-dev libssl-dev zlib1g-dev libzstd-dev libsnappy-dev" \
   runtimeDeps="krb5-kdc libsasl2-modules-gssapi-mit libsasl2-2" \
       && export DEBIAN_FRONTEND=noninteractive  && apt-get update \
      && apt-get upgrade -y \

docker-image/v1.16/debian-kafka/Gemfile.lock

@@ -102,7 +102,7 @@ GEM
     ltsv (0.1.2)
     mime-types (3.5.2)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2024.0806)
+    mime-types-data (3.2024.0820)
     mini_portile2 (2.8.7)
     msgpack (1.7.2)
     multi_json (1.15.0)

docker-image/v1.16/debian-kafka/hooks/post_push

@@ -13,7 +13,7 @@ curl -Lo manifest-tool https://github.com/estesp/manifest-tool/releases/download
 chmod +x manifest-tool
 
 # Tag and push image and manifest list for each additional tag
-for tag in {v1.16.6-debian-kafka-amd64-1.0,v1.16-debian-kafka-amd64-2}; do
+for tag in {v1.16.6-debian-kafka-amd64-1.1,v1.16-debian-kafka-amd64-2}; do
   docker tag $IMAGE_NAME ${repoName}:${tag}
   docker push ${repoName}:${tag}
   # Note: this will fail until both the amd64 and arm64 images have been pushed

docker-image/v1.16/debian-kafka2/Dockerfile

@@ -13,7 +13,7 @@ ENV GEM_HOME /fluentd/vendor/bundle/ruby/3.2.0
 ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1
 
 COPY Gemfile* /fluentd/
-RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev build-essential autoconf automake libtool pkg-config libsasl2-dev libssl-dev" \
+RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev build-essential autoconf automake libtool pkg-config libsasl2-dev libssl-dev zlib1g-dev libzstd-dev libsnappy-dev" \
   runtimeDeps="krb5-kdc libsasl2-modules-gssapi-mit libsasl2-2" \
       && export DEBIAN_FRONTEND=noninteractive  && apt-get update \
      && apt-get upgrade -y \

docker-image/v1.16/debian-kafka2/Gemfile.lock

@@ -102,7 +102,7 @@ GEM
     ltsv (0.1.2)
     mime-types (3.5.2)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2024.0806)
+    mime-types-data (3.2024.0820)
     mini_portile2 (2.8.7)
     msgpack (1.7.2)
     multi_json (1.15.0)

docker-image/v1.16/debian-kafka2/hooks/post_push

@@ -13,7 +13,7 @@ curl -Lo manifest-tool https://github.com/estesp/manifest-tool/releases/download
 chmod +x manifest-tool
 
 # Tag and push image and manifest list for each additional tag
-for tag in {v1.16.6-debian-kafka2-amd64-1.0,v1.16-debian-kafka2-amd64-2}; do
+for tag in {v1.16.6-debian-kafka2-amd64-1.1,v1.16-debian-kafka2-amd64-2}; do
   docker tag $IMAGE_NAME ${repoName}:${tag}
   docker push ${repoName}:${tag}
   # Note: this will fail until both the amd64 and arm64 images have been pushed

templates/Dockerfile.erb

@@ -29,7 +29,7 @@ ENV GEM_HOME /fluentd/vendor/bundle/ruby/<%= ruby_version %>.0
 ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1
 
 COPY Gemfile* /fluentd/
-RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev<% if target == "graylog" %> build-essential patch zlib1g-dev liblzma-dev<% elsif target == "kafka" || target == "kafka2" %> build-essential autoconf automake libtool pkg-config libsasl2-dev libssl-dev<% elsif target == "s3" %> curl<% end %><% if requires_git %> git<% end %>" \
+RUN buildDeps="sudo make gcc g++ libc-dev libffi-dev<% if target == "graylog" %> build-essential patch zlib1g-dev liblzma-dev<% elsif target == "kafka" || target == "kafka2" %> build-essential autoconf automake libtool pkg-config libsasl2-dev libssl-dev zlib1g-dev libzstd-dev libsnappy-dev<% elsif target == "s3" %> curl<% end %><% if requires_git %> git<% end %>" \
   runtimeDeps="<% if target == "kafka" || target == "kafka2" %>krb5-kdc libsasl2-modules-gssapi-mit libsasl2-2<% elsif target == "azureblob" %>pkg-config libxslt-dev libxml2-dev<% end %>" \
      <% if target == "kafka" || target == "kafka2" %> && export DEBIAN_FRONTEND=noninteractive <% end %> && apt-get update \
      && apt-get upgrade -y \