typhoeus backend broken due to GCC runtime requirement

[mindw]

(check apply)

• [*] read the contribution guideline
• [*] (optional) already reported 3rd party upstream repository or mailing list if you use k8s addon or helm charts.

Problem

Resolved with ethon 0.14.0 : typhoeus/ethon#191

As of 2021-4-12 (ethon 0.13.0 release) Using the typhoeus backend causes a runtime failure. The issue is reported here to potentially save time for users and maintainers this excellent plugin.

Relevant issues:
typhoeus/ethon#186
typhoeus/ethon#189
typhoeus/ethon#190
typhoeus/ethon#166

fluentd-agent /usr/local/bundle/gems/ffi-1.15.0/lib/ffi/tools/const_generator.rb:131:in `block in calculate': Compilation error generating constants : (RuntimeError)
fluentd-agent        sh: 1: gcc: not found
fluentd-agent        from /usr/local/lib/ruby/2.7.0/tempfile.rb:291:in `open'
fluentd-agent        from /usr/local/bundle/gems/ffi-1.15.0/lib/ffi/tools/const_generator.rb:110:in `calculate'
fluentd-agent        from /usr/local/bundle/gems/ffi-1.15.0/lib/ffi/tools/const_generator.rb:49:in `initialize'
fluentd-agent        from /usr/local/bundle/gems/ethon-0.13.0/lib/ethon/libc.rb:17:in `new'
fluentd-agent        from /usr/local/bundle/gems/ethon-0.13.0/lib/ethon/libc.rb:17:in `<module:Libc>'
fluentd-agent        from /usr/local/bundle/gems/ethon-0.13.0/lib/ethon/libc.rb:7:in `<module:Ethon>'
fluentd-agent        from /usr/local/bundle/gems/ethon-0.13.0/lib/ethon/libc.rb:2:in `<top (required)>'
fluentd-agent        from /usr/local/lib/ruby/2.7.0/rubygems/core_ext/kernel_require.rb:83:in `require'
fluentd-agent        from /usr/local/lib/ruby/2.7.0/rubygems/core_ext/kernel_require.rb:83:in `require'
fluentd-agent        from /usr/local/bundle/gems/ethon-0.13.0/lib/ethon.rb:16:in `<top (required)>'
fluentd-agent        from /usr/local/lib/ruby/2.7.0/rubygems/core_ext/kernel_require.rb:83:in `require'
fluentd-agent        from /usr/local/lib/ruby/2.7.0/rubygems/core_ext/kernel_require.rb:83:in `require'
fluentd-agent        from /usr/local/bundle/gems/typhoeus-1.4.0/lib/typhoeus.rb:2:in `<top (required)>'
fluentd-agent        from /usr/local/lib/ruby/2.7.0/rubygems/core_ext/kernel_require.rb:158:in `require'
fluentd-agent        from /usr/local/lib/ruby/2.7.0/rubygems/core_ext/kernel_require.rb:158:in `rescue in require'
fluentd-agent        from /usr/local/lib/ruby/2.7.0/rubygems/core_ext/kernel_require.rb:147:in `require'
fluentd-agent        from /usr/local/bundle/gems/fluent-plugin-elasticsearch-5.0.3/lib/fluent/plugin/out_elasticsearch.rb:475:in `backend_options'
fluentd-agent        from /usr/local/bundle/gems/fluent-plugin-elasticsearch-5.0.3/lib/fluent/plugin/out_elasticsearch.rb:210:in `configure'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/plugin.rb:178:in `configure'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/plugin/multi_output.rb:74:in `block in configure'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/plugin/multi_output.rb:63:in `each'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/plugin/multi_output.rb:63:in `configure'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/plugin/out_copy.rb:39:in `configure'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/plugin.rb:178:in `configure'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/agent.rb:132:in `add_match'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/agent.rb:74:in `block in configure'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/agent.rb:64:in `each'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/agent.rb:64:in `configure'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/label.rb:31:in `configure'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/root_agent.rb:143:in `block in configure'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/root_agent.rb:143:in `each'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/root_agent.rb:143:in `configure'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/engine.rb:105:in `configure'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/engine.rb:80:in `run_configure'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/supervisor.rb:648:in `run_supervisor'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/command/fluentd.rb:345:in `<top (required)>'
fluentd-agent        from /usr/local/lib/ruby/2.7.0/rubygems/core_ext/kernel_require.rb:83:in `require'
fluentd-agent        from /usr/local/lib/ruby/2.7.0/rubygems/core_ext/kernel_require.rb:83:in `require'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/bin/fluentd:8:in `<top (required)>'
fluentd-agent        from /usr/local/bundle/bin/fluentd:23:in `load'
fluentd-agent        from /usr/local/bundle/bin/fluentd:23:in `<main>'
fluentd-agent /usr/local/lib/ruby/2.7.0/rubygems/core_ext/kernel_require.rb:83:in `require': cannot load such file -- typhoeus (LoadError)
fluentd-agent        from /usr/local/lib/ruby/2.7.0/rubygems/core_ext/kernel_require.rb:83:in `require'
fluentd-agent        from /usr/local/bundle/gems/fluent-plugin-elasticsearch-5.0.3/lib/fluent/plugin/out_elasticsearch.rb:475:in `backend_options'
fluentd-agent        from /usr/local/bundle/gems/fluent-plugin-elasticsearch-5.0.3/lib/fluent/plugin/out_elasticsearch.rb:210:in `configure'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/plugin.rb:178:in `configure'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/plugin/multi_output.rb:74:in `block in configure'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/plugin/multi_output.rb:63:in `each'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/plugin/multi_output.rb:63:in `configure'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/plugin/out_copy.rb:39:in `configure'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/plugin.rb:178:in `configure'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/agent.rb:132:in `add_match'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/agent.rb:74:in `block in configure'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/agent.rb:64:in `each'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/agent.rb:64:in `configure'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/label.rb:31:in `configure'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/root_agent.rb:143:in `block in configure'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/root_agent.rb:143:in `each'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/root_agent.rb:143:in `configure'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/engine.rb:105:in `configure'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/engine.rb:80:in `run_configure'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/supervisor.rb:648:in `run_supervisor'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/lib/fluent/command/fluentd.rb:345:in `<top (required)>'
fluentd-agent        from /usr/local/lib/ruby/2.7.0/rubygems/core_ext/kernel_require.rb:83:in `require'
fluentd-agent        from /usr/local/lib/ruby/2.7.0/rubygems/core_ext/kernel_require.rb:83:in `require'
fluentd-agent        from /usr/local/bundle/gems/fluentd-1.12.3/bin/fluentd:8:in `<top (required)>'
fluentd-agent        from /usr/local/bundle/bin/fluentd:23:in `load'
fluentd-agent        from /usr/local/bundle/bin/fluentd:23:in `<main>'

Steps to replicate

• Build a fluentd docker image with typhoeus .
• check ethon is 0.13.0
• set typhoeus backend in the output,
• start up fluentd.

Expected Behavior or What you need to ask

FluentD starts successfully.

Using Fluentd and ES plugin versions

• Linux
• Docker / Kubernetes
• FluentD v1.12.3
• ES plugin 5.0.3
  • paste boot log of fluentd or td-agent
  • paste result of fluent-gem list, td-agent-gem list or your Gemfile.lock

[cosmo0920]

Hmm..., it's curious breaking change....

Could you share your Dockerfile without sensitive information?
Current official Fluentd images does not ship with gcc toolchains:
https://github.com/fluent/fluentd-docker-image/blob/master/v1.12/debian/Dockerfile#L39-L41
https://github.com/fluent/fluentd-docker-image/blob/master/v1.12/debian/Dockerfile#L15-L17

Could you build your custom image which includes gcc toolchain and its libraries?

[mindw]

Hmm..., it's curious breaking change....

Could you share your Dockerfile without sensitive information?
Current official Fluentd images does not ship with gcc toolchains:
https://github.com/fluent/fluentd-docker-image/blob/master/v1.12/debian/Dockerfile#L39-L41
https://github.com/fluent/fluentd-docker-image/blob/master/v1.12/debian/Dockerfile#L15-L17

Could you build your custom image which includes gcc toolchain and its libraries?
@cosmo0920 - Not sure what will that check? the filesystem needs also be made writable. The issue only happens when switching backends. And ethon is 0.13.0.

Main build files -
Dockerfile:

FROM ruby:2.7-slim-buster as builder

ARG FLUENTD_VERSION="latest"
ARG JEMALLOC_VERSION="latest"

# Ensure there are enough file descriptors for running FluentD.
RUN ulimit -n 65536

ARG DEBIAN_FRONTEND=noninteractive

SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"]

COPY build.sh /tmp/build.sh
RUN chmod +x /tmp/build.sh
RUN /tmp/build.sh

FROM ruby:2.7-slim-buster

ARG DEBIAN_FRONTEND=noninteractive

# Ensure there are enough file descriptors for running FluentD.
RUN ulimit -n 65536

COPY --from=builder /usr/local/bundle/ /usr/local/bundle
COPY --from=builder /usr/local/lib/libjemalloc.so /usr/local/lib/libjemalloc.so
COPY entrypoint.sh /entrypoint.sh
COPY fluent.conf /etc/fluent/fluent.conf
COPY plugins /fluentd/plugins

SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"]

RUN chmod +x /entrypoint.sh

RUN apt-get update && \
    apt-get install -y --no-install-recommends curl net-tools && \
    apt-get clean -y && \
    rm -rf \
        /var/cache/debconf/* \
        /var/lib/apt/lists/* \
        /var/log/* \
        /var/tmp/* \
        rm -rf /tmp/*

# make loading the simple sniffer easier
RUN sniffer=$( gem contents fluent-plugin-elasticsearch|grep elasticsearch_simple_sniffer.rb ) ; \
    cp $sniffer /fluentd/plugins

#ENV LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1
ENV LD_PRELOAD /usr/local/lib/libjemalloc.so

CMD ["/entrypoint.sh"]

Build.sh:

#!/bin/bash

set -eux

FLUENTD_VERSION=${FLUENTD_VERSION}
JEMALLOC_VERSION=${JEMALLOC_VERSION}
BUILD_DEPS='gcc g++ make git'

echo 'gem: --no-document' >> /etc/gemrc

apt-get update
apt-get install -y --no-install-recommends curl net-tools ${BUILD_DEPS}

# install jemalloc
JEMALLOC=jemalloc-${JEMALLOC_VERSION}
curl -O -L https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/${JEMALLOC}.tar.bz2
tar xf ${JEMALLOC}.tar.bz2
cd ${JEMALLOC}
./configure --disable-debug --disable-static
make -j 8
make install
cd ..
rm -rf ${JEMALLOC} ${JEMALLOC}.tar.bz2 /usr/local/lib/libjemalloc*.a

export FLUENTD_DISABLE_BUNDLER_INJECTION=1

# TODO: migrate to bundler

# install fluentd & plugins
# pin eton to a working version https://github.com/typhoeus/ethon/issues/190
gem install json_pure oj logfmt ethon:'0.12.0' typhoeus async-http
gem install elasticsearch-xpack:'>= 7.12.0'
gem install fluentd -v "${FLUENTD_VERSION}"
#gem install rake

gem install fluent-plugin-elasticsearch -v 5.0.3

gem install fluent-plugin-systemd -v 1.0.4
gem install fluent-plugin-concat -v 2.4.0

gem install fluent-plugin-grok-parser -v 2.6.2
gem install fluent-plugin-rewrite -v 0.1.1
gem install fluent-plugin-rewrite-tag-filter -v 2.4.0
gem install fluent-plugin-route -v 1.0.0
gem install fluent-plugin-multi-format-parser -v 1.0.0
gem install fluent-plugin-record-modifier -v 2.1.0
gem install fluent-plugin-detect-exceptions -v 0.0.13
gem install fluent-plugin-genhashvalue -v 1.1
gem install fluent-plugin-prometheus -v 2.0.1
gem install fluent-plugin-remote_syslog -v 1.0.0
gem install fluent-plugin-kinesis -v 3.3.0
gem install fluent-plugin-s3 -v 1.6.0
gem install fluent-plugin-kubernetes_metadata_filter -v 2.6.0

gem install specific_install

# disabled due to missing
# https://github.com/ryotarai/fluent-plugin-cloudwatch-logs/pull/74
#gem install fluent-plugin-cloudwatch-logs -v 0.13.4
gem specific_install https://github.com/mindw/fluent-plugin-cloudwatch-logs

gem uninstall specific_install
gem cleanup