fluent-plugin-loki: Restructuring and CI

.drone/drone.jsonnet

@@ -84,6 +84,30 @@ local fluentbit() = pipeline('fluent-bit-amd64') + arch_image('amd64', 'latest,m
   depends_on: ['check'],
 };
 
+local fluentd() = pipeline('fluentd-amd64') + arch_image('amd64', 'latest,master') {
+  steps+: [
+    // dry run for everything that is not tag or master
+    docker('amd64', 'fluentd') {
+      depends_on: ['image-tag'],
+      when: condition('exclude').tagMaster,
+      settings+: {
+        dry_run: true,
+        repo: 'grafana/fluent-plugin-loki',
+      },
+    },
+  ] + [
+    // publish for tag or master
+    docker('amd64', 'fluent-bit') {
+      depends_on: ['image-tag'],
+      when: condition('include').tagMaster,
+      settings+: {
+        repo: 'grafana/fluent-plugin-loki',
+      },
+    },
+  ],
+  depends_on: ['check'],
+};
+
 local multiarch_image(arch) = pipeline('docker-' + arch) + arch_image(arch) {
   steps+: [
     // dry run for everything that is not tag or master
@@ -181,6 +205,7 @@ local manifest(apps) = pipeline('manifest') {
   for arch in archs
 ] + [
   fluentbit(),
+  fluentd(),
 ] + [
   manifest(['promtail', 'loki', 'loki-canary']) {
     trigger: condition('include').tagMaster,

.drone/drone.yml

@@ -1,3 +1,4 @@
+# Generated by jsonnet from .drone/drone.jsonnet
 ---
 kind: pipeline
 name: check
@@ -514,6 +515,62 @@ steps:
 depends_on:
 - check
 
+---
+kind: pipeline
+name: fluentd-amd64
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: image-tag
+  image: alpine
+  commands:
+  - apk add --no-cache bash git
+  - git fetch origin --tags
+  - echo $(./tools/image-tag)-amd64 > .tags
+  - echo ",latest,master" >> .tags
+
+- name: build-fluentd-image
+  image: plugins/docker
+  settings:
+    dockerfile: cmd/fluentd/Dockerfile
+    dry_run: true
+    password:
+      from_secret: docker_password
+    repo: grafana/fluent-plugin-loki
+    username:
+      from_secret: docker_username
+  when:
+    ref:
+      exclude:
+      - refs/heads/master
+      - refs/heads/k??
+      - refs/tags/v*
+  depends_on:
+  - image-tag
+
+- name: publish-fluent-bit-image
+  image: plugins/docker
+  settings:
+    dockerfile: cmd/fluent-bit/Dockerfile
+    password:
+      from_secret: docker_password
+    repo: grafana/fluent-plugin-loki
+    username:
+      from_secret: docker_username
+  when:
+    ref:
+    - refs/heads/master
+    - refs/heads/k??
+    - refs/tags/v*
+  depends_on:
+  - image-tag
+
+depends_on:
+- check
+
 ---
 kind: pipeline
 name: manifest

Makefile

@@ -3,6 +3,7 @@
 .PHONY: helm helm-install helm-upgrade helm-publish helm-debug helm-clean
 .PHONY: docker-driver docker-driver-clean docker-driver-enable docker-driver-push
 .PHONY: fluent-bit-image, fluent-bit-push, fluent-bit-test
+.PHONY: fluentd-image, fluentd-push, fluentd-test
 .PHONY: push-images push-latest save-images load-images promtail-image loki-image build-image
 .PHONY: bigtable-backup, push-bigtable-backup
 .PHONY: benchmark-store, drone, check-mod
@@ -414,6 +415,26 @@ fluent-bit-test:
 	docker run -v /var/log:/var/log -e LOG_PATH="/var/log/*.log" -e LOKI_URL="$(LOKI_URL)" \
 	 $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG)
 
+
+##################
+# fluentd plugin #
+##################
+fluentd-plugin:
+	gem install bundler --version 1.16.2
+	bundle config silence_root_warning true
+	bundle install --gemfile=cmd/fluentd/Gemfile --path=cmd/fluentd/vendor/bundle
+
+fluentd-image:
+	$(SUDO) docker build -t $(IMAGE_PREFIX)/fluent-plugin-loki:$(IMAGE_TAG) -f cmd/fluentd/Dockerfile .
+
+fluentd-push:
+	$(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/fluent-plugin-loki:$(IMAGE_TAG)
+
+fluentd-test: LOKI_URL ?= http://localhost:3100/loki/api/
+fluentd-test:
+	docker run -v /var/log:/var/log -e LOG_PATH="/var/log/*.log" -e LOKI_URL="$(LOKI_URL)" \
+	 $(IMAGE_PREFIX)/fluent-plugin-loki:$(IMAGE_TAG)
+
 ########################
 # Bigtable Backup Tool #
 ########################
@@ -432,7 +453,7 @@ push-bigtable-backup: bigtable-backup
 # Images #
 ##########
 
-images: promtail-image loki-image loki-canary-image docker-driver fluent-bit-image
+images: promtail-image loki-image loki-canary-image docker-driver fluent-bit-image fluentd-image
 
 print-images:
 	$(info $(patsubst %,%:$(IMAGE_TAG),$(IMAGE_NAMES)))
@@ -522,6 +543,7 @@ ifeq ($(BUILD_IN_CONTAINER),true)
 		$(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@;
 else
 	drone jsonnet --stream --format -V __build-image-version=$(BUILD_IMAGE_VERSION) --source .drone/drone.jsonnet --target .drone/drone.yml
+	sed -i '1s/^/\# Generated by jsonnet from .drone\/drone.jsonnet\n/' .drone/drone.yml
 endif
 
 

cmd/fluentd/Dockerfile

@@ -0,0 +1,35 @@
+FROM ruby:2.6 as build
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    sudo make gcc g++ libc-dev ruby-dev golang
+
+COPY . /src/loki
+WORKDIR /src/loki
+RUN make BUILD_IN_CONTAINER=false fluentd-plugin
+
+FROM fluent/fluentd:v1.9.2-debian-1.0
+COPY --from=build /src/loki/cmd/fluentd/lib/fluent/plugin/out_loki.rb /fluentd/plugins/out_loki.rb
+
+## skip runtime bundler installation
+ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1
+ENV DEBIAN_FRONTEND=noninteractive
+
+USER root
+RUN apt-get update && apt-get install -y --no-install-recommends libjemalloc2
+
+COPY cmd/fluentd/docker/Gemfile* /fluentd/
+COPY cmd/fluentd/docker/entrypoint.sh /fluentd/entrypoint.sh
+COPY cmd/fluentd/docker/conf/ /fluentd/etc/loki/
+
+ENV FLUENTD_CONF="/fluentd/etc/loki/fluentd.conf"
+ENV FLUENTD_OPT=""
+
+ENV LOKI_URL "https://logs-us-west1.grafana.net"
+
+# See https://packages.debian.org/stretch/amd64/libjemalloc1/filelist
+ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libjemalloc.so.2"
+
+# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib
+ENTRYPOINT ["/fluentd/entrypoint.sh"]

fluentd/fluent-plugin-grafana-loki/README.md → cmd/fluentd/README.md

@@ -1,6 +1,8 @@
-# fluent-plugin-grafana-loki
+# FluentD output plugin
 
-[Fluentd](https://fluentd.org/) output plugin to ship logs to a Loki server. See [docs/client/fluentd/README.md](../../docs/clients/fluentd/README.md) for detailed information.
+[Fluentd](https://fluentd.org/) is a data collector for unified logging layer, it can be configured with the Loki output plugin, provided in this folder, to ship logs to Loki.
+
+See [docs/client/fluentd/README.md](../../docs/clients/fluentd/README.md) for detailed information.
 
 ## Development
 

fluentd/fluent-plugin-grafana-loki/docker/entrypoint.sh → cmd/fluentd/docker/entrypoint.sh

@@ -1,3 +1,3 @@
-#!/usr/bin/dumb-init /bin/sh
+#!/bin/bash
 
-exec fluentd -c ${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT}
+exec fluentd -c ${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT}