fluent-plugin-loki: Restructuring and CI

.drone/drone.jsonnet

@@ -84,6 +84,30 @@ local fluentbit() = pipeline('fluent-bit-amd64') + arch_image('amd64', 'latest,m
   depends_on: ['check'],
 };
 
+local fluentd() = pipeline('fluentd-amd64') + arch_image('amd64', 'latest,master') {
+  steps+: [
+    // dry run for everything that is not tag or master
+    docker('amd64', 'fluentd') {
+      depends_on: ['image-tag'],
+      when: condition('exclude').tagMaster,
+      settings+: {
+        dry_run: true,
+        repo: 'grafana/fluent-plugin-loki',
+      },
+    },
+  ] + [
+    // publish for tag or master
+    docker('amd64', 'fluentd') {
+      depends_on: ['image-tag'],
+      when: condition('include').tagMaster,
+      settings+: {
+        repo: 'grafana/fluent-plugin-loki',
+      },
+    },
+  ],
+  depends_on: ['check'],
+};
+
 local multiarch_image(arch) = pipeline('docker-' + arch) + arch_image(arch) {
   steps+: [
     // dry run for everything that is not tag or master
@@ -181,6 +205,7 @@ local manifest(apps) = pipeline('manifest') {
   for arch in archs
 ] + [
   fluentbit(),
+  fluentd(),
 ] + [
   manifest(['promtail', 'loki', 'loki-canary']) {
     trigger: condition('include').tagMaster,

.drone/drone.yml

@@ -1,3 +1,4 @@
+# Generated by jsonnet from .drone/drone.jsonnet
 ---
 kind: pipeline
 name: check
@@ -514,6 +515,62 @@ steps:
 depends_on:
 - check
 
+---
+kind: pipeline
+name: fluentd-amd64
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: image-tag
+  image: alpine
+  commands:
+  - apk add --no-cache bash git
+  - git fetch origin --tags
+  - echo $(./tools/image-tag)-amd64 > .tags
+  - echo ",latest,master" >> .tags
+
+- name: build-fluentd-image
+  image: plugins/docker
+  settings:
+    dockerfile: cmd/fluentd/Dockerfile
+    dry_run: true
+    password:
+      from_secret: docker_password
+    repo: grafana/fluent-plugin-loki
+    username:
+      from_secret: docker_username
+  when:
+    ref:
+      exclude:
+      - refs/heads/master
+      - refs/heads/k??
+      - refs/tags/v*
+  depends_on:
+  - image-tag
+
+- name: publish-fluentd-image
+  image: plugins/docker
+  settings:
+    dockerfile: cmd/fluentd/Dockerfile
+    password:
+      from_secret: docker_password
+    repo: grafana/fluent-plugin-loki
+    username:
+      from_secret: docker_username
+  when:
+    ref:
+    - refs/heads/master
+    - refs/heads/k??
+    - refs/tags/v*
+  depends_on:
+  - image-tag
+
+depends_on:
+- check
+
 ---
 kind: pipeline
 name: manifest

Makefile

@@ -3,6 +3,7 @@
 .PHONY: helm helm-install helm-upgrade helm-publish helm-debug helm-clean
 .PHONY: docker-driver docker-driver-clean docker-driver-enable docker-driver-push
 .PHONY: fluent-bit-image, fluent-bit-push, fluent-bit-test
+.PHONY: fluentd-image, fluentd-push, fluentd-test
 .PHONY: push-images push-latest save-images load-images promtail-image loki-image build-image
 .PHONY: bigtable-backup, push-bigtable-backup
 .PHONY: benchmark-store, drone, check-mod
@@ -414,6 +415,25 @@ fluent-bit-test:
 	docker run -v /var/log:/var/log -e LOG_PATH="/var/log/*.log" -e LOKI_URL="$(LOKI_URL)" \
 	 $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG)
 
+
+##################
+# fluentd plugin #
+##################
+fluentd-plugin:
+	gem install bundler --version 1.16.2
+	bundle config silence_root_warning true
+	bundle install --gemfile=cmd/fluentd/Gemfile --path=cmd/fluentd/vendor/bundle
+
+fluentd-image:
+	$(SUDO) docker build -t $(IMAGE_PREFIX)/fluent-plugin-loki:$(IMAGE_TAG) -f cmd/fluentd/Dockerfile .
+
+fluentd-push:
+	$(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/fluent-plugin-loki:$(IMAGE_TAG)
+
+fluentd-test: LOKI_URL ?= http://localhost:3100/loki/api/
+fluentd-test:
+	LOKI_URL="$(LOKI_URL)" docker-compose -f cmd/fluentd/docker/docker-compose.yml up --build #$(IMAGE_PREFIX)/fluent-plugin-loki:$(IMAGE_TAG)
+
 ########################
 # Bigtable Backup Tool #
 ########################
@@ -432,7 +452,7 @@ push-bigtable-backup: bigtable-backup
 # Images #
 ##########
 
-images: promtail-image loki-image loki-canary-image docker-driver fluent-bit-image
+images: promtail-image loki-image loki-canary-image docker-driver fluent-bit-image fluentd-image
 
 print-images:
 	$(info $(patsubst %,%:$(IMAGE_TAG),$(IMAGE_NAMES)))
@@ -522,6 +542,7 @@ ifeq ($(BUILD_IN_CONTAINER),true)
 		$(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@;
 else
 	drone jsonnet --stream --format -V __build-image-version=$(BUILD_IMAGE_VERSION) --source .drone/drone.jsonnet --target .drone/drone.yml
+	sed -i '1s/^/\# Generated by jsonnet from .drone\/drone.jsonnet\n/' .drone/drone.yml
 endif
 
 

cmd/fluentd/Dockerfile

@@ -0,0 +1,22 @@
+FROM ruby:2.6 as build
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    sudo make gcc g++ libc-dev ruby-dev golang
+
+COPY . /src/loki
+WORKDIR /src/loki
+RUN make BUILD_IN_CONTAINER=false fluentd-plugin
+
+FROM fluent/fluentd:v1.9.2-debian-1.0
+ENV LOKI_URL "https://logs-prod-us-central1.grafana.net"
+
+COPY --from=build /src/loki/cmd/fluentd/lib/fluent/plugin/out_loki.rb /fluentd/plugins/out_loki.rb
+
+COPY cmd/fluentd/docker/Gemfile /fluentd/
+COPY cmd/fluentd/docker/conf/loki.conf /fluentd/etc/loki.conf
+
+USER root
+RUN sed -i '$i''  @include loki.conf' /fluentd/etc/fluent.conf
+USER fluent

cmd/fluentd/README.md

@@ -0,0 +1,78 @@
+# FluentD output plugin
+
+[Fluentd](https://fluentd.org/) is a data collector for unified logging layer, it can be configured with the Loki output plugin, provided in this folder, to ship logs to Loki.
+
+See [docs/client/fluentd/README.md](../../docs/clients/fluentd/README.md) for detailed information.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `fluent-plugin-grafana-loki.gemspec`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+To create the gem: `gem build fluent-plugin-grafana-loki.gemspec`
+
+Useful additions:
+  `gem install rubocop`
+
+## Testing
+
+Start Loki using:
+```
+docker run -it -p 3100:3100 grafana/loki:latest
+```
+
+Verify that Loki accept and stores logs:
+```
+curl -H "Content-Type: application/json" -XPOST -s "http://localhost:3100/loki/api/v1/push" --data-raw "{\"streams\": [{\"stream\": {\"job\": \"test\"}, \"values\": [[\"$(date +%s)000000000\", \"fizzbuzz\"]]}]}"
+curl "http://localhost:3100/loki/api/v1/query_range" --data-urlencode 'query={job="test"}' --data-urlencode 'step=300' | jq .data.result
+```
+The expected output is:
+```
+[
+  {
+    "stream": {
+      "job": "test"
+    },
+    "values": [
+      [
+        "1588337198000000000",
+        "fizzbuzz"
+      ]
+    ]
+  }
+]
+```
+
+Start FluentBit + FluentD using:
+```
+LOKI_URL=http://{{ IP }}:3100 make fluentd-test
+```
+
+Verify that syslogs are being feeded into Loki:
+```
+curl "http://localhost:3100/loki/api/v1/query_range" --data-urlencode 'query={job="fluentd"}' --data-urlencode 'step=300' | jq .data.result
+```
+The expected output is:
+```
+[
+  {
+    "stream": {
+      "job": "fluentd"
+    },
+    "values": [
+      [
+        "1588336950379591919",
+        "log=\"May  1 14:42:30 ibuprofen avahi-daemon[859]: New relevant interface vethb503225.IPv6 for mDNS.\""
+      ],
+      ...
+    ]
+  }
+]
+```
+
+## Copyright
+
+* Copyright(c) 2018- Grafana Labs
+* License
+  * Apache License, Version 2.0

cmd/fluentd/docker/Gemfile

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+gem 'fluent-plugin-multi-format-parser', '~>1.0.0'
+gem 'fluentd', '1.9.0'

cmd/fluentd/docker/conf/loki.conf

@@ -0,0 +1,9 @@
+<match loki.**>
+  @type loki
+  url "#{ENV['LOKI_URL']}"
+  extra_labels {"job":"fluentd"}
+  <buffer>
+    flush_interval 10s
+    flush_at_shutdown true
+  </buffer>
+</match>

cmd/fluentd/docker/docker-compose.yml

@@ -0,0 +1,22 @@
+version: '3'
+services:
+  # Receive forwarded logs and send to /fluentd/logs/data.log and loki
+  fluentd:
+    build:
+      context: ../../..
+      dockerfile: cmd/fluentd/Dockerfile
+    image: fluentd:loki
+    volumes:
+    - ./fluentd.conf:/fluentd/etc/fluent.conf
+    environment:
+      - LOKI_URL
+  # Read /var/log/syslog and send it to fluentd
+  fluentbit:
+    image: fluent/fluent-bit:1.0
+    command: "/fluent-bit/bin/fluent-bit -c /srv/fluent-bit.conf"
+    user: root
+    volumes:
+    - ./fluent-bit.conf:/srv/fluent-bit.conf
+    - /var/log/syslog:/var/log/syslog:ro
+    depends_on:
+    - fluentd

cmd/fluentd/docker/fluent-bit.conf

@@ -0,0 +1,18 @@
+[SERVICE]
+    Flush        1
+    Daemon       Off
+
+[INPUT]
+    Name        tail
+    Path        /var/log/syslog
+    Tag         loki.output
+
+#[OUTPUT]
+#    Name   stdout
+#    Match  *
+
+[OUTPUT]
+    Name          forward
+    Match         *
+    Host          fluentd
+    Port          24224

cmd/fluentd/docker/fluentd.conf

@@ -0,0 +1,8 @@
+<source>
+    @type forward
+    port 24224
+</source>
+<filter **>
+    @type stdout
+</filter>
+@include loki.conf