diff --git a/.drone/drone.jsonnet b/.drone/drone.jsonnet index 90a1175def15..96064196e6e8 100644 --- a/.drone/drone.jsonnet +++ b/.drone/drone.jsonnet @@ -84,6 +84,30 @@ local fluentbit() = pipeline('fluent-bit-amd64') + arch_image('amd64', 'latest,m depends_on: ['check'], }; +local fluentd() = pipeline('fluentd-amd64') + arch_image('amd64', 'latest,master') { + steps+: [ + // dry run for everything that is not tag or master + docker('amd64', 'fluentd') { + depends_on: ['image-tag'], + when: condition('exclude').tagMaster, + settings+: { + dry_run: true, + repo: 'grafana/fluent-plugin-loki', + }, + }, + ] + [ + // publish for tag or master + docker('amd64', 'fluentd') { + depends_on: ['image-tag'], + when: condition('include').tagMaster, + settings+: { + repo: 'grafana/fluent-plugin-loki', + }, + }, + ], + depends_on: ['check'], +}; + local multiarch_image(arch) = pipeline('docker-' + arch) + arch_image(arch) { steps+: [ // dry run for everything that is not tag or master @@ -181,6 +205,7 @@ local manifest(apps) = pipeline('manifest') { for arch in archs ] + [ fluentbit(), + fluentd(), ] + [ manifest(['promtail', 'loki', 'loki-canary']) { trigger: condition('include').tagMaster, diff --git a/.drone/drone.yml b/.drone/drone.yml index 6d28f12524c3..4bdfa5b55d57 100644 --- a/.drone/drone.yml +++ b/.drone/drone.yml @@ -1,3 +1,4 @@ +# Generated by jsonnet from .drone/drone.jsonnet --- kind: pipeline name: check @@ -514,6 +515,62 @@ steps: depends_on: - check +--- +kind: pipeline +name: fluentd-amd64 + +platform: + os: linux + arch: amd64 + +steps: +- name: image-tag + image: alpine + commands: + - apk add --no-cache bash git + - git fetch origin --tags + - echo $(./tools/image-tag)-amd64 > .tags + - echo ",latest,master" >> .tags + +- name: build-fluentd-image + image: plugins/docker + settings: + dockerfile: cmd/fluentd/Dockerfile + dry_run: true + password: + from_secret: docker_password + repo: grafana/fluent-plugin-loki + username: + from_secret: docker_username + when: + ref: + exclude: + - refs/heads/master + - refs/heads/k?? + - refs/tags/v* + depends_on: + - image-tag + +- name: publish-fluentd-image + image: plugins/docker + settings: + dockerfile: cmd/fluentd/Dockerfile + password: + from_secret: docker_password + repo: grafana/fluent-plugin-loki + username: + from_secret: docker_username + when: + ref: + - refs/heads/master + - refs/heads/k?? + - refs/tags/v* + depends_on: + - image-tag + +depends_on: +- check + --- kind: pipeline name: manifest diff --git a/Makefile b/Makefile index e40df874006b..43f2fdbed747 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ .PHONY: helm helm-install helm-upgrade helm-publish helm-debug helm-clean .PHONY: docker-driver docker-driver-clean docker-driver-enable docker-driver-push .PHONY: fluent-bit-image, fluent-bit-push, fluent-bit-test +.PHONY: fluentd-image, fluentd-push, fluentd-test .PHONY: push-images push-latest save-images load-images promtail-image loki-image build-image .PHONY: bigtable-backup, push-bigtable-backup .PHONY: benchmark-store, drone, check-mod @@ -414,6 +415,25 @@ fluent-bit-test: docker run -v /var/log:/var/log -e LOG_PATH="/var/log/*.log" -e LOKI_URL="$(LOKI_URL)" \ $(IMAGE_PREFIX)/fluent-bit-plugin-loki:$(IMAGE_TAG) + +################## +# fluentd plugin # +################## +fluentd-plugin: + gem install bundler --version 1.16.2 + bundle config silence_root_warning true + bundle install --gemfile=cmd/fluentd/Gemfile --path=cmd/fluentd/vendor/bundle + +fluentd-image: + $(SUDO) docker build -t $(IMAGE_PREFIX)/fluent-plugin-loki:$(IMAGE_TAG) -f cmd/fluentd/Dockerfile . + +fluentd-push: + $(SUDO) $(PUSH_OCI) $(IMAGE_PREFIX)/fluent-plugin-loki:$(IMAGE_TAG) + +fluentd-test: LOKI_URL ?= http://localhost:3100/loki/api/ +fluentd-test: + LOKI_URL="$(LOKI_URL)" docker-compose -f cmd/fluentd/docker/docker-compose.yml up --build #$(IMAGE_PREFIX)/fluent-plugin-loki:$(IMAGE_TAG) + ######################## # Bigtable Backup Tool # ######################## @@ -432,7 +452,7 @@ push-bigtable-backup: bigtable-backup # Images # ########## -images: promtail-image loki-image loki-canary-image docker-driver fluent-bit-image +images: promtail-image loki-image loki-canary-image docker-driver fluent-bit-image fluentd-image print-images: $(info $(patsubst %,%:$(IMAGE_TAG),$(IMAGE_NAMES))) @@ -522,6 +542,7 @@ ifeq ($(BUILD_IN_CONTAINER),true) $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; else drone jsonnet --stream --format -V __build-image-version=$(BUILD_IMAGE_VERSION) --source .drone/drone.jsonnet --target .drone/drone.yml + sed -i '1s/^/\# Generated by jsonnet from .drone\/drone.jsonnet\n/' .drone/drone.yml endif diff --git a/fluentd/fluent-plugin-grafana-loki/.gitignore b/cmd/fluentd/.gitignore similarity index 100% rename from fluentd/fluent-plugin-grafana-loki/.gitignore rename to cmd/fluentd/.gitignore diff --git a/fluentd/fluent-plugin-grafana-loki/.rspec b/cmd/fluentd/.rspec similarity index 100% rename from fluentd/fluent-plugin-grafana-loki/.rspec rename to cmd/fluentd/.rspec diff --git a/fluentd/fluent-plugin-grafana-loki/.rubocop.yml b/cmd/fluentd/.rubocop.yml similarity index 100% rename from fluentd/fluent-plugin-grafana-loki/.rubocop.yml rename to cmd/fluentd/.rubocop.yml diff --git a/cmd/fluentd/Dockerfile b/cmd/fluentd/Dockerfile new file mode 100644 index 000000000000..885fd91a31c5 --- /dev/null +++ b/cmd/fluentd/Dockerfile @@ -0,0 +1,22 @@ +FROM ruby:2.6 as build + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends \ + sudo make gcc g++ libc-dev ruby-dev golang + +COPY . /src/loki +WORKDIR /src/loki +RUN make BUILD_IN_CONTAINER=false fluentd-plugin + +FROM fluent/fluentd:v1.9.2-debian-1.0 +ENV LOKI_URL "https://logs-prod-us-central1.grafana.net" + +COPY --from=build /src/loki/cmd/fluentd/lib/fluent/plugin/out_loki.rb /fluentd/plugins/out_loki.rb + +COPY cmd/fluentd/docker/Gemfile /fluentd/ +COPY cmd/fluentd/docker/conf/loki.conf /fluentd/etc/loki.conf + +USER root +RUN sed -i '$i'' @include loki.conf' /fluentd/etc/fluent.conf +USER fluent diff --git a/fluentd/fluent-plugin-grafana-loki/Gemfile b/cmd/fluentd/Gemfile similarity index 100% rename from fluentd/fluent-plugin-grafana-loki/Gemfile rename to cmd/fluentd/Gemfile diff --git a/fluentd/fluent-plugin-grafana-loki/LICENSE b/cmd/fluentd/LICENSE similarity index 100% rename from fluentd/fluent-plugin-grafana-loki/LICENSE rename to cmd/fluentd/LICENSE diff --git a/cmd/fluentd/README.md b/cmd/fluentd/README.md new file mode 100644 index 000000000000..49e42d76b58c --- /dev/null +++ b/cmd/fluentd/README.md @@ -0,0 +1,78 @@ +# FluentD output plugin + +[Fluentd](https://fluentd.org/) is a data collector for unified logging layer, it can be configured with the Loki output plugin, provided in this folder, to ship logs to Loki. + +See [docs/client/fluentd/README.md](../../docs/clients/fluentd/README.md) for detailed information. + +## Development + +After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. + +To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `fluent-plugin-grafana-loki.gemspec`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org). + +To create the gem: `gem build fluent-plugin-grafana-loki.gemspec` + +Useful additions: + `gem install rubocop` + +## Testing + +Start Loki using: +``` +docker run -it -p 3100:3100 grafana/loki:latest +``` + +Verify that Loki accept and stores logs: +``` +curl -H "Content-Type: application/json" -XPOST -s "http://localhost:3100/loki/api/v1/push" --data-raw "{\"streams\": [{\"stream\": {\"job\": \"test\"}, \"values\": [[\"$(date +%s)000000000\", \"fizzbuzz\"]]}]}" +curl "http://localhost:3100/loki/api/v1/query_range" --data-urlencode 'query={job="test"}' --data-urlencode 'step=300' | jq .data.result +``` +The expected output is: +``` +[ + { + "stream": { + "job": "test" + }, + "values": [ + [ + "1588337198000000000", + "fizzbuzz" + ] + ] + } +] +``` + +Start FluentBit + FluentD using: +``` +LOKI_URL=http://{{ IP }}:3100 make fluentd-test +``` + +Verify that syslogs are being feeded into Loki: +``` +curl "http://localhost:3100/loki/api/v1/query_range" --data-urlencode 'query={job="fluentd"}' --data-urlencode 'step=300' | jq .data.result +``` +The expected output is: +``` +[ + { + "stream": { + "job": "fluentd" + }, + "values": [ + [ + "1588336950379591919", + "log=\"May 1 14:42:30 ibuprofen avahi-daemon[859]: New relevant interface vethb503225.IPv6 for mDNS.\"" + ], + ... + ] + } +] +``` + +## Copyright + +* Copyright(c) 2018- Grafana Labs +* License + * Apache License, Version 2.0 diff --git a/fluentd/fluent-plugin-grafana-loki/Rakefile b/cmd/fluentd/Rakefile similarity index 100% rename from fluentd/fluent-plugin-grafana-loki/Rakefile rename to cmd/fluentd/Rakefile diff --git a/fluentd/fluent-plugin-grafana-loki/bin/console b/cmd/fluentd/bin/console similarity index 100% rename from fluentd/fluent-plugin-grafana-loki/bin/console rename to cmd/fluentd/bin/console diff --git a/fluentd/fluent-plugin-grafana-loki/bin/setup b/cmd/fluentd/bin/setup similarity index 100% rename from fluentd/fluent-plugin-grafana-loki/bin/setup rename to cmd/fluentd/bin/setup diff --git a/cmd/fluentd/docker/Gemfile b/cmd/fluentd/docker/Gemfile new file mode 100644 index 000000000000..6766bbcd7767 --- /dev/null +++ b/cmd/fluentd/docker/Gemfile @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +source 'https://rubygems.org' + +gem 'fluent-plugin-multi-format-parser', '~>1.0.0' +gem 'fluentd', '1.9.0' diff --git a/cmd/fluentd/docker/conf/loki.conf b/cmd/fluentd/docker/conf/loki.conf new file mode 100644 index 000000000000..62a51356e10d --- /dev/null +++ b/cmd/fluentd/docker/conf/loki.conf @@ -0,0 +1,9 @@ + + @type loki + url "#{ENV['LOKI_URL']}" + extra_labels {"job":"fluentd"} + + flush_interval 10s + flush_at_shutdown true + + diff --git a/cmd/fluentd/docker/docker-compose.yml b/cmd/fluentd/docker/docker-compose.yml new file mode 100644 index 000000000000..08a0f249cece --- /dev/null +++ b/cmd/fluentd/docker/docker-compose.yml @@ -0,0 +1,22 @@ +version: '3' +services: + # Receive forwarded logs and send to /fluentd/logs/data.log and loki + fluentd: + build: + context: ../../.. + dockerfile: cmd/fluentd/Dockerfile + image: fluentd:loki + volumes: + - ./fluentd.conf:/fluentd/etc/fluent.conf + environment: + - LOKI_URL + # Read /var/log/syslog and send it to fluentd + fluentbit: + image: fluent/fluent-bit:1.0 + command: "/fluent-bit/bin/fluent-bit -c /srv/fluent-bit.conf" + user: root + volumes: + - ./fluent-bit.conf:/srv/fluent-bit.conf + - /var/log/syslog:/var/log/syslog:ro + depends_on: + - fluentd diff --git a/cmd/fluentd/docker/fluent-bit.conf b/cmd/fluentd/docker/fluent-bit.conf new file mode 100644 index 000000000000..3800bdbf3795 --- /dev/null +++ b/cmd/fluentd/docker/fluent-bit.conf @@ -0,0 +1,18 @@ +[SERVICE] + Flush 1 + Daemon Off + +[INPUT] + Name tail + Path /var/log/syslog + Tag loki.output + +#[OUTPUT] +# Name stdout +# Match * + +[OUTPUT] + Name forward + Match * + Host fluentd + Port 24224 diff --git a/cmd/fluentd/docker/fluentd.conf b/cmd/fluentd/docker/fluentd.conf new file mode 100644 index 000000000000..23b065c069f5 --- /dev/null +++ b/cmd/fluentd/docker/fluentd.conf @@ -0,0 +1,8 @@ + + @type forward + port 24224 + + + @type stdout + +@include loki.conf diff --git a/fluentd/fluent-plugin-grafana-loki/fluent-plugin-grafana-loki.gemspec b/cmd/fluentd/fluent-plugin-grafana-loki.gemspec similarity index 100% rename from fluentd/fluent-plugin-grafana-loki/fluent-plugin-grafana-loki.gemspec rename to cmd/fluentd/fluent-plugin-grafana-loki.gemspec diff --git a/fluentd/fluent-plugin-grafana-loki/lib/fluent/plugin/out_loki.rb b/cmd/fluentd/lib/fluent/plugin/out_loki.rb similarity index 100% rename from fluentd/fluent-plugin-grafana-loki/lib/fluent/plugin/out_loki.rb rename to cmd/fluentd/lib/fluent/plugin/out_loki.rb diff --git a/fluentd/fluent-plugin-grafana-loki/spec/gems/fluent/plugin/data/syslog b/cmd/fluentd/spec/gems/fluent/plugin/data/syslog similarity index 100% rename from fluentd/fluent-plugin-grafana-loki/spec/gems/fluent/plugin/data/syslog rename to cmd/fluentd/spec/gems/fluent/plugin/data/syslog diff --git a/fluentd/fluent-plugin-grafana-loki/spec/gems/fluent/plugin/data/syslog2 b/cmd/fluentd/spec/gems/fluent/plugin/data/syslog2 similarity index 100% rename from fluentd/fluent-plugin-grafana-loki/spec/gems/fluent/plugin/data/syslog2 rename to cmd/fluentd/spec/gems/fluent/plugin/data/syslog2 diff --git a/fluentd/fluent-plugin-grafana-loki/spec/gems/fluent/plugin/loki_output_spec.rb b/cmd/fluentd/spec/gems/fluent/plugin/loki_output_spec.rb similarity index 100% rename from fluentd/fluent-plugin-grafana-loki/spec/gems/fluent/plugin/loki_output_spec.rb rename to cmd/fluentd/spec/gems/fluent/plugin/loki_output_spec.rb diff --git a/fluentd/fluent-plugin-grafana-loki/spec/spec_helper.rb b/cmd/fluentd/spec/spec_helper.rb similarity index 100% rename from fluentd/fluent-plugin-grafana-loki/spec/spec_helper.rb rename to cmd/fluentd/spec/spec_helper.rb diff --git a/fluentd/fluent-plugin-grafana-loki/Dockerfile b/fluentd/fluent-plugin-grafana-loki/Dockerfile deleted file mode 100644 index 796d489ba432..000000000000 --- a/fluentd/fluent-plugin-grafana-loki/Dockerfile +++ /dev/null @@ -1,41 +0,0 @@ -FROM fluent/fluentd:v1.9.2-debian-1.0 - -USER root -WORKDIR /home/fluent -ENV PATH /fluentd/vendor/bundle/ruby/2.6.0/bin:$PATH -ENV GEM_PATH /fluentd/vendor/bundle/ruby/2.6.0 -ENV GEM_HOME /fluentd/vendor/bundle/ruby/2.6.0 -# skip runtime bundler installation -ENV FLUENTD_DISABLE_BUNDLER_INJECTION 1 - -COPY docker/Gemfile* /fluentd/ -RUN buildDeps="sudo make gcc g++ libc-dev ruby-dev" \ - && apt-get update \ - && apt-get install -y --no-install-recommends \ - $buildDeps libsystemd0 net-tools libjemalloc2 \ - && gem install bundler --version 1.16.2 \ - && bundle config silence_root_warning true \ - && bundle install --gemfile=/fluentd/Gemfile --path=/fluentd/vendor/bundle \ - && sudo gem sources --clear-all \ - && SUDO_FORCE_REMOVE=yes \ - apt-get purge -y --auto-remove \ - -o APT::AutoRemove::RecommendsImportant=false \ - $buildDeps \ - && rm -rf /var/lib/apt/lists/* \ - /home/fluent/.gem/ruby/2.6.0/cache/*.gem \ - /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem - -COPY docker/entrypoint.sh /fluentd/entrypoint.sh -COPY lib/fluent/plugin/out_loki.rb /fluentd/plugins/out_loki.rb -COPY docker/conf/ /fluentd/etc/loki/ - -ENV FLUENTD_CONF="/fluentd/etc/loki/fluentd.conf" -ENV FLUENTD_OPT="" - -ENV LOKI_URL "https://logs-us-west1.grafana.net" - -# See https://packages.debian.org/stretch/amd64/libjemalloc1/filelist -ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libjemalloc.so.2" - -# Overwrite ENTRYPOINT to run fluentd as root for /var/log / /var/lib -ENTRYPOINT ["/fluentd/entrypoint.sh"] diff --git a/fluentd/fluent-plugin-grafana-loki/README.md b/fluentd/fluent-plugin-grafana-loki/README.md deleted file mode 100644 index 21554b72254f..000000000000 --- a/fluentd/fluent-plugin-grafana-loki/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# fluent-plugin-grafana-loki - -[Fluentd](https://fluentd.org/) output plugin to ship logs to a Loki server. See [docs/client/fluentd/README.md](../../docs/clients/fluentd/README.md) for detailed information. - -## Development - -After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. - -To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `fluent-plugin-grafana-loki.gemspec`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org). - -To create the gem: `gem build fluent-plugin-grafana-loki.gemspec` - -Useful additions: - `gem install rubocop` - -## Copyright - -* Copyright(c) 2018- Grafana Labs -* License - * Apache License, Version 2.0 diff --git a/fluentd/fluent-plugin-grafana-loki/docker/Gemfile b/fluentd/fluent-plugin-grafana-loki/docker/Gemfile deleted file mode 100644 index 6b8e2a504030..000000000000 --- a/fluentd/fluent-plugin-grafana-loki/docker/Gemfile +++ /dev/null @@ -1,9 +0,0 @@ -# frozen_string_literal: true - -source 'https://rubygems.org' - -gem 'fluent-plugin-kubernetes_metadata_filter', '~> 2.1.6' -gem 'fluent-plugin-multi-format-parser', '~>1.0.0' -gem 'fluent-plugin-prometheus', '~>1.4.0' -gem 'fluent-plugin-systemd', '~> 1.0.1' -gem 'fluentd', '1.9.0' diff --git a/fluentd/fluent-plugin-grafana-loki/docker/conf/fluentd-1.conf b/fluentd/fluent-plugin-grafana-loki/docker/conf/fluentd-1.conf deleted file mode 100644 index 798038f644d0..000000000000 --- a/fluentd/fluent-plugin-grafana-loki/docker/conf/fluentd-1.conf +++ /dev/null @@ -1,44 +0,0 @@ -# /tmp/fluentd.conf - - - @type tail - path /var/log/sntpc.log - pos_file /var/log/td-agent/sntpc.log.pos - tag tail.sntpc.log - path_key filename - - @type none - - - - - @type tail - path /var/log/docker.log - pos_file /var/log/td-agent/docker.log.pos - tag tail.docker.log - path_key filename - - @type none - - - - - @type tail - path /var/log/kmsg.log - pos_file /var/log/td-agent/kmsg.log.pos - tag tail.kmsg.log - path_key filename - - @type none - - - - - @type record_transformer - - hostname "#{Socket.gethostname}" - tag ${tag} - - - -@include loki.conf \ No newline at end of file diff --git a/fluentd/fluent-plugin-grafana-loki/docker/conf/fluentd-2.conf b/fluentd/fluent-plugin-grafana-loki/docker/conf/fluentd-2.conf deleted file mode 100644 index 2060a32bfdf5..000000000000 --- a/fluentd/fluent-plugin-grafana-loki/docker/conf/fluentd-2.conf +++ /dev/null @@ -1,54 +0,0 @@ -# /tmp/fluentd.conf - - @type tail - path /var/log/syslog - pos_file /var/log/td-agent/syslog.pos - tag tail.syslog - path_key filename - - @type syslog - - - - - @type tail - path /var/log/sntpc.log - pos_file /var/log/td-agent/sntpc.log.pos - tag tail.sntpc.log - path_key filename - - @type none - - - - - @type tail - path /var/log/docker.log - pos_file /var/log/td-agent/docker.log.pos - tag tail.docker.log - path_key filename - - @type none - - - - - @type tail - path /var/log/kmsg.log - pos_file /var/log/td-agent/kmsg.log.pos - tag tail.kmsg.log - path_key filename - - @type none - - - - - @type record_transformer - - hostname "#{Socket.gethostname}" - tag ${tag} - - - -@include loki.conf diff --git a/fluentd/fluent-plugin-grafana-loki/docker/conf/fluentd.conf b/fluentd/fluent-plugin-grafana-loki/docker/conf/fluentd.conf deleted file mode 100644 index b69df0f8303b..000000000000 --- a/fluentd/fluent-plugin-grafana-loki/docker/conf/fluentd.conf +++ /dev/null @@ -1,10 +0,0 @@ -# /tmp/fluentd.conf - -@include systemd.conf - -# -# @type stdout -# format json -# - -@include loki.conf diff --git a/fluentd/fluent-plugin-grafana-loki/docker/conf/loki.conf b/fluentd/fluent-plugin-grafana-loki/docker/conf/loki.conf deleted file mode 100644 index 11d96f8ba45c..000000000000 --- a/fluentd/fluent-plugin-grafana-loki/docker/conf/loki.conf +++ /dev/null @@ -1,12 +0,0 @@ - - @type loki - url "#{ENV['LOKI_URL']}" - username "#{ENV['LOKI_USERNAME']}" - password "#{ENV['LOKI_PASSWORD']}" - extra_labels {"env":"dev"} - label_keys "job,instance,level" - drop_single_key true - flush_interval 10s - flush_at_shutdown true - buffer_chunk_limit 1m - diff --git a/fluentd/fluent-plugin-grafana-loki/docker/conf/systemd.conf b/fluentd/fluent-plugin-grafana-loki/docker/conf/systemd.conf deleted file mode 100644 index 0137172ba171..000000000000 --- a/fluentd/fluent-plugin-grafana-loki/docker/conf/systemd.conf +++ /dev/null @@ -1,33 +0,0 @@ -# Logs from systemd-journal - - @type systemd - @id in_systemd_kubelet - tag journald - - @type local - persistent true - path /var/log/fluentd-journald-cursor.json - - read_from_head false - - fields_lowercase false - field_map {"_PID": "PID", "MESSAGE": "message", "SYSLOG_IDENTIFIER": "identifier", "_HOSTNAME": "HOSTNAME", "PRIORITY": "PRIORITY"} - field_map_strict true - - - - - @type record_transformer - enable_ruby true - remove_keys ["PRIORITY", "HOSTNAME"] - - # add a "job" label - job default/journald - - # re-label the HOSTNAME to "instance" - instance ${record["HOSTNAME"]} - - # change priority from an int to a string. - level ${record["PRIORITY"] ? record["PRIORITY"] == "0" ? "EMERG" : record["PRIORITY"] == "1" ? "ALERT" : record["PRIORITY"] == "2" ? "CRIT" : record["PRIORITY"] == "3" ? "ERROR" : record["PRIORITY"] == "4" ? "WARN" : record["PRIORITY"] == "5" ? "NOTICE" : record["PRIORITY"] == "6" ? "INFO" : record["PRIORITY"] == "7" ? "DEBUG" : record["PRIORITY"] : "unset"} - - \ No newline at end of file diff --git a/fluentd/fluent-plugin-grafana-loki/docker/docker-compose.yml b/fluentd/fluent-plugin-grafana-loki/docker/docker-compose.yml deleted file mode 100644 index d39f0b0804ec..000000000000 --- a/fluentd/fluent-plugin-grafana-loki/docker/docker-compose.yml +++ /dev/null @@ -1,21 +0,0 @@ -version: '3' -services: - fluentd: - build: ../ - image: fluentd:loki - command: - - "fluentd" - - "-v" - - "-p" - - "/fluentd/plugins" - - "-c" - - "/fluentd/etc/fluentd.conf" - volumes: - - ./conf:/fluentd/etc - - /var/log:/var/log - - ../lib/fluent/plugin:/fluentd/plugins - environment: - LOKI_USERNAME: "${LOKI_USERNAME}" - LOKI_PASSWORD: ${LOKI_PASSWORD} - FLUENT_CONF: /fluentd/etc/fluentd.conf - FLUENT_UID: '0' diff --git a/fluentd/fluent-plugin-grafana-loki/docker/entrypoint.sh b/fluentd/fluent-plugin-grafana-loki/docker/entrypoint.sh deleted file mode 100755 index 892dd2786e21..000000000000 --- a/fluentd/fluent-plugin-grafana-loki/docker/entrypoint.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/dumb-init /bin/sh - -exec fluentd -c ${FLUENTD_CONF} -p /fluentd/plugins --gemfile /fluentd/Gemfile ${FLUENTD_OPT} \ No newline at end of file