grafana · owen-d · Jul 23, 2020 · Jul 22, 2020
diff --git a/Makefile b/Makefile
@@ -435,7 +435,7 @@ fluentd-push:
 
 fluentd-test: LOKI_URL ?= http://localhost:3100/loki/api/
 fluentd-test:
-	LOKI_URL="$(LOKI_URL)" docker-compose -f cmd/fluentd/docker/docker-compose.yml up --build #$(IMAGE_PREFIX)/fluent-plugin-loki:$(IMAGE_TAG)
+	LOKI_URL="$(LOKI_URL)" docker-compose -f cmd/fluentd/docker/docker-compose.yml up --build $(IMAGE_PREFIX)/fluent-plugin-loki:$(IMAGE_TAG)
 
 ##################
 # logstash plugin #

diff --git a/cmd/fluentd/README.md b/cmd/fluentd/README.md
@@ -2,7 +2,7 @@
 
 [Fluentd](https://fluentd.org/) is a data collector for unified logging layer, it can be configured with the Loki output plugin, provided in this folder, to ship logs to Loki.
 
-See [docs/client/fluentd/README.md](../../docs/clients/fluentd/README.md) for detailed information.
+See [docs/client/fluentd/README.md](../../docs/sources/clients/fluentd/_index.md) for detailed information.
 
 ## Development
 
@@ -18,17 +18,21 @@ Useful additions:
 ## Testing
 
 Start Loki using:
-```
+
+```bash
 docker run -it -p 3100:3100 grafana/loki:latest
 ```
 
 Verify that Loki accept and stores logs:
-```
+
+```bash
 curl -H "Content-Type: application/json" -XPOST -s "http://localhost:3100/loki/api/v1/push" --data-raw "{\"streams\": [{\"stream\": {\"job\": \"test\"}, \"values\": [[\"$(date +%s)000000000\", \"fizzbuzz\"]]}]}"
 curl "http://localhost:3100/loki/api/v1/query_range" --data-urlencode 'query={job="test"}' --data-urlencode 'step=300' | jq .data.result
 ```
+
 The expected output is:
-```
+
+```json
 [
   {
     "stream": {
@@ -44,17 +48,21 @@ The expected output is:
 ]
 ```
 
-Start FluentBit + Fluentd using:
-```
+Start and send test logs with Fluentd using:
+
+```bash
 LOKI_URL=http://{{ IP }}:3100 make fluentd-test
 ```
 
 Verify that syslogs are being feeded into Loki:
-```
+
+```bash
 curl "http://localhost:3100/loki/api/v1/query_range" --data-urlencode 'query={job="fluentd"}' --data-urlencode 'step=300' | jq .data.result
 ```
+
 The expected output is:
-```
+
+```json
 [
   {
     "stream": {

diff --git a/cmd/fluentd/lib/fluent/plugin/out_loki.rb b/cmd/fluentd/lib/fluent/plugin/out_loki.rb
@@ -36,7 +36,7 @@ class LogPostError < StandardError; end
       DEFAULT_BUFFER_TYPE = 'memory'
 
       desc 'url of loki server'
-      config_param :url, :string, default: 'https://logs-us-west1.grafana.net'
+      config_param :url, :string, default: 'https://logs-prod-us-central1.grafana.net'
 
       desc 'BasicAuth credentials'
       config_param :username, :string, default: nil

diff --git a/docs/sources/clients/fluentd/_index.md b/docs/sources/clients/fluentd/_index.md
@@ -1,27 +1,77 @@
 ---
-title: Fulentd
+title: Fluentd
 ---
-# Fluentd
+# Fluentd Loki Output Plugin
 
 Loki has a [Fluentd](https://www.fluentd.org/) output plugin called
 `fluent-plugin-grafana-loki` that enables shipping logs to a private Loki
-instance or [Grafana Cloud](https://grafana.com/oss/loki).
+instance or [Grafana Cloud](https://grafana.com/products/cloud/).
+
+The code source of the plugin is located in our [public repository](https://github.com/grafana/loki/tree/master/cmd/fluentd).
 
 ## Installation
 
+### Local
+
+To install the plugin use [fluent-gem](https://docs.fluentd.org/deployment/plugin-management):
+
+```bash
+fluent-gem install fluent-plugin-grafana-loki
 ```
-$ gem install fluent-plugin-grafana-loki
+
+## Docker Image
+
+There is a Docker image `grafana/fluent-plugin-loki:master` which contains [default configuration files](https://github.com/grafana/loki/tree/master/fluentd/fluent-plugin-loki/docker/conf). By default, fluentd containers use the configurations but you can also specify your `fluentd.conf` with `FLUENTD_CONF` environment variable.
+
+This image also uses `LOKI_URL`, `LOKI_USERNAME`, and `LOKI_PASSWORD` environment variables to specify the Loki's endpoint, user, and password (you can leave the USERNAME and PASSWORD blank if they're not used).
+
+This image will start an instance of Fluentd to forward incoming logs to the specified Loki url. As an alternate, containerized applications can also use [docker driver plugin](../docker-driver/) to ship logs without needing Fluentd.
+
+### Example
+
+A Docker Compose configuration that will work looks like:
+
+```yaml
+services:
+  fluentd:
+    image: grafana/fluent-plugin-loki:master
+    command:
+      - "fluentd"
+      - "-v"
+      - "-p"
+      - "/fluentd/plugins"
+    environment:
+      LOKI_URL: http://loki:3100
+      LOKI_USERNAME:
+      LOKI_PASSWORD:
+    deploy:
+      mode: global
+    configs:
+      - source: loki_config
+        target: /fluentd/etc/loki/loki.conf
+    networks:
+      - loki
+    volumes:
+      - host_logs:/var/log
+      # Needed for journald log ingestion:
+      - /etc/machine-id:/etc/machine-id
+      - /dev/log:/dev/log
+      - /var/run/systemd/journal/:/var/run/systemd/journal/
+    logging:
+      options:
+         tag: infra.monitoring
 ```
 
 ## Usage
 
 **Note**: use either `<label>...</label>` or `extra_labels` to set at least one label!
 
 In your Fluentd configuration, use `@type loki`. Additional configuration is optional, default values would look like this:
-```
+
+```conf
 <match **>
   @type loki
-  url "https://logs-us-west1.grafana.net"
+  url "https://logs-prod-us-central1.grafana.net"
   username "#{ENV['LOKI_USERNAME']}"
   password "#{ENV['LOKI_PASSWORD']}"
   extra_labels {"env":"dev"}
@@ -31,10 +81,11 @@ In your Fluentd configuration, use `@type loki`. Additional configuration is opt
 </match>
 ```
 
-### Using labels
+### Adding labels
 
 Simple label from top level attribute
-```
+
+```conf
 <match mytag>
   @type loki
   # ...
@@ -47,7 +98,7 @@ Simple label from top level attribute
 
 You can rewrite the label keys as well as the following
 
-```
+```conf
 <match mytag>
   @type loki
   # ...
@@ -58,9 +109,9 @@ You can rewrite the label keys as well as the following
 </match>
 ```
 
-You can use record accessor syntax for nested field. https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-record_accessor#syntax
+You can use [record accessor](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-record_accessor#syntax) syntax for nested field.
 
-```
+```conf
 <match mytag>
   @type loki
   # ...
@@ -76,7 +127,8 @@ You can use record accessor syntax for nested field. https://docs.fluentd.org/pl
 As Kubernetes labels are a list of nested key-value pairs there is a separate option to extract them.
 Note that special characters like "`. - /`" will be overwritten with `_`.
 Use with the `remove_keys kubernetes` option to eliminate metadata from the log.
-```
+
+```conf
 <match mytag>
   @type loki
   # ...
@@ -89,12 +141,15 @@ Use with the `remove_keys kubernetes` option to eliminate metadata from the log.
 </match>
 ```
 
+> You can also include automatically all kubernetes labels by using `extract_kubernetes_labels true` in your configuration.
+
 ### Multi-worker usage
 
 Loki doesn't currently support out-of-order inserts - if you try to insert a log entry an earlier timestamp after a log entry with identical labels but a later timestamp, the insert will fail with `HTTP status code: 500, message: rpc error: code = Unknown desc = Entry out of order`. Therefore, in order to use this plugin in a multi worker Fluentd setup, you'll need to include the worker ID in the labels or otherwise [ensure log streams are always sent to the same worker](https://docs.fluentd.org/deployment/multi-process-workers#less-than-worker-n-greater-than-directive).
 
 For example, using [fluent-plugin-record-modifier](https://github.com/repeatedly/fluent-plugin-record-modifier):
-```
+
+```conf
 <filter mytag>
     @type record_modifier
     <record>
@@ -120,73 +175,33 @@ times for their unique label sets.
 
 This plugin automatically adds a `fluentd_thread` label with the name of the buffer flush thread when `flush_thread_count` > 1.
 
-## Docker Image
-
-There is a Docker image `grafana/fluent-plugin-loki:master` which contains [default configuration files](https://github.com/grafana/loki/tree/master/fluentd/fluent-plugin-loki/docker/conf). By default, fluentd containers use the configurations but you can also specify your `fluentd.conf` with `FLUENTD_CONF` environment variable.
-
-This image also uses `LOKI_URL`, `LOKI_USERNAME`, and `LOKI_PASSWORD` environment variables to specify the Loki's endpoint, user, and password (you can leave the USERNAME and PASSWORD blank if they're not used).
-
-This image will start an instance of Fluentd to forward incoming logs to the specified Loki url. As an alternate, containerized applications can also use [docker driver plugin](../docker-driver/) to ship logs without needing Fluentd.
-
-### Example
-
-A Docker Compose configuration that will work looks like:
-
-```
-services:
-  fluentd:
-    image: grafana/fluent-plugin-loki:master
-    command:
-      - "fluentd"
-      - "-v"
-      - "-p"
-      - "/fluentd/plugins"
-    environment:
-      LOKI_URL: http://loki:3100
-      LOKI_USERNAME:
-      LOKI_PASSWORD:
-    deploy:
-      mode: global
-    configs:
-      - source: loki_config
-        target: /fluentd/etc/loki/loki.conf
-    networks:
-      - loki
-    volumes:
-      - host_logs:/var/log
-      # Needed for journald log ingestion:
-      - /etc/machine-id:/etc/machine-id
-      - /dev/log:/dev/log
-      - /var/run/systemd/journal/:/var/run/systemd/journal/
-    logging:
-      options:
-         tag: infra.monitoring
-```
-
 ## Configuration
 
 ### url
-The url of the Loki server to send logs to.  When sending data the publish path (../api/prom/push`) will automatically be appended.
-By default the url is set to `https://logs-us-west1.grafana.net`, the url of the Grafana Labs preview (hosted Loki)[https://grafana.com/loki] service.
+
+The url of the Loki server to send logs to.  When sending data the publish path (`../api/loki/v1/push`) will automatically be appended.
+By default the url is set to `https://logs-prod-us-central1.grafana.net`, the url of the Grafana Labs [hosted Loki][https://grafana.com/products/cloud/] service.
 
 #### Proxy Support
 
-Starting with version 0.8.0, this gem uses excon, which supports proxy with environment variables - https://github.com/excon/excon#proxy-support
+Starting with version 0.8.0, this gem uses [excon, which supports proxy with environment variables](https://github.com/excon/excon#proxy-support).
 
 ### username / password
+
 Specify a username and password if the Loki server requires authentication.
 If using the GrafanaLab's hosted Loki, the username needs to be set to your instanceId and the password should be a Grafana.com api key.
 
 ### tenant
+
 Loki is a multi-tenant log storage platform and all requests sent must include a tenant.  For some installations the tenant will be set automatically by an authenticating proxy.  Otherwise you can define a tenant to be passed through.
 The tenant can be any string value.
 
 The tenant field also supports placeholders, so it can dynamically change based on tag and record fields. Each placeholder must be added as a buffer chunk key. The following is an example of setting the tenant based on a k8s pod label:
 
-```
+```conf
 <match **>
   @type loki
-  url "https://logs-us-west1.grafana.net"
+  url "https://logs-prod-us-central1.grafana.net"
   tenant ${$.kubernetes.labels.tenant}
   # ...
   <buffer $.kubernetes.labels.tenant>
@@ -197,9 +212,10 @@ The tenant field also supports placeholders, so it can dynamically change based
 ```
 
 ### client certificate verification
+
 Specify a pair of client certificate and private key with `cert` and `key` if a reverse proxy with client certificate verification is configured in front of Loki. `ca_cert` can also be specified if the server uses custom certificate authority.
 
-```
+```conf
 <match **>
   @type loki
 
@@ -214,9 +230,10 @@ Specify a pair of client certificate and private key with `cert` and `key` if a
 ```
 
 ### Server certificate verification
+
 A flag to disable a server certificate verification. By default the `insecure_tls` is set to false.
 
-```
+```conf
 <match **>
   @type loki
 
@@ -229,19 +246,21 @@ A flag to disable a server certificate verification. By default the `insecure_tl
 ```
 
 ### output format
+
 Loki is intended to index and group log streams using only a small set of labels.  It is not intended for full-text indexing.  When sending logs to Loki the majority of log message will be sent as a single log "line".
 
 There are few configurations settings to control the output format.
- - extra_labels: (default: nil) set of labels to include with every Loki stream. eg `{"env":"dev", "datacenter": "dc1"}`
- - remove_keys: (default: nil) comma separated list of needless record keys to remove. All other keys will be placed into the log line. You can use [record_accessor syntax](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-record_accessor#syntax).
- - line_format: format to use when flattening the record to a log line. Valid values are "json" or "key_value". If set to "json" the log line sent to Loki will be the fluentd record (excluding any keys extracted out as labels) dumped as json. If set to "key_value", the log line will be each item in the record concatenated together (separated by a single space) in the format `<key>=<value>`.
- - drop_single_key: if set to true and a record only has 1 key after extracting `<label></label>` blocks, set the log line to the value and discard the key.
+
+- extra_labels: (default: nil) set of labels to include with every Loki stream. eg `{"env":"dev", "datacenter": "dc1"}`
+- remove_keys: (default: nil) comma separated list of needless record keys to remove. All other keys will be placed into the log line. You can use [record_accessor syntax](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-record_accessor#syntax).
+- line_format: format to use when flattening the record to a log line. Valid values are "json" or "key_value". If set to "json" the log line sent to Loki will be the fluentd record (excluding any keys extracted out as labels) dumped as json. If set to "key_value", the log line will be each item in the record concatenated together (separated by a single space) in the format `<key>=<value>`.
+- drop_single_key: if set to true and a record only has 1 key after extracting `<label></label>` blocks, set the log line to the value and discard the key.
 
 ### Buffer options
 
 `fluentd-plugin-loki` extends [Fluentd's builtin Output plugin](https://docs.fluentd.org/v1.0/articles/output-plugin-overview) and use `compat_parameters` plugin helper. It adds the following options:
 
-```
+```conf
 buffer_type memory
 flush_interval 10s
 retry_limit 17