pkg/loki: support multiple instances of promtail (BREAKING CHANGE)

CHANGELOG.md

@@ -4,6 +4,11 @@ can be found at [#317](https://github.com/grafana/agent/issues/317).
 
 # Master (unreleased)
 
+- [FEATURE] BREAKING CHANGE: Support for multiple Loki Promtail instances has
+  been added, using the same `configs` array used by the Prometheus subsystem.
+  Read the new [migration guide](./docs/migration-guide.md) before upgrading!
+  (@rfratto)
+
 - [FEATURE] Added [ElasticSearch exporter](https://github.com/justwatchcom/elasticsearch_exporter)
   integration. (@colega)
 

cmd/agent/main.go

@@ -69,7 +69,7 @@ func main() {
 	}
 
 	if cfg.Loki.Enabled {
-		lokiLogs, err = loki.New(cfg.Loki, util.Logger)
+		lokiLogs, err = loki.New(prometheus.DefaultRegisterer, cfg.Loki, util.Logger)
 		if err != nil {
 			level.Error(util.Logger).Log("msg", "failed to create loki log collection instance", "err", err)
 			os.Exit(1)

docs/README.md

@@ -29,7 +29,8 @@ main problems faced by users of large deployments of Prometheus:
         3. [Loki Config/Migrating from Promtail](./getting-started.md#loki-configmigrating-from-promtail)
     5. [Running](./getting-started.md#running)
 3. [Configuration Reference](./configuration-reference.md)
-4. [API](./api.md)
-5. [Scraping Service Mode](./scraping-service.md)
-6. [Operation Guide](./operation-guide.md)
-7. [Maintainers Guide](./maintaining.md)
+4. [Migration Guide](./migration-guide.md)
+5. [API](./api.md)
+6. [Scraping Service Mode](./scraping-service.md)
+7. [Operation Guide](./operation-guide.md)
+8. [Maintainers Guide](./maintaining.md)

docs/configuration-reference.md

@@ -1869,17 +1869,52 @@ metadata_config:
 
 ### loki_config
 
-The `loki_config` block configures how the Agent collects logs and sends them to a Loki push API endpoint. `loki_config` is identical to how Promtail is configured, except deprecated
-fields have been removed and the server_config is not supported.
+The `loki_config` block configures how the Agent collects logs and sends them to
+a Loki push API endpoint. `loki_config` is identical to how Promtail is
+configured, except deprecated fields have been removed and the server_config is
+not supported.
 
 Please refer to the
 [Promtail documentation](https://github.com/grafana/loki/tree/master/docs/sources/clients/promtail#client_config)
 for the supported values for these fields.
 
 ```yaml
+# Directory to store Loki Promtail positions files in. Positions files are
+# required to read logs, and are used to store the last read offset of log
+# sources. The positions files will be stored in
+# <positions_directory>/<loki_instance_config.name>.yml.
+#
+# Optional only if every config has a positions.filename manually provided.
+[positions_directory: <string>]
+
+# Loki Promtail instances to run for log collection.
+configs:
+  - [<loki_instance_config>]
+```
+
+### loki_instance_config
+
+The `loki_instance_config` block is an individual instance of Promtail with its
+own set of scrape rules and where to forward logs. It is identical to how
+Promtail is configured, except deprecated fields have been removed and the
+`server_config` block is not supported.
+
+Please refer to the
+[Promtail documentation](https://github.com/grafana/loki/tree/master/docs/sources/clients/promtail#client_config)
+for the supported values for these fields.
+
+```yaml
+# Name of this config. Required, and must be unique across all Loki configs.
+# The name of the config will be the value of a loki_config label for all
+# Loki Promtail metrics.
+name: <string>
+
 clients:
   - [<promtail.client_config>]
 
+# Optional configuration for where to store the positions files. If
+# positions.filename is left empty, the file will be stored in
+# <loki_config.positions_directory>/<loki_instance_config.name>.yml.
 [positions: <promtail.position_config>]
 
 scrape_configs:
@@ -2876,7 +2911,7 @@ which is an embedded version of
 [`elasticsearch_exporter`](https://github.com/justwatchcom/elasticsearch_exporter). This allows for
 the collection of metrics from ElasticSearch servers.
 
-Note that currently, an Agent can only collect metrics from a single ElasticSearch server. 
+Note that currently, an Agent can only collect metrics from a single ElasticSearch server.
 However, the exporter is able to collect the metrics from all nodes through that server configured.
 
 Full reference of options:
@@ -2916,10 +2951,10 @@ Full reference of options:
   #
   # Exporter-specific configuration options
   #
-  
+
   # HTTP API address of an Elasticsearch node.
   [ address : <string> | default = "http://localhost:9200" ]
-  
+
   # Timeout for trying to get stats from Elasticsearch.
   [ timeout: <duration> | default = "5s" ]
 
@@ -2934,10 +2969,10 @@ Full reference of options:
 
   # Export stats for settings of all indices of the cluster.
   [ indices_settings: <boolean> ]
-  
+
   # Export stats for cluster settings.
   [ cluster_settings: <boolean> ]
-  
+
   # Export stats for shards in the cluster (implies indices).
   [ shards: <boolean> ]
 
@@ -2955,7 +2990,7 @@ Full reference of options:
 
   # Path to PEM file that contains the corresponding cert for the private key to connect to Elasticsearch.
   [ client_cert: <string> ]
-  
+
   # Skip SSL verification when connecting to Elasticsearch.
   [ ssl_skip_verify: <boolean> ]
 ```

docs/getting-started.md

@@ -203,7 +203,9 @@ prometheus:
   # AGENT PROMETHEUS SETTINGS
 
 loki:
-  # PASTE YOUR PROMTAIL CONFIG INSIDE OF HERE
+  configs:
+  - name: default
+    # PASTE YOUR PROMTAIL CONFIG INSIDE OF HERE
 
 tempo:
   # AGENT TEMPO SETTINGS
@@ -233,17 +235,19 @@ prometheus:
         - url: http://localhost:9009/api/prom/push
 
 loki:
-  positions:
-    filename: /tmp/positions.yaml
-  scrape_configs:
-    - job_name: varlogs
-      static_configs:
-        - targets: [localhost]
-          labels:
-            job: varlogs
-            __path__: /var/log/*log
-  clients:
-    - url: http://localhost:3100/loki/api/v1/push
+  configs:
+  - name: default
+    positions:
+      filename: /tmp/positions.yaml
+    scrape_configs:
+      - job_name: varlogs
+        static_configs:
+          - targets: [localhost]
+            labels:
+              job: varlogs
+              __path__: /var/log/*log
+    clients:
+      - url: http://localhost:3100/loki/api/v1/push
 
 tempo:
   receivers:

docs/migration-guide.md

@@ -0,0 +1,58 @@
+# Migration Guide
+
+This is a guide detailing all breaking changes that have happened in prior
+releases and how to migrate to newer versions.
+
+# v0.12.0
+
+## Loki Promtail Config Change
+
+The Loki Promtail config (`loki` in the config file) has been changed to store
+configs within a `configs` list. This allows for defining multiple Loki
+Promtail instances for collecting logs and forwarding them to different Loki
+servers.
+
+To migrate, add a `configs:` array and move your existing config inside of it.
+Give the element a `name: default` field.
+
+Each config must have a unique non-empty name. `default` is recommended for users
+that don't have other configs. The name of the config will be added as a
+`loki_config` label for Loki Promtail metrics.
+
+Example old config:
+
+```yaml
+loki:
+  positions:
+    filename: /tmp/positions.yaml
+  clients:
+    - url: http://loki:3100/loki/api/v1/push
+  scrape_configs:
+  - job_name: system
+    static_configs:
+      - targets:
+        - localhost
+        labels:
+          job: varlogs
+          __path__: /var/log/*log
+```
+
+Example migrated config:
+
+```yaml
+loki:
+  configs:
+  - name: default
+    positions:
+      filename: /tmp/positions.yaml
+    clients:
+      - url: http://loki:3100/loki/api/v1/push
+    scrape_configs:
+    - job_name: system
+      static_configs:
+        - targets:
+          - localhost
+          labels:
+            job: varlogs
+            __path__: /var/log/*log
+```

cmd/agent/agent-integrations-config.yaml → example/docker-compose/agent/config/agent-local.yaml

@@ -1,3 +1,4 @@
+# agent-local is intended to be used with docker-compose.integrations.yaml.
 server:
   log_level: info
   http_listen_port: 12345
@@ -6,6 +7,29 @@ prometheus:
   wal_directory: /tmp/agent
   global:
     scrape_interval: 5s
+  configs:
+    - name: default
+      scrape_configs:
+      - job_name: cortex
+        static_configs:
+          - targets: ['localhost:9009']
+      - job_name: loki
+        static_configs:
+          - targets: ['localhost:3100']
+
+loki:
+  positions_directory: /tmp/loki-positions
+  configs:
+  - name: default
+    clients:
+      - url: http://localhost:3100/loki/api/v1/push
+    scrape_configs:
+    - job_name: system
+      static_configs:
+      - targets: ['localhost']
+        labels:
+          job: varlogs
+          __path__: /var/log/*log
 
 integrations:
   agent:
@@ -45,4 +69,3 @@ integrations:
     address: http://localhost:9200
   prometheus_remote_write:
     - url: http://localhost:9009/api/prom/push
-

example/docker-compose/agent/config/agent.yaml

@@ -46,18 +46,20 @@ prometheus:
         - url: http://cortex:9009/api/prom/push
 
 loki:
-  positions:
-    filename: /tmp/positions.yaml
-  clients:
-    - url: http://loki:3100/loki/api/v1/push
-  scrape_configs:
-  - job_name: system
-    static_configs:
-      - targets:
-        - localhost
-        labels:
-          job: varlogs
-          __path__: /var/log/*log
+  configs:
+  - name: default
+    positions:
+      filename: /tmp/positions.yaml
+    clients:
+      - url: http://loki:3100/loki/api/v1/push
+    scrape_configs:
+    - job_name: system
+      static_configs:
+        - targets:
+          - localhost
+          labels:
+            job: varlogs
+            __path__: /var/log/*log
 
 tempo:
   receivers:

example/docker-compose/docker-compose.integrations.yaml

@@ -4,6 +4,9 @@
 # - integration scrape targets
 #
 # that enable local testing of all components of the Agent.
+#
+# The agent/config/agent-local.yaml file holds a config to 
+# scrape all of these services.
 version: "2"
 services:
 
@@ -109,4 +112,4 @@ services:
 
 volumes:
   elasticsearch_data:
-    driver: local
+    driver: local

pkg/config/config.go

@@ -63,7 +63,6 @@ func (c *Config) RegisterFlags(f *flag.FlagSet) {
 	c.Server.RegisterInstrumentation = true
 	c.Prometheus.RegisterFlags(f)
 	c.Server.RegisterFlags(f)
-	c.Loki.RegisterFlags(f)
 }
 
 // LoadFile reads a file and passes the contents to Load