Add logging support to operator

CHANGELOG.md

@@ -8,6 +8,8 @@
 
 - [FEATURE] Put Tests requiring Network Access behind a -online flag (@flokli)
 
+- [FEATURE] Add logging support to the Grafana Agent Operator. (@rfratto)
+
 - [ENHANCEMENT] The Grafana Agent Operator will now default to deploying
   the matching release version of the Grafana Agent instead of v0.14.0.
   (@rfratto)
@@ -20,7 +22,7 @@
   the $HOSTNAME variable in agent config.  (@dfrankel33)
 
 - [BUGFIX] Regex capture groups like `${1}` will now be kept intact when
-  using `-config.expand-env`.
+  using `-config.expand-env`. (@rfratto)
 
 - [BUGFIX] The directory of the logs positions file will now properly be created
   on startup for all instances.

Makefile

@@ -133,7 +133,7 @@ endif
 # CRDs #
 ########
 
-crds:
+crds: build-image/.uptodate
 ifeq ($(BUILD_IN_CONTAINER),true)
 	mkdir -p $(shell pwd)/.pkg
 	mkdir -p $(shell pwd)/.cache

cmd/agent-operator/agent-example-config.yaml

@@ -6,7 +6,7 @@ metadata:
   labels:
     app: grafana-agent-example
 spec:
-  image: grafana/agent:v0.14.0
+  image: grafana/agent:v0.18.2
   logLevel: info
   serviceAccountName: grafana-agent
   storage:
@@ -15,6 +15,10 @@ spec:
         resources:
           requests:
             storage: 1Gi
+  logs:
+    instanceSelector:
+      matchLabels:
+        agent: grafana-agent-example
   prometheus:
     instanceSelector:
       matchLabels:
@@ -47,19 +51,56 @@ spec:
 
 ---
 
+apiVersion: monitoring.grafana.com/v1alpha1
+kind: LogsInstance
+metadata:
+  name: primary
+  namespace: default
+  labels:
+    agent: grafana-agent-example
+spec:
+  clients:
+  - url: http://loki:8080/loki/api/v1/push
+
+  # Supply an empty namespace selector to look in all namespaces.
+  podLogsNamespaceSelector: {}
+  podLogsSelector:
+    matchLabels:
+      instance: primary
+
+---
+
+# Have the Agent monitor itself.
 apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
+kind: PodMonitor
+metadata:
+  name: grafana-agents
+  namespace: default
+  labels:
+    instance: primary
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: grafana-agent
+  podMetricsEndpoints:
+    - port: http-metrics
+
+---
+
+# Have the Agent get logs from itself.
+apiVersion: monitoring.grafana.com/v1alpha1
+kind: PodLogs
 metadata:
-  name: kube-dns
-  namespace: kube-system
+  name: grafana-agents
+  namespace: default
   labels:
     instance: primary
 spec:
   selector:
     matchLabels:
-      k8s-app: kube-dns
-  endpoints:
-  - port: metrics
+      app.kubernetes.io/name: grafana-agent
+  pipelineStages:
+    - cri: {}
 
 #
 # Pretend credentials

cmd/agent-operator/example-grafana.yaml

@@ -0,0 +1,135 @@
+## example-grafana.yaml contains a tiny Grafana deployment used for testing the
+## Grafana Agent Operator.
+
+---
+
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: grafana
+  namespace: default
+  labels:
+    app: grafana
+spec:
+  serviceName: grafana
+  replicas: 1
+  selector:
+    matchLabels:
+      app: grafana
+  template:
+    metadata:
+      labels:
+        app: grafana
+    spec:
+      containers:
+      - name: grafana
+        image: grafana/grafana:8.0.4
+        args:
+          - --homepath=/usr/share/grafana
+          - --config=/etc/grafana-config/grafana.ini
+        ports:
+          - name: http
+            containerPort: 8080
+        volumeMounts:
+          - mountPath: /etc/grafana/provisioning/datasources
+            name: grafana-datasources
+            readOnly: true
+          - mountPath: /etc/grafana-config
+            name: config
+            readOnly: true
+      volumes:
+        - name: config
+          configMap:
+            name: grafana-config
+        - name: grafana-datasources
+          configMap:
+            name: grafana-datasources
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: grafana
+  namespace: default
+  labels:
+    app: grafana
+spec:
+  selector:
+    app: grafana
+  ports:
+  - name: http
+    port: 8080
+    targetPort: 8080
+
+---
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-config
+  namespace: default
+  labels:
+    app: grafana
+data:
+  grafana.ini: |
+    [server]
+    http_port = 8080
+    root_url = http://grafana.k3d.localhost:30080/
+
+    [auth.anonymous]
+    enabled = true
+    org_role = "Admin"
+
+    [analytics]
+    reporting_enabled = false
+
+    [users]
+    default_theme = 'dark'
+
+    [explore]
+    enabled = true
+
+---
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-datasources
+  namespace: default
+  labels:
+    app: grafana
+data:
+  loki.yml: |
+    apiVersion: 1
+    datasources:
+    - name: loki
+      type: loki
+      access: proxy
+      url: http://loki.default.svc.cluster.local:8080
+      isDefault: true
+      version: 1
+      editable: false
+      jsonData:
+        httpMethod: GET
+
+---
+
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: grafana
+  namespace: default
+  labels:
+    app: grafana
+spec:
+  rules:
+  - host: grafana.k3d.localhost
+    http:
+      paths:
+        - path: /
+          pathType: Prefix
+          backend:
+            service:
+              name: grafana
+              port: { name: 'http' }

cmd/agent-operator/example-loki.yaml

@@ -0,0 +1,112 @@
+## example-loki.yaml contains a tiny Loki deployment used for testing the
+## Grafana Agent Operator.
+
+---
+
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: loki
+  namespace: default
+  labels:
+    app: loki
+spec:
+  serviceName: loki
+  replicas: 1
+  selector:
+    matchLabels:
+      app: loki
+  template:
+    metadata:
+      labels:
+        app: loki
+    spec:
+      containers:
+      - name: loki
+        image: grafana/loki:2.2.1
+        args:
+          - --config.file=/etc/loki/config.yml
+          - --server.http-listen-port=8080
+        ports:
+          - name: http-metrics
+            containerPort: 9090
+        volumeMounts:
+          - mountPath: /etc/loki
+            name: config
+            readOnly: true
+          - mountPath: /var/lib/data
+            name: data
+      volumes:
+        - name: config
+          configMap:
+            name: loki-config
+        - name: data
+          emptyDir: {}
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: loki
+  namespace: default
+  labels:
+    app: loki
+spec:
+  selector:
+    app: loki
+  ports:
+  - name: http-metrics
+    port: 8080
+    targetPort: 8080
+
+---
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: loki-config
+  namespace: default
+  labels:
+    app: loki
+data:
+  config.yml: |
+    auth_enabled: false
+    server:
+      graceful_shutdown_timeout: 5m
+      http_server_idle_timeout: 120s
+      grpc_server_max_recv_msg_size: 25165824
+    limits_config:
+      enforce_metric_name: false
+      reject_old_samples: true
+      reject_old_samples_max_age: 24h
+    ingester:
+      chunk_idle_period: 5m
+      chunk_retain_period: 30s
+      max_transfer_retries: 1
+      lifecycler:
+        address: 127.0.0.1
+        final_sleep: 0s
+        ring:
+          kvstore:
+            store: inmemory
+          replication_factor: 1
+    schema_config:
+      configs:
+      - from: 2021-07-16
+        store: boltdb
+        object_store: filesystem
+        schema: v11
+        index:
+          prefix: index_
+          period: 24h
+    storage_config:
+      boltdb:
+        directory: /var/lib/data/index
+      filesystem:
+        directory: /var/lib/data/chunks
+    chunk_store_config:
+      max_look_back_period: 0
+    table_manager:
+      retention_deletes_enabled: true
+      retention_period: 48h

pkg/operator/apis/monitoring/v1alpha1/types_logs.go

@@ -153,8 +153,6 @@ type LogsInstanceList struct {
 }
 
 // +kubebuilder:object:root=true
-// +kubebuilder:resource:path="pod-logs"
-// +kubebuilder:resource:singular="pod-logs"
 // +kubebuilder:resource:categories="agent-operator"
 
 // PodLogs defines how to collect logs for a pod.
@@ -365,7 +363,17 @@ type MatchStageSpec struct {
 
 	// Nested set of pipeline stages to execute when action: keep and the log
 	// line matches selector.
-	Stages []*PipelineStageSpec `json:"stages,omitempty"`
+	//
+	// An example value for stages may be:
+	//
+	//   stages: |
+	//     - json: {}
+	//     - labelAllow: [foo, bar]
+	//
+	// Note that stages is a string because SIG API Machinery does not
+	// support recursive types, and so it cannot be validated for correctness. Be
+	// careful not to mistype anything.
+	Stages string `json:"stages,omitempty"`
 }
 
 // MetricsStageSpec is an action stage that allows for defining and updating