Development Docker Compose Setup (#3941)

* Adds the first version of a docker compose dev stack.

Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com>

* Add index-gateway and code source.

Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com>

* Better docker ignore.

Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com>

* better bash comment

Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com>

* Update tools/dev/loki-boltdb-storage-s3/README.md

Co-authored-by: Karsten Jeschkies <k@jeschkies.xyz>

* lint bash scripts.

Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com>

Co-authored-by: Karsten Jeschkies <k@jeschkies.xyz>

tools/dev/loki-boltdb-storage-s3/.dockerignore

@@ -0,0 +1 @@
+.data-*

tools/dev/loki-boltdb-storage-s3/.gitignore

@@ -0,0 +1,3 @@
+loki
+.data*/
+.src/

tools/dev/loki-boltdb-storage-s3/README.md

@@ -0,0 +1,69 @@
+# Loki Docker-Compose
+
+This folder contains the necessary to run the Loki distributed setup locally on docker-compose.
+
+It runs the current code base in the repository this means you can debug new features and iterate very quickly by simply restarting the compose project.
+
+To start the stack simply run:
+
+```bash
+./tools/dev/loki-boltdb-storage-s3/compose-up.sh
+```
+
+You can then access grafana locally with http://localhost:3000 (default account admin/admin). The grafana container should already have the datasource to correctly query the frontend.
+
+To tear it down use:
+
+```bash
+./tools/dev/loki-boltdb-storage-s3/compose-down.sh
+```
+
+> On MacOS :apple: docker can get stuck when restarting the stack, you can restart docker to workaround the problem :shrug:
+
+## Configuration
+
+The configuration of the stack is in the [`config/`](./config/) folder.
+The stack currently runs boltdb-shipper storage on minio (via s3 compatible API). All containers data are stored under `.data-.*` if you need it for debugging purpose.
+
+Logs from all containers are ingested via the [loki docker driver](https://grafana.com/docs/loki/latest/clients/docker-driver/).
+
+## Remote Debugging
+
+You can also remote debug all Loki containers which are built in debug more. Each container automatically runs [`dlv`](https://github.com/go-delve/delve) headless and then start Loki process.
+You'll need to grab the `dlv` port from the [docker-compose file](./docker-compose.yml) for the container you want to debug. (a different one is used per container.)
+
+### dlv
+
+For example, if command line tooling is your thing, you can debug `ingester-1` with dlv by simply running:
+
+```bash
+dlv connect 127.0.0.1:18002
+```
+
+### vs-code
+
+If you use vs-code, you can add this snippet bellow in your [`launch.json`](https://code.visualstudio.com/docs/editor/debugging) file:
+
+```json
+{
+    "name": "Launch Loki remote",
+    "type": "go",
+    "request": "attach",
+    "mode": "remote",
+    "substitutePath": [
+        {
+            "from": "${workspaceFolder}",
+            "to": "${workspaceFolder}",
+        },
+    ],
+    "port": 18002,
+    "host": "127.0.0.1",
+    "cwd": "${workspaceFolder}/tools/dev/loki-boltdb-storage-s3/loki",
+    "remotePath": "/loki/loki",
+    "showLog": true,
+    "trace": "log",
+    "logOutput": "rpc",
+},
+```
+
+Then you can debug `ingester-1` with the `Launch Loki remote` configuration within the debugging tab.

tools/dev/loki-boltdb-storage-s3/compose-down.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+
+docker-compose -f "${SCRIPT_DIR}"/docker-compose.yml down  --remove-orphans

tools/dev/loki-boltdb-storage-s3/compose-up.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+
+SRC_DEST="${SCRIPT_DIR}/.src/"
+# sync all sources for dlv
+rm -Rf "${SRC_DEST}"
+mkdir "${SRC_DEST}"
+for d in cmd pkg vendor clients
+do
+    cp -Rf "${SCRIPT_DIR}/../../../${d}/" "${SRC_DEST}/${d}/"
+done
+
+# build loki -gcflags "all=-N -l" disables optimizations that allow for better run with combination with Delve debugger.
+CGO_ENABLED=0 GOOS=linux go build -mod=vendor -gcflags "all=-N -l" -o "${SCRIPT_DIR}/loki" "${SCRIPT_DIR}/../../../cmd/loki"
+# ## install loki driver to send logs
+docker plugin install grafana/loki-docker-driver:latest --alias loki-compose --grant-all-permissions || true
+# build the compose image
+docker compose -f "${SCRIPT_DIR}"/docker-compose.yml build distributor
+# cleanup sources
+rm -Rf "${SRC_DEST}"
+docker compose -f "${SCRIPT_DIR}"/docker-compose.yml up "$@"

tools/dev/loki-boltdb-storage-s3/config/datasource.yaml

@@ -0,0 +1,6 @@
+apiVersion: 1
+datasources:
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://query-frontend:8007

tools/dev/loki-boltdb-storage-s3/config/loki.yaml

@@ -0,0 +1,142 @@
+auth_enabled: false
+chunk_store_config:
+    chunk_cache_config:
+        memcached:
+            batch_size: 256
+            parallelism: 10
+        memcached_client:
+            addresses: memcached:11211
+compactor:
+    compaction_interval: 1m
+    retention_delete_worker_count: 500
+    retention_enabled: true
+    shared_store: s3
+    working_directory: /data/compactor
+distributor:
+    ring:
+        kvstore:
+            consul:
+                consistent_reads: false
+                host: consul:8500
+                http_client_timeout: 20s
+                watch_burst_size: 1
+                watch_rate_limit: 1
+            store: consul
+frontend:
+    compress_responses: true
+    log_queries_longer_than: 5s
+    max_outstanding_per_tenant: 512
+frontend_worker:
+    frontend_address: query-frontend:9007
+    grpc_client_config:
+        max_send_msg_size: 1.048576e+08
+    parallelism: 6
+ingester:
+    chunk_block_size: 262144
+    chunk_encoding: snappy
+    chunk_idle_period: 15m
+    chunk_retain_period: 6m
+    chunk_target_size: 1.572864e+06
+    flush_op_timeout: 30s
+    lifecycler:
+        heartbeat_period: 10s
+        interface_names:
+          - eth0
+        join_after: 30s
+        num_tokens: 512
+        ring:
+            heartbeat_timeout: 10m
+            kvstore:
+                consul:
+                    consistent_reads: true
+                    host: consul:8500
+                    http_client_timeout: 20s
+                store: consul
+            replication_factor: 3
+    max_transfer_retries: 0
+    sync_min_utilization: 0.2
+    sync_period: 15m
+    wal:
+        dir: /data/wal
+        enabled: true
+        replay_memory_ceiling: 7GB
+ingester_client:
+    grpc_client_config:
+        max_recv_msg_size: 6.7108864e+07
+    remote_timeout: 1s
+limits_config:
+    cardinality_limit: 100000
+    enforce_metric_name: false
+    ingestion_burst_size_mb: 5
+    ingestion_rate_mb: 2
+    ingestion_rate_strategy: global
+    max_concurrent_tail_requests: 5
+    max_global_streams_per_user: 5000
+    max_label_name_length: 1024
+    max_label_names_per_series: 15
+    max_label_value_length: 2048
+    max_line_size: 65536
+    max_query_length: 721h
+    max_query_lookback: 744h
+    max_query_parallelism: 32
+    max_streams_matchers_per_query: 128
+    max_streams_per_user: 0
+    reject_old_samples: true
+    reject_old_samples_max_age: 168h
+    split_queries_by_interval: 15m
+querier:
+    engine:
+        timeout: 5m
+    query_ingesters_within: 2h
+    query_timeout: 5m
+query_range:
+    align_queries_with_step: true
+    cache_results: true
+    max_retries: 5
+    parallelise_shardable_queries: true
+    results_cache:
+        cache:
+            memcached_client:
+                addresses: memcached:11211
+    split_queries_by_interval: 30m
+schema_config:
+    configs:
+      - from: "2020-07-30"
+        index:
+            period: 24h
+            prefix: loki_boltdb_shipper_index_
+        object_store: s3
+        schema: v11
+        store: boltdb-shipper
+server:
+    graceful_shutdown_timeout: 5s
+    grpc_server_max_concurrent_streams: 1000
+    grpc_server_max_recv_msg_size: 1.048576e+08
+    grpc_server_max_send_msg_size: 1.048576e+08
+    http_listen_port: 3100
+    http_server_idle_timeout: 120s
+    http_server_write_timeout: 6m
+    log_level: debug
+storage_config:
+    boltdb_shipper:
+        active_index_directory: /data/index
+        cache_location: /data/boltdb-cache
+        index_gateway_client:
+            server_address: index-gateway:9008
+        shared_store: s3
+    aws:
+      s3: s3://loki:supersecret@minio:9000/loki
+      s3forcepathstyle: true
+      insecure: true
+      endpoint: minio:9000
+    index_queries_cache_config:
+        memcached:
+            batch_size: 256
+            parallelism: 10
+        memcached_client:
+            addresses: memcached:11211
+table_manager:
+    creation_grace_period: 3h
+    poll_interval: 10m
+    retention_deletes_enabled: false
+    retention_period: 0

tools/dev/loki-boltdb-storage-s3/dev.dockerfile

@@ -0,0 +1,11 @@
+FROM golang:1.16
+ENV CGO_ENABLED=0
+RUN go get github.com/go-delve/delve/cmd/dlv
+
+FROM alpine:3.13
+
+RUN     mkdir /loki
+WORKDIR /loki
+ADD     ./loki ./
+ADD     ./.src ./src
+COPY --from=0 /go/bin/dlv ./