From 254746007930355964a9ac56a456be3133e38087 Mon Sep 17 00:00:00 2001
From: Cyril Tovena <cyril.tovena@gmail.com>
Date: Thu, 1 Jul 2021 20:33:18 +0200
Subject: [PATCH 1/6] Adds the first version of a docker compose dev stack.

Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com>
---
 .../dev/loki-boltdb-storage-s3/.dockerignore  |   4 +
 tools/dev/loki-boltdb-storage-s3/.gitignore   |   2 +
 .../loki-boltdb-storage-s3/compose-down.sh    |   5 +
 .../dev/loki-boltdb-storage-s3/compose-up.sh  |  12 ++
 .../loki-boltdb-storage-s3/config/loki.yaml   | 141 ++++++++++++
 .../dev/loki-boltdb-storage-s3/dev.dockerfile |  10 +
 .../loki-boltdb-storage-s3/docker-compose.yml | 200 ++++++++++++++++++
 7 files changed, 374 insertions(+)
 create mode 100644 tools/dev/loki-boltdb-storage-s3/.dockerignore
 create mode 100644 tools/dev/loki-boltdb-storage-s3/.gitignore
 create mode 100755 tools/dev/loki-boltdb-storage-s3/compose-down.sh
 create mode 100755 tools/dev/loki-boltdb-storage-s3/compose-up.sh
 create mode 100644 tools/dev/loki-boltdb-storage-s3/config/loki.yaml
 create mode 100644 tools/dev/loki-boltdb-storage-s3/dev.dockerfile
 create mode 100644 tools/dev/loki-boltdb-storage-s3/docker-compose.yml

diff --git a/tools/dev/loki-boltdb-storage-s3/.dockerignore b/tools/dev/loki-boltdb-storage-s3/.dockerignore
new file mode 100644
index 000000000000..085f68e84d4a
--- /dev/null
+++ b/tools/dev/loki-boltdb-storage-s3/.dockerignore
@@ -0,0 +1,4 @@
+.data-configstore
+.data-ingester-1
+.data-ingester-2
+.data-minio
diff --git a/tools/dev/loki-boltdb-storage-s3/.gitignore b/tools/dev/loki-boltdb-storage-s3/.gitignore
new file mode 100644
index 000000000000..0259a3579211
--- /dev/null
+++ b/tools/dev/loki-boltdb-storage-s3/.gitignore
@@ -0,0 +1,2 @@
+loki
+.data*/
diff --git a/tools/dev/loki-boltdb-storage-s3/compose-down.sh b/tools/dev/loki-boltdb-storage-s3/compose-down.sh
new file mode 100755
index 000000000000..39f37303e9f2
--- /dev/null
+++ b/tools/dev/loki-boltdb-storage-s3/compose-down.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+SCRIPT_DIR=$(cd `dirname $0` && pwd)
+
+docker-compose -f ${SCRIPT_DIR}/docker-compose.yml down
diff --git a/tools/dev/loki-boltdb-storage-s3/compose-up.sh b/tools/dev/loki-boltdb-storage-s3/compose-up.sh
new file mode 100755
index 000000000000..8cdf5c0b7f78
--- /dev/null
+++ b/tools/dev/loki-boltdb-storage-s3/compose-up.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+SCRIPT_DIR=$(cd `dirname $0` && pwd)
+
+# -gcflags "all=-N -l" disables optimizations that allow for better run with combination with Delve debugger.
+# GOARCH is not changed.
+CGO_ENABLED=0 GOOS=linux go build -mod=vendor -gcflags "all=-N -l" -o ${SCRIPT_DIR}/loki ${SCRIPT_DIR}/../../../cmd/loki
+docker plugin install grafana/loki-docker-driver:latest --alias loki-compose --grant-all-permissions || true
+# docker-compose -f ${SCRIPT_DIR}/docker-compose.yml build distributor
+docker-compose -f ${SCRIPT_DIR}/docker-compose.yml up $@
diff --git a/tools/dev/loki-boltdb-storage-s3/config/loki.yaml b/tools/dev/loki-boltdb-storage-s3/config/loki.yaml
new file mode 100644
index 000000000000..296d007d1d67
--- /dev/null
+++ b/tools/dev/loki-boltdb-storage-s3/config/loki.yaml
@@ -0,0 +1,141 @@
+auth_enabled: false
+chunk_store_config:
+    chunk_cache_config:
+        memcached:
+            batch_size: 256
+            parallelism: 10
+        memcached_client:
+            addresses: memcached:11211
+compactor:
+    compaction_interval: 1m
+    retention_delete_worker_count: 500
+    retention_enabled: true
+    shared_store: gcs
+    working_directory: /data/compactor
+distributor:
+    ring:
+        kvstore:
+            consul:
+                consistent_reads: false
+                host: consul:8500
+                http_client_timeout: 20s
+                watch_burst_size: 1
+                watch_rate_limit: 1
+            store: consul
+frontend:
+    compress_responses: true
+    log_queries_longer_than: 5s
+    max_outstanding_per_tenant: 512
+frontend_worker:
+    frontend_address: query-frontend:9007
+    grpc_client_config:
+        max_send_msg_size: 1.048576e+08
+    parallelism: 6
+ingester:
+    chunk_block_size: 262144
+    chunk_encoding: snappy
+    chunk_idle_period: 15m
+    chunk_retain_period: 6m
+    chunk_target_size: 1.572864e+06
+    flush_op_timeout: 30s
+    lifecycler:
+        heartbeat_period: 10s
+        interface_names:
+          - eth0
+        join_after: 30s
+        num_tokens: 512
+        ring:
+            heartbeat_timeout: 10m
+            kvstore:
+                consul:
+                    consistent_reads: true
+                    host: consul:8500
+                    http_client_timeout: 20s
+                store: consul
+            replication_factor: 3
+    max_transfer_retries: 0
+    sync_min_utilization: 0.2
+    sync_period: 15m
+    wal:
+        dir: /data/wal
+        enabled: true
+        replay_memory_ceiling: 7GB
+ingester_client:
+    grpc_client_config:
+        max_recv_msg_size: 6.7108864e+07
+    remote_timeout: 1s
+limits_config:
+    cardinality_limit: 100000
+    enforce_metric_name: false
+    ingestion_burst_size_mb: 5
+    ingestion_rate_mb: 2
+    ingestion_rate_strategy: global
+    max_concurrent_tail_requests: 5
+    max_global_streams_per_user: 5000
+    max_label_name_length: 1024
+    max_label_names_per_series: 15
+    max_label_value_length: 2048
+    max_line_size: 65536
+    max_query_length: 721h
+    max_query_lookback: 744h
+    max_query_parallelism: 32
+    max_streams_matchers_per_query: 128
+    max_streams_per_user: 0
+    reject_old_samples: true
+    reject_old_samples_max_age: 168h
+    split_queries_by_interval: 15m
+querier:
+    engine:
+        timeout: 5m
+    query_ingesters_within: 2h
+    query_timeout: 5m
+query_range:
+    align_queries_with_step: true
+    cache_results: true
+    max_retries: 5
+    results_cache:
+        cache:
+            memcached_client:
+                addresses: memcached:11211
+    split_queries_by_interval: 30m
+schema_config:
+    configs:
+      - from: "2020-07-30"
+        index:
+            period: 24h
+            prefix: loki_boltdb_shipper_index_
+        object_store: s3
+        schema: v11
+        store: boltdb-shipper
+server:
+    graceful_shutdown_timeout: 5s
+    grpc_server_max_concurrent_streams: 1000
+    grpc_server_max_recv_msg_size: 1.048576e+08
+    grpc_server_max_send_msg_size: 1.048576e+08
+    http_listen_port: 3100
+    http_server_idle_timeout: 120s
+    http_server_write_timeout: 6m
+    log_level: debug
+storage_config:
+    boltdb_shipper:
+        active_index_directory: /data/index
+        cache_location: /data/boltdb-cache
+        # todo index_gateway_client:
+        #     server_address: dns:///index-gateway.loki-boltdb-shipper.svc.cluster.local:9095
+        shared_store: s3
+    aws:
+      s3: s3://loki:supersecret@minio:9000/loki
+      s3forcepathstyle: true
+      insecure: true
+      endpoint: minio:9000
+    index_queries_cache_config:
+        memcached:
+            batch_size: 256
+            parallelism: 10
+        memcached_client:
+            addresses: memcached:11211
+table_manager:
+    creation_grace_period: 3h
+    poll_interval: 10m
+    retention_deletes_enabled: false
+    retention_period: 0
diff --git a/tools/dev/loki-boltdb-storage-s3/dev.dockerfile b/tools/dev/loki-boltdb-storage-s3/dev.dockerfile
new file mode 100644
index 000000000000..407850d83d31
--- /dev/null
+++ b/tools/dev/loki-boltdb-storage-s3/dev.dockerfile
@@ -0,0 +1,10 @@
+FROM golang:1.16
+ENV CGO_ENABLED=0
+RUN go get github.com/go-delve/delve/cmd/dlv
+
+FROM alpine:3.13
+
+RUN     mkdir /loki
+WORKDIR /loki
+ADD     ./loki ./
+COPY --from=0 /go/bin/dlv ./
diff --git a/tools/dev/loki-boltdb-storage-s3/docker-compose.yml b/tools/dev/loki-boltdb-storage-s3/docker-compose.yml
new file mode 100644
index 000000000000..6174b5cf1254
--- /dev/null
+++ b/tools/dev/loki-boltdb-storage-s3/docker-compose.yml
@@ -0,0 +1,200 @@
+version: '3.4'
+services:
+  consul:
+    logging: &logging
+      driver: loki-compose
+      options:
+        loki-url: "http://localhost:8001/loki/api/v1/push"
+    image: consul
+    command: [ "agent", "-dev" ,"-client=0.0.0.0", "-log-level=info" ]
+    ports:
+      - 8500:8500
+
+  minio:
+    logging:
+      <<: *logging
+    image: minio/minio
+    entrypoint: sh
+    command: -c 'mkdir -p /data/loki && /usr/bin/minio server /data'
+    environment:
+      - MINIO_ACCESS_KEY=loki
+      - MINIO_SECRET_KEY=supersecret
+    ports:
+      - 9000:9000
+    volumes:
+      - .data-minio:/data:delegated
+
+  memcached:
+    logging:
+      <<: *logging
+    image: memcached:1.6
+
+  jaeger:
+    logging:
+      <<: *logging
+    image: jaegertracing/all-in-one
+    ports:
+      - 16686:16686
+      - "14268"
+
+  distributor:
+    logging:
+      <<: *logging
+    build:
+      context:    .
+      dockerfile: dev.dockerfile
+    image: loki
+    command: ["sh", "-c", "sleep 3 && exec ./dlv exec ./loki --listen=:18001 --headless=true --api-version=2 --accept-multiclient --continue -- -config.file=./config/loki.yaml -target=distributor -server.http-listen-port=8001 -server.grpc-listen-port=9001"]
+    depends_on:
+      - consul
+      - minio
+    environment:
+      - JAEGER_AGENT_HOST=jaeger
+      - JAEGER_AGENT_PORT=6831
+      - JAEGER_TAGS=app=distributor
+      - JAEGER_SAMPLER_TYPE=const
+      - JAEGER_SAMPLER_PARAM=1
+    ports:
+      - 8001:8001
+      - 18001:18001
+    volumes:
+      - ./config:/loki/config
+
+  ingester-1:
+    logging:
+      <<: *logging
+    build:
+      context:    .
+      dockerfile: dev.dockerfile
+    image: loki
+    command: ["sh", "-c", "sleep 3 && exec ./dlv exec ./loki --listen=:18002 --headless=true --api-version=2 --accept-multiclient --continue -- -config.file=./config/loki.yaml -target=ingester -server.http-listen-port=8002 -server.grpc-listen-port=9002"]
+    depends_on:
+      - consul
+      - minio
+    environment:
+      - JAEGER_AGENT_HOST=jaeger
+      - JAEGER_AGENT_PORT=6831
+      - JAEGER_TAGS=app=ingester-1
+      - JAEGER_SAMPLER_TYPE=const
+      - JAEGER_SAMPLER_PARAM=1
+    ports:
+      - 8002:8002
+      - 18002:18002
+    volumes:
+      - ./config:/loki/config
+      - .data-ingester-1:/data:delegated
+
+  ingester-2:
+    logging:
+      <<: *logging
+    build:
+      context:    .
+      dockerfile: dev.dockerfile
+    image: loki
+    command: ["sh", "-c", "sleep 3 && exec ./dlv exec ./loki --listen=:18003 --headless=true --api-version=2 --accept-multiclient --continue -- -config.file=./config/loki.yaml -target=ingester -server.http-listen-port=8003 -server.grpc-listen-port=9003"]
+    depends_on:
+      - consul
+      - minio
+    environment:
+      - JAEGER_AGENT_HOST=jaeger
+      - JAEGER_AGENT_PORT=6831
+      - JAEGER_TAGS=app=ingester-2
+      - JAEGER_SAMPLER_TYPE=const
+      - JAEGER_SAMPLER_PARAM=1
+    ports:
+      - 8003:8003
+      - 18003:18003
+    volumes:
+      - ./config:/loki/config
+      - .data-ingester-2:/data:delegated
+
+  querier:
+    logging:
+      <<: *logging
+    build:
+      context:    .
+      dockerfile: dev.dockerfile
+    image: loki
+    command: ["sh", "-c", "sleep 3 && exec ./dlv exec ./loki --listen=:18004 --headless=true --api-version=2 --accept-multiclient --continue -- -config.file=./config/loki.yaml -target=querier -server.http-listen-port=8004 -server.grpc-listen-port=9004"]
+    depends_on:
+      - consul
+      - minio
+      - query-frontend
+    environment:
+      - JAEGER_AGENT_HOST=jaeger
+      - JAEGER_AGENT_PORT=6831
+      - JAEGER_TAGS=app=querier
+      - JAEGER_SAMPLER_TYPE=const
+      - JAEGER_SAMPLER_PARAM=1
+    ports:
+      - 8004:8004
+      - 18004:18004
+    volumes:
+      - ./config:/loki/config
+
+  # store-gateway-1:
+  #   build:
+  #     context:    .
+  #     dockerfile: dev.dockerfile
+  #   image: cortex
+  #   command: ["sh", "-c", "sleep 3 && exec ./dlv exec ./cortex --listen=:18008 --headless=true --api-version=2 --accept-multiclient --continue -- -config.file=./config/cortex.yaml -target=store-gateway -server.http-listen-port=8008 -server.grpc-listen-port=9008"]
+  #   depends_on:
+  #     - consul
+  #     - minio
+  #   environment:
+  #     - JAEGER_AGENT_HOST=jaeger
+  #     - JAEGER_AGENT_PORT=6831
+  #     - JAEGER_TAGS=app=store-gateway-1
+  #     - JAEGER_SAMPLER_TYPE=const
+  #     - JAEGER_SAMPLER_PARAM=1
+  #   ports:
+  #     - 8008:8008
+  #     - 18008:18008
+  #   volumes:
+  #     - ./config:/cortex/config
+
+  compactor:
+    logging:
+      <<: *logging
+    build:
+      context:    .
+      dockerfile: dev.dockerfile
+    image: cortex
+    command: ["sh", "-c", "sleep 3 && exec ./dlv exec ./loki --listen=:18006 --headless=true --api-version=2 --accept-multiclient --continue -- -config.file=./config/loki.yaml -target=compactor -server.http-listen-port=8006 -server.grpc-listen-port=9006"]
+    depends_on:
+      - consul
+      - minio
+    environment:
+      - JAEGER_AGENT_HOST=jaeger
+      - JAEGER_AGENT_PORT=6831
+      - JAEGER_TAGS=app=compactor
+      - JAEGER_SAMPLER_TYPE=const
+      - JAEGER_SAMPLER_PARAM=1
+    ports:
+      - 8006:8006
+      - 18006:18006
+    volumes:
+      - ./config:/loki/config
+
+  query-frontend:
+    logging:
+      <<: *logging
+    build:
+      context:    .
+      dockerfile: dev.dockerfile
+    image: loki
+    command: ["sh", "-c", "sleep 3 && exec ./dlv exec ./loki --listen=:18007 --headless=true --api-version=2 --accept-multiclient --continue -- -config.file=./config/loki.yaml -target=query-frontend -server.http-listen-port=8007 -server.grpc-listen-port=9007 -log.level=debug"]
+    depends_on:
+      - consul
+      - minio
+    environment:
+      - JAEGER_AGENT_HOST=jaeger
+      - JAEGER_AGENT_PORT=6831
+      - JAEGER_TAGS=app=query-frontend
+      - JAEGER_SAMPLER_TYPE=const
+      - JAEGER_SAMPLER_PARAM=1
+    ports:
+      - 8007:8007
+      - 18007:18007
+    volumes:
+      - ./config:/loki/config

From d097aa4ccfae264b2c17d619656020a085b694f6 Mon Sep 17 00:00:00 2001
From: Cyril Tovena <cyril.tovena@gmail.com>
Date: Mon, 5 Jul 2021 11:51:13 +0200
Subject: [PATCH 2/6] Add index-gateway and code source.

Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com>
---
 tools/dev/loki-boltdb-storage-s3/.gitignore   |  1 +
 tools/dev/loki-boltdb-storage-s3/README.md    | 69 +++++++++++++++++++
 .../dev/loki-boltdb-storage-s3/compose-up.sh  | 16 ++++-
 .../config/datasource.yaml                    |  6 ++
 .../loki-boltdb-storage-s3/config/loki.yaml   |  7 +-
 .../dev/loki-boltdb-storage-s3/dev.dockerfile |  1 +
 .../loki-boltdb-storage-s3/docker-compose.yml | 63 +++++++++++------
 7 files changed, 137 insertions(+), 26 deletions(-)
 create mode 100644 tools/dev/loki-boltdb-storage-s3/README.md
 create mode 100644 tools/dev/loki-boltdb-storage-s3/config/datasource.yaml

diff --git a/tools/dev/loki-boltdb-storage-s3/.gitignore b/tools/dev/loki-boltdb-storage-s3/.gitignore
index 0259a3579211..3d2fe38dc4fe 100644
--- a/tools/dev/loki-boltdb-storage-s3/.gitignore
+++ b/tools/dev/loki-boltdb-storage-s3/.gitignore
@@ -1,2 +1,3 @@
 loki
 .data*/
+.src/
diff --git a/tools/dev/loki-boltdb-storage-s3/README.md b/tools/dev/loki-boltdb-storage-s3/README.md
new file mode 100644
index 000000000000..165f16c2e981
--- /dev/null
+++ b/tools/dev/loki-boltdb-storage-s3/README.md
@@ -0,0 +1,69 @@
+# Loki Docker-Compose
+
+This  folder contains the necessary to run the Loki distributed setup locally on docker-compose.
+
+It runs the current code base in the repository this means you can debug new features and iterate very quickly by simply restarting the compose project.
+
+To start the stack simply run:
+
+```bash
+./tools/dev/loki-boltdb-storage-s3/compose-up.sh
+```
+
+You can then access grafana locally with http://localhost:3000 (default account admin/admin). The grafana container should already have the datasource to correctly query the frontend.
+
+To tear it down use:
+
+```bash
+./tools/dev/loki-boltdb-storage-s3/compose-down.sh
+```
+
+> On MacOS :apple: docker can get stuck when restarting the stack, you can restart docker to workaround the problem :shrug:
+
+## Configuration
+
+The configuration of the stack is in the [`config/`](./config/) folder.
+The stack currently runs boltdb-shipper storage on minio (via s3 compatible API). All containers data are stored under `.data-.*` if you need it for debugging purpose.
+
+Logs from all containers are ingested via the [loki docker driver](https://grafana.com/docs/loki/latest/clients/docker-driver/).
+
+## Remote Debugging
+
+You can also remote debug all Loki containers which are built in debug more. Each container automatically runs [`dlv`](https://github.com/go-delve/delve) headless and then start Loki process.
+You'll need to grab the `dlv` port from the [docker-compose file](./docker-compose.yml) for the container you want to debug. (a different one is used per container.)
+
+### dlv
+
+For example, if command line tooling is your thing, you can debug `ingester-1` with dlv by simply running:
+
+```bash
+dlv connect 127.0.0.1:18002
+```
+
+### vs-code
+
+If you use vs-code, you can add this snippet bellow in your [`launch.json`](https://code.visualstudio.com/docs/editor/debugging) file:
+
+```json
+{
+    "name": "Launch Loki remote",
+    "type": "go",
+    "request": "attach",
+    "mode": "remote",
+    "substitutePath": [
+        {
+            "from": "${workspaceFolder}",
+            "to": "${workspaceFolder}",
+        },
+    ],
+    "port": 18002,
+    "host": "127.0.0.1",
+    "cwd": "${workspaceFolder}/tools/dev/loki-boltdb-storage-s3/loki",
+    "remotePath": "/loki/loki",
+    "showLog": true,
+    "trace": "log",
+    "logOutput": "rpc",
+},
+```
+
+Then you can debug `ingester-1` with the `Launch Loki remote` configuration within the debugging tab.
diff --git a/tools/dev/loki-boltdb-storage-s3/compose-up.sh b/tools/dev/loki-boltdb-storage-s3/compose-up.sh
index 8cdf5c0b7f78..1d033e99e453 100755
--- a/tools/dev/loki-boltdb-storage-s3/compose-up.sh
+++ b/tools/dev/loki-boltdb-storage-s3/compose-up.sh
@@ -6,7 +6,21 @@ SCRIPT_DIR=$(cd `dirname $0` && pwd)
 
 # -gcflags "all=-N -l" disables optimizations that allow for better run with combination with Delve debugger.
 # GOARCH is not changed.
+SRC_DEST=${SCRIPT_DIR}/.src/
+# sync all sources for dlv
+rm -Rf ${SRC_DEST}
+mkdir ${SRC_DEST}
+for d in cmd pkg vendor clients
+do
+    cp -Rf ${SCRIPT_DIR}/../../../${d}/ ${SRC_DEST}/${d}/
+done
+
+# ## build loki
 CGO_ENABLED=0 GOOS=linux go build -mod=vendor -gcflags "all=-N -l" -o ${SCRIPT_DIR}/loki ${SCRIPT_DIR}/../../../cmd/loki
+# ## install loki driver to send logs
 docker plugin install grafana/loki-docker-driver:latest --alias loki-compose --grant-all-permissions || true
-# docker-compose -f ${SCRIPT_DIR}/docker-compose.yml build distributor
+# ## build the compose image
+docker-compose -f ${SCRIPT_DIR}/docker-compose.yml build distributor
+# ## cleanup sources
+rm -Rf ${SRC_DEST}
 docker-compose -f ${SCRIPT_DIR}/docker-compose.yml up $@
diff --git a/tools/dev/loki-boltdb-storage-s3/config/datasource.yaml b/tools/dev/loki-boltdb-storage-s3/config/datasource.yaml
new file mode 100644
index 000000000000..1b103960f1e6
--- /dev/null
+++ b/tools/dev/loki-boltdb-storage-s3/config/datasource.yaml
@@ -0,0 +1,6 @@
+apiVersion: 1
+datasources:
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://query-frontend:8007
diff --git a/tools/dev/loki-boltdb-storage-s3/config/loki.yaml b/tools/dev/loki-boltdb-storage-s3/config/loki.yaml
index 296d007d1d67..9b67b6ee3f59 100644
--- a/tools/dev/loki-boltdb-storage-s3/config/loki.yaml
+++ b/tools/dev/loki-boltdb-storage-s3/config/loki.yaml
@@ -10,7 +10,7 @@ compactor:
     compaction_interval: 1m
     retention_delete_worker_count: 500
     retention_enabled: true
-    shared_store: gcs
+    shared_store: s3
     working_directory: /data/compactor
 distributor:
     ring:
@@ -93,6 +93,7 @@ query_range:
     align_queries_with_step: true
     cache_results: true
     max_retries: 5
+    parallelise_shardable_queries: true
     results_cache:
         cache:
             memcached_client:
@@ -120,8 +121,8 @@ storage_config:
     boltdb_shipper:
         active_index_directory: /data/index
         cache_location: /data/boltdb-cache
-        # todo index_gateway_client:
-        #     server_address: dns:///index-gateway.loki-boltdb-shipper.svc.cluster.local:9095
+        index_gateway_client:
+            server_address: index-gateway:9008
         shared_store: s3
     aws:
       s3: s3://loki:supersecret@minio:9000/loki
diff --git a/tools/dev/loki-boltdb-storage-s3/dev.dockerfile b/tools/dev/loki-boltdb-storage-s3/dev.dockerfile
index 407850d83d31..fe6293ff5329 100644
--- a/tools/dev/loki-boltdb-storage-s3/dev.dockerfile
+++ b/tools/dev/loki-boltdb-storage-s3/dev.dockerfile
@@ -7,4 +7,5 @@ FROM alpine:3.13
 RUN     mkdir /loki
 WORKDIR /loki
 ADD     ./loki ./
+ADD     ./.src ./src
 COPY --from=0 /go/bin/dlv ./
diff --git a/tools/dev/loki-boltdb-storage-s3/docker-compose.yml b/tools/dev/loki-boltdb-storage-s3/docker-compose.yml
index 6174b5cf1254..cb23dba8a34e 100644
--- a/tools/dev/loki-boltdb-storage-s3/docker-compose.yml
+++ b/tools/dev/loki-boltdb-storage-s3/docker-compose.yml
@@ -46,8 +46,9 @@ services:
     image: loki
     command: ["sh", "-c", "sleep 3 && exec ./dlv exec ./loki --listen=:18001 --headless=true --api-version=2 --accept-multiclient --continue -- -config.file=./config/loki.yaml -target=distributor -server.http-listen-port=8001 -server.grpc-listen-port=9001"]
     depends_on:
+      - ingester-1
+      - ingester-2
       - consul
-      - minio
     environment:
       - JAEGER_AGENT_HOST=jaeger
       - JAEGER_AGENT_PORT=6831
@@ -131,27 +132,30 @@ services:
       - 18004:18004
     volumes:
       - ./config:/loki/config
+      - .data-querier:/data:delegated
 
-  # store-gateway-1:
-  #   build:
-  #     context:    .
-  #     dockerfile: dev.dockerfile
-  #   image: cortex
-  #   command: ["sh", "-c", "sleep 3 && exec ./dlv exec ./cortex --listen=:18008 --headless=true --api-version=2 --accept-multiclient --continue -- -config.file=./config/cortex.yaml -target=store-gateway -server.http-listen-port=8008 -server.grpc-listen-port=9008"]
-  #   depends_on:
-  #     - consul
-  #     - minio
-  #   environment:
-  #     - JAEGER_AGENT_HOST=jaeger
-  #     - JAEGER_AGENT_PORT=6831
-  #     - JAEGER_TAGS=app=store-gateway-1
-  #     - JAEGER_SAMPLER_TYPE=const
-  #     - JAEGER_SAMPLER_PARAM=1
-  #   ports:
-  #     - 8008:8008
-  #     - 18008:18008
-  #   volumes:
-  #     - ./config:/cortex/config
+  index-gateway:
+    logging:
+      <<: *logging
+    build:
+      context:    .
+      dockerfile: dev.dockerfile
+    image: loki
+    command: ["sh", "-c", "sleep 3 && exec ./dlv exec ./loki --listen=:18008 --headless=true --api-version=2 --accept-multiclient --continue -- -config.file=./config/loki.yaml -target=index-gateway -server.http-listen-port=8008 -server.grpc-listen-port=9008 -boltdb.shipper.query-ready-num-days=30"]
+    depends_on:
+      - consul
+      - minio
+    environment:
+      - JAEGER_AGENT_HOST=jaeger
+      - JAEGER_AGENT_PORT=6831
+      - JAEGER_TAGS=app=index-gateway
+      - JAEGER_SAMPLER_TYPE=const
+      - JAEGER_SAMPLER_PARAM=1
+    ports:
+      - 8008:8008
+      - 18008:18008
+    volumes:
+      - ./config:/loki/config
 
   compactor:
     logging:
@@ -159,7 +163,7 @@ services:
     build:
       context:    .
       dockerfile: dev.dockerfile
-    image: cortex
+    image: loki
     command: ["sh", "-c", "sleep 3 && exec ./dlv exec ./loki --listen=:18006 --headless=true --api-version=2 --accept-multiclient --continue -- -config.file=./config/loki.yaml -target=compactor -server.http-listen-port=8006 -server.grpc-listen-port=9006"]
     depends_on:
       - consul
@@ -175,6 +179,7 @@ services:
       - 18006:18006
     volumes:
       - ./config:/loki/config
+      - .data-compactor:/data:delegated
 
   query-frontend:
     logging:
@@ -198,3 +203,17 @@ services:
       - 18007:18007
     volumes:
       - ./config:/loki/config
+
+  grafana:
+    logging:
+      <<: *logging
+    image: grafana/grafana
+    depends_on:
+      - query-frontend
+      - querier
+    environment:
+      - GF_PATHS_PROVISIONING=/etc/config/grafana/provisioning
+    ports:
+      - 3000:3000
+    volumes:
+      - ./config/datasource.yaml:/etc/config/grafana/provisioning/datasources/ds.yaml

From 64220cf6143bda6f3ab2c05fd168ddc5161a920d Mon Sep 17 00:00:00 2001
From: Cyril Tovena <cyril.tovena@gmail.com>
Date: Mon, 5 Jul 2021 11:53:00 +0200
Subject: [PATCH 3/6] Better docker ignore.

Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com>
---
 tools/dev/loki-boltdb-storage-s3/.dockerignore | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/tools/dev/loki-boltdb-storage-s3/.dockerignore b/tools/dev/loki-boltdb-storage-s3/.dockerignore
index 085f68e84d4a..2157ef4cf22d 100644
--- a/tools/dev/loki-boltdb-storage-s3/.dockerignore
+++ b/tools/dev/loki-boltdb-storage-s3/.dockerignore
@@ -1,4 +1 @@
-.data-configstore
-.data-ingester-1
-.data-ingester-2
-.data-minio
+.data-*

From acf31411184598ac71fcea06f3daf3b8b1b3edb7 Mon Sep 17 00:00:00 2001
From: Cyril Tovena <cyril.tovena@gmail.com>
Date: Mon, 5 Jul 2021 11:54:34 +0200
Subject: [PATCH 4/6] better bash comment

Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com>
---
 tools/dev/loki-boltdb-storage-s3/compose-up.sh | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/tools/dev/loki-boltdb-storage-s3/compose-up.sh b/tools/dev/loki-boltdb-storage-s3/compose-up.sh
index 1d033e99e453..400735a0f75c 100755
--- a/tools/dev/loki-boltdb-storage-s3/compose-up.sh
+++ b/tools/dev/loki-boltdb-storage-s3/compose-up.sh
@@ -4,8 +4,6 @@ set -e
 
 SCRIPT_DIR=$(cd `dirname $0` && pwd)
 
-# -gcflags "all=-N -l" disables optimizations that allow for better run with combination with Delve debugger.
-# GOARCH is not changed.
 SRC_DEST=${SCRIPT_DIR}/.src/
 # sync all sources for dlv
 rm -Rf ${SRC_DEST}
@@ -15,12 +13,12 @@ do
     cp -Rf ${SCRIPT_DIR}/../../../${d}/ ${SRC_DEST}/${d}/
 done
 
-# ## build loki
+# build loki -gcflags "all=-N -l" disables optimizations that allow for better run with combination with Delve debugger.
 CGO_ENABLED=0 GOOS=linux go build -mod=vendor -gcflags "all=-N -l" -o ${SCRIPT_DIR}/loki ${SCRIPT_DIR}/../../../cmd/loki
 # ## install loki driver to send logs
 docker plugin install grafana/loki-docker-driver:latest --alias loki-compose --grant-all-permissions || true
-# ## build the compose image
+# build the compose image
 docker-compose -f ${SCRIPT_DIR}/docker-compose.yml build distributor
-# ## cleanup sources
+# cleanup sources
 rm -Rf ${SRC_DEST}
 docker-compose -f ${SCRIPT_DIR}/docker-compose.yml up $@

From cc84f4fa78fa7c8b3857d1c0553196d9f90ca180 Mon Sep 17 00:00:00 2001
From: Cyril Tovena <cyril.tovena@gmail.com>
Date: Tue, 6 Jul 2021 10:48:48 +0200
Subject: [PATCH 5/6] Update tools/dev/loki-boltdb-storage-s3/README.md

Co-authored-by: Karsten Jeschkies <k@jeschkies.xyz>
---
 tools/dev/loki-boltdb-storage-s3/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/dev/loki-boltdb-storage-s3/README.md b/tools/dev/loki-boltdb-storage-s3/README.md
index 165f16c2e981..0925225cddca 100644
--- a/tools/dev/loki-boltdb-storage-s3/README.md
+++ b/tools/dev/loki-boltdb-storage-s3/README.md
@@ -1,6 +1,6 @@
 # Loki Docker-Compose
 
-This  folder contains the necessary to run the Loki distributed setup locally on docker-compose.
+This folder contains the necessary to run the Loki distributed setup locally on docker-compose.
 
 It runs the current code base in the repository this means you can debug new features and iterate very quickly by simply restarting the compose project.
 

From 0c8aca40681075310b7ab6118b2077d51a49c173 Mon Sep 17 00:00:00 2001
From: Cyril Tovena <cyril.tovena@gmail.com>
Date: Tue, 6 Jul 2021 13:32:15 +0200
Subject: [PATCH 6/6] lint bash scripts.

Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com>
---
 .../dev/loki-boltdb-storage-s3/compose-down.sh |  4 ++--
 tools/dev/loki-boltdb-storage-s3/compose-up.sh | 18 +++++++++---------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/tools/dev/loki-boltdb-storage-s3/compose-down.sh b/tools/dev/loki-boltdb-storage-s3/compose-down.sh
index 39f37303e9f2..7e20021c792c 100755
--- a/tools/dev/loki-boltdb-storage-s3/compose-down.sh
+++ b/tools/dev/loki-boltdb-storage-s3/compose-down.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
 
-SCRIPT_DIR=$(cd `dirname $0` && pwd)
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
 
-docker-compose -f ${SCRIPT_DIR}/docker-compose.yml down
+docker-compose -f "${SCRIPT_DIR}"/docker-compose.yml down  --remove-orphans
diff --git a/tools/dev/loki-boltdb-storage-s3/compose-up.sh b/tools/dev/loki-boltdb-storage-s3/compose-up.sh
index 400735a0f75c..df1077c4a0de 100755
--- a/tools/dev/loki-boltdb-storage-s3/compose-up.sh
+++ b/tools/dev/loki-boltdb-storage-s3/compose-up.sh
@@ -2,23 +2,23 @@
 
 set -e
 
-SCRIPT_DIR=$(cd `dirname $0` && pwd)
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
 
-SRC_DEST=${SCRIPT_DIR}/.src/
+SRC_DEST="${SCRIPT_DIR}/.src/"
 # sync all sources for dlv
-rm -Rf ${SRC_DEST}
-mkdir ${SRC_DEST}
+rm -Rf "${SRC_DEST}"
+mkdir "${SRC_DEST}"
 for d in cmd pkg vendor clients
 do
-    cp -Rf ${SCRIPT_DIR}/../../../${d}/ ${SRC_DEST}/${d}/
+    cp -Rf "${SCRIPT_DIR}/../../../${d}/" "${SRC_DEST}/${d}/"
 done
 
 # build loki -gcflags "all=-N -l" disables optimizations that allow for better run with combination with Delve debugger.
-CGO_ENABLED=0 GOOS=linux go build -mod=vendor -gcflags "all=-N -l" -o ${SCRIPT_DIR}/loki ${SCRIPT_DIR}/../../../cmd/loki
+CGO_ENABLED=0 GOOS=linux go build -mod=vendor -gcflags "all=-N -l" -o "${SCRIPT_DIR}/loki" "${SCRIPT_DIR}/../../../cmd/loki"
 # ## install loki driver to send logs
 docker plugin install grafana/loki-docker-driver:latest --alias loki-compose --grant-all-permissions || true
 # build the compose image
-docker-compose -f ${SCRIPT_DIR}/docker-compose.yml build distributor
+docker compose -f "${SCRIPT_DIR}"/docker-compose.yml build distributor
 # cleanup sources
-rm -Rf ${SRC_DEST}
-docker-compose -f ${SCRIPT_DIR}/docker-compose.yml up $@
+rm -Rf "${SRC_DEST}"
+docker compose -f "${SCRIPT_DIR}"/docker-compose.yml up "$@"