diff --git a/tools/dev/loki-boltdb-storage-s3/.dockerignore b/tools/dev/loki-boltdb-storage-s3/.dockerignore
new file mode 100644
index 000000000000..2157ef4cf22d
--- /dev/null
+++ b/tools/dev/loki-boltdb-storage-s3/.dockerignore
@@ -0,0 +1 @@
+.data-*
diff --git a/tools/dev/loki-boltdb-storage-s3/.gitignore b/tools/dev/loki-boltdb-storage-s3/.gitignore
new file mode 100644
index 000000000000..3d2fe38dc4fe
--- /dev/null
+++ b/tools/dev/loki-boltdb-storage-s3/.gitignore
@@ -0,0 +1,3 @@
+loki
+.data*/
+.src/
diff --git a/tools/dev/loki-boltdb-storage-s3/README.md b/tools/dev/loki-boltdb-storage-s3/README.md
new file mode 100644
index 000000000000..0925225cddca
--- /dev/null
+++ b/tools/dev/loki-boltdb-storage-s3/README.md
@@ -0,0 +1,69 @@
+# Loki Docker-Compose
+
+This folder contains the necessary to run the Loki distributed setup locally on docker-compose.
+
+It runs the current code base in the repository this means you can debug new features and iterate very quickly by simply restarting the compose project.
+
+To start the stack simply run:
+
+```bash
+./tools/dev/loki-boltdb-storage-s3/compose-up.sh
+```
+
+You can then access grafana locally with http://localhost:3000 (default account admin/admin). The grafana container should already have the datasource to correctly query the frontend.
+
+To tear it down use:
+
+```bash
+./tools/dev/loki-boltdb-storage-s3/compose-down.sh
+```
+
+> On MacOS :apple: docker can get stuck when restarting the stack, you can restart docker to workaround the problem :shrug:
+
+## Configuration
+
+The configuration of the stack is in the [`config/`](./config/) folder.
+The stack currently runs boltdb-shipper storage on minio (via s3 compatible API). All containers data are stored under `.data-.*` if you need it for debugging purpose.
+
+Logs from all containers are ingested via the [loki docker driver](https://grafana.com/docs/loki/latest/clients/docker-driver/).
+
+## Remote Debugging
+
+You can also remote debug all Loki containers which are built in debug more. Each container automatically runs [`dlv`](https://github.com/go-delve/delve) headless and then start Loki process.
+You'll need to grab the `dlv` port from the [docker-compose file](./docker-compose.yml) for the container you want to debug. (a different one is used per container.)
+
+### dlv
+
+For example, if command line tooling is your thing, you can debug `ingester-1` with dlv by simply running:
+
+```bash
+dlv connect 127.0.0.1:18002
+```
+
+### vs-code
+
+If you use vs-code, you can add this snippet bellow in your [`launch.json`](https://code.visualstudio.com/docs/editor/debugging) file:
+
+```json
+{
+    "name": "Launch Loki remote",
+    "type": "go",
+    "request": "attach",
+    "mode": "remote",
+    "substitutePath": [
+        {
+            "from": "${workspaceFolder}",
+            "to": "${workspaceFolder}",
+        },
+    ],
+    "port": 18002,
+    "host": "127.0.0.1",
+    "cwd": "${workspaceFolder}/tools/dev/loki-boltdb-storage-s3/loki",
+    "remotePath": "/loki/loki",
+    "showLog": true,
+    "trace": "log",
+    "logOutput": "rpc",
+},
+```
+
+Then you can debug `ingester-1` with the `Launch Loki remote` configuration within the debugging tab.
diff --git a/tools/dev/loki-boltdb-storage-s3/compose-down.sh b/tools/dev/loki-boltdb-storage-s3/compose-down.sh
new file mode 100755
index 000000000000..7e20021c792c
--- /dev/null
+++ b/tools/dev/loki-boltdb-storage-s3/compose-down.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+
+docker-compose -f "${SCRIPT_DIR}"/docker-compose.yml down  --remove-orphans
diff --git a/tools/dev/loki-boltdb-storage-s3/compose-up.sh b/tools/dev/loki-boltdb-storage-s3/compose-up.sh
new file mode 100755
index 000000000000..df1077c4a0de
--- /dev/null
+++ b/tools/dev/loki-boltdb-storage-s3/compose-up.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+
+SRC_DEST="${SCRIPT_DIR}/.src/"
+# sync all sources for dlv
+rm -Rf "${SRC_DEST}"
+mkdir "${SRC_DEST}"
+for d in cmd pkg vendor clients
+do
+    cp -Rf "${SCRIPT_DIR}/../../../${d}/" "${SRC_DEST}/${d}/"
+done
+
+# build loki -gcflags "all=-N -l" disables optimizations that allow for better run with combination with Delve debugger.
+CGO_ENABLED=0 GOOS=linux go build -mod=vendor -gcflags "all=-N -l" -o "${SCRIPT_DIR}/loki" "${SCRIPT_DIR}/../../../cmd/loki"
+# ## install loki driver to send logs
+docker plugin install grafana/loki-docker-driver:latest --alias loki-compose --grant-all-permissions || true
+# build the compose image
+docker compose -f "${SCRIPT_DIR}"/docker-compose.yml build distributor
+# cleanup sources
+rm -Rf "${SRC_DEST}"
+docker compose -f "${SCRIPT_DIR}"/docker-compose.yml up "$@"
diff --git a/tools/dev/loki-boltdb-storage-s3/config/datasource.yaml b/tools/dev/loki-boltdb-storage-s3/config/datasource.yaml
new file mode 100644
index 000000000000..1b103960f1e6
--- /dev/null
+++ b/tools/dev/loki-boltdb-storage-s3/config/datasource.yaml
@@ -0,0 +1,6 @@
+apiVersion: 1
+datasources:
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://query-frontend:8007
diff --git a/tools/dev/loki-boltdb-storage-s3/config/loki.yaml b/tools/dev/loki-boltdb-storage-s3/config/loki.yaml
new file mode 100644
index 000000000000..9b67b6ee3f59
--- /dev/null
+++ b/tools/dev/loki-boltdb-storage-s3/config/loki.yaml
@@ -0,0 +1,142 @@
+auth_enabled: false
+chunk_store_config:
+    chunk_cache_config:
+        memcached:
+            batch_size: 256
+            parallelism: 10
+        memcached_client:
+            addresses: memcached:11211
+compactor:
+    compaction_interval: 1m
+    retention_delete_worker_count: 500
+    retention_enabled: true
+    shared_store: s3
+    working_directory: /data/compactor
+distributor:
+    ring:
+        kvstore:
+            consul:
+                consistent_reads: false
+                host: consul:8500
+                http_client_timeout: 20s
+                watch_burst_size: 1
+                watch_rate_limit: 1
+            store: consul
+frontend:
+    compress_responses: true
+    log_queries_longer_than: 5s
+    max_outstanding_per_tenant: 512
+frontend_worker:
+    frontend_address: query-frontend:9007
+    grpc_client_config:
+        max_send_msg_size: 1.048576e+08
+    parallelism: 6
+ingester:
+    chunk_block_size: 262144
+    chunk_encoding: snappy
+    chunk_idle_period: 15m
+    chunk_retain_period: 6m
+    chunk_target_size: 1.572864e+06
+    flush_op_timeout: 30s
+    lifecycler:
+        heartbeat_period: 10s
+        interface_names:
+          - eth0
+        join_after: 30s
+        num_tokens: 512
+        ring:
+            heartbeat_timeout: 10m
+            kvstore:
+                consul:
+                    consistent_reads: true
+                    host: consul:8500
+                    http_client_timeout: 20s
+                store: consul
+            replication_factor: 3
+    max_transfer_retries: 0
+    sync_min_utilization: 0.2
+    sync_period: 15m
+    wal:
+        dir: /data/wal
+        enabled: true
+        replay_memory_ceiling: 7GB
+ingester_client:
+    grpc_client_config:
+        max_recv_msg_size: 6.7108864e+07
+    remote_timeout: 1s
+limits_config:
+    cardinality_limit: 100000
+    enforce_metric_name: false
+    ingestion_burst_size_mb: 5
+    ingestion_rate_mb: 2
+    ingestion_rate_strategy: global
+    max_concurrent_tail_requests: 5
+    max_global_streams_per_user: 5000
+    max_label_name_length: 1024
+    max_label_names_per_series: 15
+    max_label_value_length: 2048
+    max_line_size: 65536
+    max_query_length: 721h
+    max_query_lookback: 744h
+    max_query_parallelism: 32
+    max_streams_matchers_per_query: 128
+    max_streams_per_user: 0
+    reject_old_samples: true
+    reject_old_samples_max_age: 168h
+    split_queries_by_interval: 15m
+querier:
+    engine:
+        timeout: 5m
+    query_ingesters_within: 2h
+    query_timeout: 5m
+query_range:
+    align_queries_with_step: true
+    cache_results: true
+    max_retries: 5
+    parallelise_shardable_queries: true
+    results_cache:
+        cache:
+            memcached_client:
+                addresses: memcached:11211
+    split_queries_by_interval: 30m
+schema_config:
+    configs:
+      - from: "2020-07-30"
+        index:
+            period: 24h
+            prefix: loki_boltdb_shipper_index_
+        object_store: s3
+        schema: v11
+        store: boltdb-shipper
+server:
+    graceful_shutdown_timeout: 5s
+    grpc_server_max_concurrent_streams: 1000
+    grpc_server_max_recv_msg_size: 1.048576e+08
+    grpc_server_max_send_msg_size: 1.048576e+08
+    http_listen_port: 3100
+    http_server_idle_timeout: 120s
+    http_server_write_timeout: 6m
+    log_level: debug
+storage_config:
+    boltdb_shipper:
+        active_index_directory: /data/index
+        cache_location: /data/boltdb-cache
+        index_gateway_client:
+            server_address: index-gateway:9008
+        shared_store: s3
+    aws:
+      s3: s3://loki:supersecret@minio:9000/loki
+      s3forcepathstyle: true
+      insecure: true
+      endpoint: minio:9000
+    index_queries_cache_config:
+        memcached:
+            batch_size: 256
+            parallelism: 10
+        memcached_client:
+            addresses: memcached:11211
+table_manager:
+    creation_grace_period: 3h
+    poll_interval: 10m
+    retention_deletes_enabled: false
+    retention_period: 0
diff --git a/tools/dev/loki-boltdb-storage-s3/dev.dockerfile b/tools/dev/loki-boltdb-storage-s3/dev.dockerfile
new file mode 100644
index 000000000000..fe6293ff5329
--- /dev/null
+++ b/tools/dev/loki-boltdb-storage-s3/dev.dockerfile
@@ -0,0 +1,11 @@
+FROM golang:1.16
+ENV CGO_ENABLED=0
+RUN go get github.com/go-delve/delve/cmd/dlv
+
+FROM alpine:3.13
+
+RUN     mkdir /loki
+WORKDIR /loki
+ADD     ./loki ./
+ADD     ./.src ./src
+COPY --from=0 /go/bin/dlv ./
diff --git a/tools/dev/loki-boltdb-storage-s3/docker-compose.yml b/tools/dev/loki-boltdb-storage-s3/docker-compose.yml
new file mode 100644
index 000000000000..cb23dba8a34e
--- /dev/null
+++ b/tools/dev/loki-boltdb-storage-s3/docker-compose.yml
@@ -0,0 +1,219 @@
+version: '3.4'
+services:
+  consul:
+    logging: &logging
+      driver: loki-compose
+      options:
+        loki-url: "http://localhost:8001/loki/api/v1/push"
+    image: consul
+    command: [ "agent", "-dev" ,"-client=0.0.0.0", "-log-level=info" ]
+    ports:
+      - 8500:8500
+
+  minio:
+    logging:
+      <<: *logging
+    image: minio/minio
+    entrypoint: sh
+    command: -c 'mkdir -p /data/loki && /usr/bin/minio server /data'
+    environment:
+      - MINIO_ACCESS_KEY=loki
+      - MINIO_SECRET_KEY=supersecret
+    ports:
+      - 9000:9000
+    volumes:
+      - .data-minio:/data:delegated
+
+  memcached:
+    logging:
+      <<: *logging
+    image: memcached:1.6
+
+  jaeger:
+    logging:
+      <<: *logging
+    image: jaegertracing/all-in-one
+    ports:
+      - 16686:16686
+      - "14268"
+
+  distributor:
+    logging:
+      <<: *logging
+    build:
+      context:    .
+      dockerfile: dev.dockerfile
+    image: loki
+    command: ["sh", "-c", "sleep 3 && exec ./dlv exec ./loki --listen=:18001 --headless=true --api-version=2 --accept-multiclient --continue -- -config.file=./config/loki.yaml -target=distributor -server.http-listen-port=8001 -server.grpc-listen-port=9001"]
+    depends_on:
+      - ingester-1
+      - ingester-2
+      - consul
+    environment:
+      - JAEGER_AGENT_HOST=jaeger
+      - JAEGER_AGENT_PORT=6831
+      - JAEGER_TAGS=app=distributor
+      - JAEGER_SAMPLER_TYPE=const
+      - JAEGER_SAMPLER_PARAM=1
+    ports:
+      - 8001:8001
+      - 18001:18001
+    volumes:
+      - ./config:/loki/config
+
+  ingester-1:
+    logging:
+      <<: *logging
+    build:
+      context:    .
+      dockerfile: dev.dockerfile
+    image: loki
+    command: ["sh", "-c", "sleep 3 && exec ./dlv exec ./loki --listen=:18002 --headless=true --api-version=2 --accept-multiclient --continue -- -config.file=./config/loki.yaml -target=ingester -server.http-listen-port=8002 -server.grpc-listen-port=9002"]
+    depends_on:
+      - consul
+      - minio
+    environment:
+      - JAEGER_AGENT_HOST=jaeger
+      - JAEGER_AGENT_PORT=6831
+      - JAEGER_TAGS=app=ingester-1
+      - JAEGER_SAMPLER_TYPE=const
+      - JAEGER_SAMPLER_PARAM=1
+    ports:
+      - 8002:8002
+      - 18002:18002
+    volumes:
+      - ./config:/loki/config
+      - .data-ingester-1:/data:delegated
+
+  ingester-2:
+    logging:
+      <<: *logging
+    build:
+      context:    .
+      dockerfile: dev.dockerfile
+    image: loki
+    command: ["sh", "-c", "sleep 3 && exec ./dlv exec ./loki --listen=:18003 --headless=true --api-version=2 --accept-multiclient --continue -- -config.file=./config/loki.yaml -target=ingester -server.http-listen-port=8003 -server.grpc-listen-port=9003"]
+    depends_on:
+      - consul
+      - minio
+    environment:
+      - JAEGER_AGENT_HOST=jaeger
+      - JAEGER_AGENT_PORT=6831
+      - JAEGER_TAGS=app=ingester-2
+      - JAEGER_SAMPLER_TYPE=const
+      - JAEGER_SAMPLER_PARAM=1
+    ports:
+      - 8003:8003
+      - 18003:18003
+    volumes:
+      - ./config:/loki/config
+      - .data-ingester-2:/data:delegated
+
+  querier:
+    logging:
+      <<: *logging
+    build:
+      context:    .
+      dockerfile: dev.dockerfile
+    image: loki
+    command: ["sh", "-c", "sleep 3 && exec ./dlv exec ./loki --listen=:18004 --headless=true --api-version=2 --accept-multiclient --continue -- -config.file=./config/loki.yaml -target=querier -server.http-listen-port=8004 -server.grpc-listen-port=9004"]
+    depends_on:
+      - consul
+      - minio
+      - query-frontend
+    environment:
+      - JAEGER_AGENT_HOST=jaeger
+      - JAEGER_AGENT_PORT=6831
+      - JAEGER_TAGS=app=querier
+      - JAEGER_SAMPLER_TYPE=const
+      - JAEGER_SAMPLER_PARAM=1
+    ports:
+      - 8004:8004
+      - 18004:18004
+    volumes:
+      - ./config:/loki/config
+      - .data-querier:/data:delegated
+
+  index-gateway:
+    logging:
+      <<: *logging
+    build:
+      context:    .
+      dockerfile: dev.dockerfile
+    image: loki
+    command: ["sh", "-c", "sleep 3 && exec ./dlv exec ./loki --listen=:18008 --headless=true --api-version=2 --accept-multiclient --continue -- -config.file=./config/loki.yaml -target=index-gateway -server.http-listen-port=8008 -server.grpc-listen-port=9008 -boltdb.shipper.query-ready-num-days=30"]
+    depends_on:
+      - consul
+      - minio
+    environment:
+      - JAEGER_AGENT_HOST=jaeger
+      - JAEGER_AGENT_PORT=6831
+      - JAEGER_TAGS=app=index-gateway
+      - JAEGER_SAMPLER_TYPE=const
+      - JAEGER_SAMPLER_PARAM=1
+    ports:
+      - 8008:8008
+      - 18008:18008
+    volumes:
+      - ./config:/loki/config
+
+  compactor:
+    logging:
+      <<: *logging
+    build:
+      context:    .
+      dockerfile: dev.dockerfile
+    image: loki
+    command: ["sh", "-c", "sleep 3 && exec ./dlv exec ./loki --listen=:18006 --headless=true --api-version=2 --accept-multiclient --continue -- -config.file=./config/loki.yaml -target=compactor -server.http-listen-port=8006 -server.grpc-listen-port=9006"]
+    depends_on:
+      - consul
+      - minio
+    environment:
+      - JAEGER_AGENT_HOST=jaeger
+      - JAEGER_AGENT_PORT=6831
+      - JAEGER_TAGS=app=compactor
+      - JAEGER_SAMPLER_TYPE=const
+      - JAEGER_SAMPLER_PARAM=1
+    ports:
+      - 8006:8006
+      - 18006:18006
+    volumes:
+      - ./config:/loki/config
+      - .data-compactor:/data:delegated
+
+  query-frontend:
+    logging:
+      <<: *logging
+    build:
+      context:    .
+      dockerfile: dev.dockerfile
+    image: loki
+    command: ["sh", "-c", "sleep 3 && exec ./dlv exec ./loki --listen=:18007 --headless=true --api-version=2 --accept-multiclient --continue -- -config.file=./config/loki.yaml -target=query-frontend -server.http-listen-port=8007 -server.grpc-listen-port=9007 -log.level=debug"]
+    depends_on:
+      - consul
+      - minio
+    environment:
+      - JAEGER_AGENT_HOST=jaeger
+      - JAEGER_AGENT_PORT=6831
+      - JAEGER_TAGS=app=query-frontend
+      - JAEGER_SAMPLER_TYPE=const
+      - JAEGER_SAMPLER_PARAM=1
+    ports:
+      - 8007:8007
+      - 18007:18007
+    volumes:
+      - ./config:/loki/config
+
+  grafana:
+    logging:
+      <<: *logging
+    image: grafana/grafana
+    depends_on:
+      - query-frontend
+      - querier
+    environment:
+      - GF_PATHS_PROVISIONING=/etc/config/grafana/provisioning
+    ports:
+      - 3000:3000
+    volumes:
+      - ./config/datasource.yaml:/etc/config/grafana/provisioning/datasources/ds.yaml