diff --git a/.github/renovate.json b/.github/renovate.json
index e5a99e5c55f4f..f817e9acb8ca7 100644
--- a/.github/renovate.json
+++ b/.github/renovate.json
@@ -30,5 +30,7 @@
     "prConcurrentLimit": 10,
     "rebaseWhen": "conflicted",
     "branchPrefix": "deps-update/",
-    "postUpdateOptions": ["gomodTidy"]
+    "postUpdateOptions": ["gomodTidy"],
+    "semanticCommitType": "fix",
+    "semanticCommitScope": "deps"
   }
diff --git a/docs/sources/alert/_index.md b/docs/sources/alert/_index.md
index ead7c325e73c4..1d99e56d1a560 100644
--- a/docs/sources/alert/_index.md
+++ b/docs/sources/alert/_index.md
@@ -66,13 +66,13 @@ groups:
         annotations:
             summary: High request latency
   - name: credentials_leak
-    rules: 
+    rules:
       - alert: http-credentials-leaked
-        annotations: 
+        annotations:
           message: "{{ $labels.job }} is leaking http basic auth credentials."
         expr: 'sum by (cluster, job, pod) (count_over_time({namespace="prod"} |~ "http(s?)://(\\w+):(\\w+)@" [5m]) > 0)'
         for: 10m
-        labels: 
+        labels:
           severity: critical
 ```
 
@@ -160,7 +160,7 @@ Here is an example of a remote-write configuration for sending data to a local P
 ```yaml
 ruler:
   ... other settings ...
-  
+
   remote_write:
     enabled: true
     client:
@@ -186,13 +186,13 @@ We don't always control the source code of applications we run. Load balancers a
 Sometimes you want to know whether _any_ instance of something has occurred. Alerting based on logs can be a great way to handle this, such as finding examples of leaked authentication credentials:
 ```yaml
 - name: credentials_leak
-  rules: 
+  rules:
     - alert: http-credentials-leaked
-      annotations: 
+      annotations:
         message: "{{ $labels.job }} is leaking http basic auth credentials."
       expr: 'sum by (cluster, job, pod) (count_over_time({namespace="prod"} |~ "http(s?)://(\\w+):(\\w+)@" [5m]) > 0)'
       for: 10m
-      labels: 
+      labels:
         severity: critical
 ```
 
@@ -208,76 +208,29 @@ As an example, we can use LogQL v2 to help Loki to monitor _itself_, alerting us
 
 ## Interacting with the Ruler
 
-### Cortextool
-Because the rule files are identical to Prometheus rule files, we can interact with the Loki Ruler via [`cortextool`](https://github.com/grafana/cortex-tools#rules). The CLI is in early development, but it works with both Loki and Cortex. Pass the `--backend=loki` option when using it with Loki.
-
-{{% admonition type="note" %}}
-Not all commands in cortextool currently support Loki.
-{{% /admonition %}}
+### Lokitool
+Because the rule files are identical to Prometheus rule files, we can interact with the Loki Ruler via `lokitool`.
 
 {{% admonition type="note" %}}
-cortextool was intended to run against multi-tenant Loki, commands need an `--id=` flag set to the Loki instance ID or set the environment variable `CORTEX_TENANT_ID`.  If Loki is running in single tenant mode, the required ID is `fake`.
+lokitool is intended to run against multi-tenant Loki.  The commands need an `--id=` flag set to the Loki instance ID or set the environment variable `LOKI_TENANT_ID`.  If Loki is running in single tenant mode, the required ID is `fake`.
 {{% /admonition %}}
 
 An example workflow is included below:
 
 ```sh
 # lint the rules.yaml file ensuring it's valid and reformatting it if necessary
-cortextool rules lint --backend=loki ./output/rules.yaml
+lokitool rules lint ./output/rules.yaml
 
 # diff rules against the currently managed ruleset in Loki
-cortextool rules diff --rule-dirs=./output --backend=loki
+lokitool rules diff --rule-dirs=./output
 
 # ensure the remote ruleset matches your local ruleset, creating/updating/deleting remote rules which differ from your local specification.
-cortextool rules sync --rule-dirs=./output --backend=loki
+lokitool rules sync --rule-dirs=./output
 
 # print the remote ruleset
-cortextool rules print --backend=loki
+lokitool rules print
 ```
 
-### Cortextool Github Actions
-There is also a [github action](https://github.com/grafana/cortex-rules-action) available for `cortex-tool`, so you can add it into your CI/CD pipelines!
-
-For instance, you can sync rules on master builds via
-```yaml
-name: sync-cortex-rules-and-alerts
-on:
-  push:
-    branches:
-      - master
-env:
-  CORTEX_ADDRESS: '<fill me in>'
-  CORTEX_TENANT_ID: '<fill me in>'
-  CORTEX_API_KEY: ${{ secrets.API_KEY }}
-  RULES_DIR: 'output/'
-jobs:
-  sync-loki-alerts:
-    runs-on: ubuntu-18.04
-    steps:
-      - name: Lint Rules
-        uses: grafana/cortex-rules-action@v0.4.0
-        env:
-          ACTION: 'lint'
-        with:
-          args: --backend=loki
-      - name: Diff rules
-        uses: grafana/cortex-rules-action@v0.4.0
-        env:
-          ACTION: 'diff'
-        with:
-          args: --backend=loki
-      - name: Sync rules
-        if: ${{ !contains(steps.diff-rules.outputs.detailed, 'no changes detected') }}
-        uses: grafana/cortex-rules-action@v0.4.0
-        env:
-          ACTION: 'sync'
-        with:
-          args: --backend=loki
-      - name: Print rules
-        uses: grafana/cortex-rules-action@v0.4.0
-        env:
-          ACTION: 'print'
-```
 ### Terraform
 
 With the [Terraform provider for Loki](https://registry.terraform.io/providers/fgouteroux/loki/latest), you can manage alerts and recording rules in Terraform HCL format:
diff --git a/docs/sources/query/template_functions.md b/docs/sources/query/template_functions.md
index 0effe4c01ac75..862f302ab7659 100644
--- a/docs/sources/query/template_functions.md
+++ b/docs/sources/query/template_functions.md
@@ -11,10 +11,15 @@ weight: 30
 
 The [text template](https://golang.org/pkg/text/template) format used in `| line_format` and `| label_format` support the usage of functions.
 
+{{% admonition type="note" %}}
+In the examples below we use backticks to quote the template strings. This is because some template strings contain double quotes, and using backticks lets us avoid escaping the double quotes.
+If you are using a different quoting style, you may need to escape the double quotes.
+{{% /admonition %}}
+
 All labels are added as variables in the template engine. They can be referenced using they label name prefixed by a `.`(e.g `.label_name`). For example the following template will output the value of the path label:
 
 ```template
-{{ .path }}
+`{{ .path }}`
 ```
 
 Additionally you can also access the log line using the [`__line__`](#__line__) function and the timestamp using the [`__timestamp__`](#__timestamp__) function.
@@ -25,7 +30,7 @@ In a chained pipeline, the result of each command is passed as the last argument
 Example:
 
 ```template
-{{ .path | replace " " "_" | trunc 5 | upper }}
+`{{ .path | replace " " "_" | trunc 5 | upper }}`
 ```
 
 For function that returns a `bool` such as `contains`, `eq`, `hasPrefix` and `hasSuffix`, you can apply `AND` / `OR` and nested `if` logic.
@@ -33,9 +38,9 @@ For function that returns a `bool` such as `contains`, `eq`, `hasPrefix` and `ha
 Example:
 
 ```template
-{{ if and (contains "he" "hello") (contains "llo" "hello") }} yes {{end}}
-{{ if or (contains "he" "hello") (contains("llo" "hello") }} yes {{end}}
-{{ if contains .err "ErrTimeout" }} timeout {{else if contains "he" "hello"}} yes {{else}} no {{end}}
+`{{ if and (contains "he" "hello") (contains "llo" "hello") }} yes {{end}}`
+`{{ if or (contains "he" "hello") (contains("llo" "hello") }} yes {{end}}`
+`{{ if contains .err "ErrTimeout" }} timeout {{else if contains "he" "hello"}} yes {{else}} no {{end}}`
 ```
 
 ## __line__
@@ -47,7 +52,7 @@ Signature: `line() string`
 Examples:
 
 ```template
-"{{ __line__ | lower }}"
+`{{ __line__ | lower }}`
 `{{ __line__ }}`
 ```
 
@@ -58,7 +63,7 @@ This function returns the current log lines timestamp.
 Signature: `timestamp() time.Time`
 
 ```template
-"{{ __timestamp__ }}"
+`{{ __timestamp__ }}`
 `{{ __timestamp__ | date "2006-01-02T15:04:05.00Z-07:00" }}`
 `{{ __timestamp__ | unixEpoch }}`
 ```
@@ -92,7 +97,7 @@ Signature: `lower(string) string`
 Examples:
 
 ```template
-"{{ .request_method | lower }}"
+`{{ .request_method | lower }}`
 `{{ lower  "HELLO"}}`
 ```
 
@@ -107,7 +112,7 @@ Signature: `upper(string) string`
 Examples:
 
 ```template
-"{{ .request_method | upper }}"
+`{ .request_method | upper }}`
 `{{ upper  "hello"}}`
 ```
 
@@ -122,7 +127,7 @@ Signature: `title(string) string`
 Examples:
 
 ```template
-"{{.request_method | title}}"
+`{{.request_method | title}}`
 `{{ title "hello world"}}`
 ```
 
@@ -137,7 +142,7 @@ Signature: `trunc(count int,value string) string`
 Examples:
 
 ```template
-"{{ .path | trunc 2 }}"
+`{{ .path | trunc 2 }}`
 `{{ trunc 5 "hello world"}}`   // output: hello
 `{{ trunc -5 "hello world"}}`  // output: world
 ```
@@ -155,7 +160,7 @@ Otherwise, this calls value[start, end].
 Examples:
 
 ```template
-"{{ .path | substr 2 5 }}"
+`{{ .path | substr 2 5 }}`
 `{{ substr 0 5 "hello world"}}`  // output: hello
 `{{ substr 6 11 "hello world"}}` // output: world
 ```
@@ -175,8 +180,8 @@ It takes three arguments:
 Examples:
 
 ```template
-{{ .cluster | replace "-cluster" "" }}
-{{ replace "hello" "world" "hello world" }}
+`{{ .cluster | replace "-cluster" "" }}`
+`{{ replace "hello" "world" "hello world" }}`
 ```
 
 The last example will return `world world`.
@@ -190,8 +195,8 @@ Signature: `trim(string) string`
 Examples:
 
 ```template
-{{ .ip | trim }}
-{{ trim "   hello    " }} // output: hello
+`{{ .ip | trim }}`
+`{{ trim "   hello    " }}` // output: hello
 ```
 
 ## trimAll
@@ -203,8 +208,8 @@ Signature: `trimAll(chars string,src string) string`
 Examples:
 
 ```template
-{{ .path | trimAll "/" }}
-{{ trimAll "$" "$5.00" }} // output: 5.00
+`{{ .path | trimAll "/" }}`
+`{{ trimAll "$" "$5.00" }}` // output: 5.00
 ```
 
 ## trimSuffix
@@ -216,8 +221,8 @@ Signature: `trimSuffix(suffix string, src string) string`
 Examples:
 
 ```template
-{{  .path | trimSuffix "/" }}
-{{ trimSuffix "-" "hello-" }} // output: hello
+`{{  .path | trimSuffix "/" }}`
+`{{ trimSuffix "-" "hello-" }}` // output: hello
 ```
 
 ## trimPrefix
@@ -229,8 +234,8 @@ Signature: `trimPrefix(prefix string, src string) string`
 Examples:
 
 ```template
-{{  .path | trimPrefix "/" }}
-{{ trimPrefix "-" "-hello" }} // output: hello
+`{{  .path | trimPrefix "/" }}`
+`{{ trimPrefix "-" "-hello" }}` // output: hello
 ```
 
 ## alignLeft
@@ -268,7 +273,7 @@ Signature: `indent(spaces int,src string) string`
 Example:
 
 ```template
-{{ indent 4 .query }}
+`{{ indent 4 .query }}`
 ```
 
 This indents each line contained in the `.query` by four (4) spaces.
@@ -282,7 +287,7 @@ Signature: `nindent(spaces int,src string) string`
 Example:
 
 ```template
-{{ nindent 4 .query }}
+`{{ nindent 4 .query }}`
 ```
 
 This will indent every line of text by 4 space characters and add a new line to the beginning.
@@ -296,20 +301,20 @@ Signature: `repeat(c int,value string) string`
 Example:
 
 ```template
-{{ repeat 3 "hello" }} // output: hellohellohello
+`{{ repeat 3 "hello" }}` // output: hellohellohello
 ```
 
 ## contains
 
 Use this function to test to see if one string is contained inside of another.
 
-Signature: `contains(s string, src string) bool`
+Signature: `contains(s string, src string,) bool`
 
 Examples:
 
 ```template
-{{ if contains .err "ErrTimeout" }} timeout {{end}}
-{{ if contains "he" "hello" }} yes {{end}}
+`{{ if contains "ErrTimeout" .err }} timeout {{end}}`
+`{{ if contains "he" "hello" }} yes {{end}}`
 ```
 
 ## eq
@@ -321,8 +326,8 @@ Signature: `eq(s string, src string) bool`
 Examples:
 
 ```template
-{{ if eq .err "ErrTimeout" }} timeout {{end}}
-{{ if eq "he" "hello" }} yes {{end}}
+`{{ if eq "ErrTimeout" .err }} timeout {{end}}`
+`{{ if eq "hello" "hello" }} yes {{end}}`
 ```
 
 ## hasPrefix and hasSuffix
@@ -337,8 +342,8 @@ Signatures:
 Examples:
 
 ```template
-{{ if hasSuffix .err "Timeout" }} timeout {{end}}
-{{ if hasPrefix "he" "hello" }} yes {{end}}
+`{{ if hasSuffix .err "Timeout" }} timeout {{end}}`
+`{{ if hasPrefix "he" "hello" }} yes {{end}}`
 ```
 
 ## add
@@ -350,7 +355,7 @@ Signature: `func(i ...interface{}) int64`
 Example:
 
 ```template
-{{ add 3 2 5 }} // output: 10
+`{{ add 3 2 5 }}` // output: 10
 ```
 
 ## sub
@@ -362,7 +367,7 @@ Signature: `func(a, b interface{}) int64`
 Example:
 
 ```template
-{{ sub 5 2 }} // output: 3
+`{{ sub 5 2 }}` // output: 3
 ```
 
 ## mul
@@ -374,7 +379,7 @@ Signature: `func(a interface{}, v ...interface{}) int64`
 Example:
 
 ```template
-{{ mul 5 2 3}} // output: 30
+`{{ mul 5 2 3}}` // output: 30
 ```
 
 ## div
@@ -386,7 +391,7 @@ Signature: `func(a, b interface{}) int64`
 Example:
 
 ```template
-{{ div 10 2}} // output: 5
+`{{ div 10 2}}` // output: 5
 ```
 
 ## addf
@@ -398,7 +403,7 @@ Signature: `func(i ...interface{}) float64`
 Example:
 
 ```template
-{{ addf 3.5 2 5 }} // output: 10.5
+`{{ addf 3.5 2 5 }}` // output: 10.5
 ```
 
 ## subf
@@ -410,7 +415,7 @@ Signature: `func(a interface{}, v ...interface{}) float64`
 Example:
 
 ```template
-{{ subf  5.5 2 1.5 }} // output: 2
+`{{ subf  5.5 2 1.5 }}` // output: 2
 ```
 
 ## mulf
@@ -422,7 +427,7 @@ Signature: `func(a interface{}, v ...interface{}) float64`
 Example:
 
 ```template
-{{ mulf 5.5 2 2.5 }} // output: 27.5
+`{{ mulf 5.5 2 2.5 }}` // output: 27.5
 ```
 
 ## divf
@@ -434,7 +439,7 @@ Signature: `func(a interface{}, v ...interface{}) float64`
 Example:
 
 ```template
-{{ divf 10 2 4}} // output: 1.25
+`{{ divf 10 2 4}}` // output: 1.25
 ```
 
 ## mod
@@ -446,7 +451,7 @@ Signature: `func(a, b interface{}) int64`
 Example:
 
 ```template
-{{ mod 10 3}} // output: 1
+`{{ mod 10 3}}` // output: 1
 ```
 
 ## max
@@ -458,7 +463,7 @@ Signature: `max(a interface{}, i ...interface{}) int64`
 Example:
 
 ```template
-{{ max 1 2 3 }} //output 3
+`{{ max 1 2 3 }}` //output 3
 ```
 
 ## min
@@ -470,7 +475,7 @@ Signature: `min(a interface{}, i ...interface{}) int64`
 Example:
 
 ```template
-{{ min 1 2 3 }} //output 1
+`{{ min 1 2 3 }}`//output 1
 ```
 
 ## maxf
@@ -482,7 +487,7 @@ Signature: `maxf(a interface{}, i ...interface{}) float64`
 Example:
 
 ```template
-{{ maxf 1 2.5 3 }} //output 3
+`{{ maxf 1 2.5 3 }}` //output 3
 ```
 
 ## minf
@@ -494,7 +499,7 @@ Signature: `minf(a interface{}, i ...interface{}) float64`
 Example:
 
 ```template
-{{ minf 1 2.5 3 }} //output 1.5
+`{{ minf 1 2.5 3 }}` //output 1.5
 ```
 
 ## ceil
@@ -506,7 +511,7 @@ Signature: `ceil(a interface{}) float64`
 Example:
 
 ```template
-{{ ceil 123.001 }} //output 124.0
+`{{ ceil 123.001 }}` //output 124.0
 ```
 
 ## floor
@@ -518,7 +523,7 @@ Signature: `floor(a interface{}) float64`
 Example:
 
 ```template
-{{ floor 123.9999 }} //output 123.0
+`{{ floor 123.9999 }}` //output 123.0
 ```
 
 ## round
@@ -530,7 +535,7 @@ Signature: `round(a interface{}, p int, rOpt ...float64) float64`
 Example:
 
 ```template
-{{ round 123.555555 3 }} //output 123.556
+`{{ round 123.555555 3 }}` //output 123.556
 ```
 
 We can also provide a `roundOn` number as third parameter
@@ -538,7 +543,7 @@ We can also provide a `roundOn` number as third parameter
 Example:
 
 ```template
-{{ round 123.88571428571 5 .2 }} //output 123.88572
+`{{ round 123.88571428571 5 .2 }}` //output 123.88572
 ```
 
 With default `roundOn` of `.5` the above value would be `123.88571`
@@ -552,7 +557,7 @@ Signature: `toInt(v interface{}) int`
 Example:
 
 ```template
-{{ "3" | int }} //output 3
+`{{ "3" | int }}` //output 3
 ```
 
 ## float64
@@ -564,7 +569,7 @@ Signature: `toFloat64(v interface{}) float64`
 Example:
 
 ```template
-{{ "3.5" | float64 }} //output 3.5
+`{{ "3.5" | float64 }}` //output 3.5
 ```
 
 ## fromJson
@@ -576,13 +581,13 @@ Signature: `fromJson(v string) interface{}`
 Example:
 
 ```template
-fromJson "{\"foo\": 55}"
+`{{fromJson "{\"foo\": 55}"}}`
 ```
 
 Example of a query to print a newline per queries stored as a json array in the log line:
 
 ```logql
-{job="loki/querier"} |= "finish in prometheus" | logfmt | line_format "{{ range $q := fromJson .queries }} {{ $q.query }} {{ end }}"
+{job="loki/querier"} |= "finish in prometheus" | logfmt | line_format `{{ range $q := fromJson .queries }} {{ $q.query }} {{ end }}`
 ```
 
 ## now
@@ -594,7 +599,7 @@ Signature: `Now() time.Time`
 Example:
 
 ```template
-{{ now }}
+`{{ now }}`
 ```
 
 ## toDate
@@ -610,8 +615,8 @@ Signature: `toDate(fmt, str string) time.Time`
 Examples: 
 
 ```template
-{{ toDate "2006-01-02" "2021-11-02" }}
-{{ .foo | toDate "2006-01-02T15:04:05.999999999Z" }}
+`{{ toDate "2006-01-02" "2021-11-02" }}`
+`{{ .foo | toDate "2006-01-02T15:04:05.999999999Z" }}`
 ```
 
 ## toDateInZone
@@ -627,8 +632,8 @@ Signature: `toDateInZone(fmt, zone, str string) time.Time`
 Examples:
 
 ```template
-{{ toDateInZone "2006-01-02" "UTC" "2021-11-02" }}
-{{ .foo | toDateInZone "2006-01-02T15:04:05.999999999Z" "UTC" }}
+`{{ toDateInZone "2006-01-02" "UTC" "2021-11-02" }}`
+`{{ .foo | toDateInZone "2006-01-02T15:04:05.999999999Z" "UTC" }}`
 ```
 
 ## date
@@ -640,7 +645,7 @@ Signature: `date(fmt string, date interface{}) string`
 Example:
 
 ```template
-{{ date "2006-01-02" now }}
+`{{ date "2006-01-02" now }}`
 ```
 
 ## unixEpoch
@@ -652,13 +657,13 @@ Signature: `unixEpoch(date time.Time) string`
 Examples:
 
 ```template
-{{ unixEpoch now }}
-{{ .foo | toDateInZone "2006-01-02T15:04:05.999999999Z" "UTC" | unixEpoch }}
+`{{ unixEpoch now }}`
+`{{ .foo | toDateInZone "2006-01-02T15:04:05.999999999Z" "UTC" | unixEpoch }}`
 ```
 
 Example of a query to filter Loki querier jobs which create time is 1 day before:
 ```logql
-{job="loki/querier"} | label_format nowEpoch=`{{(unixEpoch now)}}`,createDateEpoch=`{{unixEpoch (toDate "2006-01-02" .createDate)}}` | label_format dateTimeDiff="{{sub .nowEpoch .createDateEpoch}}" | dateTimeDiff > 86400
+{job="loki/querier"} | label_format nowEpoch=`{{(unixEpoch now)}}`,createDateEpoch=`{{unixEpoch (toDate "2006-01-02" .createDate)}}` | label_format dateTimeDiff=`{{sub .nowEpoch .createDateEpoch}}` | dateTimeDiff > 86400
 ```
 
 ## unixEpochMillis
@@ -670,8 +675,8 @@ Signature: `unixEpochMillis(date time.Time) string`
 Examples:
 
 ```template
-{{ unixEpochMillis now }}
-{{ .foo | toDateInZone "2006-01-02T15:04:05.999999999Z" "UTC" | unixEpochMillis }}
+`{{ unixEpochMillis now }}`
+`{{ .foo | toDateInZone "2006-01-02T15:04:05.999999999Z" "UTC" | unixEpochMillis }}`
 ```
 
 ## unixEpochNanos
@@ -683,8 +688,8 @@ Signature: `unixEpochNanos(date time.Time) string`
 Examples:
 
 ```template
-{{ unixEpochNanos now }}
-{{ .foo | toDateInZone "2006-01-02T15:04:05.999999999Z" "UTC" | unixEpochNanos }}
+`{{ unixEpochNanos now }}`
+`{{ .foo | toDateInZone "2006-01-02T15:04:05.999999999Z" "UTC" | unixEpochNanos }}`
 ```
 
 ## unixToTime
@@ -709,8 +714,8 @@ Signature: `default(d string, src string) string`
 Examples:
 
 ```template
-{{ default "-" "" }} // output: -
-{{ default "-" "foo" }} // output: foo
+`{{ default "-" "" }}` // output: -
+`{{ default "-" "foo" }}` // output: foo
 ```
 
 Example of a query to print a `-` if the `http_request_headers_x_forwarded_for` label is empty:
@@ -727,8 +732,8 @@ Signature: `count(regex string, src string) int`
 Examples:
 
 ```template
-{{ count "a|b" "abab" }} // output: 4
-{{ count "o" "foo" }}    // output: 2
+`{{ count "a|b" "abab" }}` // output: 4
+`{{ count "o" "foo" }}`    // output: 2
 ```
 
 Example of a query to print how many times XYZ occurs in a line:
@@ -745,7 +750,7 @@ Signature: `urlencode(string) string`
 Examples:
 
 ```template
-"{{ .request_url | urlencode }}"
+`{{ .request_url | urlencode }}`
 `{{ urlencode  .request_url}}`
 ```
 
@@ -758,7 +763,7 @@ Signature: `urldecode(string) string`
 Examples:
 
 ```template
-"{{ .request_url | urldecode }}"
+`{{ .request_url | urldecode }}`
 `{{ urldecode  .request_url}}`
 ```
 
@@ -771,7 +776,7 @@ Signature: `b64enc(string) string`
 Examples:
 
 ```template
-"{{ .foo | b64enc }}"
+`{{ .foo | b64enc }}`
 `{{ b64enc  .foo }}`
 ```
 
@@ -784,7 +789,7 @@ Signature: `b64dec(string) string`
 Examples:
 
 ```template
-"{{ .foo | b64dec }}"
+`{{ .foo | b64dec }}`
 `{{ b64dec  .foo }}`
 ```
 
@@ -797,7 +802,7 @@ Signature: `bytes(string) string`
 Examples:
 
 ```template
-"{{ .foo | bytes }}"
+`{{ .foo | bytes }}`
 `{{ bytes .foo }}`
 ```
 
@@ -808,7 +813,7 @@ An alias for `duration_seconds`
 Examples:
 
 ```template
-"{{ .foo | duration }}"
+`{{ .foo | duration }}`
 `{{ duration .foo }}`
 ```
 
@@ -821,6 +826,6 @@ Signature: `duration_seconds(string) float64`
 Examples:
 
 ```template
-"{{ .foo | duration_seconds }}"
+`{{ .foo | duration_seconds }}`
 `{{ duration_seconds .foo }}`
 ```
diff --git a/docs/sources/send-data/promtail/installation.md b/docs/sources/send-data/promtail/installation.md
index 2daa12638083d..4d2359e94c171 100644
--- a/docs/sources/send-data/promtail/installation.md
+++ b/docs/sources/send-data/promtail/installation.md
@@ -23,43 +23,52 @@ See the instructions [here](https://grafana.com/docs/loki/<LOKI_VERSION>/setup/i
 
 ## Install using Docker
 
-```bash
-# modify tag to most recent version
-docker pull grafana/promtail:2.9.2
-```
+1. Make sure to modify the tag to the most recent version.
+
+    ```bash
+    docker pull grafana/promtail:3.0.0
+    ```
+
+1. Create your Promtail configuration file in a file called `promtail-config.yaml`. Refer to the [Promtail configuration reference](https://grafana.com/docs/loki/<LOKI_VERSION>/send-data/promtail/configuration/) for more details.
+
+1. Note that you will need to replace `<local-path>` in the commands with your local path.
+
+    ```bash
+    docker run -v <local-path>:/mnt/config -v /var/log:/var/log --link loki grafana/promtail:3.0.0 --config.file=/mnt/config/promtail-config.yaml
+    ```
 
 ## Install using Helm
 
-Make sure that Helm is installed.
-See [Installing Helm](https://helm.sh/docs/intro/install/).
-Then you can add Grafana's chart repository to Helm:
+1. Make sure that Helm is installed. See [Installing Helm](https://helm.sh/docs/intro/install/).
 
-```bash
-helm repo add grafana https://grafana.github.io/helm-charts
-```
+1. Then you can add Grafana's chart repository to Helm:
 
-And the chart repository can be updated by running:
+    ```bash
+    helm repo add grafana https://grafana.github.io/helm-charts
+    ```
 
-```bash
-helm repo update
-```
+1. Update the chart repository:
 
-Create the configuration file `values.yaml`. The example below illustrates a connection to the locally deployed loki server:
+    ```bash
+    helm repo update
+    ```
 
-```yaml
-config:
-  # publish data to loki
-  clients:
-    - url: http://loki-gateway/loki/api/v1/push
-      tenant_id: 1
-```
+1. Create the configuration file `values.yaml`. The example below illustrates a connection to the locally deployed loki server:
 
-Finally, Promtail can be deployed with:
+    ```yaml
+    config:
+    # publish data to loki
+      clients:
+        - url: http://loki-gateway/loki/api/v1/push
+          tenant_id: 1
+    ```
 
-```bash
-# The default helm configuration deploys promtail as a daemonSet (recommended)
-helm upgrade --values values.yaml --install promtail grafana/promtail
-```
+1. Finally, Promtail can be deployed with:
+
+    ```bash
+    # The default helm configuration deploys promtail as a daemonSet (recommended)
+    helm upgrade --values values.yaml --install promtail grafana/promtail
+    ```
 
 ## Install as Kubernetes daemonSet (recommended)
 
diff --git a/docs/sources/setup/install/docker.md b/docs/sources/setup/install/docker.md
index 8e9007c0fdf43..de2006250a7c3 100644
--- a/docs/sources/setup/install/docker.md
+++ b/docs/sources/setup/install/docker.md
@@ -9,55 +9,120 @@ weight: 400
 # Install Loki with Docker or Docker Compose
 
 You can install Loki and Promtail with Docker or Docker Compose if you are evaluating, testing, or developing Loki.
-For production, Grafana recommends installing with Tanka or Helm.
+For production, Grafana recommends installing with Helm or Tanka.
 
-The configuration acquired with these installation instructions run Loki as a single binary.
+The configuration files associated with these installation instructions run Loki as a single binary.
 
 ## Prerequisites
 
 - [Docker](https://docs.docker.com/install)
 - [Docker Compose](https://docs.docker.com/compose/install) (optional, only needed for the Docker Compose install method)
 
-## Install with Docker
+## Install with Docker on Linux
 
-**Linux**
+1. Create a directory called `loki`. Make `loki` your current working directory:
 
-Copy and paste the commands below into your command line.
+    ```bash
+    mkdir loki
+    cd loki
+    ```
 
-```bash
-wget https://raw.githubusercontent.com/grafana/loki/v2.9.2/cmd/loki/loki-local-config.yaml -O loki-config.yaml
-docker run --name loki -d -v $(pwd):/mnt/config -p 3100:3100 grafana/loki:2.9.2 -config.file=/mnt/config/loki-config.yaml
-wget https://raw.githubusercontent.com/grafana/loki/v2.9.2/clients/cmd/promtail/promtail-docker-config.yaml -O promtail-config.yaml
-docker run --name promtail -d -v $(pwd):/mnt/config -v /var/log:/var/log --link loki grafana/promtail:2.9.2 -config.file=/mnt/config/promtail-config.yaml
-```
+1. Copy and paste the following commands into your command line to download `loki-local-config.yaml` and `promtail-docker-config.yaml` to your `loki` directory.
+
+    ```bash
+    wget https://raw.githubusercontent.com/grafana/loki/v3.0.0/cmd/loki/loki-local-config.yaml -O loki-config.yaml
+    wget https://raw.githubusercontent.com/grafana/loki/v3.0.0/clients/cmd/promtail/promtail-docker-config.yaml -O promtail-config.yaml
+    ```
+
+1. Copy and paste the following commands into your command line to start the Docker containers using the configuration files you downloaded in the previous step.
+
+    ```bash
+    docker run --name loki -d -v $(pwd):/mnt/config -p 3100:3100 grafana/loki:3.0.0 -config.file=/mnt/config/loki-config.yaml
+    docker run --name promtail -d -v $(pwd):/mnt/config -v /var/log:/var/log --link loki grafana/promtail:3.0.0 -config.file=/mnt/config/promtail-config.yaml
+    ```
+
+    {{< admonition type="note" >}}
+    The image is configured to run by default as user `loki` with  UID `10001` and GID `10001`. You can use a different user, specially if you are using bind mounts, by specifying the UID with a `docker run` command and using `--user=UID` with a numeric UID suited to your needs.
+    {{< /admonition >}}
+
+1. Verify that your containers are running:
 
-When finished, `loki-config.yaml` and `promtail-config.yaml` are downloaded in the directory you chose. Docker containers are running Loki and Promtail using those config files.
+    ```bash
+    docker container ls
+    ```
 
-Navigate to http://localhost:3100/metrics to view the metrics and http://localhost:3100/ready for readiness.
+    You should see something similar to the following:
 
-The image is configured to run by default as user loki with  UID `10001` and GID `10001`. You can use a different user, specially if you are using bind mounts, by specifying the UID with a `docker run` command and using `--user=UID` with numeric UID suited to your needs.
+    ```bash
 
-**Windows**
+    CONTAINER ID   IMAGE                    COMMAND                  CREATED              STATUS              PORTS                                       NAMES
+    9485de9ad351   grafana/promtail:3.0.0   "/usr/bin/promtail -…"   About a minute ago   Up About a minute                                               promtail
+    cece1df84519   grafana/loki:3.0.0       "/usr/bin/loki -conf…"   About a minute ago   Up About a minute   0.0.0.0:3100->3100/tcp, :::3100->3100/tcp   loki
+    ```
 
-Copy and paste the commands below into your terminal. Note that you will need to replace the `<placeholders>` in the commands with your local path.
+1. Verify that Loki is up and running.
+
+    - To view readiness, navigate to http://localhost:3100/ready.
+    - To view metrics, navigate to http://localhost:3100/metrics.
+
+## Install with Docker on Windows
+
+1. Copy and paste the following commands into your command line to download `loki-local-config.yaml` and `promtail-docker-config.yaml` to your `loki` directory. Note that you will need to replace the `<local-path>` in the commands with your local path.
 
 ```bash
 cd "<local-path>"
-wget https://raw.githubusercontent.com/grafana/loki/v2.9.2/cmd/loki/loki-local-config.yaml -O loki-config.yaml
-docker run --name loki -v <local-path>:/mnt/config -p 3100:3100 grafana/loki:2.9.2 --config.file=/mnt/config/loki-config.yaml
-wget https://raw.githubusercontent.com/grafana/loki/v2.9.2/clients/cmd/promtail/promtail-docker-config.yaml -O promtail-config.yaml
-docker run -v <local-path>:/mnt/config -v /var/log:/var/log --link loki grafana/promtail:2.9.2 --config.file=/mnt/config/promtail-config.yaml
+wget https://raw.githubusercontent.com/grafana/loki/v3.0.0/cmd/loki/loki-local-config.yaml -O loki-config.yaml
+wget https://raw.githubusercontent.com/grafana/loki/v3.0.0/clients/cmd/promtail/promtail-docker-config.yaml -O promtail-config.yaml
 ```
 
-When finished, `loki-config.yaml` and `promtail-config.yaml` are downloaded in the directory you chose. Docker containers are running Loki and Promtail using those config files.
+1. Copy and paste the following commands into your command line to start the Docker containers using the configuration files you downloaded in the previous step. Note that you will need to replace the `<local-path>` in the commands with your local path.
+
+```bash
+docker run --name loki -v <local-path>:/mnt/config -p 3100:3100 grafana/loki:3.0.0 --config.file=/mnt/config/loki-config.yaml
+docker run -v <local-path>:/mnt/config -v /var/log:/var/log --link loki grafana/promtail:3.0.0 --config.file=/mnt/config/promtail-config.yaml
+```
 
-Navigate to http://localhost:3100/metrics to view the output.
+1. Verify that Loki is up and running.
+
+    - To view readiness, navigate to http://localhost:3100/ready.
+    - To view metrics, navigate to http://localhost:3100/metrics.
 
 ## Install with Docker Compose
 
 Run the following commands in your command line. They work for Windows or Linux systems.
 
-```bash
-wget https://raw.githubusercontent.com/grafana/loki/v2.9.2/production/docker-compose.yaml -O docker-compose.yaml
-docker-compose -f docker-compose.yaml up
-```
+1. Create a directory called `loki`. Make `loki` your current working directory:
+
+    ```bash
+    mkdir loki
+    cd loki
+    ```
+
+1. Copy and paste the following command into your command line to download the `docker-compose` file.
+
+    ```bash
+    wget https://raw.githubusercontent.com/grafana/loki/v3.0.0/production/docker-compose.yaml -O docker-compose.yaml
+    ```
+
+1. With `loki` as the current working directory, run the following 'docker-compose` command:
+
+    ```bash
+    docker-compose -f docker-compose.yaml up
+    ```
+
+    You should see something similar to the following:
+
+    ```bash
+    ✔ Container mydevice-minio-1     Started              0.0s
+    ✔ Container mydevice-flog-1      Started              0.0s
+    ✔ Container mydevice-write-1     Started              0.0s
+    ✔ Container mydevice-read-1      Started              0.0s
+    ✔ Container mydevice-gateway-1   Started              0.0s
+    ✔ Container mydevice-grafana-1   Started              0.0s
+    ✔ Container mydevice-promtail-1  Started              0.0s
+    ```
+
+1. Verify that Loki is up and running.
+
+    - To view readiness, navigate to http://localhost:3100/ready.
+    - To view metrics, navigate to http://localhost:3100/metrics.
diff --git a/docs/sources/setup/install/helm/reference.md b/docs/sources/setup/install/helm/reference.md
index dc14aa4f032de..de875439bf2b8 100644
--- a/docs/sources/setup/install/helm/reference.md
+++ b/docs/sources/setup/install/helm/reference.md
@@ -2630,7 +2630,7 @@ null
     "tolerations": []
   },
   "useExternalLicense": false,
-  "version": "v3.0.0"
+  "version": "v3.0.1"
 }
 </pre>
 </td>
diff --git a/docs/sources/shared/configuration.md b/docs/sources/shared/configuration.md
index 99bca7d2e2cd0..208def0cdd521 100644
--- a/docs/sources/shared/configuration.md
+++ b/docs/sources/shared/configuration.md
@@ -5280,6 +5280,26 @@ bloom_shipper:
   # component.
   # The CLI flags prefix for this block configuration is: bloom.metas-cache
   [metas_cache: <cache_config>]
+
+  metas_lru_cache:
+    # In-memory LRU cache for bloom metas. Whether embedded cache is enabled.
+    # CLI flag: -bloom.metas-lru-cache.enabled
+    [enabled: <boolean> | default = false]
+
+    # In-memory LRU cache for bloom metas. Maximum memory size of the cache in
+    # MB.
+    # CLI flag: -bloom.metas-lru-cache.max-size-mb
+    [max_size_mb: <int> | default = 100]
+
+    # In-memory LRU cache for bloom metas. Maximum number of entries in the
+    # cache.
+    # CLI flag: -bloom.metas-lru-cache.max-size-items
+    [max_size_items: <int> | default = 0]
+
+    # In-memory LRU cache for bloom metas. The time to live for items in the
+    # cache before they get purged.
+    # CLI flag: -bloom.metas-lru-cache.ttl
+    [ttl: <duration> | default = 1h]
 ```
 
 ### swift_storage_config
diff --git a/go.mod b/go.mod
index 67c1302ab124a..06969fdeed2e1 100644
--- a/go.mod
+++ b/go.mod
@@ -75,7 +75,7 @@ require (
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f
 	github.com/ncw/swift v1.0.53
 	github.com/oklog/run v1.1.0
-	github.com/oklog/ulid v1.3.1
+	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e
 	github.com/opentracing-contrib/go-stdlib v1.0.0
 	github.com/opentracing/opentracing-go v1.2.0
diff --git a/pkg/bloomgateway/bloomgateway.go b/pkg/bloomgateway/bloomgateway.go
index 487b87bac352f..b4ccd0bee075d 100644
--- a/pkg/bloomgateway/bloomgateway.go
+++ b/pkg/bloomgateway/bloomgateway.go
@@ -278,10 +278,7 @@ func (g *Gateway) FilterChunkRefs(ctx context.Context, req *logproto.FilterChunk
 	tasks := make([]Task, 0, len(seriesByDay))
 	responses := make([][]v1.Output, 0, len(seriesByDay))
 	for _, seriesForDay := range seriesByDay {
-		task, err := NewTask(ctx, tenantID, seriesForDay, filters, blocks)
-		if err != nil {
-			return nil, err
-		}
+		task := newTask(ctx, tenantID, seriesForDay, filters, blocks)
 		// TODO(owen-d): include capacity in constructor?
 		task.responses = responsesPool.Get(len(seriesForDay.series))
 		tasks = append(tasks, task)
@@ -298,7 +295,6 @@ func (g *Gateway) FilterChunkRefs(ctx context.Context, req *logproto.FilterChunk
 	for _, task := range tasks {
 		task := task
 		task.enqueueTime = time.Now()
-		level.Info(logger).Log("msg", "enqueue task", "task", task.ID, "table", task.table, "series", len(task.series))
 
 		// TODO(owen-d): gracefully handle full queues
 		if err := g.queue.Enqueue(tenantID, nil, task, func() {
@@ -329,7 +325,6 @@ func (g *Gateway) FilterChunkRefs(ctx context.Context, req *logproto.FilterChunk
 			stats.Status = "cancel"
 			return nil, errors.Wrap(ctx.Err(), "request failed")
 		case task := <-tasksCh:
-			level.Info(logger).Log("msg", "task done", "task", task.ID, "err", task.Err())
 			if task.Err() != nil {
 				stats.Status = labelFailure
 				return nil, errors.Wrap(task.Err(), "request failed")
diff --git a/pkg/bloomgateway/client.go b/pkg/bloomgateway/client.go
index af4b453882417..8a01514cdf2f1 100644
--- a/pkg/bloomgateway/client.go
+++ b/pkg/bloomgateway/client.go
@@ -14,6 +14,7 @@ import (
 	ringclient "github.com/grafana/dskit/ring/client"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
+	"go.uber.org/atomic"
 	"golang.org/x/exp/slices"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/health/grpc_health_v1"
@@ -238,20 +239,8 @@ func (c *GatewayClient) FilterChunks(ctx context.Context, _ string, interval blo
 		}
 	}
 
-	if len(servers) > 0 {
-		// cache locality score (higher is better):
-		// `% keyspace / % instances`. Ideally converges to 1 (querying x% of keyspace requires x% of instances),
-		// but can be less if the keyspace is not evenly distributed across instances. Ideal operation will see the range of
-		// `1-2/num_instances` -> `1`, where the former represents slight
-		// overlap on instances to the left and right of the range.
-		pctKeyspace := float64(lastFp-firstFp) / float64(math.MaxUint64)
-		pctInstances := float64(len(servers)) / float64(max(1, len(c.pool.Addrs())))
-		cacheLocalityScore := pctKeyspace / pctInstances
-		c.metrics.cacheLocalityScore.Observe(cacheLocalityScore)
-	}
-
 	results := make([][]*logproto.GroupedChunkRefs, len(servers))
-	count := 0
+	count := atomic.NewInt64(0)
 	err := concurrency.ForEachJob(ctx, len(servers), len(servers), func(ctx context.Context, i int) error {
 		rs := servers[i]
 
@@ -269,10 +258,24 @@ func (c *GatewayClient) FilterChunks(ctx context.Context, _ string, interval blo
 			}
 			resp, err := client.FilterChunkRefs(ctx, req)
 			if err != nil {
-				return err
+				// We don't want a single bloom-gw failure to fail the entire query,
+				// so instrument & move on
+				level.Error(c.logger).Log(
+					"msg", "filter failed for instance, skipping",
+					"addr", rs.addr,
+					"series", len(rs.groups),
+					"blocks", len(rs.blocks),
+					"err", err,
+				)
+				// filter none of the results on failed request
+				c.metrics.clientRequests.WithLabelValues(typeError).Inc()
+				results[i] = rs.groups
+			} else {
+				c.metrics.clientRequests.WithLabelValues(typeSuccess).Inc()
+				results[i] = resp.ChunkRefs
 			}
-			results[i] = resp.ChunkRefs
-			count += len(resp.ChunkRefs)
+
+			count.Add(int64(len(results[i])))
 			return nil
 		})
 	})
@@ -281,7 +284,7 @@ func (c *GatewayClient) FilterChunks(ctx context.Context, _ string, interval blo
 		return nil, err
 	}
 
-	buf := make([]*logproto.GroupedChunkRefs, 0, count)
+	buf := make([]*logproto.GroupedChunkRefs, 0, int(count.Load()))
 	return mergeSeries(results, buf)
 }
 
diff --git a/pkg/bloomgateway/metrics.go b/pkg/bloomgateway/metrics.go
index 0d408991b40c2..0885bc2ae7cb4 100644
--- a/pkg/bloomgateway/metrics.go
+++ b/pkg/bloomgateway/metrics.go
@@ -15,21 +15,25 @@ type metrics struct {
 	*serverMetrics
 }
 
+const (
+	typeSuccess = "success"
+	typeError   = "error"
+)
+
 type clientMetrics struct {
-	cacheLocalityScore prometheus.Histogram
-	requestLatency     *prometheus.HistogramVec
-	clients            prometheus.Gauge
+	clientRequests *prometheus.CounterVec
+	requestLatency *prometheus.HistogramVec
+	clients        prometheus.Gauge
 }
 
 func newClientMetrics(registerer prometheus.Registerer) *clientMetrics {
 	return &clientMetrics{
-		cacheLocalityScore: promauto.With(registerer).NewHistogram(prometheus.HistogramOpts{
+		clientRequests: promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{
 			Namespace: constants.Loki,
 			Subsystem: "bloom_gateway_client",
-			Name:      "cache_locality_score",
-			Help:      "Cache locality score of the bloom filter, as measured by % of keyspace touched / % of bloom_gws required",
-			Buckets:   prometheus.LinearBuckets(0.01, 0.2, 5),
-		}),
+			Name:      "requests_total",
+			Help:      "Total number of requests made to the bloom gateway",
+		}, []string{"type"}),
 		requestLatency: promauto.With(registerer).NewHistogramVec(prometheus.HistogramOpts{
 			Namespace: constants.Loki,
 			Subsystem: "bloom_gateway_client",
diff --git a/pkg/bloomgateway/multiplexing.go b/pkg/bloomgateway/multiplexing.go
index 84516788d001a..b3d80d4a0fb08 100644
--- a/pkg/bloomgateway/multiplexing.go
+++ b/pkg/bloomgateway/multiplexing.go
@@ -2,11 +2,9 @@ package bloomgateway
 
 import (
 	"context"
-	"math/rand"
 	"sync"
 	"time"
 
-	"github.com/oklog/ulid"
 	"github.com/prometheus/common/model"
 
 	"github.com/grafana/loki/v3/pkg/logproto"
@@ -20,10 +18,6 @@ const (
 	Day = 24 * time.Hour
 )
 
-var (
-	entropy = rand.New(rand.NewSource(time.Now().UnixNano()))
-)
-
 type tokenSettings struct {
 	nGramLen int
 }
@@ -45,10 +39,8 @@ func (e *wrappedError) Set(err error) {
 
 // Task is the data structure that is enqueued to the internal queue and dequeued by query workers
 type Task struct {
-	// ID is a lexcographically sortable unique identifier of the task
-	ID ulid.ULID
-	// Tenant is the tenant ID
-	Tenant string
+	// tenant is the tenant ID
+	tenant string
 
 	// channel to write partial responses to
 	resCh chan v1.Output
@@ -79,18 +71,9 @@ type Task struct {
 	enqueueTime time.Time
 }
 
-// NewTask returns a new Task that can be enqueued to the task queue.
-// In addition, it returns a result and an error channel, as well
-// as an error if the instantiation fails.
-func NewTask(ctx context.Context, tenantID string, refs seriesWithInterval, filters []syntax.LineFilterExpr, blocks []bloomshipper.BlockRef) (Task, error) {
-	key, err := ulid.New(ulid.Now(), entropy)
-	if err != nil {
-		return Task{}, err
-	}
-
-	task := Task{
-		ID:       key,
-		Tenant:   tenantID,
+func newTask(ctx context.Context, tenantID string, refs seriesWithInterval, filters []syntax.LineFilterExpr, blocks []bloomshipper.BlockRef) Task {
+	return Task{
+		tenant:   tenantID,
 		err:      new(wrappedError),
 		resCh:    make(chan v1.Output),
 		filters:  filters,
@@ -101,7 +84,6 @@ func NewTask(ctx context.Context, tenantID string, refs seriesWithInterval, filt
 		ctx:      ctx,
 		done:     make(chan struct{}),
 	}
-	return task, nil
 }
 
 // Bounds implements Bounded
@@ -130,9 +112,8 @@ func (t Task) CloseWithError(err error) {
 
 // Copy returns a copy of the existing task but with a new slice of grouped chunk refs
 func (t Task) Copy(series []*logproto.GroupedChunkRefs) Task {
-	// do not copy ID to distinguish it as copied task
 	return Task{
-		Tenant:   t.Tenant,
+		tenant:   t.tenant,
 		err:      t.err,
 		resCh:    t.resCh,
 		filters:  t.filters,
diff --git a/pkg/bloomgateway/multiplexing_test.go b/pkg/bloomgateway/multiplexing_test.go
index a64ac6cbe4044..5f71d3c9623df 100644
--- a/pkg/bloomgateway/multiplexing_test.go
+++ b/pkg/bloomgateway/multiplexing_test.go
@@ -31,8 +31,7 @@ func TestTask(t *testing.T) {
 			},
 		}
 		swb := partitionRequest(req)[0]
-		task, err := NewTask(context.Background(), "tenant", swb, nil, nil)
-		require.NoError(t, err)
+		task := newTask(context.Background(), "tenant", swb, nil, nil)
 		from, through := task.Bounds()
 		require.Equal(t, ts.Add(-1*time.Hour), from)
 		require.Equal(t, ts, through)
@@ -45,8 +44,7 @@ func createTasksForRequests(t *testing.T, tenant string, requests ...*logproto.F
 	tasks := make([]Task, 0, len(requests))
 	for _, r := range requests {
 		for _, swb := range partitionRequest(r) {
-			task, err := NewTask(context.Background(), tenant, swb, nil, nil)
-			require.NoError(t, err)
+			task := newTask(context.Background(), tenant, swb, nil, nil)
 			tasks = append(tasks, task)
 		}
 	}
@@ -63,7 +61,7 @@ func TestTask_RequestIterator(t *testing.T) {
 			interval: bloomshipper.Interval{Start: 0, End: math.MaxInt64},
 			series:   []*logproto.GroupedChunkRefs{},
 		}
-		task, _ := NewTask(context.Background(), tenant, swb, []syntax.LineFilterExpr{}, nil)
+		task := newTask(context.Background(), tenant, swb, []syntax.LineFilterExpr{}, nil)
 		it := task.RequestIter(tokenizer)
 		// nothing to iterate over
 		require.False(t, it.Next())
diff --git a/pkg/bloomgateway/processor.go b/pkg/bloomgateway/processor.go
index d94b305a9b26b..947296d5712c4 100644
--- a/pkg/bloomgateway/processor.go
+++ b/pkg/bloomgateway/processor.go
@@ -40,7 +40,7 @@ func (p *processor) run(ctx context.Context, tasks []Task) error {
 }
 
 func (p *processor) runWithBounds(ctx context.Context, tasks []Task, bounds v1.MultiFingerprintBounds) error {
-	tenant := tasks[0].Tenant
+	tenant := tasks[0].tenant
 	level.Info(p.logger).Log(
 		"msg", "process tasks with bounds",
 		"tenant", tenant,
@@ -157,7 +157,7 @@ func (p *processor) processBlock(_ context.Context, blockQuerier *v1.BlockQuerie
 	for _, task := range tasks {
 		if sp := opentracing.SpanFromContext(task.ctx); sp != nil {
 			md, _ := blockQuerier.Metadata()
-			blk := bloomshipper.BlockRefFrom(task.Tenant, task.table.String(), md)
+			blk := bloomshipper.BlockRefFrom(task.tenant, task.table.String(), md)
 			sp.LogKV("process block", blk.String(), "series", len(task.series))
 		}
 
diff --git a/pkg/bloomgateway/processor_test.go b/pkg/bloomgateway/processor_test.go
index 04856225e18f2..b86dbf8006b78 100644
--- a/pkg/bloomgateway/processor_test.go
+++ b/pkg/bloomgateway/processor_test.go
@@ -126,7 +126,7 @@ func TestProcessor(t *testing.T) {
 		}
 
 		t.Log("series", len(swb.series))
-		task, _ := NewTask(ctx, "fake", swb, filters, nil)
+		task := newTask(ctx, "fake", swb, filters, nil)
 		tasks := []Task{task}
 
 		results := atomic.NewInt64(0)
@@ -178,7 +178,7 @@ func TestProcessor(t *testing.T) {
 		}
 
 		t.Log("series", len(swb.series))
-		task, _ := NewTask(ctx, "fake", swb, filters, blocks)
+		task := newTask(ctx, "fake", swb, filters, blocks)
 		tasks := []Task{task}
 
 		results := atomic.NewInt64(0)
@@ -227,7 +227,7 @@ func TestProcessor(t *testing.T) {
 		}
 
 		t.Log("series", len(swb.series))
-		task, _ := NewTask(ctx, "fake", swb, filters, nil)
+		task := newTask(ctx, "fake", swb, filters, nil)
 		tasks := []Task{task}
 
 		results := atomic.NewInt64(0)
diff --git a/pkg/bloomgateway/querier.go b/pkg/bloomgateway/querier.go
index 58415a1fd6673..0e523817dc70a 100644
--- a/pkg/bloomgateway/querier.go
+++ b/pkg/bloomgateway/querier.go
@@ -5,7 +5,6 @@ import (
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
-	"github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
@@ -16,6 +15,7 @@ import (
 	v1 "github.com/grafana/loki/v3/pkg/storage/bloom/v1"
 	"github.com/grafana/loki/v3/pkg/storage/stores/shipper/bloomshipper"
 	"github.com/grafana/loki/v3/pkg/util/constants"
+	"github.com/grafana/loki/v3/pkg/util/spanlogger"
 )
 
 type querierMetrics struct {
@@ -90,8 +90,9 @@ func (bq *BloomQuerier) FilterChunkRefs(ctx context.Context, tenant string, from
 	if !bq.limits.BloomGatewayEnabled(tenant) || len(chunkRefs) == 0 || len(v1.ExtractTestableLineFilters(queryPlan.AST)) == 0 {
 		return chunkRefs, nil
 	}
-	sp, ctx := opentracing.StartSpanFromContext(ctx, "bloomquerier.FilterChunkRefs")
-	defer sp.Finish()
+
+	logger, ctx := spanlogger.NewWithLogger(ctx, bq.logger, "bloomquerier.FilterChunkRefs")
+	defer logger.Finish()
 
 	grouped := groupedChunksRefPool.Get(len(chunkRefs))
 	defer groupedChunksRefPool.Put(grouped)
@@ -141,8 +142,7 @@ func (bq *BloomQuerier) FilterChunkRefs(ctx context.Context, tenant string, from
 	postFilterChunks := len(result)
 	postFilterSeries := len(deduped)
 
-	level.Debug(bq.logger).Log(
-		"operation", "bloomquerier.FilterChunkRefs",
+	level.Debug(logger).Log(
 		"tenant", tenant,
 		"from", from.Time(),
 		"through", through.Time(),
@@ -153,6 +153,7 @@ func (bq *BloomQuerier) FilterChunkRefs(ctx context.Context, tenant string, from
 		"preFilterSeries", preFilterSeries,
 		"postFilterSeries", postFilterSeries,
 		"filteredSeries", preFilterSeries-postFilterSeries,
+		"operation", "bloomquerier.FilterChunkRefs",
 	)
 
 	bq.metrics.chunksTotal.Add(float64(preFilterChunks))
diff --git a/pkg/bloomgateway/worker.go b/pkg/bloomgateway/worker.go
index fab243f29613a..6b234db27189c 100644
--- a/pkg/bloomgateway/worker.go
+++ b/pkg/bloomgateway/worker.go
@@ -100,7 +100,6 @@ func (w *worker) running(_ context.Context) error {
 				w.queue.ReleaseRequests(items)
 				return errors.Errorf("failed to cast dequeued item to Task: %v", item)
 			}
-			level.Debug(w.logger).Log("msg", "dequeued task", "task", task.ID)
 			_ = w.pending.Dec()
 			w.metrics.queueDuration.WithLabelValues(w.id).Observe(time.Since(task.enqueueTime).Seconds())
 			FromContext(task.ctx).AddQueueTime(time.Since(task.enqueueTime))
diff --git a/pkg/loghttp/detected.go b/pkg/loghttp/detected.go
index d255bf6124a75..632ac7cd02410 100644
--- a/pkg/loghttp/detected.go
+++ b/pkg/loghttp/detected.go
@@ -11,4 +11,5 @@ type DetectedField struct {
 	Label       string                     `json:"label,omitempty"`
 	Type        logproto.DetectedFieldType `json:"type,omitempty"`
 	Cardinality uint64                     `json:"cardinality,omitempty"`
+	Parser      string                     `json:"parser,omitempty"`
 }
diff --git a/pkg/loghttp/query.go b/pkg/loghttp/query.go
index 8b135602b7f07..89ad4e00a79c0 100644
--- a/pkg/loghttp/query.go
+++ b/pkg/loghttp/query.go
@@ -537,6 +537,9 @@ func ParseIndexShardsQuery(r *http.Request) (*RangeQuery, datasize.ByteSize, err
 		return nil, 0, err
 	}
 	targetBytes, err := parseBytes(r, "targetBytesPerShard", true)
+	if targetBytes <= 0 {
+		return nil, 0, errors.New("targetBytesPerShard must be a positive value")
+	}
 	return parsed, targetBytes, err
 }
 
diff --git a/pkg/logproto/logproto.pb.go b/pkg/logproto/logproto.pb.go
index 774af1fb709b1..ac9bd37a06186 100644
--- a/pkg/logproto/logproto.pb.go
+++ b/pkg/logproto/logproto.pb.go
@@ -2818,7 +2818,8 @@ type DetectedField struct {
 	Label       string            `protobuf:"bytes,1,opt,name=label,proto3" json:"label,omitempty"`
 	Type        DetectedFieldType `protobuf:"bytes,2,opt,name=type,proto3,casttype=DetectedFieldType" json:"type,omitempty"`
 	Cardinality uint64            `protobuf:"varint,3,opt,name=cardinality,proto3" json:"cardinality,omitempty"`
-	Sketch      []byte            `protobuf:"bytes,4,opt,name=sketch,proto3" json:"sketch,omitempty"`
+	Parser      string            `protobuf:"bytes,4,opt,name=parser,proto3" json:"parser,omitempty"`
+	Sketch      []byte            `protobuf:"bytes,5,opt,name=sketch,proto3" json:"sketch,omitempty"`
 }
 
 func (m *DetectedField) Reset()      { *m = DetectedField{} }
@@ -2874,6 +2875,13 @@ func (m *DetectedField) GetCardinality() uint64 {
 	return 0
 }
 
+func (m *DetectedField) GetParser() string {
+	if m != nil {
+		return m.Parser
+	}
+	return ""
+}
+
 func (m *DetectedField) GetSketch() []byte {
 	if m != nil {
 		return m.Sketch
@@ -3097,174 +3105,174 @@ func init() {
 func init() { proto.RegisterFile("pkg/logproto/logproto.proto", fileDescriptor_c28a5f14f1f4c79a) }
 
 var fileDescriptor_c28a5f14f1f4c79a = []byte{
-	// 2658 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x1a, 0x4d, 0x8c, 0x5b, 0x47,
-	0xd9, 0xcf, 0x7e, 0xf6, 0xda, 0x9f, 0xbd, 0x9b, 0xcd, 0xac, 0x93, 0x58, 0x9b, 0xd4, 0x6f, 0x3b,
-	0x82, 0x36, 0x34, 0xe9, 0xba, 0x49, 0x69, 0x49, 0x53, 0x4a, 0x89, 0x77, 0x9b, 0x6d, 0xd2, 0x6d,
-	0x9a, 0xce, 0xa6, 0x69, 0x41, 0x54, 0xd5, 0x8b, 0x3d, 0xeb, 0x7d, 0x8a, 0xfd, 0x9e, 0xf3, 0xde,
-	0xb8, 0xe9, 0xde, 0x90, 0x38, 0x23, 0x2a, 0x71, 0x00, 0x2e, 0x08, 0x24, 0x24, 0x10, 0xa8, 0x17,
-	0xc4, 0x11, 0xc1, 0x85, 0x43, 0xb9, 0x95, 0x5b, 0xd5, 0x83, 0xa1, 0xdb, 0x0b, 0xda, 0x53, 0x25,
-	0x24, 0x0e, 0x3d, 0xa1, 0xf9, 0x7b, 0x6f, 0xde, 0x5b, 0x9b, 0xd4, 0xdb, 0xa0, 0x92, 0x8b, 0x3d,
-	0xf3, 0xcd, 0x37, 0xdf, 0xcc, 0xf7, 0x33, 0xdf, 0x9f, 0x0d, 0x27, 0x87, 0xb7, 0x7b, 0xad, 0x7e,
-	0xd0, 0x1b, 0x86, 0x01, 0x0b, 0xe2, 0xc1, 0xaa, 0xf8, 0x44, 0x65, 0x3d, 0x5f, 0xae, 0xf7, 0x82,
-	0x5e, 0x20, 0x71, 0xf8, 0x48, 0xae, 0x2f, 0x3b, 0xbd, 0x20, 0xe8, 0xf5, 0x69, 0x4b, 0xcc, 0x6e,
-	0x8d, 0xb6, 0x5b, 0xcc, 0x1b, 0xd0, 0x88, 0xb9, 0x83, 0xa1, 0x42, 0x58, 0x51, 0xd4, 0xef, 0xf4,
-	0x07, 0x41, 0x97, 0xf6, 0x5b, 0x11, 0x73, 0x59, 0x24, 0x3f, 0x15, 0xc6, 0x12, 0xc7, 0x18, 0x8e,
-	0xa2, 0x1d, 0xf1, 0x21, 0x81, 0xf8, 0x0f, 0x16, 0x1c, 0xdb, 0x74, 0x6f, 0xd1, 0xfe, 0x8d, 0xe0,
-	0xa6, 0xdb, 0x1f, 0xd1, 0x88, 0xd0, 0x68, 0x18, 0xf8, 0x11, 0x45, 0x6b, 0x50, 0xea, 0xf3, 0x85,
-	0xa8, 0x61, 0xad, 0x14, 0x4e, 0x57, 0xcf, 0x9f, 0x59, 0x8d, 0xaf, 0x3c, 0x71, 0x83, 0x84, 0x46,
-	0x2f, 0xf8, 0x2c, 0xdc, 0x25, 0x6a, 0xeb, 0xf2, 0x4d, 0xa8, 0x1a, 0x60, 0xb4, 0x08, 0x85, 0xdb,
-	0x74, 0xb7, 0x61, 0xad, 0x58, 0xa7, 0x2b, 0x84, 0x0f, 0xd1, 0x39, 0x28, 0xbe, 0xcd, 0xc9, 0x34,
-	0xf2, 0x2b, 0xd6, 0xe9, 0xea, 0xf9, 0x93, 0xc9, 0x21, 0xaf, 0xf9, 0xde, 0x9d, 0x11, 0x15, 0xbb,
-	0xd5, 0x41, 0x12, 0xf3, 0x62, 0xfe, 0x82, 0x85, 0xcf, 0xc0, 0xd1, 0x03, 0xeb, 0xe8, 0x38, 0x94,
-	0x04, 0x86, 0xbc, 0x71, 0x85, 0xa8, 0x19, 0xae, 0x03, 0xda, 0x62, 0x21, 0x75, 0x07, 0xc4, 0x65,
-	0xfc, 0xbe, 0x77, 0x46, 0x34, 0x62, 0xf8, 0x65, 0x58, 0x4a, 0x41, 0x15, 0xdb, 0x4f, 0x43, 0x35,
-	0x4a, 0xc0, 0x8a, 0xf7, 0x7a, 0x72, 0xad, 0x64, 0x0f, 0x31, 0x11, 0xf1, 0xcf, 0x2d, 0x80, 0x64,
-	0x0d, 0x35, 0x01, 0xe4, 0xea, 0x8b, 0x6e, 0xb4, 0x23, 0x18, 0xb6, 0x89, 0x01, 0x41, 0x67, 0xe1,
-	0x68, 0x32, 0xbb, 0x16, 0x6c, 0xed, 0xb8, 0x61, 0x57, 0xc8, 0xc0, 0x26, 0x07, 0x17, 0x10, 0x02,
-	0x3b, 0x74, 0x19, 0x6d, 0x14, 0x56, 0xac, 0xd3, 0x05, 0x22, 0xc6, 0x9c, 0x5b, 0x46, 0x7d, 0xd7,
-	0x67, 0x0d, 0x5b, 0x88, 0x53, 0xcd, 0x38, 0x9c, 0xeb, 0x97, 0x46, 0x8d, 0xe2, 0x8a, 0x75, 0x7a,
-	0x9e, 0xa8, 0x19, 0xfe, 0x77, 0x01, 0x6a, 0xaf, 0x8e, 0x68, 0xb8, 0xab, 0x04, 0x80, 0x9a, 0x50,
-	0x8e, 0x68, 0x9f, 0x76, 0x58, 0x10, 0x4a, 0x8d, 0xb4, 0xf3, 0x0d, 0x8b, 0xc4, 0x30, 0x54, 0x87,
-	0x62, 0xdf, 0x1b, 0x78, 0x4c, 0x5c, 0x6b, 0x9e, 0xc8, 0x09, 0xba, 0x08, 0xc5, 0x88, 0xb9, 0x21,
-	0x13, 0x77, 0xa9, 0x9e, 0x5f, 0x5e, 0x95, 0x86, 0xb9, 0xaa, 0x0d, 0x73, 0xf5, 0x86, 0x36, 0xcc,
-	0x76, 0xf9, 0xfd, 0xb1, 0x93, 0x7b, 0xf7, 0xef, 0x8e, 0x45, 0xe4, 0x16, 0xf4, 0x34, 0x14, 0xa8,
-	0xdf, 0x15, 0xf7, 0xfd, 0xbc, 0x3b, 0xf9, 0x06, 0x74, 0x0e, 0x2a, 0x5d, 0x2f, 0xa4, 0x1d, 0xe6,
-	0x05, 0xbe, 0xe0, 0x6a, 0xe1, 0xfc, 0x52, 0xa2, 0x91, 0x75, 0xbd, 0x44, 0x12, 0x2c, 0x74, 0x16,
-	0x4a, 0x11, 0x17, 0x5d, 0xd4, 0x98, 0xe3, 0xb6, 0xd0, 0xae, 0xef, 0x8f, 0x9d, 0x45, 0x09, 0x39,
-	0x1b, 0x0c, 0x3c, 0x46, 0x07, 0x43, 0xb6, 0x4b, 0x14, 0x0e, 0x7a, 0x0c, 0xe6, 0xba, 0xb4, 0x4f,
-	0xb9, 0xc2, 0xcb, 0x42, 0xe1, 0x8b, 0x06, 0x79, 0xb1, 0x40, 0x34, 0x02, 0x7a, 0x13, 0xec, 0x61,
-	0xdf, 0xf5, 0x1b, 0x15, 0xc1, 0xc5, 0x42, 0x82, 0x78, 0xbd, 0xef, 0xfa, 0xed, 0x67, 0x3e, 0x1a,
-	0x3b, 0x4f, 0xf5, 0x3c, 0xb6, 0x33, 0xba, 0xb5, 0xda, 0x09, 0x06, 0xad, 0x5e, 0xe8, 0x6e, 0xbb,
-	0xbe, 0xdb, 0xea, 0x07, 0xb7, 0xbd, 0xd6, 0xdb, 0x4f, 0xb6, 0xf8, 0x1b, 0xbc, 0x33, 0xa2, 0xa1,
-	0x47, 0xc3, 0x16, 0x27, 0xb3, 0x2a, 0x54, 0xc2, 0xb7, 0x12, 0x41, 0x16, 0x5d, 0xe5, 0xf6, 0x17,
-	0x84, 0x74, 0x6d, 0x67, 0xe4, 0xdf, 0x8e, 0x1a, 0x20, 0x4e, 0x39, 0x91, 0x9c, 0x22, 0xe0, 0x84,
-	0x6e, 0x6f, 0x84, 0xc1, 0x68, 0xd8, 0x3e, 0xb2, 0x3f, 0x76, 0x4c, 0x7c, 0x62, 0x4e, 0xae, 0xda,
-	0xe5, 0xd2, 0xe2, 0x1c, 0x7e, 0xaf, 0x00, 0x68, 0xcb, 0x1d, 0x0c, 0xfb, 0x74, 0x26, 0xf5, 0xc7,
-	0x8a, 0xce, 0x1f, 0x5a, 0xd1, 0x85, 0x59, 0x15, 0x9d, 0x68, 0xcd, 0x9e, 0x4d, 0x6b, 0xc5, 0xcf,
-	0xab, 0xb5, 0xd2, 0xff, 0xbd, 0xd6, 0x70, 0x03, 0x6c, 0x4e, 0x99, 0x3b, 0xcb, 0xd0, 0xbd, 0x2b,
-	0x74, 0x53, 0x23, 0x7c, 0x88, 0x37, 0xa1, 0x24, 0xf9, 0x42, 0xcb, 0x59, 0xe5, 0xa5, 0xdf, 0x6d,
-	0xa2, 0xb8, 0x82, 0x56, 0xc9, 0x62, 0xa2, 0x92, 0x82, 0x10, 0x36, 0xfe, 0xa3, 0x05, 0xf3, 0xca,
-	0x22, 0x94, 0xef, 0xbb, 0x05, 0x73, 0xd2, 0xf7, 0x68, 0xbf, 0x77, 0x22, 0xeb, 0xf7, 0x2e, 0x75,
-	0xdd, 0x21, 0xa3, 0x61, 0xbb, 0xf5, 0xfe, 0xd8, 0xb1, 0x3e, 0x1a, 0x3b, 0x8f, 0x4e, 0x13, 0x9a,
-	0x8e, 0x35, 0xda, 0x5f, 0x6a, 0xc2, 0xe8, 0x8c, 0xb8, 0x1d, 0x8b, 0x94, 0x59, 0x1d, 0x59, 0x95,
-	0x21, 0xea, 0x8a, 0xdf, 0xa3, 0x11, 0xa7, 0x6c, 0x73, 0x8b, 0x20, 0x12, 0x87, 0xb3, 0x79, 0xd7,
-	0x0d, 0x7d, 0xcf, 0xef, 0x45, 0x8d, 0x82, 0xf0, 0xe9, 0xf1, 0x1c, 0xff, 0xd4, 0x82, 0xa5, 0x94,
-	0x59, 0x2b, 0x26, 0x2e, 0x40, 0x29, 0xe2, 0x9a, 0xd2, 0x3c, 0x18, 0x46, 0xb1, 0x25, 0xe0, 0xed,
-	0x05, 0x75, 0xf9, 0x92, 0x9c, 0x13, 0x85, 0x7f, 0xff, 0xae, 0xf6, 0x17, 0x0b, 0x6a, 0x22, 0x30,
-	0xe9, 0xb7, 0x86, 0xc0, 0xf6, 0xdd, 0x01, 0x55, 0xaa, 0x12, 0x63, 0x23, 0x5a, 0xf1, 0xe3, 0xca,
-	0x3a, 0x5a, 0xcd, 0xea, 0x60, 0xad, 0x43, 0x3b, 0x58, 0x2b, 0x79, 0x77, 0x75, 0x28, 0x72, 0xf3,
-	0xde, 0x15, 0xce, 0xb5, 0x42, 0xe4, 0x04, 0x3f, 0x0a, 0xf3, 0x8a, 0x0b, 0x25, 0xda, 0x69, 0x01,
-	0x76, 0x00, 0x25, 0xa9, 0x09, 0xf4, 0x15, 0xa8, 0xc4, 0x89, 0x89, 0xe0, 0xb6, 0xd0, 0x2e, 0xed,
-	0x8f, 0x9d, 0x3c, 0x8b, 0x48, 0xb2, 0x80, 0x1c, 0x33, 0xe8, 0x5b, 0xed, 0xca, 0xfe, 0xd8, 0x91,
-	0x00, 0x15, 0xe2, 0xd1, 0x29, 0xb0, 0x77, 0x78, 0xdc, 0xe4, 0x22, 0xb0, 0xdb, 0xe5, 0xfd, 0xb1,
-	0x23, 0xe6, 0x44, 0x7c, 0xe2, 0x0d, 0xa8, 0x6d, 0xd2, 0x9e, 0xdb, 0xd9, 0x55, 0x87, 0xd6, 0x35,
-	0x39, 0x7e, 0xa0, 0xa5, 0x69, 0x3c, 0x0c, 0xb5, 0xf8, 0xc4, 0xb7, 0x06, 0x91, 0x7a, 0x0d, 0xd5,
-	0x18, 0xf6, 0x72, 0x84, 0x7f, 0x66, 0x81, 0xb2, 0x01, 0x84, 0x8d, 0x6c, 0x87, 0xfb, 0x42, 0xd8,
-	0x1f, 0x3b, 0x0a, 0xa2, 0x93, 0x19, 0xf4, 0x2c, 0xcc, 0x45, 0xe2, 0x44, 0x4e, 0x2c, 0x6b, 0x5a,
-	0x62, 0xa1, 0x7d, 0x84, 0x9b, 0xc8, 0xfe, 0xd8, 0xd1, 0x88, 0x44, 0x0f, 0xd0, 0x6a, 0x2a, 0x21,
-	0x90, 0x8c, 0x2d, 0xec, 0x8f, 0x1d, 0x03, 0x6a, 0x26, 0x08, 0xf8, 0x33, 0x0b, 0xaa, 0x37, 0x5c,
-	0x2f, 0x36, 0xa1, 0x86, 0x56, 0x51, 0xe2, 0xab, 0x25, 0x80, 0x5b, 0x62, 0x97, 0xf6, 0xdd, 0xdd,
-	0xcb, 0x41, 0x28, 0xe8, 0xce, 0x93, 0x78, 0x9e, 0xc4, 0x70, 0x7b, 0x62, 0x0c, 0x2f, 0xce, 0xee,
-	0xda, 0xff, 0xb7, 0x8e, 0xf4, 0xaa, 0x5d, 0xce, 0x2f, 0x16, 0xf0, 0x7b, 0x16, 0xd4, 0x24, 0xf3,
-	0xca, 0xf2, 0xbe, 0x07, 0x25, 0x29, 0x1b, 0xc1, 0xfe, 0x7f, 0x71, 0x4c, 0x67, 0x66, 0x71, 0x4a,
-	0x8a, 0x26, 0x7a, 0x1e, 0x16, 0xba, 0x61, 0x30, 0x1c, 0xd2, 0xee, 0x96, 0x72, 0x7f, 0xf9, 0xac,
-	0xfb, 0x5b, 0x37, 0xd7, 0x49, 0x06, 0x1d, 0xff, 0xd5, 0x82, 0x79, 0xe5, 0x4c, 0x94, 0xba, 0x62,
-	0x11, 0x5b, 0x87, 0x8e, 0x9e, 0xf9, 0x59, 0xa3, 0xe7, 0x71, 0x28, 0xf5, 0x78, 0x7c, 0xd1, 0x0e,
-	0x49, 0xcd, 0x66, 0x8b, 0xaa, 0xf8, 0x2a, 0x2c, 0x68, 0x56, 0xa6, 0x78, 0xd4, 0xe5, 0xac, 0x47,
-	0xbd, 0xd2, 0xa5, 0x3e, 0xf3, 0xb6, 0xbd, 0xd8, 0x47, 0x2a, 0x7c, 0xfc, 0x23, 0x0b, 0x16, 0xb3,
-	0x28, 0x68, 0x3d, 0x53, 0x58, 0x3c, 0x32, 0x9d, 0x9c, 0x59, 0x53, 0x68, 0xd2, 0xaa, 0xb2, 0x78,
-	0xea, 0x5e, 0x95, 0x45, 0xdd, 0x74, 0x32, 0x15, 0xe5, 0x15, 0xf0, 0x4f, 0x2c, 0x98, 0x4f, 0xe9,
-	0x12, 0x5d, 0x00, 0x7b, 0x3b, 0x0c, 0x06, 0x33, 0x29, 0x4a, 0xec, 0x40, 0x5f, 0x87, 0x3c, 0x0b,
-	0x66, 0x52, 0x53, 0x9e, 0x05, 0x5c, 0x4b, 0x8a, 0xfd, 0x82, 0xcc, 0xdb, 0xe5, 0x0c, 0x3f, 0x05,
-	0x15, 0xc1, 0xd0, 0x75, 0xd7, 0x0b, 0x27, 0x06, 0x8c, 0xc9, 0x0c, 0x3d, 0x0b, 0x47, 0xa4, 0x33,
-	0x9c, 0xbc, 0xb9, 0x36, 0x69, 0x73, 0x4d, 0x6f, 0x3e, 0x09, 0x45, 0x91, 0x74, 0xf0, 0x2d, 0x5d,
-	0x97, 0xb9, 0x7a, 0x0b, 0x1f, 0xe3, 0x63, 0xb0, 0xc4, 0xdf, 0x20, 0x0d, 0xa3, 0xb5, 0x60, 0xe4,
-	0x33, 0x5d, 0x37, 0x9d, 0x85, 0x7a, 0x1a, 0xac, 0xac, 0xa4, 0x0e, 0xc5, 0x0e, 0x07, 0x08, 0x1a,
-	0xf3, 0x44, 0x4e, 0xf0, 0xaf, 0x2c, 0x40, 0x1b, 0x94, 0x89, 0x53, 0xae, 0xac, 0xc7, 0xcf, 0x63,
-	0x19, 0xca, 0x03, 0x97, 0x75, 0x76, 0x68, 0x18, 0xe9, 0xfc, 0x45, 0xcf, 0xbf, 0x8c, 0xc4, 0x13,
-	0x9f, 0x83, 0xa5, 0xd4, 0x2d, 0x15, 0x4f, 0xcb, 0x50, 0xee, 0x28, 0x98, 0x0a, 0x79, 0xf1, 0x1c,
-	0xff, 0x3e, 0x0f, 0x65, 0x9d, 0xd6, 0xa1, 0x73, 0x50, 0xdd, 0xf6, 0xfc, 0x1e, 0x0d, 0x87, 0xa1,
-	0xa7, 0x44, 0x60, 0xcb, 0x34, 0xcf, 0x00, 0x13, 0x73, 0x82, 0x1e, 0x87, 0xb9, 0x51, 0x44, 0xc3,
-	0xb7, 0x3c, 0xf9, 0xd2, 0x2b, 0xed, 0xfa, 0xde, 0xd8, 0x29, 0xbd, 0x16, 0xd1, 0xf0, 0xca, 0x3a,
-	0x0f, 0x3e, 0x23, 0x31, 0x22, 0xf2, 0xbb, 0x8b, 0x5e, 0x52, 0x66, 0x2a, 0x12, 0xb8, 0xf6, 0x37,
-	0xf8, 0xf5, 0x33, 0xae, 0x6e, 0x18, 0x06, 0x03, 0xca, 0x76, 0xe8, 0x28, 0x6a, 0x75, 0x82, 0xc1,
-	0x20, 0xf0, 0x5b, 0xa2, 0x13, 0x20, 0x98, 0xe6, 0x11, 0x94, 0x6f, 0x57, 0x96, 0x7b, 0x03, 0xe6,
-	0xd8, 0x4e, 0x18, 0x8c, 0x7a, 0x3b, 0x22, 0x30, 0x14, 0xda, 0x17, 0x67, 0xa7, 0xa7, 0x29, 0x10,
-	0x3d, 0x40, 0x0f, 0x73, 0x69, 0xd1, 0xce, 0xed, 0x68, 0x34, 0x90, 0xb5, 0x67, 0xbb, 0xb8, 0x3f,
-	0x76, 0xac, 0xc7, 0x49, 0x0c, 0xc6, 0x97, 0x60, 0x3e, 0x95, 0x0a, 0xa3, 0x27, 0xc0, 0x0e, 0xe9,
-	0xb6, 0x76, 0x05, 0xe8, 0x60, 0xc6, 0x2c, 0xa3, 0x3f, 0xc7, 0x21, 0xe2, 0x13, 0xff, 0x30, 0x0f,
-	0x8e, 0x51, 0xf5, 0x5f, 0x0e, 0xc2, 0x97, 0x29, 0x0b, 0xbd, 0xce, 0x35, 0x77, 0x40, 0xb5, 0x79,
-	0x39, 0x50, 0x1d, 0x08, 0xe0, 0x5b, 0xc6, 0x2b, 0x82, 0x41, 0x8c, 0x87, 0x1e, 0x02, 0x10, 0xcf,
-	0x4e, 0xae, 0xcb, 0x07, 0x55, 0x11, 0x10, 0xb1, 0xbc, 0x96, 0x12, 0x76, 0x6b, 0x46, 0xe1, 0x28,
-	0x21, 0x5f, 0xc9, 0x0a, 0x79, 0x66, 0x3a, 0xb1, 0x64, 0xcd, 0xe7, 0x52, 0x4c, 0x3f, 0x17, 0xfc,
-	0x37, 0x0b, 0x9a, 0x9b, 0xfa, 0xe6, 0x87, 0x14, 0x87, 0xe6, 0x37, 0x7f, 0x9f, 0xf8, 0x2d, 0x7c,
-	0x31, 0x7e, 0x71, 0x13, 0x60, 0xd3, 0xf3, 0xe9, 0x65, 0xaf, 0xcf, 0x68, 0x38, 0xa1, 0x10, 0xfa,
-	0x71, 0x21, 0xf1, 0x2a, 0x84, 0x6e, 0x6b, 0x3e, 0xd7, 0x0c, 0x57, 0x7e, 0x3f, 0xd8, 0xc8, 0xdf,
-	0x47, 0xb5, 0x15, 0x32, 0x5e, 0xce, 0x87, 0xb9, 0x6d, 0xc1, 0x9e, 0x8c, 0xca, 0xa9, 0x1e, 0x53,
-	0xc2, 0x7b, 0xfb, 0x5b, 0xea, 0xf0, 0xa7, 0xef, 0x91, 0x54, 0x89, 0xce, 0x5f, 0x2b, 0xda, 0xf5,
-	0x99, 0xfb, 0x8e, 0xb1, 0x9f, 0xe8, 0x43, 0x90, 0xab, 0xf2, 0xb6, 0xe2, 0xc4, 0xbc, 0xed, 0x39,
-	0x75, 0xcc, 0x17, 0xc9, 0xdd, 0xf0, 0x73, 0x89, 0x13, 0x15, 0x4a, 0x51, 0x4e, 0xf4, 0x91, 0x7b,
-	0x3d, 0x71, 0xf5, 0xb0, 0xff, 0x64, 0xc1, 0xe2, 0x06, 0x65, 0xe9, 0x3c, 0xea, 0x01, 0x52, 0x29,
-	0x7e, 0x11, 0x8e, 0x1a, 0xf7, 0x57, 0xdc, 0x3f, 0x99, 0x49, 0x9e, 0x8e, 0x25, 0xfc, 0x5f, 0xf1,
-	0xbb, 0xf4, 0x1d, 0x55, 0x93, 0xa6, 0xf3, 0xa6, 0xeb, 0x50, 0x35, 0x16, 0xd1, 0xa5, 0x4c, 0xc6,
-	0xb4, 0x94, 0x69, 0xc5, 0xf2, 0xa8, 0xdf, 0xae, 0x2b, 0x9e, 0x64, 0xe5, 0xa9, 0xf2, 0xe1, 0x38,
-	0xbb, 0xd8, 0x02, 0x24, 0xd4, 0x25, 0xc8, 0x9a, 0xf1, 0x4d, 0x40, 0x5f, 0x8a, 0x53, 0xa7, 0x78,
-	0x8e, 0x1e, 0x06, 0x3b, 0x0c, 0xee, 0xea, 0x54, 0x78, 0x3e, 0x39, 0x92, 0x04, 0x77, 0x89, 0x58,
-	0xc2, 0xcf, 0x42, 0x81, 0x04, 0x77, 0x51, 0x13, 0x20, 0x74, 0xfd, 0x1e, 0xbd, 0x19, 0x17, 0x61,
-	0x35, 0x62, 0x40, 0xa6, 0xe4, 0x1e, 0x6b, 0x70, 0xd4, 0xbc, 0x91, 0x54, 0xf7, 0x2a, 0xcc, 0xbd,
-	0x3a, 0x32, 0xc5, 0x55, 0xcf, 0x88, 0x4b, 0xd6, 0xfa, 0x1a, 0x89, 0xdb, 0x0c, 0x24, 0x70, 0x74,
-	0x0a, 0x2a, 0xcc, 0xbd, 0xd5, 0xa7, 0xd7, 0x12, 0x37, 0x97, 0x00, 0xf8, 0x2a, 0xaf, 0x1f, 0x6f,
-	0x1a, 0x49, 0x54, 0x02, 0x40, 0x8f, 0xc1, 0x62, 0x72, 0xe7, 0xeb, 0x21, 0xdd, 0xf6, 0xde, 0x11,
-	0x1a, 0xae, 0x91, 0x03, 0x70, 0x74, 0x1a, 0x8e, 0x24, 0xb0, 0x2d, 0x91, 0xac, 0xd8, 0x02, 0x35,
-	0x0b, 0xe6, 0xb2, 0x11, 0xec, 0xbe, 0x70, 0x67, 0xe4, 0xf6, 0xc5, 0xe3, 0xab, 0x11, 0x03, 0x82,
-	0xff, 0x6c, 0xc1, 0x51, 0xa9, 0x6a, 0xe6, 0xb2, 0x07, 0xd2, 0xea, 0x7f, 0x6d, 0x01, 0x32, 0x39,
-	0x50, 0xa6, 0xf5, 0x55, 0xb3, 0x97, 0xc4, 0xb3, 0xa1, 0xaa, 0x28, 0x8b, 0x25, 0x28, 0x69, 0x07,
-	0x61, 0x28, 0x75, 0x64, 0xcf, 0x4c, 0x34, 0xbf, 0x65, 0xdd, 0x2d, 0x21, 0x44, 0x7d, 0x23, 0x07,
-	0x8a, 0xb7, 0x76, 0x19, 0x8d, 0x54, 0xd5, 0x2c, 0xda, 0x05, 0x02, 0x40, 0xe4, 0x17, 0x3f, 0x8b,
-	0xfa, 0x4c, 0x58, 0x8d, 0x9d, 0x9c, 0xa5, 0x40, 0x44, 0x0f, 0xf0, 0xef, 0xf2, 0x30, 0x7f, 0x33,
-	0xe8, 0x8f, 0x92, 0xc0, 0xf8, 0x20, 0x05, 0x8c, 0x54, 0x29, 0x5f, 0xd4, 0xa5, 0x3c, 0x02, 0x3b,
-	0x62, 0x74, 0x28, 0x2c, 0xab, 0x40, 0xc4, 0x18, 0x61, 0xa8, 0x31, 0x37, 0xec, 0x51, 0x26, 0x0b,
-	0xa4, 0x46, 0x49, 0x64, 0xae, 0x29, 0x18, 0x5a, 0x81, 0xaa, 0xdb, 0xeb, 0x85, 0xb4, 0xe7, 0x32,
-	0xda, 0xde, 0x6d, 0xcc, 0x89, 0xc3, 0x4c, 0x10, 0x7e, 0x03, 0x16, 0xb4, 0xb0, 0x94, 0x4a, 0x9f,
-	0x80, 0xb9, 0xb7, 0x05, 0x64, 0x42, 0x6b, 0x4d, 0xa2, 0x2a, 0x37, 0xa6, 0xd1, 0xd2, 0x3f, 0x21,
-	0xe8, 0x3b, 0xe3, 0xab, 0x50, 0x92, 0xe8, 0xe8, 0x94, 0x59, 0xe6, 0xc8, 0x4c, 0x8f, 0xcf, 0x55,
-	0xcd, 0x82, 0xa1, 0x24, 0x09, 0x29, 0xc5, 0x0b, 0xdb, 0x90, 0x10, 0xa2, 0xbe, 0xf1, 0xbf, 0x2c,
-	0x38, 0xb6, 0x4e, 0x19, 0xed, 0x30, 0xda, 0xbd, 0xec, 0xd1, 0x7e, 0xf7, 0x4b, 0xad, 0xc0, 0xe3,
-	0x3e, 0x5a, 0xc1, 0xe8, 0xa3, 0x71, 0xbf, 0xd3, 0xf7, 0x7c, 0xba, 0x69, 0x34, 0x62, 0x12, 0x00,
-	0xf7, 0x10, 0xdb, 0xfc, 0xe2, 0x72, 0x59, 0xfe, 0x66, 0x63, 0x40, 0x62, 0x0d, 0x97, 0x12, 0x0d,
-	0xe3, 0x1f, 0x58, 0x70, 0x3c, 0xcb, 0xb5, 0x52, 0x52, 0x0b, 0x4a, 0x62, 0xf3, 0x84, 0x16, 0x6e,
-	0x6a, 0x07, 0x51, 0x68, 0xe8, 0x42, 0xea, 0x7c, 0xf1, 0x5b, 0x4f, 0xbb, 0xb1, 0x3f, 0x76, 0xea,
-	0x09, 0xd4, 0xe8, 0x12, 0x18, 0xb8, 0xf8, 0x17, 0xbc, 0x96, 0x36, 0x69, 0x0a, 0x7d, 0x73, 0xfb,
-	0x52, 0xbe, 0x57, 0x4e, 0xd0, 0xd7, 0xc0, 0x66, 0xbb, 0x43, 0xe5, 0x72, 0xdb, 0xc7, 0x3e, 0x1b,
-	0x3b, 0x47, 0x53, 0xdb, 0x6e, 0xec, 0x0e, 0x29, 0x11, 0x28, 0xdc, 0x2c, 0x3b, 0x6e, 0xd8, 0xf5,
-	0x7c, 0xb7, 0xef, 0x31, 0x29, 0x46, 0x9b, 0x98, 0x20, 0xd1, 0xcc, 0xb8, 0x4d, 0x59, 0x47, 0x26,
-	0xd5, 0x35, 0xd5, 0xcc, 0x10, 0x90, 0x54, 0x33, 0x43, 0x40, 0xf0, 0x2f, 0x0d, 0xf3, 0x90, 0x96,
-	0x7f, 0x48, 0xf3, 0xb0, 0x0e, 0x6d, 0x1e, 0xd6, 0x3d, 0xcc, 0x03, 0x7f, 0x27, 0xd1, 0xa5, 0xbe,
-	0xa2, 0xd2, 0xe5, 0xf3, 0xb0, 0xd0, 0x4d, 0xad, 0x4c, 0xd7, 0xa9, 0x6c, 0xd4, 0x66, 0xd0, 0xf1,
-	0x46, 0xa2, 0x20, 0x01, 0x99, 0xa2, 0xa0, 0x8c, 0xd4, 0xf3, 0x07, 0xa4, 0xfe, 0xd8, 0x23, 0x50,
-	0x89, 0x7f, 0x66, 0x43, 0x55, 0x98, 0xbb, 0xfc, 0x0a, 0x79, 0xfd, 0x12, 0x59, 0x5f, 0xcc, 0xa1,
-	0x1a, 0x94, 0xdb, 0x97, 0xd6, 0x5e, 0x12, 0x33, 0xeb, 0xfc, 0x6f, 0x4b, 0x3a, 0x80, 0x87, 0xe8,
-	0x9b, 0x50, 0x94, 0x51, 0xf9, 0x78, 0x72, 0x5d, 0xf3, 0x17, 0xa8, 0xe5, 0x13, 0x07, 0xe0, 0x92,
-	0x6f, 0x9c, 0x7b, 0xc2, 0x42, 0xd7, 0xa0, 0x2a, 0x80, 0xaa, 0xc7, 0x7b, 0x2a, 0xdb, 0x6a, 0x4d,
-	0x51, 0x7a, 0x68, 0xca, 0xaa, 0x41, 0xef, 0x22, 0x14, 0xa5, 0x08, 0x8e, 0x67, 0x92, 0xa7, 0x09,
-	0xb7, 0x49, 0x75, 0xbd, 0x71, 0x0e, 0x3d, 0x03, 0xf6, 0x0d, 0xd7, 0xeb, 0x23, 0x23, 0x77, 0x33,
-	0x5a, 0xb3, 0xcb, 0xc7, 0xb3, 0x60, 0xe3, 0xd8, 0xe7, 0xe2, 0x0e, 0xf3, 0x89, 0x6c, 0x9b, 0x4b,
-	0x6f, 0x6f, 0x1c, 0x5c, 0x88, 0x4f, 0x7e, 0x45, 0xf6, 0x41, 0x75, 0xb3, 0x05, 0x3d, 0x94, 0x3e,
-	0x2a, 0xd3, 0x9b, 0x59, 0x6e, 0x4e, 0x5b, 0x8e, 0x09, 0x6e, 0x42, 0xd5, 0x68, 0x74, 0x98, 0x62,
-	0x3d, 0xd8, 0xa5, 0x31, 0xc5, 0x3a, 0xa1, 0x3b, 0x82, 0x73, 0x68, 0x03, 0xca, 0x3c, 0xe3, 0x15,
-	0x3f, 0x88, 0x9c, 0xcc, 0x26, 0xb6, 0x46, 0x42, 0xb3, 0x7c, 0x6a, 0xf2, 0x62, 0x4c, 0xe8, 0xdb,
-	0x50, 0xd9, 0xa0, 0x4c, 0x45, 0x85, 0x13, 0xd9, 0xb0, 0x32, 0x41, 0x52, 0xe9, 0xd0, 0x84, 0x73,
-	0xe8, 0x0d, 0x91, 0x7c, 0xa7, 0x9d, 0x22, 0x72, 0xa6, 0x38, 0xbf, 0xf8, 0x5e, 0x2b, 0xd3, 0x11,
-	0x62, 0xca, 0xaf, 0xa7, 0x28, 0xab, 0xf8, 0xe9, 0x4c, 0x79, 0x82, 0x31, 0x65, 0xe7, 0x1e, 0x7f,
-	0x97, 0xc0, 0xb9, 0xf3, 0x6f, 0xea, 0x7f, 0x0c, 0xac, 0xbb, 0xcc, 0x45, 0xaf, 0xc0, 0x82, 0x90,
-	0x65, 0xfc, 0x97, 0x82, 0x94, 0xcd, 0x1f, 0xf8, 0xff, 0x42, 0xca, 0xe6, 0x0f, 0xfe, 0x8f, 0x01,
-	0xe7, 0xda, 0x6f, 0x7e, 0xf0, 0x71, 0x33, 0xf7, 0xe1, 0xc7, 0xcd, 0xdc, 0xa7, 0x1f, 0x37, 0xad,
-	0xef, 0xef, 0x35, 0xad, 0xdf, 0xec, 0x35, 0xad, 0xf7, 0xf7, 0x9a, 0xd6, 0x07, 0x7b, 0x4d, 0xeb,
-	0x1f, 0x7b, 0x4d, 0xeb, 0x9f, 0x7b, 0xcd, 0xdc, 0xa7, 0x7b, 0x4d, 0xeb, 0xdd, 0x4f, 0x9a, 0xb9,
-	0x0f, 0x3e, 0x69, 0xe6, 0x3e, 0xfc, 0xa4, 0x99, 0xfb, 0xee, 0xa3, 0xf7, 0x2e, 0x34, 0xa5, 0xa3,
-	0x2b, 0x89, 0xaf, 0x27, 0xff, 0x13, 0x00, 0x00, 0xff, 0xff, 0x2d, 0xd2, 0x02, 0x97, 0xd7, 0x22,
-	0x00, 0x00,
+	// 2671 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x1a, 0x4d, 0x8c, 0x1b, 0x57,
+	0xd9, 0x63, 0x8f, 0xbd, 0xf6, 0x67, 0xef, 0x66, 0xf3, 0xd6, 0x49, 0xac, 0x4d, 0xea, 0xd9, 0x3e,
+	0x41, 0x1b, 0x9a, 0x74, 0xdd, 0xa4, 0xb4, 0xa4, 0x29, 0xa5, 0xc4, 0xbb, 0xcd, 0x36, 0xe9, 0x36,
+	0x4d, 0xdf, 0xa6, 0x69, 0x41, 0x54, 0xd5, 0xc4, 0x7e, 0xeb, 0x1d, 0xc5, 0x9e, 0x71, 0x66, 0x9e,
+	0x9b, 0xee, 0x0d, 0x89, 0x33, 0xa2, 0x12, 0x07, 0xe0, 0x82, 0x84, 0x84, 0x04, 0x02, 0xf5, 0x82,
+	0x38, 0x70, 0x40, 0x70, 0xe1, 0x50, 0x6e, 0xe5, 0x56, 0xf5, 0x60, 0xe8, 0xf6, 0x82, 0xf6, 0x54,
+	0x09, 0x89, 0x43, 0x4f, 0xe8, 0xfd, 0xcd, 0xbc, 0x99, 0xb5, 0x49, 0xbd, 0x0d, 0x2a, 0xb9, 0xd8,
+	0xf3, 0xbe, 0xf7, 0xbd, 0xef, 0xbd, 0xef, 0xe7, 0x7d, 0x7f, 0x33, 0x70, 0x72, 0x78, 0xbb, 0xd7,
+	0xea, 0x07, 0xbd, 0x61, 0x18, 0xb0, 0x20, 0x7e, 0x58, 0x15, 0xbf, 0xa8, 0xac, 0xc7, 0xcb, 0xf5,
+	0x5e, 0xd0, 0x0b, 0x24, 0x0e, 0x7f, 0x92, 0xf3, 0xcb, 0x4e, 0x2f, 0x08, 0x7a, 0x7d, 0xda, 0x12,
+	0xa3, 0x5b, 0xa3, 0xed, 0x16, 0xf3, 0x06, 0x34, 0x62, 0xee, 0x60, 0xa8, 0x10, 0x56, 0x14, 0xf5,
+	0x3b, 0xfd, 0x41, 0xd0, 0xa5, 0xfd, 0x56, 0xc4, 0x5c, 0x16, 0xc9, 0x5f, 0x85, 0xb1, 0xc4, 0x31,
+	0x86, 0xa3, 0x68, 0x47, 0xfc, 0x48, 0x20, 0xfe, 0xbd, 0x05, 0xc7, 0x36, 0xdd, 0x5b, 0xb4, 0x7f,
+	0x23, 0xb8, 0xe9, 0xf6, 0x47, 0x34, 0x22, 0x34, 0x1a, 0x06, 0x7e, 0x44, 0xd1, 0x1a, 0x94, 0xfa,
+	0x7c, 0x22, 0x6a, 0x58, 0x2b, 0x85, 0xd3, 0xd5, 0xf3, 0x67, 0x56, 0xe3, 0x23, 0x4f, 0x5c, 0x20,
+	0xa1, 0xd1, 0x0b, 0x3e, 0x0b, 0x77, 0x89, 0x5a, 0xba, 0x7c, 0x13, 0xaa, 0x06, 0x18, 0x2d, 0x42,
+	0xe1, 0x36, 0xdd, 0x6d, 0x58, 0x2b, 0xd6, 0xe9, 0x0a, 0xe1, 0x8f, 0xe8, 0x1c, 0x14, 0xdf, 0xe6,
+	0x64, 0x1a, 0xf9, 0x15, 0xeb, 0x74, 0xf5, 0xfc, 0xc9, 0x64, 0x93, 0xd7, 0x7c, 0xef, 0xce, 0x88,
+	0x8a, 0xd5, 0x6a, 0x23, 0x89, 0x79, 0x31, 0x7f, 0xc1, 0xc2, 0x67, 0xe0, 0xe8, 0x81, 0x79, 0x74,
+	0x1c, 0x4a, 0x02, 0x43, 0x9e, 0xb8, 0x42, 0xd4, 0x08, 0xd7, 0x01, 0x6d, 0xb1, 0x90, 0xba, 0x03,
+	0xe2, 0x32, 0x7e, 0xde, 0x3b, 0x23, 0x1a, 0x31, 0xfc, 0x32, 0x2c, 0xa5, 0xa0, 0x8a, 0xed, 0xa7,
+	0xa1, 0x1a, 0x25, 0x60, 0xc5, 0x7b, 0x3d, 0x39, 0x56, 0xb2, 0x86, 0x98, 0x88, 0xf8, 0xe7, 0x16,
+	0x40, 0x32, 0x87, 0x9a, 0x00, 0x72, 0xf6, 0x45, 0x37, 0xda, 0x11, 0x0c, 0xdb, 0xc4, 0x80, 0xa0,
+	0xb3, 0x70, 0x34, 0x19, 0x5d, 0x0b, 0xb6, 0x76, 0xdc, 0xb0, 0x2b, 0x64, 0x60, 0x93, 0x83, 0x13,
+	0x08, 0x81, 0x1d, 0xba, 0x8c, 0x36, 0x0a, 0x2b, 0xd6, 0xe9, 0x02, 0x11, 0xcf, 0x9c, 0x5b, 0x46,
+	0x7d, 0xd7, 0x67, 0x0d, 0x5b, 0x88, 0x53, 0x8d, 0x38, 0x9c, 0xeb, 0x97, 0x46, 0x8d, 0xe2, 0x8a,
+	0x75, 0x7a, 0x9e, 0xa8, 0x11, 0xfe, 0x77, 0x01, 0x6a, 0xaf, 0x8e, 0x68, 0xb8, 0xab, 0x04, 0x80,
+	0x9a, 0x50, 0x8e, 0x68, 0x9f, 0x76, 0x58, 0x10, 0x4a, 0x8d, 0xb4, 0xf3, 0x0d, 0x8b, 0xc4, 0x30,
+	0x54, 0x87, 0x62, 0xdf, 0x1b, 0x78, 0x4c, 0x1c, 0x6b, 0x9e, 0xc8, 0x01, 0xba, 0x08, 0xc5, 0x88,
+	0xb9, 0x21, 0x13, 0x67, 0xa9, 0x9e, 0x5f, 0x5e, 0x95, 0x86, 0xb9, 0xaa, 0x0d, 0x73, 0xf5, 0x86,
+	0x36, 0xcc, 0x76, 0xf9, 0xfd, 0xb1, 0x93, 0x7b, 0xf7, 0xef, 0x8e, 0x45, 0xe4, 0x12, 0xf4, 0x34,
+	0x14, 0xa8, 0xdf, 0x15, 0xe7, 0xfd, 0xbc, 0x2b, 0xf9, 0x02, 0x74, 0x0e, 0x2a, 0x5d, 0x2f, 0xa4,
+	0x1d, 0xe6, 0x05, 0xbe, 0xe0, 0x6a, 0xe1, 0xfc, 0x52, 0xa2, 0x91, 0x75, 0x3d, 0x45, 0x12, 0x2c,
+	0x74, 0x16, 0x4a, 0x11, 0x17, 0x5d, 0xd4, 0x98, 0xe3, 0xb6, 0xd0, 0xae, 0xef, 0x8f, 0x9d, 0x45,
+	0x09, 0x39, 0x1b, 0x0c, 0x3c, 0x46, 0x07, 0x43, 0xb6, 0x4b, 0x14, 0x0e, 0x7a, 0x0c, 0xe6, 0xba,
+	0xb4, 0x4f, 0xb9, 0xc2, 0xcb, 0x42, 0xe1, 0x8b, 0x06, 0x79, 0x31, 0x41, 0x34, 0x02, 0x7a, 0x13,
+	0xec, 0x61, 0xdf, 0xf5, 0x1b, 0x15, 0xc1, 0xc5, 0x42, 0x82, 0x78, 0xbd, 0xef, 0xfa, 0xed, 0x67,
+	0x3e, 0x1a, 0x3b, 0x4f, 0xf5, 0x3c, 0xb6, 0x33, 0xba, 0xb5, 0xda, 0x09, 0x06, 0xad, 0x5e, 0xe8,
+	0x6e, 0xbb, 0xbe, 0xdb, 0xea, 0x07, 0xb7, 0xbd, 0xd6, 0xdb, 0x4f, 0xb6, 0xf8, 0x1d, 0xbc, 0x33,
+	0xa2, 0xa1, 0x47, 0xc3, 0x16, 0x27, 0xb3, 0x2a, 0x54, 0xc2, 0x97, 0x12, 0x41, 0x16, 0x5d, 0xe5,
+	0xf6, 0x17, 0x84, 0x74, 0x6d, 0x67, 0xe4, 0xdf, 0x8e, 0x1a, 0x20, 0x76, 0x39, 0x91, 0xec, 0x22,
+	0xe0, 0x84, 0x6e, 0x6f, 0x84, 0xc1, 0x68, 0xd8, 0x3e, 0xb2, 0x3f, 0x76, 0x4c, 0x7c, 0x62, 0x0e,
+	0xae, 0xda, 0xe5, 0xd2, 0xe2, 0x1c, 0x7e, 0xaf, 0x00, 0x68, 0xcb, 0x1d, 0x0c, 0xfb, 0x74, 0x26,
+	0xf5, 0xc7, 0x8a, 0xce, 0x1f, 0x5a, 0xd1, 0x85, 0x59, 0x15, 0x9d, 0x68, 0xcd, 0x9e, 0x4d, 0x6b,
+	0xc5, 0xcf, 0xab, 0xb5, 0xd2, 0xff, 0xbd, 0xd6, 0x70, 0x03, 0x6c, 0x4e, 0x99, 0x3b, 0xcb, 0xd0,
+	0xbd, 0x2b, 0x74, 0x53, 0x23, 0xfc, 0x11, 0x6f, 0x42, 0x49, 0xf2, 0x85, 0x96, 0xb3, 0xca, 0x4b,
+	0xdf, 0xdb, 0x44, 0x71, 0x05, 0xad, 0x92, 0xc5, 0x44, 0x25, 0x05, 0x21, 0x6c, 0xfc, 0x47, 0x0b,
+	0xe6, 0x95, 0x45, 0x28, 0xdf, 0x77, 0x0b, 0xe6, 0xa4, 0xef, 0xd1, 0x7e, 0xef, 0x44, 0xd6, 0xef,
+	0x5d, 0xea, 0xba, 0x43, 0x46, 0xc3, 0x76, 0xeb, 0xfd, 0xb1, 0x63, 0x7d, 0x34, 0x76, 0x1e, 0x9d,
+	0x26, 0x34, 0x1d, 0x6b, 0xb4, 0xbf, 0xd4, 0x84, 0xd1, 0x19, 0x71, 0x3a, 0x16, 0x29, 0xb3, 0x3a,
+	0xb2, 0x2a, 0x43, 0xd4, 0x15, 0xbf, 0x47, 0x23, 0x4e, 0xd9, 0xe6, 0x16, 0x41, 0x24, 0x0e, 0x67,
+	0xf3, 0xae, 0x1b, 0xfa, 0x9e, 0xdf, 0x8b, 0x1a, 0x05, 0xe1, 0xd3, 0xe3, 0x31, 0xfe, 0xa9, 0x05,
+	0x4b, 0x29, 0xb3, 0x56, 0x4c, 0x5c, 0x80, 0x52, 0xc4, 0x35, 0xa5, 0x79, 0x30, 0x8c, 0x62, 0x4b,
+	0xc0, 0xdb, 0x0b, 0xea, 0xf0, 0x25, 0x39, 0x26, 0x0a, 0xff, 0xfe, 0x1d, 0xed, 0x2f, 0x16, 0xd4,
+	0x44, 0x60, 0xd2, 0x77, 0x0d, 0x81, 0xed, 0xbb, 0x03, 0xaa, 0x54, 0x25, 0x9e, 0x8d, 0x68, 0xc5,
+	0xb7, 0x2b, 0xeb, 0x68, 0x35, 0xab, 0x83, 0xb5, 0x0e, 0xed, 0x60, 0xad, 0xe4, 0xde, 0xd5, 0xa1,
+	0xc8, 0xcd, 0x7b, 0x57, 0x38, 0xd7, 0x0a, 0x91, 0x03, 0xfc, 0x28, 0xcc, 0x2b, 0x2e, 0x94, 0x68,
+	0xa7, 0x05, 0xd8, 0x01, 0x94, 0xa4, 0x26, 0xd0, 0x57, 0xa0, 0x12, 0x27, 0x26, 0x82, 0xdb, 0x42,
+	0xbb, 0xb4, 0x3f, 0x76, 0xf2, 0x2c, 0x22, 0xc9, 0x04, 0x72, 0xcc, 0xa0, 0x6f, 0xb5, 0x2b, 0xfb,
+	0x63, 0x47, 0x02, 0x54, 0x88, 0x47, 0xa7, 0xc0, 0xde, 0xe1, 0x71, 0x93, 0x8b, 0xc0, 0x6e, 0x97,
+	0xf7, 0xc7, 0x8e, 0x18, 0x13, 0xf1, 0x8b, 0x37, 0xa0, 0xb6, 0x49, 0x7b, 0x6e, 0x67, 0x57, 0x6d,
+	0x5a, 0xd7, 0xe4, 0xf8, 0x86, 0x96, 0xa6, 0xf1, 0x30, 0xd4, 0xe2, 0x1d, 0xdf, 0x1a, 0x44, 0xea,
+	0x36, 0x54, 0x63, 0xd8, 0xcb, 0x11, 0xfe, 0x99, 0x05, 0xca, 0x06, 0x10, 0x36, 0xb2, 0x1d, 0xee,
+	0x0b, 0x61, 0x7f, 0xec, 0x28, 0x88, 0x4e, 0x66, 0xd0, 0xb3, 0x30, 0x17, 0x89, 0x1d, 0x39, 0xb1,
+	0xac, 0x69, 0x89, 0x89, 0xf6, 0x11, 0x6e, 0x22, 0xfb, 0x63, 0x47, 0x23, 0x12, 0xfd, 0x80, 0x56,
+	0x53, 0x09, 0x81, 0x64, 0x6c, 0x61, 0x7f, 0xec, 0x18, 0x50, 0x33, 0x41, 0xc0, 0x9f, 0x59, 0x50,
+	0xbd, 0xe1, 0x7a, 0xb1, 0x09, 0x35, 0xb4, 0x8a, 0x12, 0x5f, 0x2d, 0x01, 0xdc, 0x12, 0xbb, 0xb4,
+	0xef, 0xee, 0x5e, 0x0e, 0x42, 0x41, 0x77, 0x9e, 0xc4, 0xe3, 0x24, 0x86, 0xdb, 0x13, 0x63, 0x78,
+	0x71, 0x76, 0xd7, 0xfe, 0xbf, 0x75, 0xa4, 0x57, 0xed, 0x72, 0x7e, 0xb1, 0x80, 0xdf, 0xb3, 0xa0,
+	0x26, 0x99, 0x57, 0x96, 0xf7, 0x3d, 0x28, 0x49, 0xd9, 0x08, 0xf6, 0xff, 0x8b, 0x63, 0x3a, 0x33,
+	0x8b, 0x53, 0x52, 0x34, 0xd1, 0xf3, 0xb0, 0xd0, 0x0d, 0x83, 0xe1, 0x90, 0x76, 0xb7, 0x94, 0xfb,
+	0xcb, 0x67, 0xdd, 0xdf, 0xba, 0x39, 0x4f, 0x32, 0xe8, 0xf8, 0xaf, 0x16, 0xcc, 0x2b, 0x67, 0xa2,
+	0xd4, 0x15, 0x8b, 0xd8, 0x3a, 0x74, 0xf4, 0xcc, 0xcf, 0x1a, 0x3d, 0x8f, 0x43, 0xa9, 0xc7, 0xe3,
+	0x8b, 0x76, 0x48, 0x6a, 0x34, 0x5b, 0x54, 0xc5, 0x57, 0x61, 0x41, 0xb3, 0x32, 0xc5, 0xa3, 0x2e,
+	0x67, 0x3d, 0xea, 0x95, 0x2e, 0xf5, 0x99, 0xb7, 0xed, 0xc5, 0x3e, 0x52, 0xe1, 0xe3, 0x1f, 0x59,
+	0xb0, 0x98, 0x45, 0x41, 0xeb, 0x99, 0xc2, 0xe2, 0x91, 0xe9, 0xe4, 0xcc, 0x9a, 0x42, 0x93, 0x56,
+	0x95, 0xc5, 0x53, 0xf7, 0xaa, 0x2c, 0xea, 0xa6, 0x93, 0xa9, 0x28, 0xaf, 0x80, 0x7f, 0x62, 0xc1,
+	0x7c, 0x4a, 0x97, 0xe8, 0x02, 0xd8, 0xdb, 0x61, 0x30, 0x98, 0x49, 0x51, 0x62, 0x05, 0xfa, 0x3a,
+	0xe4, 0x59, 0x30, 0x93, 0x9a, 0xf2, 0x2c, 0xe0, 0x5a, 0x52, 0xec, 0x17, 0x64, 0xde, 0x2e, 0x47,
+	0xf8, 0x29, 0xa8, 0x08, 0x86, 0xae, 0xbb, 0x5e, 0x38, 0x31, 0x60, 0x4c, 0x66, 0xe8, 0x59, 0x38,
+	0x22, 0x9d, 0xe1, 0xe4, 0xc5, 0xb5, 0x49, 0x8b, 0x6b, 0x7a, 0xf1, 0x49, 0x28, 0x8a, 0xa4, 0x83,
+	0x2f, 0xe9, 0xba, 0xcc, 0xd5, 0x4b, 0xf8, 0x33, 0x3e, 0x06, 0x4b, 0xfc, 0x0e, 0xd2, 0x30, 0x5a,
+	0x0b, 0x46, 0x3e, 0xd3, 0x75, 0xd3, 0x59, 0xa8, 0xa7, 0xc1, 0xca, 0x4a, 0xea, 0x50, 0xec, 0x70,
+	0x80, 0xa0, 0x31, 0x4f, 0xe4, 0x00, 0xff, 0xd2, 0x02, 0xb4, 0x41, 0x99, 0xd8, 0xe5, 0xca, 0x7a,
+	0x7c, 0x3d, 0x96, 0xa1, 0x3c, 0x70, 0x59, 0x67, 0x87, 0x86, 0x91, 0xce, 0x5f, 0xf4, 0xf8, 0xcb,
+	0x48, 0x3c, 0xf1, 0x39, 0x58, 0x4a, 0x9d, 0x52, 0xf1, 0xb4, 0x0c, 0xe5, 0x8e, 0x82, 0xa9, 0x90,
+	0x17, 0x8f, 0xf1, 0xef, 0xf2, 0x50, 0xd6, 0x69, 0x1d, 0x3a, 0x07, 0xd5, 0x6d, 0xcf, 0xef, 0xd1,
+	0x70, 0x18, 0x7a, 0x4a, 0x04, 0xb6, 0x4c, 0xf3, 0x0c, 0x30, 0x31, 0x07, 0xe8, 0x71, 0x98, 0x1b,
+	0x45, 0x34, 0x7c, 0xcb, 0x93, 0x37, 0xbd, 0xd2, 0xae, 0xef, 0x8d, 0x9d, 0xd2, 0x6b, 0x11, 0x0d,
+	0xaf, 0xac, 0xf3, 0xe0, 0x33, 0x12, 0x4f, 0x44, 0xfe, 0x77, 0xd1, 0x4b, 0xca, 0x4c, 0x45, 0x02,
+	0xd7, 0xfe, 0x06, 0x3f, 0x7e, 0xc6, 0xd5, 0x0d, 0xc3, 0x60, 0x40, 0xd9, 0x0e, 0x1d, 0x45, 0xad,
+	0x4e, 0x30, 0x18, 0x04, 0x7e, 0x4b, 0x74, 0x02, 0x04, 0xd3, 0x3c, 0x82, 0xf2, 0xe5, 0xca, 0x72,
+	0x6f, 0xc0, 0x1c, 0xdb, 0x09, 0x83, 0x51, 0x6f, 0x47, 0x04, 0x86, 0x42, 0xfb, 0xe2, 0xec, 0xf4,
+	0x34, 0x05, 0xa2, 0x1f, 0xd0, 0xc3, 0x5c, 0x5a, 0xb4, 0x73, 0x3b, 0x1a, 0x0d, 0x64, 0xed, 0xd9,
+	0x2e, 0xee, 0x8f, 0x1d, 0xeb, 0x71, 0x12, 0x83, 0xf1, 0x25, 0x98, 0x4f, 0xa5, 0xc2, 0xe8, 0x09,
+	0xb0, 0x43, 0xba, 0xad, 0x5d, 0x01, 0x3a, 0x98, 0x31, 0xcb, 0xe8, 0xcf, 0x71, 0x88, 0xf8, 0xc5,
+	0x3f, 0xcc, 0x83, 0x63, 0x54, 0xfd, 0x97, 0x83, 0xf0, 0x65, 0xca, 0x42, 0xaf, 0x73, 0xcd, 0x1d,
+	0x50, 0x6d, 0x5e, 0x0e, 0x54, 0x07, 0x02, 0xf8, 0x96, 0x71, 0x8b, 0x60, 0x10, 0xe3, 0xa1, 0x87,
+	0x00, 0xc4, 0xb5, 0x93, 0xf3, 0xf2, 0x42, 0x55, 0x04, 0x44, 0x4c, 0xaf, 0xa5, 0x84, 0xdd, 0x9a,
+	0x51, 0x38, 0x4a, 0xc8, 0x57, 0xb2, 0x42, 0x9e, 0x99, 0x4e, 0x2c, 0x59, 0xf3, 0xba, 0x14, 0xd3,
+	0xd7, 0x05, 0xff, 0xcd, 0x82, 0xe6, 0xa6, 0x3e, 0xf9, 0x21, 0xc5, 0xa1, 0xf9, 0xcd, 0xdf, 0x27,
+	0x7e, 0x0b, 0x5f, 0x8c, 0x5f, 0xdc, 0x04, 0xd8, 0xf4, 0x7c, 0x7a, 0xd9, 0xeb, 0x33, 0x1a, 0x4e,
+	0x28, 0x84, 0x7e, 0x5c, 0x48, 0xbc, 0x0a, 0xa1, 0xdb, 0x9a, 0xcf, 0x35, 0xc3, 0x95, 0xdf, 0x0f,
+	0x36, 0xf2, 0xf7, 0x51, 0x6d, 0x85, 0x8c, 0x97, 0xf3, 0x61, 0x6e, 0x5b, 0xb0, 0x27, 0xa3, 0x72,
+	0xaa, 0xc7, 0x94, 0xf0, 0xde, 0xfe, 0x96, 0xda, 0xfc, 0xe9, 0x7b, 0x24, 0x55, 0xa2, 0xf3, 0xd7,
+	0x8a, 0x76, 0x7d, 0xe6, 0xbe, 0x63, 0xac, 0x27, 0x7a, 0x13, 0xe4, 0xaa, 0xbc, 0xad, 0x38, 0x31,
+	0x6f, 0x7b, 0x4e, 0x6d, 0xf3, 0x45, 0x72, 0x37, 0xfc, 0x5c, 0xe2, 0x44, 0x85, 0x52, 0x94, 0x13,
+	0x7d, 0xe4, 0x5e, 0x57, 0x5c, 0x5d, 0xec, 0x3f, 0x59, 0xb0, 0xb8, 0x41, 0x59, 0x3a, 0x8f, 0x7a,
+	0x80, 0x54, 0x8a, 0x5f, 0x84, 0xa3, 0xc6, 0xf9, 0x15, 0xf7, 0x4f, 0x66, 0x92, 0xa7, 0x63, 0x09,
+	0xff, 0x57, 0xfc, 0x2e, 0x7d, 0x47, 0xd5, 0xa4, 0xe9, 0xbc, 0xe9, 0x3a, 0x54, 0x8d, 0x49, 0x74,
+	0x29, 0x93, 0x31, 0x2d, 0x65, 0x5a, 0xb1, 0x3c, 0xea, 0xb7, 0xeb, 0x8a, 0x27, 0x59, 0x79, 0xaa,
+	0x7c, 0x38, 0xce, 0x2e, 0xb6, 0x00, 0x09, 0x75, 0x09, 0xb2, 0x66, 0x7c, 0x13, 0xd0, 0x97, 0xe2,
+	0xd4, 0x29, 0x1e, 0xa3, 0x87, 0xc1, 0x0e, 0x83, 0xbb, 0x3a, 0x15, 0x9e, 0x4f, 0xb6, 0x24, 0xc1,
+	0x5d, 0x22, 0xa6, 0xf0, 0xb3, 0x50, 0x20, 0xc1, 0x5d, 0xd4, 0x04, 0x08, 0x5d, 0xbf, 0x47, 0x6f,
+	0xc6, 0x45, 0x58, 0x8d, 0x18, 0x90, 0x29, 0xb9, 0xc7, 0x1a, 0x1c, 0x35, 0x4f, 0x24, 0xd5, 0xbd,
+	0x0a, 0x73, 0xaf, 0x8e, 0x4c, 0x71, 0xd5, 0x33, 0xe2, 0x92, 0xb5, 0xbe, 0x46, 0xe2, 0x36, 0x03,
+	0x09, 0x1c, 0x9d, 0x82, 0x0a, 0x73, 0x6f, 0xf5, 0xe9, 0xb5, 0xc4, 0xcd, 0x25, 0x00, 0x3e, 0xcb,
+	0xeb, 0xc7, 0x9b, 0x46, 0x12, 0x95, 0x00, 0xd0, 0x63, 0xb0, 0x98, 0x9c, 0xf9, 0x7a, 0x48, 0xb7,
+	0xbd, 0x77, 0x84, 0x86, 0x6b, 0xe4, 0x00, 0x1c, 0x9d, 0x86, 0x23, 0x09, 0x6c, 0x4b, 0x24, 0x2b,
+	0xb6, 0x40, 0xcd, 0x82, 0xb9, 0x6c, 0x04, 0xbb, 0x2f, 0xdc, 0x19, 0xb9, 0x7d, 0x71, 0xf9, 0x6a,
+	0xc4, 0x80, 0xe0, 0x3f, 0x5b, 0x70, 0x54, 0xaa, 0x9a, 0xb9, 0xec, 0x81, 0xb4, 0xfa, 0x5f, 0x59,
+	0x80, 0x4c, 0x0e, 0x94, 0x69, 0x7d, 0xd5, 0xec, 0x25, 0xf1, 0x6c, 0xa8, 0x2a, 0xca, 0x62, 0x09,
+	0x4a, 0xda, 0x41, 0x18, 0x4a, 0x1d, 0xd9, 0x33, 0x13, 0xcd, 0x6f, 0x59, 0x77, 0x4b, 0x08, 0x51,
+	0xff, 0xc8, 0x81, 0xe2, 0xad, 0x5d, 0x46, 0x23, 0x55, 0x35, 0x8b, 0x76, 0x81, 0x00, 0x10, 0xf9,
+	0xc7, 0xf7, 0xa2, 0x3e, 0x13, 0x56, 0x63, 0x27, 0x7b, 0x29, 0x10, 0xd1, 0x0f, 0xf8, 0xb7, 0x79,
+	0x98, 0xbf, 0x19, 0xf4, 0x47, 0x49, 0x60, 0x7c, 0x90, 0x02, 0x46, 0xaa, 0x94, 0x2f, 0xea, 0x52,
+	0x1e, 0x81, 0x1d, 0x31, 0x3a, 0x14, 0x96, 0x55, 0x20, 0xe2, 0x19, 0x61, 0xa8, 0x31, 0x37, 0xec,
+	0x51, 0x26, 0x0b, 0xa4, 0x46, 0x49, 0x64, 0xae, 0x29, 0x18, 0x5a, 0x81, 0xaa, 0xdb, 0xeb, 0x85,
+	0xb4, 0xe7, 0x32, 0xda, 0xde, 0x6d, 0xcc, 0x89, 0xcd, 0x4c, 0x10, 0x7e, 0x03, 0x16, 0xb4, 0xb0,
+	0x94, 0x4a, 0x9f, 0x80, 0xb9, 0xb7, 0x05, 0x64, 0x42, 0x6b, 0x4d, 0xa2, 0x2a, 0x37, 0xa6, 0xd1,
+	0xd2, 0xaf, 0x10, 0xf4, 0x99, 0xf1, 0x55, 0x28, 0x49, 0x74, 0x74, 0xca, 0x2c, 0x73, 0x64, 0xa6,
+	0xc7, 0xc7, 0xaa, 0x66, 0xc1, 0x50, 0x92, 0x84, 0x94, 0xe2, 0x85, 0x6d, 0x48, 0x08, 0x51, 0xff,
+	0xf8, 0x5f, 0x16, 0x1c, 0x5b, 0xa7, 0x8c, 0x76, 0x18, 0xed, 0x5e, 0xf6, 0x68, 0xbf, 0xfb, 0xa5,
+	0x56, 0xe0, 0x71, 0x1f, 0xad, 0x60, 0xf4, 0xd1, 0xb8, 0xdf, 0xe9, 0x7b, 0x3e, 0xdd, 0x34, 0x1a,
+	0x31, 0x09, 0x80, 0x7b, 0x88, 0x6d, 0x7e, 0x70, 0x39, 0x2d, 0xdf, 0xd9, 0x18, 0x90, 0x58, 0xc3,
+	0xa5, 0x44, 0xc3, 0xf8, 0x07, 0x16, 0x1c, 0xcf, 0x72, 0xad, 0x94, 0xd4, 0x82, 0x92, 0x58, 0x3c,
+	0xa1, 0x85, 0x9b, 0x5a, 0x41, 0x14, 0x1a, 0xba, 0x90, 0xda, 0x5f, 0xbc, 0xeb, 0x69, 0x37, 0xf6,
+	0xc7, 0x4e, 0x3d, 0x81, 0x1a, 0x5d, 0x02, 0x03, 0x17, 0xff, 0x81, 0xd7, 0xd2, 0x26, 0x4d, 0xa1,
+	0x6f, 0x6e, 0x5f, 0xca, 0xf7, 0xca, 0x01, 0xfa, 0x1a, 0xd8, 0x6c, 0x77, 0xa8, 0x5c, 0x6e, 0xfb,
+	0xd8, 0x67, 0x63, 0xe7, 0x68, 0x6a, 0xd9, 0x8d, 0xdd, 0x21, 0x25, 0x02, 0x85, 0x9b, 0x65, 0xc7,
+	0x0d, 0xbb, 0x9e, 0xef, 0xf6, 0x3d, 0x26, 0xc5, 0x68, 0x13, 0x13, 0x24, 0x5e, 0x6f, 0xb9, 0x61,
+	0x44, 0x43, 0xfd, 0xda, 0x4b, 0x8e, 0x44, 0x93, 0xe3, 0x36, 0x65, 0x9d, 0x1d, 0xe9, 0x64, 0x55,
+	0x93, 0x43, 0x40, 0x52, 0x4d, 0x0e, 0x01, 0xc1, 0xbf, 0x30, 0xcc, 0x46, 0xde, 0x88, 0x43, 0x9a,
+	0x8d, 0x75, 0x68, 0xb3, 0xb1, 0xee, 0x61, 0x36, 0xf8, 0x3b, 0x89, 0x8e, 0xf5, 0x11, 0x95, 0x8e,
+	0x9f, 0x87, 0x85, 0x6e, 0x6a, 0x66, 0xba, 0xae, 0x65, 0x03, 0x37, 0x83, 0x8e, 0x37, 0x12, 0xc5,
+	0x09, 0xc8, 0x14, 0xc5, 0x65, 0xb4, 0x91, 0x3f, 0xa0, 0x8d, 0xc7, 0x1e, 0x81, 0x4a, 0xfc, 0xfa,
+	0x0d, 0x55, 0x61, 0xee, 0xf2, 0x2b, 0xe4, 0xf5, 0x4b, 0x64, 0x7d, 0x31, 0x87, 0x6a, 0x50, 0x6e,
+	0x5f, 0x5a, 0x7b, 0x49, 0x8c, 0xac, 0xf3, 0xbf, 0x29, 0xe9, 0xc0, 0x1e, 0xa2, 0x6f, 0x42, 0x51,
+	0x46, 0xeb, 0xe3, 0xc9, 0x71, 0xcd, 0x37, 0x53, 0xcb, 0x27, 0x0e, 0xc0, 0x25, 0xdf, 0x38, 0xf7,
+	0x84, 0x85, 0xae, 0x41, 0x55, 0x00, 0x55, 0xef, 0xf7, 0x54, 0xb6, 0x05, 0x9b, 0xa2, 0xf4, 0xd0,
+	0x94, 0x59, 0x83, 0xde, 0x45, 0x28, 0x4a, 0x11, 0x1c, 0xcf, 0x24, 0x55, 0x13, 0x4e, 0x93, 0xea,
+	0x86, 0xe3, 0x1c, 0x7a, 0x06, 0xec, 0x1b, 0xae, 0xd7, 0x47, 0x46, 0x4e, 0x67, 0xb4, 0x6c, 0x97,
+	0x8f, 0x67, 0xc1, 0xc6, 0xb6, 0xcf, 0xc5, 0x9d, 0xe7, 0x13, 0xd9, 0xf6, 0x97, 0x5e, 0xde, 0x38,
+	0x38, 0x11, 0xef, 0xfc, 0x8a, 0xec, 0x8f, 0xea, 0x26, 0x0c, 0x7a, 0x28, 0xbd, 0x55, 0xa6, 0x67,
+	0xb3, 0xdc, 0x9c, 0x36, 0x1d, 0x13, 0xdc, 0x84, 0xaa, 0xd1, 0x00, 0x31, 0xc5, 0x7a, 0xb0, 0x7b,
+	0x63, 0x8a, 0x75, 0x42, 0xd7, 0x04, 0xe7, 0xd0, 0x06, 0x94, 0x79, 0x26, 0x2c, 0x5e, 0x94, 0x9c,
+	0xcc, 0x26, 0xbc, 0x46, 0xa2, 0xb3, 0x7c, 0x6a, 0xf2, 0x64, 0x4c, 0xe8, 0xdb, 0x50, 0xd9, 0xa0,
+	0x4c, 0x45, 0x8b, 0x13, 0xd9, 0x70, 0x33, 0x41, 0x52, 0xe9, 0x90, 0x85, 0x73, 0xe8, 0x0d, 0x91,
+	0x94, 0xa7, 0x9d, 0x25, 0x72, 0xa6, 0x38, 0xc5, 0xf8, 0x5c, 0x2b, 0xd3, 0x11, 0x62, 0xca, 0xaf,
+	0xa7, 0x28, 0xab, 0xb8, 0xea, 0x4c, 0xb9, 0x82, 0x31, 0x65, 0xe7, 0x1e, 0x9f, 0x51, 0xe0, 0xdc,
+	0xf9, 0x37, 0xf5, 0x97, 0x04, 0xeb, 0x2e, 0x73, 0xd1, 0x2b, 0xb0, 0x20, 0x64, 0x19, 0x7f, 0x6a,
+	0x90, 0xb2, 0xf9, 0x03, 0xdf, 0x35, 0xa4, 0x6c, 0xfe, 0xe0, 0xf7, 0x0d, 0x38, 0xd7, 0x7e, 0xf3,
+	0x83, 0x8f, 0x9b, 0xb9, 0x0f, 0x3f, 0x6e, 0xe6, 0x3e, 0xfd, 0xb8, 0x69, 0x7d, 0x7f, 0xaf, 0x69,
+	0xfd, 0x7a, 0xaf, 0x69, 0xbd, 0xbf, 0xd7, 0xb4, 0x3e, 0xd8, 0x6b, 0x5a, 0xff, 0xd8, 0x6b, 0x5a,
+	0xff, 0xdc, 0x6b, 0xe6, 0x3e, 0xdd, 0x6b, 0x5a, 0xef, 0x7e, 0xd2, 0xcc, 0x7d, 0xf0, 0x49, 0x33,
+	0xf7, 0xe1, 0x27, 0xcd, 0xdc, 0x77, 0x1f, 0xbd, 0x77, 0x01, 0x2a, 0x1d, 0x5d, 0x49, 0xfc, 0x3d,
+	0xf9, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x82, 0x5c, 0x85, 0xa1, 0xef, 0x22, 0x00, 0x00,
 }
 
 func (x Direction) String() string {
@@ -4956,6 +4964,9 @@ func (this *DetectedField) Equal(that interface{}) bool {
 	if this.Cardinality != that1.Cardinality {
 		return false
 	}
+	if this.Parser != that1.Parser {
+		return false
+	}
 	if !bytes.Equal(this.Sketch, that1.Sketch) {
 		return false
 	}
@@ -5720,11 +5731,12 @@ func (this *DetectedField) GoString() string {
 	if this == nil {
 		return "nil"
 	}
-	s := make([]string, 0, 8)
+	s := make([]string, 0, 9)
 	s = append(s, "&logproto.DetectedField{")
 	s = append(s, "Label: "+fmt.Sprintf("%#v", this.Label)+",\n")
 	s = append(s, "Type: "+fmt.Sprintf("%#v", this.Type)+",\n")
 	s = append(s, "Cardinality: "+fmt.Sprintf("%#v", this.Cardinality)+",\n")
+	s = append(s, "Parser: "+fmt.Sprintf("%#v", this.Parser)+",\n")
 	s = append(s, "Sketch: "+fmt.Sprintf("%#v", this.Sketch)+",\n")
 	s = append(s, "}")
 	return strings.Join(s, "")
@@ -8677,6 +8689,13 @@ func (m *DetectedField) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 		copy(dAtA[i:], m.Sketch)
 		i = encodeVarintLogproto(dAtA, i, uint64(len(m.Sketch)))
 		i--
+		dAtA[i] = 0x2a
+	}
+	if len(m.Parser) > 0 {
+		i -= len(m.Parser)
+		copy(dAtA[i:], m.Parser)
+		i = encodeVarintLogproto(dAtA, i, uint64(len(m.Parser)))
+		i--
 		dAtA[i] = 0x22
 	}
 	if m.Cardinality != 0 {
@@ -9849,6 +9868,10 @@ func (m *DetectedField) Size() (n int) {
 	if m.Cardinality != 0 {
 		n += 1 + sovLogproto(uint64(m.Cardinality))
 	}
+	l = len(m.Parser)
+	if l > 0 {
+		n += 1 + l + sovLogproto(uint64(l))
+	}
 	l = len(m.Sketch)
 	if l > 0 {
 		n += 1 + l + sovLogproto(uint64(l))
@@ -10599,6 +10622,7 @@ func (this *DetectedField) String() string {
 		`Label:` + fmt.Sprintf("%v", this.Label) + `,`,
 		`Type:` + fmt.Sprintf("%v", this.Type) + `,`,
 		`Cardinality:` + fmt.Sprintf("%v", this.Cardinality) + `,`,
+		`Parser:` + fmt.Sprintf("%v", this.Parser) + `,`,
 		`Sketch:` + fmt.Sprintf("%v", this.Sketch) + `,`,
 		`}`,
 	}, "")
@@ -17470,6 +17494,38 @@ func (m *DetectedField) Unmarshal(dAtA []byte) error {
 				}
 			}
 		case 4:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Parser", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowLogproto
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthLogproto
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthLogproto
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Parser = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 5:
 			if wireType != 2 {
 				return fmt.Errorf("proto: wrong wireType = %d for field Sketch", wireType)
 			}
diff --git a/pkg/logproto/logproto.proto b/pkg/logproto/logproto.proto
index a29e38df01af9..f6f8c12a8fdec 100644
--- a/pkg/logproto/logproto.proto
+++ b/pkg/logproto/logproto.proto
@@ -471,7 +471,8 @@ message DetectedField {
   string label = 1;
   string type = 2 [(gogoproto.casttype) = "DetectedFieldType"];
   uint64 cardinality = 3;
-  bytes sketch = 4 [(gogoproto.jsontag) = "sketch,omitempty"];
+  string parser = 4;
+  bytes sketch = 5 [(gogoproto.jsontag) = "sketch,omitempty"];
 }
 
 message DetectedLabelsRequest {
diff --git a/pkg/logql/engine.go b/pkg/logql/engine.go
index eabbf1c43323a..a25a726bb3021 100644
--- a/pkg/logql/engine.go
+++ b/pkg/logql/engine.go
@@ -232,7 +232,6 @@ func (q *query) Exec(ctx context.Context) (logqlmodel.Result, error) {
 	sp, ctx := opentracing.StartSpanFromContext(ctx, "query.Exec")
 	defer sp.Finish()
 	spLogger := spanlogger.FromContext(ctx)
-	defer spLogger.Finish()
 
 	sp.LogKV(
 		"type", GetRangeType(q.params),
diff --git a/pkg/loki/modules.go b/pkg/loki/modules.go
index a4690a779f0bf..458d7c9e3f5c8 100644
--- a/pkg/loki/modules.go
+++ b/pkg/loki/modules.go
@@ -729,6 +729,14 @@ func (t *Loki) initBloomStore() (services.Service, error) {
 	var metasCache cache.Cache
 	if cache.IsCacheConfigured(bsCfg.MetasCache) {
 		metasCache, err = cache.New(bsCfg.MetasCache, reg, logger, stats.BloomMetasCache, constants.Loki)
+
+		// always enable LRU cache
+		lruCfg := bsCfg.MetasLRUCache
+		lruCfg.Enabled = true
+		lruCfg.PurgeInterval = 1 * time.Minute
+		lruCache := cache.NewEmbeddedCache("inmemory-metas-lru", lruCfg, reg, logger, stats.BloomMetasCache)
+
+		metasCache = cache.NewTiered([]cache.Cache{lruCache, metasCache})
 		if err != nil {
 			return nil, fmt.Errorf("failed to create metas cache: %w", err)
 		}
diff --git a/pkg/pattern/drain/drain.go b/pkg/pattern/drain/drain.go
index 20d9dadb6c8d0..ade8fca366b8a 100644
--- a/pkg/pattern/drain/drain.go
+++ b/pkg/pattern/drain/drain.go
@@ -268,7 +268,6 @@ func (d *Drain) Match(content string) *LogCluster {
 }
 
 func (d *Drain) getContentAsTokens(content string) []string {
-	content = strings.TrimSpace(content)
 	for _, extraDelimiter := range d.config.ExtraDelimiters {
 		content = strings.Replace(content, extraDelimiter, " ", -1)
 	}
diff --git a/pkg/pattern/drain/drain_test.go b/pkg/pattern/drain/drain_test.go
index beb09742af931..ef7754c4ed57e 100644
--- a/pkg/pattern/drain/drain_test.go
+++ b/pkg/pattern/drain/drain_test.go
@@ -6,9 +6,12 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+
+	"github.com/grafana/loki/v3/pkg/logql/log/pattern"
 )
 
 func TestDrain_TrainExtractsPatterns(t *testing.T) {
+	t.Parallel()
 	tests := []struct {
 		name      string
 		drain     *Drain
@@ -116,3 +119,118 @@ func TestDrain_TrainExtractsPatterns(t *testing.T) {
 		})
 	}
 }
+
+func TestDrain_TrainGeneratesMatchablePatterns(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name       string
+		drain      *Drain
+		inputLines []string
+	}{
+		{
+			name:  "should match each line against a pattern",
+			drain: New(DefaultConfig()),
+			inputLines: []string{
+				`test test test`,
+				`test test test`,
+				`test test test`,
+				`test test test`,
+			},
+		},
+		{
+			name:  "should also match newlines",
+			drain: New(DefaultConfig()),
+			inputLines: []string{
+				"test test test\n",
+				"test test test\n",
+				"test test test\n",
+				"test test test\n",
+			},
+		},
+	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			for _, line := range tt.inputLines {
+				tt.drain.Train(line, 0)
+			}
+			t.Log("Learned clusters", tt.drain.Clusters())
+
+			for _, line := range tt.inputLines {
+				match := tt.drain.Match(line)
+				require.NotNil(t, match, "Line should match a cluster")
+			}
+		})
+	}
+
+}
+
+func TestDrain_TrainGeneratesPatternsMatchableByLokiPatternFilter(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name       string
+		drain      *Drain
+		inputLines []string
+	}{
+		{
+			name:  "should extract patterns that all lines match",
+			drain: New(DefaultConfig()),
+			inputLines: []string{
+				`test 1 test`,
+				`test 2 test`,
+				`test 3 test`,
+				`test 4 test`,
+			},
+		},
+		{
+			name:  "should extract patterns that match if line ends with newlines",
+			drain: New(DefaultConfig()),
+			inputLines: []string{
+				"test 1 test\n",
+				"test 2 test\n",
+				"test 3 test\n",
+				"test 4 test\n",
+			},
+		},
+		{
+			name:  "should extract patterns that match if line ends with empty space",
+			drain: New(DefaultConfig()),
+			inputLines: []string{
+				"test 1 test			",
+				"test 2 test			",
+				"test 3 test			",
+				"test 4 test			",
+			},
+		},
+		{
+			name:  "should extract patterns that match if line starts with empty space",
+			drain: New(DefaultConfig()),
+			inputLines: []string{
+				"			test 1 test",
+				"			test 2 test",
+				"			test 3 test",
+				"			test 4 test",
+			},
+		},
+	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			for _, line := range tt.inputLines {
+				tt.drain.Train(line, 0)
+			}
+			require.Equal(t, 1, len(tt.drain.Clusters()))
+			cluster := tt.drain.Clusters()[0]
+			t.Log("Extracted cluster: ", cluster)
+
+			matcher, err := pattern.ParseLineFilter([]byte(cluster.String()))
+			require.NoError(t, err)
+
+			for _, line := range tt.inputLines {
+				passes := matcher.Test([]byte(line))
+				require.Truef(t, passes, "Line %q should match extracted pattern", line)
+			}
+		})
+	}
+
+}
diff --git a/pkg/querier/handler.go b/pkg/querier/handler.go
index ee5d0648ac276..bb01a0e3754b5 100644
--- a/pkg/querier/handler.go
+++ b/pkg/querier/handler.go
@@ -5,8 +5,6 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/opentracing/opentracing-go"
-
 	"github.com/grafana/loki/v3/pkg/loghttp"
 	"github.com/grafana/loki/v3/pkg/logproto"
 	"github.com/grafana/loki/v3/pkg/querier/queryrange"
@@ -24,8 +22,6 @@ func NewQuerierHandler(api *QuerierAPI) *Handler {
 }
 
 func (h *Handler) Do(ctx context.Context, req queryrangebase.Request) (queryrangebase.Response, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "queryHandler")
-	defer span.Finish()
 
 	switch concrete := req.(type) {
 	case *queryrange.LokiRequest:
diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go
index bee850fd82d69..73ea98d05fef5 100644
--- a/pkg/querier/querier.go
+++ b/pkg/querier/querier.go
@@ -1108,6 +1108,7 @@ func (q *SingleTenantQuerier) DetectedFields(ctx context.Context, req *logproto.
 			Type:        v.fieldType,
 			Cardinality: v.Estimate(),
 			Sketch:      sketch,
+			Parser:      v.parser,
 		}
 
 		fieldCount++
@@ -1124,13 +1125,19 @@ type parsedFields struct {
 	sketch         *hyperloglog.Sketch
 	isTypeDetected bool
 	fieldType      logproto.DetectedFieldType
+	parser         string
 }
 
-func newParsedFields() *parsedFields {
+func newParsedFields(parser *string) *parsedFields {
+	p := ""
+	if parser != nil {
+		p = *parser
+	}
 	return &parsedFields{
 		sketch:         hyperloglog.New(),
 		isTypeDetected: false,
 		fieldType:      logproto.DetectedFieldString,
+		parser:         p,
 	}
 }
 
@@ -1185,11 +1192,12 @@ func parseDetectedFields(ctx context.Context, limit uint32, streams logqlmodel.S
 			"msg", fmt.Sprintf("looking for detected fields in stream %d with %d lines", stream.Hash, len(stream.Entries)))
 
 		for _, entry := range stream.Entries {
-			detected := parseLine(entry.Line)
+			detected, parser := parseLine(entry.Line)
 			for k, vals := range detected {
 				df, ok := detectedFields[k]
 				if !ok && fieldCount < limit {
-					df = newParsedFields()
+
+					df = newParsedFields(parser)
 					detectedFields[k] = df
 					fieldCount++
 				}
@@ -1217,17 +1225,19 @@ func parseDetectedFields(ctx context.Context, limit uint32, streams logqlmodel.S
 	return detectedFields
 }
 
-func parseLine(line string) map[string][]string {
+func parseLine(line string) (map[string][]string, *string) {
+	parser := "logfmt"
 	logFmtParser := logql_log.NewLogfmtParser(true, false)
-	jsonParser := logql_log.NewJSONParser()
 
 	lbls := logql_log.NewBaseLabelsBuilder().ForLabels(labels.EmptyLabels(), 0)
 	_, logfmtSuccess := logFmtParser.Process(0, []byte(line), lbls)
 	if !logfmtSuccess || lbls.HasErr() {
+		parser = "json"
+		jsonParser := logql_log.NewJSONParser()
 		lbls.Reset()
 		_, jsonSuccess := jsonParser.Process(0, []byte(line), lbls)
 		if !jsonSuccess || lbls.HasErr() {
-			return map[string][]string{}
+			return map[string][]string{}, nil
 		}
 	}
 
@@ -1249,7 +1259,7 @@ func parseLine(line string) map[string][]string {
 		result[lbl] = vals
 	}
 
-	return result
+	return result, &parser
 }
 
 // streamsForFieldDetection reads the streams from the iterator and returns them sorted.
diff --git a/pkg/querier/queryrange/downstreamer.go b/pkg/querier/queryrange/downstreamer.go
index 7103a38ab3768..cf1cfc36dc4ac 100644
--- a/pkg/querier/queryrange/downstreamer.go
+++ b/pkg/querier/queryrange/downstreamer.go
@@ -143,7 +143,6 @@ func (in instance) Downstream(ctx context.Context, queries []logql.DownstreamQue
 		sp, ctx := opentracing.StartSpanFromContext(ctx, "DownstreamHandler.instance")
 		defer sp.Finish()
 		logger := spanlogger.FromContext(ctx)
-		defer logger.Finish()
 		level.Debug(logger).Log("shards", fmt.Sprintf("%+v", qry.Params.Shards()), "query", req.GetQuery(), "step", req.GetStep(), "handler", reflect.TypeOf(in.handler), "engine", "downstream")
 
 		res, err := in.handler.Do(ctx, req)
diff --git a/pkg/querier/queryrange/limits.go b/pkg/querier/queryrange/limits.go
index b557663cf2467..5d63402fc9f33 100644
--- a/pkg/querier/queryrange/limits.go
+++ b/pkg/querier/queryrange/limits.go
@@ -340,8 +340,6 @@ func (q *querySizeLimiter) guessLimitName() string {
 }
 
 func (q *querySizeLimiter) Do(ctx context.Context, r queryrangebase.Request) (queryrangebase.Response, error) {
-	span, ctx := opentracing.StartSpanFromContext(ctx, "query_size_limits")
-	defer span.Finish()
 	log := spanlogger.FromContext(ctx)
 	defer log.Finish()
 
diff --git a/pkg/querier/queryrange/shard_resolver.go b/pkg/querier/queryrange/shard_resolver.go
index 9950190ae0eb9..c8a66de690915 100644
--- a/pkg/querier/queryrange/shard_resolver.go
+++ b/pkg/querier/queryrange/shard_resolver.go
@@ -221,9 +221,7 @@ func (r *dynamicShardResolver) ShardingRanges(expr syntax.Expr, targetBytesPerSh
 	[]logproto.ChunkRefGroup,
 	error,
 ) {
-	sp, ctx := opentracing.StartSpanFromContext(r.ctx, "dynamicShardResolver.ShardingRanges")
-	defer sp.Finish()
-	log := spanlogger.FromContext(ctx)
+	log := spanlogger.FromContext(r.ctx)
 	defer log.Finish()
 
 	adjustedFrom := r.from
@@ -254,7 +252,7 @@ func (r *dynamicShardResolver) ShardingRanges(expr syntax.Expr, targetBytesPerSh
 	exprStr := expr.String()
 	// try to get shards for the given expression
 	// if it fails, fallback to linearshards based on stats
-	resp, err := r.next.Do(ctx, &logproto.ShardsRequest{
+	resp, err := r.next.Do(r.ctx, &logproto.ShardsRequest{
 		From:                adjustedFrom,
 		Through:             r.through,
 		Query:               expr.String(),
@@ -271,7 +269,7 @@ func (r *dynamicShardResolver) ShardingRanges(expr syntax.Expr, targetBytesPerSh
 	}
 
 	// accumulate stats
-	logqlstats.JoinResults(ctx, casted.Response.Statistics)
+	logqlstats.JoinResults(r.ctx, casted.Response.Statistics)
 
 	var refs int
 	for _, x := range casted.Response.ChunkGroups {
diff --git a/pkg/querier/worker/scheduler_processor.go b/pkg/querier/worker/scheduler_processor.go
index 00b08219e5dbe..a19975486ef34 100644
--- a/pkg/querier/worker/scheduler_processor.go
+++ b/pkg/querier/worker/scheduler_processor.go
@@ -29,7 +29,6 @@ import (
 	"github.com/grafana/loki/v3/pkg/querier/queryrange"
 	querier_stats "github.com/grafana/loki/v3/pkg/querier/stats"
 	"github.com/grafana/loki/v3/pkg/scheduler/schedulerpb"
-	httpgrpcutil "github.com/grafana/loki/v3/pkg/util/httpgrpc"
 	util_log "github.com/grafana/loki/v3/pkg/util/log"
 )
 
@@ -147,15 +146,6 @@ func (sp *schedulerProcessor) querierLoop(c schedulerpb.SchedulerForQuerier_Quer
 			ctx := user.InjectOrgID(ctx, request.UserID)
 
 			sp.metrics.inflightRequests.Inc()
-			tracer := opentracing.GlobalTracer()
-			// Ignore errors here. If we cannot get parent span, we just don't create new one.
-			parentSpanContext, _ := httpgrpcutil.GetParentSpanForRequest(tracer, request)
-			if parentSpanContext != nil {
-				queueSpan, spanCtx := opentracing.StartSpanFromContextWithTracer(ctx, tracer, "querier_processor_runRequest", opentracing.ChildOf(parentSpanContext))
-				defer queueSpan.Finish()
-
-				ctx = spanCtx
-			}
 			logger := util_log.WithContext(ctx, sp.log)
 
 			switch r := request.Request.(type) {
diff --git a/pkg/storage/async_store.go b/pkg/storage/async_store.go
index 8d104d702b8bb..ed3c9dab6b422 100644
--- a/pkg/storage/async_store.go
+++ b/pkg/storage/async_store.go
@@ -5,12 +5,15 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/c2h5oh/datasize"
 	"github.com/opentracing/opentracing-go"
 
 	"github.com/grafana/loki/v3/pkg/logproto"
 	"github.com/grafana/loki/v3/pkg/storage/stores"
 	"github.com/grafana/loki/v3/pkg/storage/stores/index/seriesvolume"
+	"github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb/sharding"
 
+	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/grafana/dskit/concurrency"
 	"github.com/prometheus/common/model"
@@ -281,3 +284,105 @@ func filterDuplicateChunks(scfg config.SchemaConfig, storeChunks [][]chunk.Chunk
 
 	return filteredChunkIDs
 }
+
+func (a *AsyncStore) GetShards(
+	ctx context.Context,
+	userID string,
+	from, through model.Time,
+	targetBytesPerShard uint64,
+	predicate chunk.Predicate,
+) (*logproto.ShardsResponse, error) {
+	logger := log.With(
+		util_log.WithContext(ctx, util_log.Logger),
+		"component", "asyncStore",
+	)
+
+	if !a.shouldQueryIngesters(through, model.Now()) {
+		return a.Store.GetShards(ctx, userID, from, through, targetBytesPerShard, predicate)
+	}
+
+	var (
+		shardResp *logproto.ShardsResponse
+		statsResp *stats.Stats
+	)
+
+	jobs := []func() error{
+		func() error {
+			var err error
+			shardResp, err = a.Store.GetShards(ctx, userID, from, through, targetBytesPerShard, predicate)
+			return err
+		},
+		// We can't dedupe shards by their contents, so we complement the
+		// store's response with the ingester's stats and .
+		func() error {
+			var err error
+			statsResp, err = a.ingesterQuerier.Stats(ctx, userID, from, through, predicate.Matchers...)
+			return err
+		},
+	}
+
+	if err := concurrency.ForEachJob(
+		ctx,
+		len(jobs),
+		len(jobs),
+		func(ctx context.Context, i int) error {
+			return jobs[i]()
+		},
+	); err != nil {
+		return nil, err
+	}
+
+	return mergeShardsFromIngestersAndStore(logger, shardResp, statsResp, targetBytesPerShard), nil
+}
+
+func mergeShardsFromIngestersAndStore(
+	logger log.Logger,
+	storeResp *logproto.ShardsResponse,
+	statsResp *logproto.IndexStatsResponse,
+	targetBytesPerShard uint64,
+) *logproto.ShardsResponse {
+	var storeBytes uint64
+	for _, shard := range storeResp.Shards {
+		storeBytes += shard.Stats.Bytes
+	}
+	totalBytes := storeBytes + statsResp.Bytes
+
+	defer func() {
+		level.Debug(logger).Log(
+			"msg", "resolved shards ",
+			"ingester_bytes", datasize.ByteSize(statsResp.Bytes).HumanReadable(),
+			"store_bytes", datasize.ByteSize(storeBytes).HumanReadable(),
+			"total_bytes", datasize.ByteSize(totalBytes).HumanReadable(),
+			"target_bytes", datasize.ByteSize(targetBytesPerShard).HumanReadable(),
+			"store_shards", len(storeResp.Shards),
+		)
+	}()
+
+	// edge case to avoid divide by zero later
+	if totalBytes == 0 {
+		return &logproto.ShardsResponse{
+			Shards: sharding.LinearShards(0, 0),
+		}
+	}
+
+	// If the ingesters don't have enough data to meaningfuly
+	// change the number of shards, use the store response.
+	if pct := float64(statsResp.Bytes) / float64(totalBytes); pct < 0.25 {
+		return storeResp
+	}
+
+	shards := sharding.LinearShards(int(totalBytes/targetBytesPerShard), totalBytes)
+
+	// increment the total chunks by the number seen from ingesters
+	// NB(owen-d): this isn't perfect as it mixes signals a bit by joining
+	// store chunks which _could_ possibly be filtered with ingester chunks which can't,
+	// but it's still directionally helpful
+	updatedStats := storeResp.Statistics
+	updatedStats.Index.TotalChunks += int64(statsResp.Chunks)
+	return &logproto.ShardsResponse{
+		Shards:     shards,
+		Statistics: updatedStats,
+		// explicitly nil chunkgroups when we've changed the shards+included chunkrefs from ingesters
+		ChunkGroups: nil,
+	}
+}
diff --git a/pkg/storage/async_store_test.go b/pkg/storage/async_store_test.go
index a85b33ecccefd..9cf80868c861d 100644
--- a/pkg/storage/async_store_test.go
+++ b/pkg/storage/async_store_test.go
@@ -5,7 +5,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/go-kit/log"
+
 	"github.com/grafana/loki/v3/pkg/logproto"
+	"github.com/grafana/loki/v3/pkg/logqlmodel/stats"
 
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/model/labels"
@@ -15,6 +18,7 @@ import (
 	"github.com/grafana/loki/v3/pkg/storage/chunk"
 	"github.com/grafana/loki/v3/pkg/storage/chunk/fetcher"
 	"github.com/grafana/loki/v3/pkg/storage/config"
+	"github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb/sharding"
 	"github.com/grafana/loki/v3/pkg/util"
 )
 
@@ -29,8 +33,8 @@ func newStoreMock() *storeMock {
 	return &storeMock{}
 }
 
-func (s *storeMock) GetChunks(ctx context.Context, userID string, from, through model.Time, predicate chunk.Predicate, storeChunksOverride *logproto.ChunkRefGroup) ([][]chunk.Chunk, []*fetcher.Fetcher, error) {
-	args := s.Called(ctx, userID, from, through, predicate, storeChunksOverride)
+func (s *storeMock) GetChunks(ctx context.Context, userID string, from, through model.Time, predicate chunk.Predicate, overrides *logproto.ChunkRefGroup) ([][]chunk.Chunk, []*fetcher.Fetcher, error) {
+	args := s.Called(ctx, userID, from, through, predicate, overrides)
 	return args.Get(0).([][]chunk.Chunk), args.Get(1).([]*fetcher.Fetcher), args.Error(2)
 }
 
@@ -360,3 +364,80 @@ func convertChunksToChunkIDs(s config.SchemaConfig, chunks []chunk.Chunk) []stri
 
 	return chunkIDs
 }
+
+func TestMergeShardsFromIngestersAndStore(t *testing.T) {
+	mkStats := func(bytes, chks uint64) logproto.IndexStatsResponse {
+		return logproto.IndexStatsResponse{
+			Bytes:  bytes,
+			Chunks: chks,
+		}
+	}
+
+	// creates n shards with bytesPerShard * n bytes and chks chunks
+	mkShards := func(n int, bytesPerShard uint64, chks int64) logproto.ShardsResponse {
+		return logproto.ShardsResponse{
+			Shards: sharding.LinearShards(n, bytesPerShard*uint64(n)),
+			Statistics: stats.Result{
+				Index: stats.Index{
+					TotalChunks: chks,
+				},
+			},
+		}
+	}
+
+	targetBytesPerShard := 10
+
+	for _, tc := range []struct {
+		desc     string
+		ingester logproto.IndexStatsResponse
+		store    logproto.ShardsResponse
+		exp      logproto.ShardsResponse
+	}{
+		{
+			desc:     "zero bytes returns one full shard",
+			ingester: mkStats(0, 0),
+			store:    mkShards(0, 0, 0),
+			exp:      mkShards(1, 0, 0),
+		},
+		{
+			desc:     "zero ingester bytes honors store",
+			ingester: mkStats(0, 0),
+			store:    mkShards(10, uint64(targetBytesPerShard), 10),
+			exp:      mkShards(10, uint64(targetBytesPerShard), 10),
+		},
+		{
+			desc:     "zero store bytes honors ingester",
+			ingester: mkStats(uint64(targetBytesPerShard*10), 10),
+			store:    mkShards(0, 0, 0),
+			exp:      mkShards(10, uint64(targetBytesPerShard), 10),
+		},
+		{
+			desc:     "ingester bytes below threshold ignored",
+			ingester: mkStats(uint64(targetBytesPerShard*2), 10),    // 2 shards worth from ingesters
+			store:    mkShards(10, uint64(targetBytesPerShard), 10), // 10 shards worth from store
+			exp:      mkShards(10, uint64(targetBytesPerShard), 10), // use the store's resp
+		},
+		{
+			desc:     "ingester bytes above threshold recreate shards",
+			ingester: mkStats(uint64(targetBytesPerShard*4), 10),    // 4 shards worth from ingesters
+			store:    mkShards(10, uint64(targetBytesPerShard), 10), // 10 shards worth from store
+			exp:      mkShards(14, uint64(targetBytesPerShard), 20), // regenerate 14 shards
+		},
+	} {
+
+		t.Run(tc.desc, func(t *testing.T) {
+			got := mergeShardsFromIngestersAndStore(
+				log.NewNopLogger(),
+				&tc.store,
+				&tc.ingester,
+				uint64(targetBytesPerShard),
+			)
+			require.Equal(t, tc.exp.Statistics, got.Statistics)
+			require.Equal(t, tc.exp.ChunkGroups, got.ChunkGroups)
+			require.Equal(t, tc.exp.Statistics.Index.TotalChunks, got.Statistics.Index.TotalChunks)
+			for i, shard := range tc.exp.Shards {
+				require.Equal(t, shard, got.Shards[i], "shard %d", i)
+			}
+		})
+	}
+}
diff --git a/pkg/storage/detected/fields.go b/pkg/storage/detected/fields.go
index 7ced040b37d2d..6310448216653 100644
--- a/pkg/storage/detected/fields.go
+++ b/pkg/storage/detected/fields.go
@@ -9,6 +9,7 @@ import (
 type UnmarshaledDetectedField struct {
 	Label  string
 	Type   logproto.DetectedFieldType
+	Parser string
 	Sketch *hyperloglog.Sketch
 }
 
@@ -22,6 +23,7 @@ func UnmarshalDetectedField(f *logproto.DetectedField) (*UnmarshaledDetectedFiel
 	return &UnmarshaledDetectedField{
 		Label:  f.Label,
 		Type:   f.Type,
+		Parser: f.Parser,
 		Sketch: sketch,
 	}, nil
 }
@@ -77,6 +79,7 @@ func MergeFields(
 			Label:       field.Label,
 			Type:        field.Type,
 			Cardinality: field.Sketch.Estimate(),
+			Parser:      field.Parser,
 			Sketch:      nil,
 		}
 		result = append(result, detectedField)
diff --git a/pkg/storage/detected/fields_test.go b/pkg/storage/detected/fields_test.go
index 4edd7026baf68..0e6ad800738ad 100644
--- a/pkg/storage/detected/fields_test.go
+++ b/pkg/storage/detected/fields_test.go
@@ -34,6 +34,7 @@ func Test_MergeFields(t *testing.T) {
 			Type:        logproto.DetectedFieldString,
 			Cardinality: 1,
 			Sketch:      marshalledFooSketch,
+			Parser:      "logfmt",
 		},
 		{
 			Label:       "bar",
@@ -65,6 +66,7 @@ func Test_MergeFields(t *testing.T) {
 
 		assert.Equal(t, logproto.DetectedFieldString, foo.Type)
 		assert.Equal(t, uint64(3), foo.Cardinality)
+		assert.Equal(t, "logfmt", foo.Parser)
 	})
 
 	t.Run("returns up to limit number of fields", func(t *testing.T) {
diff --git a/pkg/storage/stores/shipper/bloomshipper/client.go b/pkg/storage/stores/shipper/bloomshipper/client.go
index c637b983cb957..56e81a4d3fe40 100644
--- a/pkg/storage/stores/shipper/bloomshipper/client.go
+++ b/pkg/storage/stores/shipper/bloomshipper/client.go
@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
 	"github.com/grafana/dskit/concurrency"
 	"github.com/pkg/errors"
 	"github.com/prometheus/common/model"
@@ -22,6 +23,7 @@ import (
 	"github.com/grafana/loki/v3/pkg/storage/config"
 	"github.com/grafana/loki/v3/pkg/storage/stores/shipper/indexshipper/tsdb"
 	"github.com/grafana/loki/v3/pkg/util/encoding"
+	"github.com/grafana/loki/v3/pkg/util/spanlogger"
 )
 
 const (
@@ -388,7 +390,11 @@ func (b *BloomClient) GetMetas(ctx context.Context, refs []MetaRef) ([]Meta, err
 	err := concurrency.ForEachJob(ctx, len(refs), b.concurrency, func(ctx context.Context, idx int) error {
 		meta, err := b.GetMeta(ctx, refs[idx])
 		if err != nil {
-			return err
+			key := b.KeyResolver.Meta(refs[idx]).Addr()
+			if !b.IsObjectNotFoundErr(err) {
+				return fmt.Errorf("failed to get meta file %s: %w", key, err)
+			}
+			level.Error(b.logger).Log("msg", "failed to get meta file", "ref", key, "err", err)
 		}
 		results[idx] = meta
 		return nil
@@ -396,20 +402,22 @@ func (b *BloomClient) GetMetas(ctx context.Context, refs []MetaRef) ([]Meta, err
 	return results, err
 }
 
+// GetMeta fetches the meta file for given MetaRef from object storage and
+// decodes the JSON data into a Meta.
+// If the meta file is not found in storage or decoding fails, the empty Meta
+// is returned along with the error.
 func (b *BloomClient) GetMeta(ctx context.Context, ref MetaRef) (Meta, error) {
-	meta := Meta{
-		MetaRef: ref,
-	}
+	meta := Meta{MetaRef: ref}
 	key := b.KeyResolver.Meta(ref).Addr()
 	reader, _, err := b.client.GetObject(ctx, key)
 	if err != nil {
-		return Meta{}, fmt.Errorf("failed to get meta file%s: %w", key, err)
+		return meta, err
 	}
 	defer reader.Close()
 
 	err = json.NewDecoder(reader).Decode(&meta)
 	if err != nil {
-		return Meta{}, fmt.Errorf("failed to decode meta file %s: %w", key, err)
+		return meta, errors.Wrap(err, "failed to decode JSON")
 	}
 	return meta, nil
 }
@@ -473,12 +481,25 @@ func newCachedListOpObjectClient(oc client.ObjectClient, ttl, interval time.Dura
 }
 
 func (c *cachedListOpObjectClient) List(ctx context.Context, prefix string, delimiter string) ([]client.StorageObject, []client.StorageCommonPrefix, error) {
+	var (
+		logger   = spanlogger.FromContext(ctx)
+		start    = time.Now()
+		cacheDur time.Duration
+	)
+	defer func() {
+		logger.LogKV(
+			"cache_duration", cacheDur,
+			"total_duration", time.Since(start),
+		)
+	}()
+
 	if delimiter != "" {
 		return nil, nil, fmt.Errorf("does not support LIST calls with delimiter: %s", delimiter)
 	}
 	c.mtx.RLock()
 	cached, found := c.cache[prefix]
 	c.mtx.RUnlock()
+	cacheDur = time.Since(start)
 	if found {
 		return cached.objects, cached.prefixes, nil
 	}
diff --git a/pkg/storage/stores/shipper/bloomshipper/client_test.go b/pkg/storage/stores/shipper/bloomshipper/client_test.go
index cd77339c0932c..ec5e7015e00b4 100644
--- a/pkg/storage/stores/shipper/bloomshipper/client_test.go
+++ b/pkg/storage/stores/shipper/bloomshipper/client_test.go
@@ -107,11 +107,20 @@ func TestBloomClient_GetMetas(t *testing.T) {
 		require.Equal(t, metas, []Meta{m1, m2})
 	})
 
-	t.Run("does not exist", func(t *testing.T) {
-		metas, err := c.GetMetas(ctx, []MetaRef{{}})
-		require.Error(t, err)
-		require.True(t, c.client.IsObjectNotFoundErr(err))
-		require.Equal(t, metas, []Meta{{}})
+	t.Run("does not exist - yields empty meta", func(t *testing.T) {
+		ref := MetaRef{
+			Ref: Ref{
+				TenantID:       "tenant",
+				TableName:      "table",
+				Bounds:         v1.FingerprintBounds{},
+				StartTimestamp: 1000,
+				EndTimestamp:   2000,
+				Checksum:       1234,
+			},
+		}
+		metas, err := c.GetMetas(ctx, []MetaRef{ref})
+		require.NoError(t, err)
+		require.Equal(t, metas, []Meta{{MetaRef: ref}})
 	})
 }
 
diff --git a/pkg/storage/stores/shipper/bloomshipper/config/config.go b/pkg/storage/stores/shipper/bloomshipper/config/config.go
index 4b715da60e7c0..72d8f8557b095 100644
--- a/pkg/storage/stores/shipper/bloomshipper/config/config.go
+++ b/pkg/storage/stores/shipper/bloomshipper/config/config.go
@@ -12,11 +12,12 @@ import (
 )
 
 type Config struct {
-	WorkingDirectory    flagext.StringSliceCSV `yaml:"working_directory"`
-	MaxQueryPageSize    flagext.Bytes          `yaml:"max_query_page_size"`
-	DownloadParallelism int                    `yaml:"download_parallelism"`
-	BlocksCache         BlocksCacheConfig      `yaml:"blocks_cache"`
-	MetasCache          cache.Config           `yaml:"metas_cache"`
+	WorkingDirectory    flagext.StringSliceCSV    `yaml:"working_directory"`
+	MaxQueryPageSize    flagext.Bytes             `yaml:"max_query_page_size"`
+	DownloadParallelism int                       `yaml:"download_parallelism"`
+	BlocksCache         BlocksCacheConfig         `yaml:"blocks_cache"`
+	MetasCache          cache.Config              `yaml:"metas_cache"`
+	MetasLRUCache       cache.EmbeddedCacheConfig `yaml:"metas_lru_cache"`
 
 	// This will always be set to true when flags are registered.
 	// In tests, where config is created as literal, it can be set manually.
@@ -32,6 +33,7 @@ func (c *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
 	f.IntVar(&c.DownloadParallelism, prefix+"download-parallelism", 8, "The amount of maximum concurrent bloom blocks downloads. Usually set to 2x number of CPU cores.")
 	c.BlocksCache.RegisterFlagsWithPrefixAndDefaults(prefix+"blocks-cache.", "Cache for bloom blocks. ", f, 24*time.Hour)
 	c.MetasCache.RegisterFlagsWithPrefix(prefix+"metas-cache.", "Cache for bloom metas. ", f)
+	c.MetasLRUCache.RegisterFlagsWithPrefix(prefix+"metas-lru-cache.", "In-memory LRU cache for bloom metas. ", f)
 
 	// always cache LIST operations
 	c.CacheListOps = true
diff --git a/pkg/storage/stores/shipper/bloomshipper/fetcher.go b/pkg/storage/stores/shipper/bloomshipper/fetcher.go
index de02d1effba8c..69715158950e0 100644
--- a/pkg/storage/stores/shipper/bloomshipper/fetcher.go
+++ b/pkg/storage/stores/shipper/bloomshipper/fetcher.go
@@ -19,6 +19,7 @@ import (
 	v1 "github.com/grafana/loki/v3/pkg/storage/bloom/v1"
 	"github.com/grafana/loki/v3/pkg/storage/chunk/cache"
 	"github.com/grafana/loki/v3/pkg/util/constants"
+	"github.com/grafana/loki/v3/pkg/util/spanlogger"
 )
 
 var downloadQueueCapacity = 10000
@@ -119,6 +120,8 @@ func (f *Fetcher) Close() {
 
 // FetchMetas implements fetcher
 func (f *Fetcher) FetchMetas(ctx context.Context, refs []MetaRef) ([]Meta, error) {
+	logger := spanlogger.FromContextWithFallback(ctx, f.logger)
+
 	if ctx.Err() != nil {
 		return nil, errors.Wrap(ctx.Err(), "fetch Metas")
 	}
@@ -127,9 +130,12 @@ func (f *Fetcher) FetchMetas(ctx context.Context, refs []MetaRef) ([]Meta, error
 	for _, ref := range refs {
 		keys = append(keys, f.client.Meta(ref).Addr())
 	}
+
+	cacheStart := time.Now()
 	cacheHits, cacheBufs, _, err := f.metasCache.Fetch(ctx, keys)
+	cacheDur := time.Since(cacheStart)
 	if err != nil {
-		level.Error(f.logger).Log("msg", "failed to fetch metas from cache", "err", err)
+		level.Error(logger).Log("msg", "failed to fetch metas from cache", "err", err)
 		return nil, nil
 	}
 
@@ -138,16 +144,31 @@ func (f *Fetcher) FetchMetas(ctx context.Context, refs []MetaRef) ([]Meta, error
 		return nil, err
 	}
 
+	storageStart := time.Now()
 	fromStorage, err := f.client.GetMetas(ctx, missing)
+	storageDur := time.Since(storageStart)
 	if err != nil {
 		return nil, err
 	}
 
+	writeBackStart := time.Now()
 	err = f.writeBackMetas(ctx, fromStorage)
+	writeBackDur := time.Since(writeBackStart)
 	if err != nil {
 		return nil, err
 	}
 
+	logger.LogKV(
+		"phase", "fetch_metas",
+		"err", err,
+		"keys", len(keys),
+		"hits", len(cacheHits),
+		"misses", len(missing),
+		"cache_dur", cacheDur.String(),
+		"storage_dur", storageDur.String(),
+		"write_back_dur", writeBackDur.String(),
+	)
+
 	results := append(fromCache, fromStorage...)
 	f.metrics.metasFetched.Observe(float64(len(results)))
 	// TODO(chaudum): get metas size from storage
diff --git a/pkg/storage/stores/shipper/bloomshipper/store.go b/pkg/storage/stores/shipper/bloomshipper/store.go
index 5e1363d0cb731..9b18427bacf10 100644
--- a/pkg/storage/stores/shipper/bloomshipper/store.go
+++ b/pkg/storage/stores/shipper/bloomshipper/store.go
@@ -21,6 +21,7 @@ import (
 	"github.com/grafana/loki/v3/pkg/storage/chunk/client/util"
 	"github.com/grafana/loki/v3/pkg/storage/config"
 	"github.com/grafana/loki/v3/pkg/util/constants"
+	"github.com/grafana/loki/v3/pkg/util/spanlogger"
 )
 
 var (
@@ -114,7 +115,11 @@ func (b *bloomStoreEntry) ResolveMetas(ctx context.Context, params MetaSearchPar
 
 // FetchMetas implements store.
 func (b *bloomStoreEntry) FetchMetas(ctx context.Context, params MetaSearchParams) ([]Meta, error) {
+	logger := spanlogger.FromContext(ctx)
+
+	resolverStart := time.Now()
 	metaRefs, fetchers, err := b.ResolveMetas(ctx, params)
+	resolverDuration := time.Since(resolverStart)
 	if err != nil {
 		return nil, err
 	}
@@ -122,6 +127,16 @@ func (b *bloomStoreEntry) FetchMetas(ctx context.Context, params MetaSearchParam
 		return nil, errors.New("metaRefs and fetchers have unequal length")
 	}
 
+	var metaCt int
+	for i := range metaRefs {
+		metaCt += len(metaRefs[i])
+	}
+	logger.LogKV(
+		"msg", "resolved metas",
+		"metas", metaCt,
+		"duration", resolverDuration,
+	)
+
 	var metas []Meta
 	for i := range fetchers {
 		res, err := fetchers[i].FetchMetas(ctx, metaRefs[i])
diff --git a/pkg/storage/stores/shipper/indexshipper/tsdb/sharding/power.go b/pkg/storage/stores/shipper/indexshipper/tsdb/sharding/power.go
index 257c198ee2d75..219563e0e5358 100644
--- a/pkg/storage/stores/shipper/indexshipper/tsdb/sharding/power.go
+++ b/pkg/storage/stores/shipper/indexshipper/tsdb/sharding/power.go
@@ -22,21 +22,7 @@ type PowerOfTwoSharding struct {
 
 func (p PowerOfTwoSharding) ShardsFor(bytes uint64, maxBytesPerShard uint64) []logproto.Shard {
 	factor := GuessShardFactor(bytes, maxBytesPerShard, p.MaxShards)
-
-	if factor < 2 {
-		return []logproto.Shard{{
-			Bounds: logproto.FPBounds{
-				Min: 0,
-				Max: math.MaxUint64,
-			},
-			Stats: &stats.Stats{
-				Bytes: bytes,
-			},
-		}}
-	}
-
 	return LinearShards(factor, bytes)
-
 }
 
 // LinearShards is a sharding implementation that splits the data into
@@ -71,14 +57,13 @@ func LinearShards(n int, bytes uint64) []logproto.Shard {
 				Bytes: bytesPerShard,
 			},
 		}
-
-		// The last shard should have the remainder of the bytes
-		// and the max bound should be math.MaxUint64
-		// NB(owen-d): this can only happen when maxShards is used
-		// and the maxShards isn't a factor of 2
-		shards[len(shards)-1].Stats.Bytes += bytes % uint64(n)
-		shards[len(shards)-1].Bounds.Max = math.MaxUint64
 	}
+	// The last shard should have the remainder of the bytes
+	// and the max bound should be math.MaxUint64
+	// NB(owen-d): this can only happen when maxShards is used
+	// and the maxShards isn't a factor of 2
+	shards[len(shards)-1].Stats.Bytes += bytes % uint64(n)
+	shards[len(shards)-1].Bounds.Max = math.MaxUint64
 
 	return shards
 
diff --git a/pkg/tool/printer/printer.go b/pkg/tool/printer/printer.go
index 084d483a07a45..1a696a8ffdf94 100644
--- a/pkg/tool/printer/printer.go
+++ b/pkg/tool/printer/printer.go
@@ -17,7 +17,7 @@ import (
 	"github.com/grafana/loki/v3/pkg/tool/rules/rwrulefmt"
 )
 
-// Printer is  used for printing formatted output from the cortextool
+// Printer is  used for printing formatted output from the lokitool
 type Printer struct {
 	disableColor bool
 	colorizer    colorstring.Colorize
diff --git a/pkg/validation/limits.go b/pkg/validation/limits.go
index 0159197831645..036f5660c0929 100644
--- a/pkg/validation/limits.go
+++ b/pkg/validation/limits.go
@@ -480,6 +480,10 @@ func (l *Limits) Validate() error {
 		return err
 	}
 
+	if l.TSDBMaxBytesPerShard <= 0 {
+		return errors.New("querier.tsdb-max-bytes-per-shard must be greater than 0")
+	}
+
 	return nil
 }
 
diff --git a/pkg/validation/limits_test.go b/pkg/validation/limits_test.go
index 598a6f9033cde..2d4457c2a1191 100644
--- a/pkg/validation/limits_test.go
+++ b/pkg/validation/limits_test.go
@@ -345,6 +345,7 @@ func TestLimitsValidation(t *testing.T) {
 		desc := fmt.Sprintf("%s/%s", tc.limits.DeletionMode, tc.limits.BloomBlockEncoding)
 		t.Run(desc, func(t *testing.T) {
 			tc.limits.TSDBShardingStrategy = logql.PowerOfTwoVersion.String() // hacky but needed for test
+			tc.limits.TSDBMaxBytesPerShard = DefaultTSDBMaxBytesPerShard
 			if tc.expected == nil {
 				require.NoError(t, tc.limits.Validate())
 			} else {
diff --git a/production/helm/loki/CHANGELOG.md b/production/helm/loki/CHANGELOG.md
index 3df6e8bf34a84..8a9f00cf7753e 100644
--- a/production/helm/loki/CHANGELOG.md
+++ b/production/helm/loki/CHANGELOG.md
@@ -13,6 +13,11 @@ Entries should include a reference to the pull request that introduced the chang
 
 [//]: # (<AUTOMATED_UPDATES_LOCATOR> : do not remove this line. This locator is used by the CI pipeline to automatically create a changelog entry for each new Loki release. Add other chart versions and respective changelog entries bellow this line.)
 
+## 6.5.0
+
+- [CHANGE] Changed version of Grafana Enterprise Logs to v3.0.1
+
+
 ## 6.4.2
 
 - [BUGFIX] Fixed helm helper functions to include missing `loki.hpa.apiVersion`  #12716
diff --git a/production/helm/loki/Chart.yaml b/production/helm/loki/Chart.yaml
index 6276b6a723c4f..60ea27903dbbe 100644
--- a/production/helm/loki/Chart.yaml
+++ b/production/helm/loki/Chart.yaml
@@ -3,7 +3,7 @@ name: loki
 description: Helm chart for Grafana Loki in simple, scalable mode
 type: application
 appVersion: 3.0.0
-version: 6.4.2
+version: 6.5.0
 home: https://grafana.github.io/helm-charts
 sources:
   - https://github.com/grafana/loki
diff --git a/production/helm/loki/README.md b/production/helm/loki/README.md
index 9df3955560ec5..b5b5961421864 100644
--- a/production/helm/loki/README.md
+++ b/production/helm/loki/README.md
@@ -1,6 +1,6 @@
 # loki
 
-![Version: 6.4.2](https://img.shields.io/badge/Version-6.4.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.0.0](https://img.shields.io/badge/AppVersion-3.0.0-informational?style=flat-square)
+![Version: 6.5.0](https://img.shields.io/badge/Version-6.5.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.0.0](https://img.shields.io/badge/AppVersion-3.0.0-informational?style=flat-square)
 
 Helm chart for Grafana Loki in simple, scalable mode
 
diff --git a/production/helm/loki/templates/NOTES.txt b/production/helm/loki/templates/NOTES.txt
index 6551a427000ff..622b1a8c26160 100644
--- a/production/helm/loki/templates/NOTES.txt
+++ b/production/helm/loki/templates/NOTES.txt
@@ -1,21 +1,33 @@
 ***********************************************************************
  Welcome to Grafana Loki
  Chart version: {{ .Chart.Version }}
+ Chart Name: {{ .Chart.Name }}
  Loki version: {{ .Chart.AppVersion }}
 ***********************************************************************
 
+** Please be patient while the chart is being deployed **
+
+Tip:
+
+  Watch the deployment status using the command: kubectl get pods -w --namespace {{ $.Release.Namespace }}
+
+If pods are taking too long to schedule make sure pod affinity can be fulfilled in the current cluster.
+
+***********************************************************************
 Installed components:
+***********************************************************************
+
 {{- if .Values.monitoring.selfMonitoring.enabled }}
 * grafana-agent-operator
 {{- end }}
 {{- if eq (include "loki.deployment.isSingleBinary" .) "true" }}
-* loki 
+* loki
 {{- else -}}
 {{- if .Values.gateway.enabled }}
 * gateway
 {{- end }}
 {{- if .Values.minio.enabled }}
-* minio 
+* minio
 {{- end }}
 {{- if eq (include "loki.deployment.isScalable" .) "true" }}
 * read
@@ -34,3 +46,139 @@ Installed components:
 * query frontend
 {{- end }}
 {{- end }}
+
+
+{{- if eq (include "loki.deployment.isSingleBinary" .) "true" }}
+
+Loki has been deployed as a single binary.
+This means a single pod is handling reads and writes. You can scale that pod vertically by adding more CPU and memory resources.
+
+{{- end }}
+
+
+***********************************************************************
+Sending logs to Loki
+***********************************************************************
+
+{{- if .Values.gateway.enabled }}
+
+Loki has been configured with a gateway (nginx) to support reads and writes from a single component.
+
+{{- end }}
+
+You can send logs from inside the cluster using the cluster DNS:
+
+{{- if .Values.gateway.enabled }}
+
+http://{{ include "loki.gatewayFullname" . }}.{{ $.Release.Namespace }}.svc.cluster.local/loki/api/v1/push
+
+{{- else }}
+{{- if eq (include "loki.deployment.isSingleBinary" .) "true" }}
+
+http://{{ include "loki.singleBinaryFullname" . }}.{{ $.Release.Namespace }}.svc.cluster.local:{{ .Values.loki.server.http_listen_port }}/loki/api/v1/push
+
+{{- end}}
+{{- if eq (include "loki.deployment.isScalable" .) "true" }}
+
+http://{{ include "loki.writeFullname" . }}.{{ $.Release.Namespace }}.svc.cluster.local:{{ .Values.loki.server.http_listen_port }}/loki/api/v1/push
+
+{{- end }}
+{{- if eq (include "loki.deployment.isDistributed" .) "true" }}
+
+http://{{ include "loki.distributorFullname" . }}.{{ $.Release.Namespace }}.svc.cluster.local:3100/loki/api/v1/push
+
+{{- end }}
+{{- end }}
+
+You can test to send data from outside the cluster by port-forwarding the gateway to your local machine:
+{{- if .Values.gateway.enabled }}
+
+  kubectl port-forward --namespace {{ $.Release.Namespace }} svc/{{ include "loki.gatewayFullname" . }} 3100:{{ .Values.gateway.service.port }} &
+
+{{- else }}
+{{- if eq (include "loki.deployment.isSingleBinary" .) "true" }}
+
+  kubectl port-forward --namespace {{ $.Release.Namespace }} svc/{{ include "loki.singleBinaryFullname" . }} 3100:{{ .Values.loki.server.http_listen_port }} &
+
+{{- end}}
+{{- if eq (include "loki.deployment.isScalable" .) "true" }}
+
+  kubectl port-forward --namespace {{ $.Release.Namespace }} svc/{{ include "loki.writeFullname" . }} 3100:{{ .Values.loki.server.http_listen_port }} &
+
+{{- end }}
+{{- if eq (include "loki.deployment.isDistributed" .) "true" }}
+
+  kubectl port-forward --namespace {{ $.Release.Namespace }} svc/{{ include "loki.distributorFullname" . }} 3100:3100 &
+
+{{- end }}
+{{- end }}
+
+And then using http://127.0.0.1:3100/loki/api/v1/push URL as shown below:
+
+```
+curl -H "Content-Type: application/json" -XPOST -s "http://127.0.0.1:3100/loki/api/v1/push"  \
+--data-raw "{\"streams\": [{\"stream\": {\"job\": \"test\"}, \"values\": [[\"$(date +%s)000000000\", \"fizzbuzz\"]]}]}"
+{{- if .Values.loki.auth_enabled }} \
+-H X-Scope-OrgId:foo
+{{- end}}
+```
+
+Then verify that Loki did received the data using the following command:
+
+```
+curl "http://127.0.0.1:3100/loki/api/v1/query_range" --data-urlencode 'query={job="test"}' {{- if .Values.loki.auth_enabled }} -H X-Scope-OrgId:foo {{- end}} | jq .data.result
+```
+
+***********************************************************************
+Connecting Grafana to Loki
+***********************************************************************
+
+If Grafana operates within the cluster, you'll set up a new Loki datasource by utilizing the following URL:
+
+{{- if .Values.gateway.enabled }}
+
+http://{{ include "loki.gatewayFullname" . }}.{{ $.Release.Namespace }}.svc.cluster.local/
+
+{{- else }}
+{{- if eq (include "loki.deployment.isSingleBinary" .) "true" }}
+
+http://{{ include "loki.singleBinaryFullname" . }}.{{ $.Release.Namespace }}.svc.cluster.local:{{ .Values.loki.server.http_listen_port }}/
+
+{{- end}}
+{{- if eq (include "loki.deployment.isScalable" .) "true" }}
+
+http://{{ include "loki.readFullname" . }}.{{ $.Release.Namespace }}.svc.cluster.local:{{ .Values.loki.server.http_listen_port }}/
+
+{{- end }}
+{{- if eq (include "loki.deployment.isDistributed" .) "true" }}
+
+http://{{ include "loki.queryFrontendFullname" . }}.{{ $.Release.Namespace }}.svc.cluster.local:3100/
+
+{{- end }}
+{{- end }}
+
+
+
+{{- if .Values.loki.auth_enabled }}
+
+***********************************************************************
+Multi-tenancy
+***********************************************************************
+
+Loki is configured with auth enabled (multi-tenancy) and expects tenant headers (`X-Scope-OrgID`) to be set for all API calls.
+
+You must configure Grafana's Loki datasource using the `HTTP Headers` section with the `X-Scope-OrgID` to target a specific tenant.
+For each tenant, you can create a different datasource.
+
+The agent of your choice must also be configured to propagate this header.
+For example, when using Promtail you can use the `tenant` stage. https://grafana.com/docs/loki/latest/send-data/promtail/stages/tenant/
+
+When not provided with the `X-Scope-OrgID` while auth is enabled, Loki will reject reads and writes with a 404 status code `no org id`.
+
+You can also use a reverse proxy, to automatically add the `X-Scope-OrgID` header as suggested by https://grafana.com/docs/loki/latest/operations/authentication/
+
+For more information, read our documentation about multi-tenancy: https://grafana.com/docs/loki/latest/operations/multi-tenancy/
+
+> When using curl you can pass `X-Scope-OrgId` header using `-H X-Scope-OrgId:foo` option, where foo can be replaced with the tenant of your choice.
+
+{{- end }}
diff --git a/production/helm/loki/values.yaml b/production/helm/loki/values.yaml
index 1ca9d916e41b8..36d42cddc09c5 100644
--- a/production/helm/loki/values.yaml
+++ b/production/helm/loki/values.yaml
@@ -118,7 +118,6 @@ loki:
   # If empty, no configmap or secret will be created.
   # The value will be passed through tpl.
   generatedConfigObjectName: '{{ include "loki.name" . }}'
-
   # -- Config file contents for Loki
   # @default -- See values.yaml
   config: |
@@ -450,7 +449,7 @@ enterprise:
   # Enable enterprise features, license must be provided
   enabled: false
   # Default verion of GEL to deploy
-  version: v3.0.0
+  version: v3.0.1
   # -- Optional name of the GEL cluster, otherwise will use .Release.Name
   # The cluster name must match what is in your GEL license
   cluster_name: null
@@ -845,8 +844,6 @@ adminApi:
   tolerations: []
   # -- Grace period to allow the admin-api to shutdown before it is killed
   terminationGracePeriodSeconds: 60
-
-
 ######################################################################################################################
 #
 # Gateway and Ingress
@@ -1015,12 +1012,8 @@ gateway:
     # @default -- Either `loki.tenants` or `gateway.basicAuth.username` and `gateway.basicAuth.password`.
     htpasswd: >-
       {{ if .Values.loki.tenants }}
-
-
         {{- range $t := .Values.loki.tenants }}
       {{ htpasswd (required "All tenants must have a 'name' set" $t.name) (required "All tenants must have a 'password' set" $t.password) }}
-
-
         {{- end }}
       {{ else }} {{ htpasswd (required "'gateway.basicAuth.username' is required" .Values.gateway.basicAuth.username) (required "'gateway.basicAuth.password' is required" .Values.gateway.basicAuth.password) }} {{ end }}
     # -- Existing basic auth secret to use. Must contain '.htpasswd'
@@ -1186,7 +1179,6 @@ migrate:
     # -- If migrating from a distributed service, provide the distributed deployment's
     # memberlist service DNS so the new deployment can join its ring.
     memberlistService: ""
-
 ######################################################################################################################
 #
 # Single Binary Deployment
@@ -3117,7 +3109,6 @@ extraObjects: []
 #       {{ .name }}: {{ b64enc .password | quote }}
 #       {{- end }}
 
-
 sidecar:
   image:
     # -- The Docker registry and image for the k8s sidecar
diff --git a/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-compactor.json b/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-compactor.json
new file mode 100644
index 0000000000000..c667d0c01eccb
--- /dev/null
+++ b/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-compactor.json
@@ -0,0 +1,5191 @@
+{
+      "annotations": {
+         "list": [ ]
+      },
+      "editable": true,
+      "fiscalYearStartMonth": 0,
+      "gnetId": null,
+      "graphTooltip": 0,
+      "hideControls": false,
+      "links": [
+         {
+            "asDropdown": true,
+            "icon": "external link",
+            "includeVars": true,
+            "keepTime": true,
+            "tags": [
+               "loki"
+            ],
+            "targetBlank": false,
+            "title": "Loki Dashboards",
+            "type": "dashboards"
+         }
+      ],
+      "panels": [
+         {
+            "collapsed": false,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 0
+            },
+            "id": 111,
+            "panels": [ ],
+            "title": "Overview",
+            "type": "row"
+         },
+         {
+            "gridPos": {
+               "h": 8,
+               "w": 14,
+               "x": 0,
+               "y": 1
+            },
+            "id": 35,
+            "options": {
+               "code": {
+                  "language": "plaintext",
+                  "showLineNumbers": false,
+                  "showMiniMap": false
+               },
+               "content": "## About the Bloom Compactor\nThe compactor iterates through chunks and creates blooms out of them.\nThe size of the resulting blooms depends on the bloom filter settings, the tokenizer settings, the number of ring tokens per compactor and the total number opf compactors.\n\nCompactors are horizontally scalable and uses a ring to:\n- Shard tenants\n- Shard series fingerprints within a tenant subring.\n\nThe blooms for the series are grouped together in blocks which are flushed to object store.",
+               "mode": "markdown"
+            },
+            "pluginVersion": "11.1.0-70005",
+            "transparent": true,
+            "type": "text"
+         },
+         {
+            "datasource": {
+               "type": "prometheus",
+               "uid": "${datasource}"
+            },
+            "description": "Cell-wide compaction progress. Should increase till completion throughout each compaction period.",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "palette-classic"
+                  },
+                  "custom": {
+                     "axisBorderShow": false,
+                     "axisCenteredZero": false,
+                     "axisColorMode": "text",
+                     "axisLabel": "",
+                     "axisPlacement": "auto",
+                     "barAlignment": 0,
+                     "drawStyle": "line",
+                     "fillOpacity": 15,
+                     "gradientMode": "none",
+                     "hideFrom": {
+                        "legend": false,
+                        "tooltip": false,
+                        "viz": false
+                     },
+                     "insertNulls": false,
+                     "lineInterpolation": "linear",
+                     "lineStyle": {
+                        "fill": "solid"
+                     },
+                     "lineWidth": 1,
+                     "pointSize": 5,
+                     "scaleDistribution": {
+                        "type": "linear"
+                     },
+                     "showPoints": "auto",
+                     "spanNulls": false,
+                     "stacking": {
+                        "group": "A",
+                        "mode": "none"
+                     },
+                     "thresholdsStyle": {
+                        "mode": "off"
+                     }
+                  },
+                  "mappings": [ ],
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [
+                        {
+                           "color": "green",
+                           "value": null
+                        },
+                        {
+                           "color": "red",
+                           "value": 80
+                        }
+                     ]
+                  },
+                  "unit": "percentunit"
+               },
+               "overrides": [ ]
+            },
+            "gridPos": {
+               "h": 7,
+               "w": 12,
+               "x": 0,
+               "y": 9
+            },
+            "id": 42,
+            "options": {
+               "legend": {
+                  "calcs": [ ],
+                  "displayMode": "list",
+                  "placement": "bottom",
+                  "showLegend": true
+               },
+               "tooltip": {
+                  "maxHeight": 600,
+                  "mode": "single",
+                  "sort": "none"
+               }
+            },
+            "pluginVersion": "11.1.0-69868",
+            "targets": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "sum(loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"})\n/\nsum(count(loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}))",
+                  "hide": false,
+                  "instant": false,
+                  "legendFormat": "avg",
+                  "range": true,
+                  "refId": "A"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "quantile(\n    0.9, \n    sum by (pod) (\n        loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}\n    )\n)",
+                  "hide": false,
+                  "instant": false,
+                  "legendFormat": "p90",
+                  "range": true,
+                  "refId": "B"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "quantile(\n    0.1, \n    sum by (pod) (\n        loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}\n    )\n)",
+                  "hide": false,
+                  "instant": false,
+                  "legendFormat": "p10",
+                  "range": true,
+                  "refId": "C"
+               }
+            ],
+            "title": "Progress",
+            "type": "timeseries"
+         },
+         {
+            "datasource": {
+               "type": "prometheus",
+               "uid": "${datasource}"
+            },
+            "description": "Uncompressed size of chunks in a series VS the size of the blooms built.",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "palette-classic"
+                  },
+                  "custom": {
+                     "axisBorderShow": false,
+                     "axisCenteredZero": false,
+                     "axisColorMode": "text",
+                     "axisLabel": "",
+                     "axisPlacement": "auto",
+                     "barAlignment": 0,
+                     "drawStyle": "line",
+                     "fillOpacity": 15,
+                     "gradientMode": "none",
+                     "hideFrom": {
+                        "legend": false,
+                        "tooltip": false,
+                        "viz": false
+                     },
+                     "insertNulls": false,
+                     "lineInterpolation": "linear",
+                     "lineStyle": {
+                        "fill": "solid"
+                     },
+                     "lineWidth": 1,
+                     "pointSize": 5,
+                     "scaleDistribution": {
+                        "type": "linear"
+                     },
+                     "showPoints": "auto",
+                     "spanNulls": false,
+                     "stacking": {
+                        "group": "A",
+                        "mode": "none"
+                     },
+                     "thresholdsStyle": {
+                        "mode": "off"
+                     }
+                  },
+                  "mappings": [ ],
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [
+                        {
+                           "color": "green",
+                           "value": null
+                        }
+                     ]
+                  },
+                  "unit": "bytes"
+               },
+               "overrides": [
+                  {
+                     "matcher": {
+                        "id": "byName",
+                        "options": "Ratio"
+                     },
+                     "properties": [
+                        {
+                           "id": "unit",
+                           "value": "percentunit"
+                        }
+                     ]
+                  },
+                  {
+                     "matcher": {
+                        "id": "byName",
+                        "options": "Ratio over range"
+                     },
+                     "properties": [
+                        {
+                           "id": "unit",
+                           "value": "percentunit"
+                        }
+                     ]
+                  }
+               ]
+            },
+            "gridPos": {
+               "h": 7,
+               "w": 12,
+               "x": 12,
+               "y": 9
+            },
+            "id": 41,
+            "options": {
+               "legend": {
+                  "calcs": [ ],
+                  "displayMode": "list",
+                  "placement": "bottom",
+                  "showLegend": true
+               },
+               "tooltip": {
+                  "maxHeight": 600,
+                  "mode": "single",
+                  "sort": "none"
+               }
+            },
+            "pluginVersion": "11.1.0-69868",
+            "targets": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "sum(rate(loki_bloom_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
+                  "hide": false,
+                  "instant": false,
+                  "legendFormat": "Bloom",
+                  "range": true,
+                  "refId": "A"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "sum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
+                  "hide": false,
+                  "instant": false,
+                  "legendFormat": "Chunk",
+                  "range": true,
+                  "refId": "B"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "sum(rate(loki_bloom_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n/\nsum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
+                  "hide": false,
+                  "instant": false,
+                  "legendFormat": "Ratio",
+                  "range": true,
+                  "refId": "C"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "sum(rate(loki_bloom_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n/\nsum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
+                  "hide": false,
+                  "instant": false,
+                  "legendFormat": "Ratio over range",
+                  "range": true,
+                  "refId": "D"
+               }
+            ],
+            "title": "Chunks and Bloom size",
+            "type": "timeseries"
+         },
+         {
+            "datasource": {
+               "type": "loki",
+               "uid": "${loki_datasource}"
+            },
+            "description": "Blooms size vs uncompressed chunk size.",
+            "gridPos": {
+               "h": 7,
+               "w": 17,
+               "x": 0,
+               "y": 16
+            },
+            "id": 51,
+            "options": {
+               "dedupStrategy": "none",
+               "enableLogDetails": true,
+               "prettifyLogMessage": false,
+               "showCommonLabels": false,
+               "showLabels": false,
+               "showTime": false,
+               "sortOrder": "Descending",
+               "wrapLogMessage": false
+            },
+            "pluginVersion": "11.1.0-69868",
+            "targets": [
+               {
+                  "datasource": {
+                     "type": "loki",
+                     "uid": "${loki_datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} |= \"level=error\" |= \"component=bloom-compactor\"",
+                  "queryType": "range",
+                  "refId": "B"
+               }
+            ],
+            "title": "Errors",
+            "type": "logs"
+         },
+         {
+            "datasource": {
+               "type": "loki",
+               "uid": "${loki_datasource}"
+            },
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "fixedColor": "red",
+                     "mode": "fixed"
+                  },
+                  "custom": {
+                     "axisBorderShow": false,
+                     "axisCenteredZero": false,
+                     "axisColorMode": "text",
+                     "axisLabel": "",
+                     "axisPlacement": "auto",
+                     "barAlignment": 0,
+                     "drawStyle": "bars",
+                     "fillOpacity": 0,
+                     "gradientMode": "none",
+                     "hideFrom": {
+                        "legend": false,
+                        "tooltip": false,
+                        "viz": false
+                     },
+                     "insertNulls": false,
+                     "lineInterpolation": "linear",
+                     "lineWidth": 1,
+                     "pointSize": 3,
+                     "scaleDistribution": {
+                        "type": "linear"
+                     },
+                     "showPoints": "auto",
+                     "spanNulls": false,
+                     "stacking": {
+                        "group": "A",
+                        "mode": "none"
+                     },
+                     "thresholdsStyle": {
+                        "mode": "off"
+                     }
+                  },
+                  "mappings": [ ],
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [
+                        {
+                           "color": "green",
+                           "value": null
+                        },
+                        {
+                           "color": "red",
+                           "value": 1
+                        }
+                     ]
+                  }
+               },
+               "overrides": [ ]
+            },
+            "gridPos": {
+               "h": 7,
+               "w": 7,
+               "x": 17,
+               "y": 16
+            },
+            "id": 53,
+            "options": {
+               "legend": {
+                  "calcs": [ ],
+                  "displayMode": "list",
+                  "placement": "bottom",
+                  "showLegend": true
+               },
+               "tooltip": {
+                  "maxHeight": 600,
+                  "mode": "single",
+                  "sort": "none"
+               }
+            },
+            "pluginVersion": "11.1.0-69868",
+            "targets": [
+               {
+                  "datasource": {
+                     "type": "loki",
+                     "uid": "${loki_datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "sum(count_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} |= \"level=error\" |= \"component=bloom-compactor\" [$__auto]))",
+                  "queryType": "range",
+                  "refId": "A"
+               }
+            ],
+            "title": "Errors Rate",
+            "type": "timeseries"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 23
+            },
+            "id": 112,
+            "panels": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 15,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "percentunit"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 0,
+                     "y": 24
+                  },
+                  "id": 114,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (pod) (\n    loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "Progress per pod",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "fieldMinMax": false,
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "percentage",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "#EAB839",
+                                 "value": 0
+                              },
+                              {
+                                 "color": "green",
+                                 "value": 100
+                              }
+                           ]
+                        },
+                        "unit": "percentunit"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 12,
+                     "y": 24
+                  },
+                  "id": 115,
+                  "options": {
+                     "minVizHeight": 75,
+                     "minVizWidth": 75,
+                     "orientation": "auto",
+                     "reduceOptions": {
+                        "calcs": [
+                           "lastNotNull"
+                        ],
+                        "fields": "",
+                        "values": false
+                     },
+                     "showThresholdLabels": false,
+                     "showThresholdMarkers": false,
+                     "sizing": "auto"
+                  },
+                  "pluginVersion": "11.0.0-68102",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (pod) (\n    loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "Current Progress per pod",
+                  "type": "gauge"
+               }
+            ],
+            "title": "Progress per pod",
+            "type": "row"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 24
+            },
+            "id": 56,
+            "panels": [
+               {
+                  "description": "",
+                  "gridPos": {
+                     "h": 1,
+                     "w": 24,
+                     "x": 0,
+                     "y": 25
+                  },
+                  "id": 85,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-70005",
+                  "title": "We use tenant sharding so each compactor will process a subset of the tenants.",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "Shows the expected number of cpu cores we need to provision to build blooms as fast as we ingest data so a compaction iteration doesn't take longer than the compaction interval.\n\nWe may decide to have more to speed up compaction.",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 26
+                  },
+                  "id": 94,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "# This query shows the expected number of cpu cores we need to not fall behind\n# building blooms for data we're ingesting.\n# conceptually, the formula is:\n# (cell_bytes * space_amplification / bloom_bytes_processed_per_core)\n\n# number of replicas needed\nsum(avg_over_time(loki_cell:bytes:rate1m{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n*\n## Space amplification (how much data do we write compared to what we ingest?)\n(\n  # rep factor\n  3 *\n  sum(\n    # 1 - dedupe_ratio\n    1 - \n    sum(rate(loki_chunk_store_deduped_chunks_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (cluster, namespace)\n    /\n    sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (cluster, namespace)\n  )\n)\n/\n(\nsum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))\n/\nsum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Needed",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Available",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Required CPUs to not lag behind",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 15,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "Bps"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 26
+                  },
+                  "id": 72,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "# MB/s/core chunk data processed\nsum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}[$__rate_interval])) by (pod)\n/\nsum(rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])) by (pod)",
+                        "hide": true,
+                        "instant": false,
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "# MB/s/core chunk data processed\nsum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}[$__rate_interval]))\n/\nsum(rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Total",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "MB/s per core",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 33
+                  },
+                  "id": 1,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"cpu\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Request",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"cpu\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Limit",
+                        "range": true,
+                        "refId": "C"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n    0.99,\n    rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}[$__rate_interval])\n)",
+                        "instant": false,
+                        "legendFormat": "p99",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n    0.50,\n    rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}[$__rate_interval])\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p50",
+                        "range": true,
+                        "refId": "D"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "avg(\n    rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}[$__rate_interval])\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Avg",
+                        "range": true,
+                        "refId": "E"
+                     }
+                  ],
+                  "title": "CPU",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 33
+                  },
+                  "id": 75,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}[$__rate_interval]))",
+                        "instant": false,
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"cpu\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Request",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"cpu\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Limit",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "CPU per pod",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "bytes"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 40
+                  },
+                  "id": 76,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"memory\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Request",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Limit",
+                        "range": true,
+                        "refId": "C"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile (\n  0.99,\n  container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}\n)",
+                        "instant": false,
+                        "legendFormat": "p99",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile (\n  0.50,\n  container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p50",
+                        "range": true,
+                        "refId": "D"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "avg (\n  container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Avg",
+                        "range": true,
+                        "refId": "E"
+                     }
+                  ],
+                  "title": "Memory (workingset)",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "bytes"
+                     },
+                     "overrides": [
+                        {
+                           "__systemRef": "hideSeriesFrom",
+                           "matcher": {
+                              "id": "byNames",
+                              "options": {
+                                 "mode": "exclude",
+                                 "names": [
+                                    "bloom-compactor-106"
+                                 ],
+                                 "prefix": "All except:",
+                                 "readOnly": true
+                              }
+                           },
+                           "properties": [
+                              {
+                                 "id": "custom.hideFrom",
+                                 "value": {
+                                    "legend": false,
+                                    "tooltip": false,
+                                    "viz": true
+                                 }
+                              }
+                           ]
+                        }
+                     ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 40
+                  },
+                  "id": 5,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"})",
+                        "instant": false,
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"memory\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Request",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Limit",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Memory per pod (workingset)",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 15,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 47
+                  },
+                  "id": 27,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum (\n    increase(\n        kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[10m]\n    )\n) > 0",
+                        "instant": false,
+                        "legendFormat": "Restarts",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Container restarts",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 15,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 47
+                  },
+                  "id": 77,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "(\n    sum by (pod) (\n        increase(\n            kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[10m]\n        )\n    )\n    * on (pod) group_right\n    max by (pod, reason) (\n        kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}\n    )\n) > 0",
+                        "instant": false,
+                        "legendFormat": "{{reason}} / {{pod}}",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Container restarts reason per pod",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "Resource Usage",
+            "type": "row"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 25
+            },
+            "id": 95,
+            "panels": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "How many tokens each compactor is appending to blooms. Accounts for tokens that are not actually added to the blooms since they are already there. See the panel on the right for a drill down on the collision.\n",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "log": 2,
+                              "type": "log"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 0,
+                     "y": 55
+                  },
+                  "id": 96,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "# tokens checked per pod, millions/s\nsum(rate(loki_bloom_tokens_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))\n/\nsum(count(loki_bloom_tokens_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}))\n/ 1e6",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Per core",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(rate(loki_bloom_inserts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])) / 1e6",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Total",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Tokens rate (millions)",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "Collision type may be `false` (no collision), `cache` (found in token cache) or true (found in bloom filter).\n\nType may be either `raw` (the original ngram) or `chunk_prefixed` (the ngram with the chunk prefix)",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "percentunit"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 12,
+                     "y": 55
+                  },
+                  "id": 97,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "# tokens/s by type+collision\nsum by (collision) (\n    rate(loki_bloom_inserts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n) \n/ on () group_left\nsum (\n    rate(loki_bloom_inserts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "tokens/s by collision type",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "The sizes of the blooms created by the compactor. We build one bloom per series. The more unique ngrams and chunks the series has, the bigger their blooms will be.",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "bytes"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 0,
+                     "y": 63
+                  },
+                  "id": 98,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(\n    0.99, \n    sum by (le) (\n        rate(loki_bloom_size_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p99",
+                        "range": true,
+                        "refId": "D"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(\n    0.90, \n    sum by (le) (\n        rate(loki_bloom_size_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p90",
+                        "range": true,
+                        "refId": "E"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(\n    0.50, \n    sum by (le) (\n        rate(loki_bloom_size_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p50",
+                        "range": true,
+                        "refId": "F"
+                     }
+                  ],
+                  "title": "Bloom size",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "How many chunks are we indexing in the blooms. Either:\n- `copied` from a pre-existing bloom block, or \n- `iterated` through all its entries if processed for the first time.",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 12,
+                     "y": 63
+                  },
+                  "id": 99,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "# chunks indexed, by iteration or copied from a pre-existing bloom\nsum(rate(loki_bloom_chunks_indexed_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])) by (type)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "Chunks indexed",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "Bloom building",
+            "type": "row"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 26
+            },
+            "id": 103,
+            "panels": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 72
+                  },
+                  "id": 107,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(increase(loki_bloomcompactor_blocks_created_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Blocks",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Created Blocks",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "Compactors delete metas and blocks marked for deletion in the metas tombstones.",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 72
+                  },
+                  "id": 106,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(increase(loki_bloomcompactor_blocks_deleted_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Blocks",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Deleted Blocks",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "Number of overlapping bloom blocks reused when creating new blocks\n",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 79
+                  },
+                  "id": 109,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(increase(loki_bloomcompactor_blocks_reused_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Blocks",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Blocks reused",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "Blocks building",
+            "type": "row"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 27
+            },
+            "id": 110,
+            "panels": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 87
+                  },
+                  "id": 108,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(increase(loki_bloomcompactor_metas_created_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Metas",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Created Metas",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "Compactors delete metas and blocks marked for deletion in the metas tombstones.",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 87
+                  },
+                  "id": 105,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(increase(loki_bloomcompactor_metas_deleted_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Metas",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Deleted Metas",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "Metas building",
+            "type": "row"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 28
+            },
+            "id": 80,
+            "panels": [
+               {
+                  "description": "",
+                  "gridPos": {
+                     "h": 1,
+                     "w": 24,
+                     "x": 0,
+                     "y": 95
+                  },
+                  "id": 93,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-70005",
+                  "title": "We use tenant sharding so each compactor will process a subset of the tenants.",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 96
+                  },
+                  "id": 83,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n  0.99,\n  increase(\n    loki_bloomcompactor_tenants_started_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n  )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p99",
+                        "range": true,
+                        "refId": "C"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n  0.90,\n  increase(\n    loki_bloomcompactor_tenants_started_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n  )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p90",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n  0.50,\n  increase(\n    loki_bloomcompactor_tenants_started_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[30m]\n  )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p50",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "Tenants",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 96
+                  },
+                  "id": 84,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (pod) (\n  increase(\n    loki_bloomcompactor_tenants_started_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n  )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Tenants per pod",
+                  "type": "timeseries"
+               },
+               {
+                  "description": "",
+                  "gridPos": {
+                     "h": 1,
+                     "w": 24,
+                     "x": 0,
+                     "y": 103
+                  },
+                  "id": 86,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-70005",
+                  "title": "Number of tenant tables processed. ",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 104
+                  },
+                  "id": 88,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n  0.99,\n  increase(\n    loki_bloomcompactor_tenant_table_ranges_completed_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n  )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p99",
+                        "range": true,
+                        "refId": "C"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n  0.90,\n  increase(\n    loki_bloomcompactor_tenant_table_ranges_completed_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n  )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p90",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n  0.50,\n  increase(\n    loki_bloomcompactor_tenant_table_ranges_completed_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n  )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p50",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "Tenant Tables",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 104
+                  },
+                  "id": 89,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (pod) (\n  increase(\n    loki_bloomcompactor_tenant_table_ranges_completed_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n  )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Tenant Tables per pod",
+                  "type": "timeseries"
+               },
+               {
+                  "description": "",
+                  "gridPos": {
+                     "h": 1,
+                     "w": 24,
+                     "x": 0,
+                     "y": 111
+                  },
+                  "id": 87,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-70005",
+                  "title": "Series per compaction (includes series copied from other blocks)",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 112
+                  },
+                  "id": 81,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "# series checked per compaction\nhistogram_quantile(\n    0.99, \n    sum by (le) (\n        rate(loki_bloomcompactor_series_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p99",
+                        "range": true,
+                        "refId": "C"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "# series checked per compaction\nhistogram_quantile(\n    0.9, \n    sum by (le) (\n        rate(loki_bloomcompactor_series_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p90",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "# series checked per compaction\nhistogram_quantile(\n    0.5, \n    sum by (le) (\n        rate(loki_bloomcompactor_series_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p50",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Series",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 112
+                  },
+                  "id": 82,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (pod) (\n    rate(loki_bloomcompactor_series_per_compaction_sum{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    /\n    rate(loki_bloomcompactor_series_per_compaction_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "avg series per compaction by pod",
+                  "type": "timeseries"
+               },
+               {
+                  "description": "",
+                  "gridPos": {
+                     "h": 1,
+                     "w": 24,
+                     "x": 0,
+                     "y": 119
+                  },
+                  "id": 90,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-70005",
+                  "title": "Number of bytes from chunks added to blocks during each compaction.",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        },
+                        "unit": "bytes"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 120
+                  },
+                  "id": 91,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "# series checked per compaction\nhistogram_quantile(\n    0.99, \n    sum by (le) (\n        rate(loki_bloomcompactor_bytes_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p99",
+                        "range": true,
+                        "refId": "C"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "# series checked per compaction\nhistogram_quantile(\n    0.9, \n    sum by (le) (\n        rate(loki_bloomcompactor_bytes_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p90",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "# series checked per compaction\nhistogram_quantile(\n    0.5, \n    sum by (le) (\n        rate(loki_bloomcompactor_bytes_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p50",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Bytes",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        },
+                        "unit": "bytes"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 120
+                  },
+                  "id": 92,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (pod) (\n    rate(loki_bloomcompactor_bytes_per_compaction_sum{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    /\n    rate(loki_bloomcompactor_bytes_per_compaction_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p99",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "avg bytes per compaction by pod",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "Data processed",
+            "type": "row"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 29
+            },
+            "id": 58,
+            "panels": [
+               {
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": { },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 3,
+                     "w": 24,
+                     "x": 0,
+                     "y": 82
+                  },
+                  "id": 47,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "\nCompactors write blocks to the attached PVs before flushing them into the object store.\nIt also download chunks and index files.\n\nAfter compacting a given tenant, all the downloaded index files and chunks, as well as the already flushed blocks are deleted.",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-69747",
+                  "title": "",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "area"
+                           }
+                        },
+                        "mappings": [ ],
+                        "max": 1,
+                        "min": 0,
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 0.80000000000000004
+                              }
+                           ]
+                        },
+                        "unit": "percentunit"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 0,
+                     "y": 85
+                  },
+                  "id": 9,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n    0.99,\n    max by(persistentvolumeclaim) (\n        kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} \n        / \n        kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n    ) \n    and \n    count by(persistentvolumeclaim) (\n        kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"bloom-compactor\"}\n    )\n)",
+                        "instant": false,
+                        "legendFormat": "p99",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n    0.90,\n    max by(persistentvolumeclaim) (\n        kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} \n        / \n        kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n    ) \n    and \n    count by(persistentvolumeclaim) (\n        kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"bloom-compactor\"}\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p90",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n    0.50,\n    max by(persistentvolumeclaim) (\n        kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} \n        / \n        kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n    ) \n    and \n    count by(persistentvolumeclaim) (\n        kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"bloom-compactor\"}\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p50",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Disk Utilization",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "area"
+                           }
+                        },
+                        "mappings": [ ],
+                        "max": 1,
+                        "min": 0,
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 0.80000000000000004
+                              }
+                           ]
+                        },
+                        "unit": "percentunit"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 12,
+                     "y": 85
+                  },
+                  "id": 100,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"bloom-compactor\"})",
+                        "instant": false,
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Disk Utilization per pod",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "bytes"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 93
+                  },
+                  "id": 7,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n    0.99,\n    sum by(instance, pod, device) (\n        rate(node_disk_written_bytes_total[$__rate_interval])\n    ) \n    + ignoring(pod) group_right() \n    (\n        label_replace(\n            count by(instance, pod, device) (\n                container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n            ), \n            \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n        ) * 0\n    )\n)",
+                        "instant": false,
+                        "legendFormat": "p99",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n    0.90,\n    sum by(instance, pod, device) (\n        rate(node_disk_written_bytes_total[$__rate_interval])\n    ) \n    + ignoring(pod) group_right() \n    (\n        label_replace(\n            count by(instance, pod, device) (\n                container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n            ), \n            \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n        ) * 0\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p90",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n    0.50,\n    sum by(instance, pod, device) (\n        rate(node_disk_written_bytes_total[$__rate_interval])\n    ) \n    + ignoring(pod) group_right() \n    (\n        label_replace(\n            count by(instance, pod, device) (\n                container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n            ), \n            \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n        ) * 0\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p50",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Disk Writes",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "bytes"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 93
+                  },
+                  "id": 101,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)",
+                        "instant": false,
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Disk Writes per pod",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "bytes"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 0,
+                     "y": 100
+                  },
+                  "id": 8,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n    0.99,\n    sum by(instance, pod, device) (\n        rate(node_disk_read_bytes_total[$__rate_interval])\n    ) + ignoring(pod) group_right()\n    (\n        label_replace(\n            count by(instance, pod, device) (\n                container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n            ), \n            \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n        ) * 0\n    )\n)",
+                        "instant": false,
+                        "legendFormat": "p99",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n    0.90,\n    sum by(instance, pod, device) (\n        rate(node_disk_read_bytes_total[$__rate_interval])\n    ) + ignoring(pod) group_right()\n    (\n        label_replace(\n            count by(instance, pod, device) (\n                container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n            ), \n            \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n        ) * 0\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p90",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n    0.50,\n    sum by(instance, pod, device) (\n        rate(node_disk_read_bytes_total[$__rate_interval])\n    ) + ignoring(pod) group_right()\n    (\n        label_replace(\n            count by(instance, pod, device) (\n                container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n            ), \n            \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n        ) * 0\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p50",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Disk Reads",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "bytes"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 12,
+                     "y": 100
+                  },
+                  "id": 102,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)",
+                        "instant": false,
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Disk Reads per pod",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "Disk Usage",
+            "type": "row"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 30
+            },
+            "id": 62,
+            "panels": [
+               {
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": { },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 3,
+                     "w": 24,
+                     "x": 0,
+                     "y": 83
+                  },
+                  "id": 71,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "Once all blocks and metas are built locally, the compactor flushes them to the object store.\n\nAfter each iteration, the compactor deletes the metas and blocks marked for deletion in the tombstones.",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-69747",
+                  "title": "",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": { },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 2,
+                     "x": 0,
+                     "y": 86
+                  },
+                  "id": 63,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "---\n#### GCS\n",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-69747",
+                  "title": "",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 25,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 11,
+                     "x": 2,
+                     "y": 86
+                  },
+                  "id": 61,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} {{status_code}}",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "QPS",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 25,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "s"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 11,
+                     "x": 13,
+                     "y": 86
+                  },
+                  "id": 64,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p99",
+                        "range": true,
+                        "refId": "D"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p90",
+                        "range": true,
+                        "refId": "E"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p50",
+                        "range": true,
+                        "refId": "F"
+                     }
+                  ],
+                  "title": "Latency",
+                  "type": "timeseries"
+               },
+               {
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": { },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 2,
+                     "x": 0,
+                     "y": 93
+                  },
+                  "id": 65,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "---\n#### S3\n",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-69747",
+                  "title": "",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 25,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 11,
+                     "x": 2,
+                     "y": 93
+                  },
+                  "id": 67,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} {{status_code}}",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "QPS",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 25,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "s"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 11,
+                     "x": 13,
+                     "y": 93
+                  },
+                  "id": 69,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p99",
+                        "range": true,
+                        "refId": "D"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p90",
+                        "range": true,
+                        "refId": "E"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p50",
+                        "range": true,
+                        "refId": "F"
+                     }
+                  ],
+                  "title": "Latency",
+                  "type": "timeseries"
+               },
+               {
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": { },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 2,
+                     "x": 0,
+                     "y": 100
+                  },
+                  "id": 66,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "---\n#### Azure\nBlob Storage",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-69747",
+                  "title": "",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 25,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 11,
+                     "x": 2,
+                     "y": 100
+                  },
+                  "id": 68,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} {{status_code}}",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "QPS",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 25,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "s"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 11,
+                     "x": 13,
+                     "y": 100
+                  },
+                  "id": 70,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p99",
+                        "range": true,
+                        "refId": "D"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p90",
+                        "range": true,
+                        "refId": "E"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p50",
+                        "range": true,
+                        "refId": "F"
+                     }
+                  ],
+                  "title": "Latency",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "Object Store",
+            "type": "row"
+         }
+      ],
+      "refresh": "10s",
+      "rows": [ ],
+      "schemaVersion": 14,
+      "style": "dark",
+      "tags": [
+         "loki"
+      ],
+      "templating": {
+         "list": [
+            {
+               "current": {
+                  "text": "default",
+                  "value": "default"
+               },
+               "hide": 0,
+               "label": "Data source",
+               "name": "datasource",
+               "options": [ ],
+               "query": "prometheus",
+               "refresh": 1,
+               "regex": "",
+               "type": "datasource"
+            },
+            {
+               "allValue": null,
+               "current": {
+                  "text": "prod",
+                  "value": "prod"
+               },
+               "datasource": "$datasource",
+               "hide": 0,
+               "includeAll": false,
+               "label": "cluster",
+               "multi": false,
+               "name": "cluster",
+               "options": [ ],
+               "query": "label_values(loki_build_info, cluster)",
+               "refresh": 1,
+               "regex": "",
+               "sort": 2,
+               "tagValuesQuery": "",
+               "tags": [ ],
+               "tagsQuery": "",
+               "type": "query",
+               "useTags": false
+            },
+            {
+               "allValue": null,
+               "current": {
+                  "text": "prod",
+                  "value": "prod"
+               },
+               "datasource": "$datasource",
+               "hide": 0,
+               "includeAll": false,
+               "label": "namespace",
+               "multi": false,
+               "name": "namespace",
+               "options": [ ],
+               "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)",
+               "refresh": 1,
+               "regex": "",
+               "sort": 2,
+               "tagValuesQuery": "",
+               "tags": [ ],
+               "tagsQuery": "",
+               "type": "query",
+               "useTags": false
+            },
+            {
+               "hide": 0,
+               "label": null,
+               "name": "loki_datasource",
+               "options": [ ],
+               "query": "loki",
+               "refresh": 1,
+               "regex": "",
+               "type": "datasource"
+            }
+         ]
+      },
+      "time": {
+         "from": "now-1h",
+         "to": "now"
+      },
+      "timeRangeUpdatedDuringEditOrView": false,
+      "timepicker": {
+         "refresh_intervals": [
+            "5s",
+            "10s",
+            "30s",
+            "1m",
+            "5m",
+            "15m",
+            "30m",
+            "1h",
+            "2h",
+            "1d"
+         ],
+         "time_options": [
+            "5m",
+            "15m",
+            "1h",
+            "6h",
+            "12h",
+            "24h",
+            "2d",
+            "7d",
+            "30d"
+         ]
+      },
+      "timezone": "utc",
+      "title": "Loki / Bloom Compactor",
+      "uid": "bloom-compactor",
+      "version": 0,
+      "weekStart": ""
+   }
\ No newline at end of file
diff --git a/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-gateway.json b/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-gateway.json
new file mode 100644
index 0000000000000..27a058ae800e2
--- /dev/null
+++ b/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-gateway.json
@@ -0,0 +1,5035 @@
+{
+      "annotations": {
+         "list": [ ]
+      },
+      "editable": true,
+      "fiscalYearStartMonth": 0,
+      "gnetId": null,
+      "graphTooltip": 0,
+      "hideControls": false,
+      "links": [
+         {
+            "asDropdown": true,
+            "icon": "external link",
+            "includeVars": true,
+            "keepTime": true,
+            "tags": [
+               "loki"
+            ],
+            "targetBlank": false,
+            "title": "Loki Dashboards",
+            "type": "dashboards"
+         }
+      ],
+      "liveNow": false,
+      "panels": [
+         {
+            "collapsed": false,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 0
+            },
+            "id": 73,
+            "panels": [ ],
+            "title": "Overview",
+            "type": "row"
+         },
+         {
+            "datasource": {
+               "type": "prometheus",
+               "uid": "${datasource}"
+            },
+            "description": "Percentage of chunks that are filtered by using bloom filters",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "thresholds",
+                     "seriesBy": "last"
+                  },
+                  "custom": {
+                     "axisBorderShow": false,
+                     "axisCenteredZero": false,
+                     "axisColorMode": "text",
+                     "axisLabel": "",
+                     "axisPlacement": "auto",
+                     "barAlignment": 0,
+                     "drawStyle": "line",
+                     "fillOpacity": 0,
+                     "gradientMode": "none",
+                     "hideFrom": {
+                        "legend": false,
+                        "tooltip": false,
+                        "viz": false
+                     },
+                     "insertNulls": false,
+                     "lineInterpolation": "linear",
+                     "lineStyle": {
+                        "fill": "solid"
+                     },
+                     "lineWidth": 2,
+                     "pointSize": 5,
+                     "scaleDistribution": {
+                        "type": "linear"
+                     },
+                     "showPoints": "auto",
+                     "spanNulls": false,
+                     "stacking": {
+                        "group": "A",
+                        "mode": "none"
+                     },
+                     "thresholdsStyle": {
+                        "mode": "area"
+                     }
+                  },
+                  "mappings": [ ],
+                  "max": 1,
+                  "min": 0,
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [
+                        {
+                           "color": "red",
+                           "value": null
+                        },
+                        {
+                           "color": "orange",
+                           "value": 0.5
+                        },
+                        {
+                           "color": "yellow",
+                           "value": 0.75
+                        },
+                        {
+                           "color": "green",
+                           "value": 0.90000000000000002
+                        }
+                     ]
+                  },
+                  "unit": "percentunit"
+               },
+               "overrides": [ ]
+            },
+            "gridPos": {
+               "h": 6,
+               "w": 6,
+               "x": 0,
+               "y": 1
+            },
+            "id": 23,
+            "options": {
+               "legend": {
+                  "calcs": [ ],
+                  "displayMode": "list",
+                  "placement": "bottom",
+                  "showLegend": true
+               },
+               "tooltip": {
+                  "maxHeight": 600,
+                  "mode": "single",
+                  "sort": "none"
+               }
+            },
+            "targets": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "sum(rate(loki_bloom_gateway_filtered_chunks_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_chunks_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))",
+                  "instant": false,
+                  "legendFormat": "Chunks",
+                  "range": true,
+                  "refId": "A"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "exemplar": false,
+                  "expr": "sum(rate(loki_bloom_gateway_filtered_series_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_series_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))",
+                  "hide": false,
+                  "instant": false,
+                  "legendFormat": "Series",
+                  "range": true,
+                  "refId": "B"
+               }
+            ],
+            "title": "Filter ratio",
+            "type": "timeseries"
+         },
+         {
+            "datasource": {
+               "type": "prometheus",
+               "uid": "${datasource}"
+            },
+            "description": "Percentage of chunks that are filtered by using bloom filters",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "thresholds"
+                  },
+                  "mappings": [ ],
+                  "max": 1,
+                  "min": 0,
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [
+                        {
+                           "color": "red",
+                           "value": null
+                        },
+                        {
+                           "color": "orange",
+                           "value": 0.5
+                        },
+                        {
+                           "color": "yellow",
+                           "value": 0.75
+                        },
+                        {
+                           "color": "green",
+                           "value": 0.90000000000000002
+                        }
+                     ]
+                  },
+                  "unit": "percentunit"
+               },
+               "overrides": [ ]
+            },
+            "gridPos": {
+               "h": 6,
+               "w": 6,
+               "x": 6,
+               "y": 1
+            },
+            "id": 75,
+            "options": {
+               "minVizHeight": 75,
+               "minVizWidth": 75,
+               "orientation": "auto",
+               "reduceOptions": {
+                  "calcs": [
+                     "lastNotNull"
+                  ],
+                  "fields": "",
+                  "values": false
+               },
+               "showThresholdLabels": false,
+               "showThresholdMarkers": true,
+               "sizing": "auto"
+            },
+            "pluginVersion": "11.1.0-70005",
+            "targets": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "exemplar": false,
+                  "expr": "sum(rate(loki_bloom_gateway_filtered_chunks_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_chunks_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))",
+                  "instant": true,
+                  "legendFormat": "Chunks",
+                  "range": false,
+                  "refId": "A"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "exemplar": false,
+                  "expr": "sum(rate(loki_bloom_gateway_filtered_series_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_series_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))",
+                  "hide": false,
+                  "instant": true,
+                  "legendFormat": "Series",
+                  "range": false,
+                  "refId": "B"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "exemplar": false,
+                  "expr": "sum(loki_bloom_gateway_filtered_chunks_sum{job=\"$namespace/bloom-gateway\"})\n/\nsum(loki_bloom_gateway_requested_chunks_sum{job=\"$namespace/bloom-gateway\"})",
+                  "hide": true,
+                  "instant": true,
+                  "legendFormat": "Chunks avg",
+                  "range": false,
+                  "refId": "C"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "exemplar": false,
+                  "expr": "sum(loki_bloom_gateway_filtered_series_sum{job=\"$namespace/bloom-gateway\"})\n/\nsum(loki_bloom_gateway_requested_series_sum{job=\"$namespace/bloom-gateway\"})",
+                  "hide": true,
+                  "instant": true,
+                  "legendFormat": "Series avg",
+                  "range": false,
+                  "refId": "D"
+               }
+            ],
+            "title": "Filter ratio",
+            "type": "gauge"
+         },
+         {
+            "datasource": {
+               "type": "prometheus",
+               "uid": "${datasource}"
+            },
+            "description": "",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "palette-classic"
+                  },
+                  "custom": {
+                     "axisBorderShow": false,
+                     "axisCenteredZero": false,
+                     "axisColorMode": "text",
+                     "axisLabel": "",
+                     "axisPlacement": "auto",
+                     "barAlignment": 0,
+                     "drawStyle": "line",
+                     "fillOpacity": 10,
+                     "gradientMode": "none",
+                     "hideFrom": {
+                        "legend": false,
+                        "tooltip": false,
+                        "viz": false
+                     },
+                     "insertNulls": false,
+                     "lineInterpolation": "linear",
+                     "lineStyle": {
+                        "fill": "solid"
+                     },
+                     "lineWidth": 1,
+                     "pointSize": 5,
+                     "scaleDistribution": {
+                        "type": "linear"
+                     },
+                     "showPoints": "auto",
+                     "spanNulls": false,
+                     "stacking": {
+                        "group": "A",
+                        "mode": "none"
+                     },
+                     "thresholdsStyle": {
+                        "mode": "off"
+                     }
+                  },
+                  "mappings": [ ],
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [
+                        {
+                           "color": "green",
+                           "value": null
+                        },
+                        {
+                           "color": "red",
+                           "value": 80
+                        }
+                     ]
+                  },
+                  "unit": "none"
+               },
+               "overrides": [
+                  {
+                     "matcher": {
+                        "id": "byName",
+                        "options": "Desired"
+                     },
+                     "properties": [
+                        {
+                           "id": "custom.fillOpacity",
+                           "value": 0
+                        },
+                        {
+                           "id": "custom.lineStyle",
+                           "value": {
+                              "dash": [
+                                 10,
+                                 10
+                              ],
+                              "fill": "dash"
+                           }
+                        },
+                        {
+                           "id": "custom.lineWidth",
+                           "value": 2
+                        }
+                     ]
+                  }
+               ]
+            },
+            "gridPos": {
+               "h": 6,
+               "w": 6,
+               "x": 12,
+               "y": 1
+            },
+            "id": 72,
+            "options": {
+               "legend": {
+                  "calcs": [ ],
+                  "displayMode": "list",
+                  "placement": "bottom",
+                  "showLegend": true
+               },
+               "tooltip": {
+                  "maxHeight": 600,
+                  "mode": "single",
+                  "sort": "none"
+               }
+            },
+            "targets": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "kube_statefulset_status_replicas_ready{cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"bloom-gateway\"}",
+                  "hide": false,
+                  "instant": false,
+                  "legendFormat": "Ready",
+                  "range": true,
+                  "refId": "D"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "sum(kube_pod_container_status_ready{container=\"bloom-gateway\", cluster=\"$cluster\", namespace=\"$namespace\"})",
+                  "hide": true,
+                  "instant": false,
+                  "legendFormat": "Running",
+                  "range": true,
+                  "refId": "A"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "kube_statefulset_replicas{cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"bloom-gateway\"}",
+                  "hide": false,
+                  "instant": false,
+                  "legendFormat": "Desired",
+                  "range": true,
+                  "refId": "B"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "kube_statefulset_status_replicas_available{cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"bloom-gateway\"}",
+                  "hide": true,
+                  "instant": false,
+                  "legendFormat": "Available",
+                  "range": true,
+                  "refId": "C"
+               }
+            ],
+            "title": "Readiness",
+            "type": "timeseries"
+         },
+         {
+            "datasource": {
+               "type": "prometheus",
+               "uid": "${datasource}"
+            },
+            "description": "",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "palette-classic"
+                  },
+                  "custom": {
+                     "axisBorderShow": false,
+                     "axisCenteredZero": false,
+                     "axisColorMode": "text",
+                     "axisLabel": "",
+                     "axisPlacement": "auto",
+                     "barAlignment": 0,
+                     "drawStyle": "line",
+                     "fillOpacity": 50,
+                     "gradientMode": "none",
+                     "hideFrom": {
+                        "legend": false,
+                        "tooltip": false,
+                        "viz": false
+                     },
+                     "insertNulls": false,
+                     "lineInterpolation": "linear",
+                     "lineStyle": {
+                        "fill": "solid"
+                     },
+                     "lineWidth": 1,
+                     "pointSize": 5,
+                     "scaleDistribution": {
+                        "type": "linear"
+                     },
+                     "showPoints": "auto",
+                     "spanNulls": false,
+                     "stacking": {
+                        "group": "A",
+                        "mode": "none"
+                     },
+                     "thresholdsStyle": {
+                        "mode": "off"
+                     }
+                  },
+                  "mappings": [ ],
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [
+                        {
+                           "color": "green",
+                           "value": null
+                        },
+                        {
+                           "color": "red",
+                           "value": 80
+                        }
+                     ]
+                  },
+                  "unit": "none"
+               },
+               "overrides": [ ]
+            },
+            "gridPos": {
+               "h": 6,
+               "w": 6,
+               "x": 18,
+               "y": 1
+            },
+            "id": 37,
+            "options": {
+               "legend": {
+                  "calcs": [ ],
+                  "displayMode": "list",
+                  "placement": "bottom",
+                  "showLegend": true
+               },
+               "tooltip": {
+                  "maxHeight": 600,
+                  "mode": "single",
+                  "sort": "none"
+               }
+            },
+            "targets": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "(\n    max by (pod, reason) (kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})\n    * on (pod) group_left\n    sum by (pod) (increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))\n) > 0",
+                  "hide": false,
+                  "instant": false,
+                  "interval": "",
+                  "legendFormat": "{{pod}} ({{reason}})",
+                  "range": true,
+                  "refId": "C"
+               }
+            ],
+            "title": "Container restarts",
+            "type": "timeseries"
+         },
+         {
+            "datasource": {
+               "type": "loki",
+               "uid": "${loki_datasource}"
+            },
+            "gridPos": {
+               "h": 9,
+               "w": 15,
+               "x": 0,
+               "y": 7
+            },
+            "id": 48,
+            "options": {
+               "dedupStrategy": "none",
+               "enableLogDetails": true,
+               "prettifyLogMessage": false,
+               "showCommonLabels": false,
+               "showLabels": false,
+               "showTime": false,
+               "sortOrder": "Descending",
+               "wrapLogMessage": true
+            },
+            "targets": [
+               {
+                  "datasource": {
+                     "type": "loki",
+                     "uid": "${loki_datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"level=error\" or \"panic:\" | logfmt",
+                  "queryType": "range",
+                  "refId": "A"
+               },
+               {
+                  "datasource": {
+                     "type": "loki",
+                     "uid": "${loki_datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"level=warn\" | logfmt",
+                  "hide": true,
+                  "queryType": "range",
+                  "refId": "B"
+               }
+            ],
+            "title": "Errors",
+            "type": "logs"
+         },
+         {
+            "datasource": {
+               "type": "loki",
+               "uid": "${loki_datasource}"
+            },
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "fixedColor": "red",
+                     "mode": "fixed"
+                  },
+                  "custom": {
+                     "axisBorderShow": false,
+                     "axisCenteredZero": false,
+                     "axisColorMode": "text",
+                     "axisLabel": "",
+                     "axisPlacement": "auto",
+                     "barAlignment": 0,
+                     "drawStyle": "bars",
+                     "fillOpacity": 0,
+                     "gradientMode": "none",
+                     "hideFrom": {
+                        "legend": false,
+                        "tooltip": false,
+                        "viz": false
+                     },
+                     "insertNulls": false,
+                     "lineInterpolation": "linear",
+                     "lineWidth": 1,
+                     "pointSize": 5,
+                     "scaleDistribution": {
+                        "log": 2,
+                        "type": "symlog"
+                     },
+                     "showPoints": "auto",
+                     "spanNulls": false,
+                     "stacking": {
+                        "group": "A",
+                        "mode": "none"
+                     },
+                     "thresholdsStyle": {
+                        "mode": "off"
+                     }
+                  },
+                  "mappings": [ ],
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [
+                        {
+                           "color": "green",
+                           "value": null
+                        },
+                        {
+                           "color": "red",
+                           "value": 1
+                        }
+                     ]
+                  }
+               },
+               "overrides": [
+                  {
+                     "matcher": {
+                        "id": "byName",
+                        "options": "warn"
+                     },
+                     "properties": [
+                        {
+                           "id": "color",
+                           "value": {
+                              "fixedColor": "orange",
+                              "mode": "fixed"
+                           }
+                        }
+                     ]
+                  },
+                  {
+                     "matcher": {
+                        "id": "byName",
+                        "options": "error"
+                     },
+                     "properties": [
+                        {
+                           "id": "color",
+                           "value": {
+                              "fixedColor": "red",
+                              "mode": "fixed"
+                           }
+                        }
+                     ]
+                  },
+                  {
+                     "matcher": {
+                        "id": "byName",
+                        "options": "panic"
+                     },
+                     "properties": [
+                        {
+                           "id": "color",
+                           "value": {
+                              "fixedColor": "semi-dark-red",
+                              "mode": "fixed"
+                           }
+                        }
+                     ]
+                  }
+               ]
+            },
+            "gridPos": {
+               "h": 9,
+               "w": 9,
+               "x": 15,
+               "y": 7
+            },
+            "id": 52,
+            "options": {
+               "legend": {
+                  "calcs": [ ],
+                  "displayMode": "list",
+                  "placement": "bottom",
+                  "showLegend": true
+               },
+               "tooltip": {
+                  "maxHeight": 600,
+                  "mode": "single",
+                  "sort": "none"
+               }
+            },
+            "targets": [
+               {
+                  "datasource": {
+                     "type": "loki",
+                     "uid": "${loki_datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "sum by (level) (count_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |~ \"level=(warn|error)\" | logfmt [$__auto]))",
+                  "legendFormat": "{{ level }}",
+                  "queryType": "range",
+                  "refId": "A"
+               },
+               {
+                  "datasource": {
+                     "type": "loki",
+                     "uid": "${loki_datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "sum (count_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"panic:\" | logfmt [$__auto]))",
+                  "hide": false,
+                  "legendFormat": "panic",
+                  "queryType": "range",
+                  "refId": "B"
+               }
+            ],
+            "title": "Errors Rate",
+            "type": "timeseries"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 16
+            },
+            "id": 56,
+            "panels": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 14,
+                     "w": 12,
+                     "x": 0,
+                     "y": 17
+                  },
+                  "id": 10,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"}[$__rate_interval]))",
+                        "instant": false,
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\", resource=\"cpu\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Request",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\", resource=\"cpu\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Limit",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "CPU",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "bytes"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 17
+                  },
+                  "id": 11,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"})",
+                        "instant": false,
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\", resource=\"memory\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Request",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Limit",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Memory (workingset)",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "bytes"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 24
+                  },
+                  "id": 81,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"})",
+                        "hide": true,
+                        "instant": false,
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\", resource=\"memory\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Request",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Limit",
+                        "range": true,
+                        "refId": "C"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"}) by (pod)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "D"
+                     }
+                  ],
+                  "title": "Memory (heap inuse)",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 8,
+                     "x": 0,
+                     "y": 31
+                  },
+                  "id": 87,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (pod) (rate(go_gc_cycles_total_gc_cycles_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "GC rate",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "s"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 8,
+                     "x": 8,
+                     "y": 31
+                  },
+                  "id": 88,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (pod) (rate(go_gc_duration_seconds_sum{container=\"bloom-gateway\"}[$__rate_interval]))\n/\nsum by (pod) (rate(go_gc_duration_seconds_count{container=\"bloom-gateway\"}[$__rate_interval]))",
+                        "hide": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "GC duration",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "s"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 8,
+                     "x": 16,
+                     "y": 31
+                  },
+                  "id": 89,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99, sum(rate(go_gc_pauses_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))",
+                        "hide": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.90, sum(rate(go_gc_pauses_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))",
+                        "hide": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "GC pauses",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "binBps"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 38
+                  },
+                  "id": 84,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by(instance, pod) (rate(node_disk_read_bytes_total[$__rate_interval]))\n+ ignoring(pod) group_right() \n(count by(instance, pod) (container_fs_reads_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}) * 0)",
+                        "hide": false,
+                        "instant": false,
+                        "interval": "",
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "D"
+                     }
+                  ],
+                  "title": "Disk reads",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "binBps"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 38
+                  },
+                  "id": 85,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by(instance, pod) (rate(node_disk_written_bytes_total[$__rate_interval]))\n+ ignoring(pod) group_right() \n(count by(instance, pod) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}) * 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "D"
+                     }
+                  ],
+                  "title": "Disk writes",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "Resource usage",
+            "type": "row"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 17
+            },
+            "id": 2,
+            "panels": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 100,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 0,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "normal"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [
+                        {
+                           "matcher": {
+                              "id": "byName",
+                              "options": "cancel"
+                           },
+                           "properties": [
+                              {
+                                 "id": "color",
+                                 "value": {
+                                    "mode": "fixed"
+                                 }
+                              }
+                           ]
+                        },
+                        {
+                           "matcher": {
+                              "id": "byName",
+                              "options": "success"
+                           },
+                           "properties": [
+                              {
+                                 "id": "color",
+                                 "value": {
+                                    "fixedColor": "green",
+                                    "mode": "fixed"
+                                 }
+                              }
+                           ]
+                        },
+                        {
+                           "matcher": {
+                              "id": "byName",
+                              "options": "error"
+                           },
+                           "properties": [
+                              {
+                                 "id": "color",
+                                 "value": {
+                                    "fixedColor": "red",
+                                    "mode": "fixed"
+                                 }
+                              }
+                           ]
+                        }
+                     ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 18
+                  },
+                  "id": 13,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (status_code) (\n    rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/bloom-gateway\", route=\"/logproto.BloomGateway/FilterChunkRefs\"}[$__rate_interval])\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "QPS",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 20,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [
+                        {
+                           "matcher": {
+                              "id": "byName",
+                              "options": "cancel"
+                           },
+                           "properties": [
+                              {
+                                 "id": "color",
+                                 "value": {
+                                    "mode": "fixed"
+                                 }
+                              }
+                           ]
+                        },
+                        {
+                           "matcher": {
+                              "id": "byName",
+                              "options": "success"
+                           },
+                           "properties": [
+                              {
+                                 "id": "color",
+                                 "value": {
+                                    "fixedColor": "green",
+                                    "mode": "fixed"
+                                 }
+                              }
+                           ]
+                        },
+                        {
+                           "matcher": {
+                              "id": "byName",
+                              "options": "error"
+                           },
+                           "properties": [
+                              {
+                                 "id": "color",
+                                 "value": {
+                                    "fixedColor": "red",
+                                    "mode": "fixed"
+                                 }
+                              }
+                           ]
+                        }
+                     ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 18
+                  },
+                  "id": 86,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (pod) (\n    rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/bloom-gateway\", route=\"/logproto.BloomGateway/FilterChunkRefs\"}[$__rate_interval])\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "QPS per Pod",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        },
+                        "unit": "s"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 25
+                  },
+                  "id": 14,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.50, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"/logproto.BloomGateway/FilterChunkRefs\"}))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{ route }} 50th percentile",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(cluster_job_route:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"/logproto.BloomGateway/FilterChunkRefs\"}) by (route)  / sum(cluster_job_route:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"/logproto.BloomGateway/FilterChunkRefs\"}) by (route) ",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{ route }} Average",
+                        "range": true,
+                        "refId": "C"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"/logproto.BloomGateway/FilterChunkRefs\"}))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{ route }} 99th percentile",
+                        "range": true,
+                        "refId": "D"
+                     }
+                  ],
+                  "title": "Latency",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        },
+                        "unit": "s"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 25
+                  },
+                  "id": 15,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99,\n  sum(\n   rate(loki_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"/logproto.BloomGateway/FilterChunkRefs\"}[$__rate_interval])\n   ) by (pod, le)\n )\n",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "Per Pod Latency (p99)",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "QPS and Latency",
+            "type": "row"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 18
+            },
+            "id": 58,
+            "panels": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 25,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "normal"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 8,
+                     "x": 0,
+                     "y": 11
+                  },
+                  "id": 16,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (pod) (loki_bloom_gateway_queue_duration_seconds_sum{cluster=\"$cluster\", namespace=\"$namespace\"})\n/\nsum by (pod) (loki_bloom_gateway_queue_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"})\n",
+                        "hide": true,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "C"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum (loki_bloom_gateway_queue_length{cluster=\"$cluster\", namespace=\"$namespace\"})",
+                        "hide": true,
+                        "instant": false,
+                        "legendFormat": "Total",
+                        "range": true,
+                        "refId": "D"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (user) (loki_bloom_gateway_queue_length{cluster=\"$cluster\", namespace=\"$namespace\"})",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{user}}",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "Queue Size",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "For how long do pending tasks stay in the queue",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 1,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "s"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 8,
+                     "x": 8,
+                     "y": 11
+                  },
+                  "id": 17,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99, sum by (le) (rate(loki_bloom_gateway_queue_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p99",
+                        "range": true,
+                        "refId": "E"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.90, sum by (le) (rate(loki_bloom_gateway_queue_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))",
+                        "hide": true,
+                        "instant": false,
+                        "legendFormat": "p90",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.50, sum by (le) (rate(loki_bloom_gateway_queue_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p50",
+                        "range": true,
+                        "refId": "C"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum (loki_bloom_gateway_queue_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\"})\n/\nsum (loki_bloom_gateway_queue_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\"})",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "avg",
+                        "range": true,
+                        "refId": "D"
+                     }
+                  ],
+                  "title": "Queue Latency",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "Inflight requests tracks all tasks both queued and in progress",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 1,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 8,
+                     "x": 16,
+                     "y": 11
+                  },
+                  "id": 22,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (quantile) (loki_bloom_gateway_inflight_tasks{cluster=\"$cluster\", namespace=\"$namespace\", quantile=\"0.99\"})",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Inflight tasks",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "Task Queue",
+            "type": "row"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 19
+            },
+            "id": 68,
+            "panels": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "s"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 8,
+                     "x": 0,
+                     "y": 12
+                  },
+                  "id": 69,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99, sum(rate(loki_bloom_gateway_process_duration_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))",
+                        "instant": false,
+                        "legendFormat": "{{status}}-p99",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_gateway_process_duration_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{status}}-p95",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.90, sum(rate(loki_bloom_gateway_process_duration_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{status}}-p90",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Processing time for tasks (per worker iteration)",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "s"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 8,
+                     "x": 8,
+                     "y": 12
+                  },
+                  "id": 70,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99, sum(rate(loki_bloom_gateway_block_query_latency_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))",
+                        "instant": false,
+                        "legendFormat": "{{status}}-p99",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_gateway_block_query_latency_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{status}}-p95",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.90, sum(rate(loki_bloom_gateway_block_query_latency_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{status}}-p90",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Block query latency (single block)",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 8,
+                     "x": 16,
+                     "y": 12
+                  },
+                  "id": 71,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(rate(loki_bloom_gateway_tasks_dequeued_total{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval]))",
+                        "instant": false,
+                        "legendFormat": "dequeued",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (status) (rate(loki_bloom_gateway_tasks_processed_total{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "processed {{status}}",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "Tasks dequeued/processed",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "Processing",
+            "type": "row"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 20
+            },
+            "id": 59,
+            "panels": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "gridPos": {
+                     "h": 1,
+                     "w": 24,
+                     "x": 0,
+                     "y": 13
+                  },
+                  "id": 19,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-70005",
+                  "title": "We cache bloom blocks in memory to prevent the gateway from hitting the object store too often",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "fieldMinMax": false,
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "bytes"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 14
+                  },
+                  "id": 20,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(loki_embeddedcache_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", cache=\"bloom-blocks-cache\", container=\"bloom-gateway\"}) by (pod)",
+                        "hide": true,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(loki_bloom_blocks_cache_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}) by (pod)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "Cache size (per pod)",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "linearThreshold": 1000,
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "fieldMinMax": false,
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [
+                        {
+                           "matcher": {
+                              "id": "byName",
+                              "options": "Size"
+                           },
+                           "properties": [
+                              {
+                                 "id": "unit",
+                                 "value": "bytes"
+                              },
+                              {
+                                 "id": "custom.fillOpacity",
+                                 "value": 25
+                              },
+                              {
+                                 "id": "custom.lineWidth",
+                                 "value": 0
+                              }
+                           ]
+                        },
+                        {
+                           "matcher": {
+                              "id": "byName",
+                              "options": "Items"
+                           },
+                           "properties": [
+                              {
+                                 "id": "custom.fillOpacity",
+                                 "value": 25
+                              },
+                              {
+                                 "id": "custom.lineWidth",
+                                 "value": 0
+                              },
+                              {
+                                 "id": "unit",
+                                 "value": ""
+                              },
+                              {
+                                 "id": "custom.axisSoftMin",
+                                 "value": 0
+                              }
+                           ]
+                        }
+                     ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 14
+                  },
+                  "id": 83,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(loki_bloom_blocks_cache_entries{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Items",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(rate(loki_bloom_blocks_cache_added_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Added",
+                        "range": true,
+                        "refId": "G"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(rate(loki_bloom_blocks_cache_evicted_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (reason)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Evicted ({{reason}})",
+                        "range": true,
+                        "refId": "F"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(rate(loki_bloom_blocks_cache_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))",
+                        "hide": true,
+                        "instant": false,
+                        "legendFormat": "Size",
+                        "range": true,
+                        "refId": "E"
+                     }
+                  ],
+                  "title": "Cache rate",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 100,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 0,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "linearThreshold": 1000,
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "normal"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "fieldMinMax": false,
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "percentunit"
+                     },
+                     "overrides": [
+                        {
+                           "matcher": {
+                              "id": "byName",
+                              "options": "hit"
+                           },
+                           "properties": [
+                              {
+                                 "id": "color",
+                                 "value": {
+                                    "fixedColor": "green",
+                                    "mode": "fixed"
+                                 }
+                              }
+                           ]
+                        },
+                        {
+                           "matcher": {
+                              "id": "byName",
+                              "options": "miss"
+                           },
+                           "properties": [
+                              {
+                                 "id": "color",
+                                 "value": {
+                                    "fixedColor": "red",
+                                    "mode": "fixed"
+                                 }
+                              }
+                           ]
+                        }
+                     ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 24,
+                     "x": 0,
+                     "y": 21
+                  },
+                  "id": 92,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (status) (\n    rate(loki_bloom_blocks_cache_fetched_total{container=\"bloom-gateway\"}[$__rate_interval])\n)\n/ ignoring(status) group_left\nsum (\n    rate(loki_bloom_blocks_cache_fetched_total{container=\"bloom-gateway\"}[$__rate_interval])\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "Hit/Miss ratio",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "fieldMinMax": false,
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "bytes"
+                     },
+                     "overrides": [
+                        {
+                           "matcher": {
+                              "id": "byRegexp",
+                              "options": "/Size (.*)/"
+                           },
+                           "properties": [
+                              {
+                                 "id": "unit",
+                                 "value": "bytes"
+                              }
+                           ]
+                        }
+                     ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 28
+                  },
+                  "id": 76,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(loki_embeddedcache_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", cache=\"bloom-blocks-cache\", container=\"bloom-gateway\"})\n/\nsum(loki_embeddedcache_entries{cluster=\"$cluster\", namespace=\"$namespace\", cache=\"bloom-blocks-cache\", container=\"bloom-gateway\"})",
+                        "instant": false,
+                        "legendFormat": "Size",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(loki_bloom_blocks_cache_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})\n/\nsum(loki_bloom_blocks_cache_entries{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Size",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "Average item size",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "fieldMinMax": false,
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [
+                        {
+                           "matcher": {
+                              "id": "byRegexp",
+                              "options": "/.* (blocks|metas) size/"
+                           },
+                           "properties": [
+                              {
+                                 "id": "unit",
+                                 "value": "bytes"
+                              }
+                           ]
+                        }
+                     ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 28
+                  },
+                  "id": 21,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(rate(loki_bloom_store_metas_fetched_sum{cluster=\"$cluster\",namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "metas fetch rate",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(rate(loki_bloom_store_blocks_fetched_sum{cluster=\"$cluster\",namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "blocks fetch rate",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.9, sum(rate(loki_bloom_store_blocks_fetched_size_bytes_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p90 blocks size",
+                        "range": true,
+                        "refId": "C"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.9, sum(rate(loki_bloom_store_metas_fetched_size_bytes_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p90 metas size",
+                        "range": true,
+                        "refId": "D"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(1.0, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))",
+                        "hide": true,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "E"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))",
+                        "hide": true,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "F"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.5, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))",
+                        "hide": true,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "G"
+                     }
+                  ],
+                  "title": "Bloom Store",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "Blocks Cache",
+            "type": "row"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 21
+            },
+            "id": 60,
+            "panels": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "gridPos": {
+                     "h": 1,
+                     "w": 24,
+                     "x": 0,
+                     "y": 14
+                  },
+                  "id": 61,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-70005",
+                  "title": "The gateway download bloom meta files and blocks from the object store.",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "gridPos": {
+                     "h": 7,
+                     "w": 2,
+                     "x": 0,
+                     "y": 15
+                  },
+                  "id": 24,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "---\n#### GCS\n",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-70005",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 25,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "normal"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 11,
+                     "x": 2,
+                     "y": 15
+                  },
+                  "id": 25,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval]))",
+                        "instant": false,
+                        "legendFormat": "{{operation}} {{status_code}}",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "QPS",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 11,
+                     "x": 13,
+                     "y": 15
+                  },
+                  "id": 29,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))",
+                        "instant": false,
+                        "legendFormat": "{{operation}} p99",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p90",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p50",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Latency",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "gridPos": {
+                     "h": 7,
+                     "w": 2,
+                     "x": 0,
+                     "y": 22
+                  },
+                  "id": 62,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "---\n#### S3\n",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-70005",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 25,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "normal"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 11,
+                     "x": 2,
+                     "y": 22
+                  },
+                  "id": 63,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval]))",
+                        "instant": false,
+                        "legendFormat": "{{operation}} {{status_code}}",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "QPS",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 25,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "normal"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 11,
+                     "x": 13,
+                     "y": 22
+                  },
+                  "id": 64,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))",
+                        "instant": false,
+                        "legendFormat": "{{operation}} p99",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p90",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p50",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Latency",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "gridPos": {
+                     "h": 7,
+                     "w": 2,
+                     "x": 0,
+                     "y": 29
+                  },
+                  "id": 65,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "---\n#### Azure\nBlob Storage\n\n",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-70005",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 25,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "normal"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 11,
+                     "x": 2,
+                     "y": 29
+                  },
+                  "id": 66,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval]))",
+                        "instant": false,
+                        "legendFormat": "{{operation}} {{status_code}}",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "QPS",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 25,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "normal"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 11,
+                     "x": 13,
+                     "y": 29
+                  },
+                  "id": 67,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))",
+                        "instant": false,
+                        "legendFormat": "{{operation}} p99",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p90",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p50",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Latency",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "Object Store",
+            "type": "row"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 22
+            },
+            "id": 77,
+            "panels": [
+               {
+                  "datasource": {
+                     "type": "loki",
+                     "uid": "${loki_datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 0,
+                     "y": 23
+                  },
+                  "id": 78,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "loki",
+                           "uid": "${loki_datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "topk(3, sum by (tasks) (count_over_time({namespace=\"loki-dev-006\", container=\"bloom-gateway\"} |= \"process tasks with bounds\" | logfmt [5s])))",
+                        "legendFormat": "{{tasks}}",
+                        "queryType": "range",
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Process tasks with bounds",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "loki",
+                     "uid": "${loki_datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [
+                        {
+                           "__systemRef": "hideSeriesFrom",
+                           "matcher": {
+                              "id": "byNames",
+                              "options": {
+                                 "mode": "exclude",
+                                 "names": [
+                                    "max",
+                                    "avg"
+                                 ],
+                                 "prefix": "All except:",
+                                 "readOnly": true
+                              }
+                           },
+                           "properties": [
+                              {
+                                 "id": "custom.hideFrom",
+                                 "value": {
+                                    "legend": false,
+                                    "tooltip": false,
+                                    "viz": true
+                                 }
+                              }
+                           ]
+                        }
+                     ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 12,
+                     "y": 23
+                  },
+                  "id": 79,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "loki",
+                           "uid": "${loki_datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "max(max_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"fetch blocks\" | logfmt | unwrap duration(duration) [$__auto]))",
+                        "hide": false,
+                        "legendFormat": "max",
+                        "queryType": "range",
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "loki",
+                           "uid": "${loki_datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "avg(avg_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"fetch blocks\" | logfmt | unwrap duration(duration) [$__auto]))",
+                        "hide": false,
+                        "legendFormat": "avg",
+                        "queryType": "range",
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "loki",
+                           "uid": "${loki_datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "avg(avg_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"request unavailable blocks in the background\" | logfmt | missing > 0 | unwrap missing [$__auto]))",
+                        "hide": false,
+                        "legendFormat": "avg missing",
+                        "queryType": "range",
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Download enqueue duration",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "loki",
+                     "uid": "${loki_datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "fixedColor": "green",
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "fillOpacity": 80,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "lineWidth": 1,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 100
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 0,
+                     "y": 31
+                  },
+                  "id": 80,
+                  "options": {
+                     "barRadius": 0,
+                     "barWidth": 0.96999999999999997,
+                     "fullHighlight": false,
+                     "groupWidth": 0.69999999999999996,
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "orientation": "horizontal",
+                     "showValue": "auto",
+                     "stacking": "none",
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     },
+                     "xTickLabelRotation": 0,
+                     "xTickLabelSpacing": 0
+                  },
+                  "pluginVersion": "11.0.0-67814",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "loki",
+                           "uid": "${loki_datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sort_desc(topk(10, sum by (tasks) (count_over_time({namespace=\"loki-dev-006\", container=\"bloom-gateway\"} |= \"process tasks with bounds\" | logfmt [$__auto]))))",
+                        "legendFormat": "",
+                        "queryType": "instant",
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Tasks multiplexed",
+                  "type": "barchart"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [
+                        {
+                           "matcher": {
+                              "id": "byName",
+                              "options": "Enqueue latency"
+                           },
+                           "properties": [
+                              {
+                                 "id": "unit",
+                                 "value": "s"
+                              }
+                           ]
+                        }
+                     ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 12,
+                     "y": 31
+                  },
+                  "id": 82,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99, sum(rate(loki_bloom_store_download_queue_enqueue_time_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))",
+                        "hide": false,
+                        "legendFormat": "Enqueue latency",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99, sum(rate(loki_bloom_store_download_queue_size_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))",
+                        "hide": false,
+                        "legendFormat": "Size",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "Block download queue",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "Misc",
+            "type": "row"
+         }
+      ],
+      "refresh": "10s",
+      "rows": [ ],
+      "schemaVersion": 14,
+      "style": "dark",
+      "tags": [
+         "loki"
+      ],
+      "templating": {
+         "list": [
+            {
+               "current": {
+                  "text": "default",
+                  "value": "default"
+               },
+               "hide": 0,
+               "label": "Data source",
+               "name": "datasource",
+               "options": [ ],
+               "query": "prometheus",
+               "refresh": 1,
+               "regex": "",
+               "type": "datasource"
+            },
+            {
+               "allValue": null,
+               "current": {
+                  "text": "prod",
+                  "value": "prod"
+               },
+               "datasource": "$datasource",
+               "hide": 0,
+               "includeAll": false,
+               "label": "cluster",
+               "multi": false,
+               "name": "cluster",
+               "options": [ ],
+               "query": "label_values(loki_build_info, cluster)",
+               "refresh": 1,
+               "regex": "",
+               "sort": 2,
+               "tagValuesQuery": "",
+               "tags": [ ],
+               "tagsQuery": "",
+               "type": "query",
+               "useTags": false
+            },
+            {
+               "allValue": null,
+               "current": {
+                  "text": "prod",
+                  "value": "prod"
+               },
+               "datasource": "$datasource",
+               "hide": 0,
+               "includeAll": false,
+               "label": "namespace",
+               "multi": false,
+               "name": "namespace",
+               "options": [ ],
+               "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)",
+               "refresh": 1,
+               "regex": "",
+               "sort": 2,
+               "tagValuesQuery": "",
+               "tags": [ ],
+               "tagsQuery": "",
+               "type": "query",
+               "useTags": false
+            },
+            {
+               "hide": 0,
+               "label": null,
+               "name": "loki_datasource",
+               "options": [ ],
+               "query": "loki",
+               "refresh": 1,
+               "regex": "",
+               "type": "datasource"
+            }
+         ]
+      },
+      "time": {
+         "from": "now-1h",
+         "to": "now"
+      },
+      "timeRangeUpdatedDuringEditOrView": false,
+      "timepicker": {
+         "refresh_intervals": [
+            "5s",
+            "10s",
+            "30s",
+            "1m",
+            "5m",
+            "15m",
+            "30m",
+            "1h",
+            "2h",
+            "1d"
+         ],
+         "time_options": [
+            "5m",
+            "15m",
+            "1h",
+            "6h",
+            "12h",
+            "24h",
+            "2d",
+            "7d",
+            "30d"
+         ]
+      },
+      "timezone": "utc",
+      "title": "Loki / Bloom Gateway",
+      "uid": "bloom-gateway",
+      "version": 0,
+      "weekStart": ""
+   }
\ No newline at end of file
diff --git a/production/loki-mixin-compiled/dashboards/loki-bloom-compactor.json b/production/loki-mixin-compiled/dashboards/loki-bloom-compactor.json
new file mode 100644
index 0000000000000..c667d0c01eccb
--- /dev/null
+++ b/production/loki-mixin-compiled/dashboards/loki-bloom-compactor.json
@@ -0,0 +1,5191 @@
+{
+      "annotations": {
+         "list": [ ]
+      },
+      "editable": true,
+      "fiscalYearStartMonth": 0,
+      "gnetId": null,
+      "graphTooltip": 0,
+      "hideControls": false,
+      "links": [
+         {
+            "asDropdown": true,
+            "icon": "external link",
+            "includeVars": true,
+            "keepTime": true,
+            "tags": [
+               "loki"
+            ],
+            "targetBlank": false,
+            "title": "Loki Dashboards",
+            "type": "dashboards"
+         }
+      ],
+      "panels": [
+         {
+            "collapsed": false,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 0
+            },
+            "id": 111,
+            "panels": [ ],
+            "title": "Overview",
+            "type": "row"
+         },
+         {
+            "gridPos": {
+               "h": 8,
+               "w": 14,
+               "x": 0,
+               "y": 1
+            },
+            "id": 35,
+            "options": {
+               "code": {
+                  "language": "plaintext",
+                  "showLineNumbers": false,
+                  "showMiniMap": false
+               },
+               "content": "## About the Bloom Compactor\nThe compactor iterates through chunks and creates blooms out of them.\nThe size of the resulting blooms depends on the bloom filter settings, the tokenizer settings, the number of ring tokens per compactor and the total number opf compactors.\n\nCompactors are horizontally scalable and uses a ring to:\n- Shard tenants\n- Shard series fingerprints within a tenant subring.\n\nThe blooms for the series are grouped together in blocks which are flushed to object store.",
+               "mode": "markdown"
+            },
+            "pluginVersion": "11.1.0-70005",
+            "transparent": true,
+            "type": "text"
+         },
+         {
+            "datasource": {
+               "type": "prometheus",
+               "uid": "${datasource}"
+            },
+            "description": "Cell-wide compaction progress. Should increase till completion throughout each compaction period.",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "palette-classic"
+                  },
+                  "custom": {
+                     "axisBorderShow": false,
+                     "axisCenteredZero": false,
+                     "axisColorMode": "text",
+                     "axisLabel": "",
+                     "axisPlacement": "auto",
+                     "barAlignment": 0,
+                     "drawStyle": "line",
+                     "fillOpacity": 15,
+                     "gradientMode": "none",
+                     "hideFrom": {
+                        "legend": false,
+                        "tooltip": false,
+                        "viz": false
+                     },
+                     "insertNulls": false,
+                     "lineInterpolation": "linear",
+                     "lineStyle": {
+                        "fill": "solid"
+                     },
+                     "lineWidth": 1,
+                     "pointSize": 5,
+                     "scaleDistribution": {
+                        "type": "linear"
+                     },
+                     "showPoints": "auto",
+                     "spanNulls": false,
+                     "stacking": {
+                        "group": "A",
+                        "mode": "none"
+                     },
+                     "thresholdsStyle": {
+                        "mode": "off"
+                     }
+                  },
+                  "mappings": [ ],
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [
+                        {
+                           "color": "green",
+                           "value": null
+                        },
+                        {
+                           "color": "red",
+                           "value": 80
+                        }
+                     ]
+                  },
+                  "unit": "percentunit"
+               },
+               "overrides": [ ]
+            },
+            "gridPos": {
+               "h": 7,
+               "w": 12,
+               "x": 0,
+               "y": 9
+            },
+            "id": 42,
+            "options": {
+               "legend": {
+                  "calcs": [ ],
+                  "displayMode": "list",
+                  "placement": "bottom",
+                  "showLegend": true
+               },
+               "tooltip": {
+                  "maxHeight": 600,
+                  "mode": "single",
+                  "sort": "none"
+               }
+            },
+            "pluginVersion": "11.1.0-69868",
+            "targets": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "sum(loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"})\n/\nsum(count(loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}))",
+                  "hide": false,
+                  "instant": false,
+                  "legendFormat": "avg",
+                  "range": true,
+                  "refId": "A"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "quantile(\n    0.9, \n    sum by (pod) (\n        loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}\n    )\n)",
+                  "hide": false,
+                  "instant": false,
+                  "legendFormat": "p90",
+                  "range": true,
+                  "refId": "B"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "quantile(\n    0.1, \n    sum by (pod) (\n        loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}\n    )\n)",
+                  "hide": false,
+                  "instant": false,
+                  "legendFormat": "p10",
+                  "range": true,
+                  "refId": "C"
+               }
+            ],
+            "title": "Progress",
+            "type": "timeseries"
+         },
+         {
+            "datasource": {
+               "type": "prometheus",
+               "uid": "${datasource}"
+            },
+            "description": "Uncompressed size of chunks in a series VS the size of the blooms built.",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "palette-classic"
+                  },
+                  "custom": {
+                     "axisBorderShow": false,
+                     "axisCenteredZero": false,
+                     "axisColorMode": "text",
+                     "axisLabel": "",
+                     "axisPlacement": "auto",
+                     "barAlignment": 0,
+                     "drawStyle": "line",
+                     "fillOpacity": 15,
+                     "gradientMode": "none",
+                     "hideFrom": {
+                        "legend": false,
+                        "tooltip": false,
+                        "viz": false
+                     },
+                     "insertNulls": false,
+                     "lineInterpolation": "linear",
+                     "lineStyle": {
+                        "fill": "solid"
+                     },
+                     "lineWidth": 1,
+                     "pointSize": 5,
+                     "scaleDistribution": {
+                        "type": "linear"
+                     },
+                     "showPoints": "auto",
+                     "spanNulls": false,
+                     "stacking": {
+                        "group": "A",
+                        "mode": "none"
+                     },
+                     "thresholdsStyle": {
+                        "mode": "off"
+                     }
+                  },
+                  "mappings": [ ],
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [
+                        {
+                           "color": "green",
+                           "value": null
+                        }
+                     ]
+                  },
+                  "unit": "bytes"
+               },
+               "overrides": [
+                  {
+                     "matcher": {
+                        "id": "byName",
+                        "options": "Ratio"
+                     },
+                     "properties": [
+                        {
+                           "id": "unit",
+                           "value": "percentunit"
+                        }
+                     ]
+                  },
+                  {
+                     "matcher": {
+                        "id": "byName",
+                        "options": "Ratio over range"
+                     },
+                     "properties": [
+                        {
+                           "id": "unit",
+                           "value": "percentunit"
+                        }
+                     ]
+                  }
+               ]
+            },
+            "gridPos": {
+               "h": 7,
+               "w": 12,
+               "x": 12,
+               "y": 9
+            },
+            "id": 41,
+            "options": {
+               "legend": {
+                  "calcs": [ ],
+                  "displayMode": "list",
+                  "placement": "bottom",
+                  "showLegend": true
+               },
+               "tooltip": {
+                  "maxHeight": 600,
+                  "mode": "single",
+                  "sort": "none"
+               }
+            },
+            "pluginVersion": "11.1.0-69868",
+            "targets": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "sum(rate(loki_bloom_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
+                  "hide": false,
+                  "instant": false,
+                  "legendFormat": "Bloom",
+                  "range": true,
+                  "refId": "A"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "sum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
+                  "hide": false,
+                  "instant": false,
+                  "legendFormat": "Chunk",
+                  "range": true,
+                  "refId": "B"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "sum(rate(loki_bloom_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n/\nsum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
+                  "hide": false,
+                  "instant": false,
+                  "legendFormat": "Ratio",
+                  "range": true,
+                  "refId": "C"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "sum(rate(loki_bloom_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n/\nsum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
+                  "hide": false,
+                  "instant": false,
+                  "legendFormat": "Ratio over range",
+                  "range": true,
+                  "refId": "D"
+               }
+            ],
+            "title": "Chunks and Bloom size",
+            "type": "timeseries"
+         },
+         {
+            "datasource": {
+               "type": "loki",
+               "uid": "${loki_datasource}"
+            },
+            "description": "Blooms size vs uncompressed chunk size.",
+            "gridPos": {
+               "h": 7,
+               "w": 17,
+               "x": 0,
+               "y": 16
+            },
+            "id": 51,
+            "options": {
+               "dedupStrategy": "none",
+               "enableLogDetails": true,
+               "prettifyLogMessage": false,
+               "showCommonLabels": false,
+               "showLabels": false,
+               "showTime": false,
+               "sortOrder": "Descending",
+               "wrapLogMessage": false
+            },
+            "pluginVersion": "11.1.0-69868",
+            "targets": [
+               {
+                  "datasource": {
+                     "type": "loki",
+                     "uid": "${loki_datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} |= \"level=error\" |= \"component=bloom-compactor\"",
+                  "queryType": "range",
+                  "refId": "B"
+               }
+            ],
+            "title": "Errors",
+            "type": "logs"
+         },
+         {
+            "datasource": {
+               "type": "loki",
+               "uid": "${loki_datasource}"
+            },
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "fixedColor": "red",
+                     "mode": "fixed"
+                  },
+                  "custom": {
+                     "axisBorderShow": false,
+                     "axisCenteredZero": false,
+                     "axisColorMode": "text",
+                     "axisLabel": "",
+                     "axisPlacement": "auto",
+                     "barAlignment": 0,
+                     "drawStyle": "bars",
+                     "fillOpacity": 0,
+                     "gradientMode": "none",
+                     "hideFrom": {
+                        "legend": false,
+                        "tooltip": false,
+                        "viz": false
+                     },
+                     "insertNulls": false,
+                     "lineInterpolation": "linear",
+                     "lineWidth": 1,
+                     "pointSize": 3,
+                     "scaleDistribution": {
+                        "type": "linear"
+                     },
+                     "showPoints": "auto",
+                     "spanNulls": false,
+                     "stacking": {
+                        "group": "A",
+                        "mode": "none"
+                     },
+                     "thresholdsStyle": {
+                        "mode": "off"
+                     }
+                  },
+                  "mappings": [ ],
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [
+                        {
+                           "color": "green",
+                           "value": null
+                        },
+                        {
+                           "color": "red",
+                           "value": 1
+                        }
+                     ]
+                  }
+               },
+               "overrides": [ ]
+            },
+            "gridPos": {
+               "h": 7,
+               "w": 7,
+               "x": 17,
+               "y": 16
+            },
+            "id": 53,
+            "options": {
+               "legend": {
+                  "calcs": [ ],
+                  "displayMode": "list",
+                  "placement": "bottom",
+                  "showLegend": true
+               },
+               "tooltip": {
+                  "maxHeight": 600,
+                  "mode": "single",
+                  "sort": "none"
+               }
+            },
+            "pluginVersion": "11.1.0-69868",
+            "targets": [
+               {
+                  "datasource": {
+                     "type": "loki",
+                     "uid": "${loki_datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "sum(count_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} |= \"level=error\" |= \"component=bloom-compactor\" [$__auto]))",
+                  "queryType": "range",
+                  "refId": "A"
+               }
+            ],
+            "title": "Errors Rate",
+            "type": "timeseries"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 23
+            },
+            "id": 112,
+            "panels": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 15,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "percentunit"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 0,
+                     "y": 24
+                  },
+                  "id": 114,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (pod) (\n    loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "Progress per pod",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "fieldMinMax": false,
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "percentage",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "#EAB839",
+                                 "value": 0
+                              },
+                              {
+                                 "color": "green",
+                                 "value": 100
+                              }
+                           ]
+                        },
+                        "unit": "percentunit"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 12,
+                     "y": 24
+                  },
+                  "id": 115,
+                  "options": {
+                     "minVizHeight": 75,
+                     "minVizWidth": 75,
+                     "orientation": "auto",
+                     "reduceOptions": {
+                        "calcs": [
+                           "lastNotNull"
+                        ],
+                        "fields": "",
+                        "values": false
+                     },
+                     "showThresholdLabels": false,
+                     "showThresholdMarkers": false,
+                     "sizing": "auto"
+                  },
+                  "pluginVersion": "11.0.0-68102",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (pod) (\n    loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "Current Progress per pod",
+                  "type": "gauge"
+               }
+            ],
+            "title": "Progress per pod",
+            "type": "row"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 24
+            },
+            "id": 56,
+            "panels": [
+               {
+                  "description": "",
+                  "gridPos": {
+                     "h": 1,
+                     "w": 24,
+                     "x": 0,
+                     "y": 25
+                  },
+                  "id": 85,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-70005",
+                  "title": "We use tenant sharding so each compactor will process a subset of the tenants.",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "Shows the expected number of cpu cores we need to provision to build blooms as fast as we ingest data so a compaction iteration doesn't take longer than the compaction interval.\n\nWe may decide to have more to speed up compaction.",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 26
+                  },
+                  "id": 94,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "# This query shows the expected number of cpu cores we need to not fall behind\n# building blooms for data we're ingesting.\n# conceptually, the formula is:\n# (cell_bytes * space_amplification / bloom_bytes_processed_per_core)\n\n# number of replicas needed\nsum(avg_over_time(loki_cell:bytes:rate1m{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n*\n## Space amplification (how much data do we write compared to what we ingest?)\n(\n  # rep factor\n  3 *\n  sum(\n    # 1 - dedupe_ratio\n    1 - \n    sum(rate(loki_chunk_store_deduped_chunks_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (cluster, namespace)\n    /\n    sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (cluster, namespace)\n  )\n)\n/\n(\nsum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))\n/\nsum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Needed",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Available",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Required CPUs to not lag behind",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 15,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "Bps"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 26
+                  },
+                  "id": 72,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "# MB/s/core chunk data processed\nsum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}[$__rate_interval])) by (pod)\n/\nsum(rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])) by (pod)",
+                        "hide": true,
+                        "instant": false,
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "# MB/s/core chunk data processed\nsum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}[$__rate_interval]))\n/\nsum(rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Total",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "MB/s per core",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 33
+                  },
+                  "id": 1,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"cpu\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Request",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"cpu\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Limit",
+                        "range": true,
+                        "refId": "C"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n    0.99,\n    rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}[$__rate_interval])\n)",
+                        "instant": false,
+                        "legendFormat": "p99",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n    0.50,\n    rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}[$__rate_interval])\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p50",
+                        "range": true,
+                        "refId": "D"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "avg(\n    rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}[$__rate_interval])\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Avg",
+                        "range": true,
+                        "refId": "E"
+                     }
+                  ],
+                  "title": "CPU",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 33
+                  },
+                  "id": 75,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}[$__rate_interval]))",
+                        "instant": false,
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"cpu\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Request",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"cpu\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Limit",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "CPU per pod",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "bytes"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 40
+                  },
+                  "id": 76,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"memory\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Request",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Limit",
+                        "range": true,
+                        "refId": "C"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile (\n  0.99,\n  container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}\n)",
+                        "instant": false,
+                        "legendFormat": "p99",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile (\n  0.50,\n  container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p50",
+                        "range": true,
+                        "refId": "D"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "avg (\n  container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Avg",
+                        "range": true,
+                        "refId": "E"
+                     }
+                  ],
+                  "title": "Memory (workingset)",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "bytes"
+                     },
+                     "overrides": [
+                        {
+                           "__systemRef": "hideSeriesFrom",
+                           "matcher": {
+                              "id": "byNames",
+                              "options": {
+                                 "mode": "exclude",
+                                 "names": [
+                                    "bloom-compactor-106"
+                                 ],
+                                 "prefix": "All except:",
+                                 "readOnly": true
+                              }
+                           },
+                           "properties": [
+                              {
+                                 "id": "custom.hideFrom",
+                                 "value": {
+                                    "legend": false,
+                                    "tooltip": false,
+                                    "viz": true
+                                 }
+                              }
+                           ]
+                        }
+                     ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 40
+                  },
+                  "id": 5,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"})",
+                        "instant": false,
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"memory\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Request",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Limit",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Memory per pod (workingset)",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 15,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 47
+                  },
+                  "id": 27,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum (\n    increase(\n        kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[10m]\n    )\n) > 0",
+                        "instant": false,
+                        "legendFormat": "Restarts",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Container restarts",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 15,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 47
+                  },
+                  "id": 77,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "(\n    sum by (pod) (\n        increase(\n            kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[10m]\n        )\n    )\n    * on (pod) group_right\n    max by (pod, reason) (\n        kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}\n    )\n) > 0",
+                        "instant": false,
+                        "legendFormat": "{{reason}} / {{pod}}",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Container restarts reason per pod",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "Resource Usage",
+            "type": "row"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 25
+            },
+            "id": 95,
+            "panels": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "How many tokens each compactor is appending to blooms. Accounts for tokens that are not actually added to the blooms since they are already there. See the panel on the right for a drill down on the collision.\n",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "log": 2,
+                              "type": "log"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 0,
+                     "y": 55
+                  },
+                  "id": 96,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "# tokens checked per pod, millions/s\nsum(rate(loki_bloom_tokens_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))\n/\nsum(count(loki_bloom_tokens_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}))\n/ 1e6",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Per core",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(rate(loki_bloom_inserts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])) / 1e6",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Total",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Tokens rate (millions)",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "Collision type may be `false` (no collision), `cache` (found in token cache) or true (found in bloom filter).\n\nType may be either `raw` (the original ngram) or `chunk_prefixed` (the ngram with the chunk prefix)",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "percentunit"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 12,
+                     "y": 55
+                  },
+                  "id": 97,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "# tokens/s by type+collision\nsum by (collision) (\n    rate(loki_bloom_inserts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n) \n/ on () group_left\nsum (\n    rate(loki_bloom_inserts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "tokens/s by collision type",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "The sizes of the blooms created by the compactor. We build one bloom per series. The more unique ngrams and chunks the series has, the bigger their blooms will be.",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "bytes"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 0,
+                     "y": 63
+                  },
+                  "id": 98,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(\n    0.99, \n    sum by (le) (\n        rate(loki_bloom_size_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p99",
+                        "range": true,
+                        "refId": "D"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(\n    0.90, \n    sum by (le) (\n        rate(loki_bloom_size_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p90",
+                        "range": true,
+                        "refId": "E"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(\n    0.50, \n    sum by (le) (\n        rate(loki_bloom_size_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p50",
+                        "range": true,
+                        "refId": "F"
+                     }
+                  ],
+                  "title": "Bloom size",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "How many chunks are we indexing in the blooms. Either:\n- `copied` from a pre-existing bloom block, or \n- `iterated` through all its entries if processed for the first time.",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 12,
+                     "y": 63
+                  },
+                  "id": 99,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "# chunks indexed, by iteration or copied from a pre-existing bloom\nsum(rate(loki_bloom_chunks_indexed_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])) by (type)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "Chunks indexed",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "Bloom building",
+            "type": "row"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 26
+            },
+            "id": 103,
+            "panels": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 72
+                  },
+                  "id": 107,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(increase(loki_bloomcompactor_blocks_created_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Blocks",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Created Blocks",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "Compactors delete metas and blocks marked for deletion in the metas tombstones.",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 72
+                  },
+                  "id": 106,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(increase(loki_bloomcompactor_blocks_deleted_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Blocks",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Deleted Blocks",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "Number of overlapping bloom blocks reused when creating new blocks\n",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 79
+                  },
+                  "id": 109,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(increase(loki_bloomcompactor_blocks_reused_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Blocks",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Blocks reused",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "Blocks building",
+            "type": "row"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 27
+            },
+            "id": 110,
+            "panels": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 87
+                  },
+                  "id": 108,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(increase(loki_bloomcompactor_metas_created_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Metas",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Created Metas",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "Compactors delete metas and blocks marked for deletion in the metas tombstones.",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 87
+                  },
+                  "id": 105,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "pluginVersion": "11.1.0-69868",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(increase(loki_bloomcompactor_metas_deleted_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Metas",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Deleted Metas",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "Metas building",
+            "type": "row"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 28
+            },
+            "id": 80,
+            "panels": [
+               {
+                  "description": "",
+                  "gridPos": {
+                     "h": 1,
+                     "w": 24,
+                     "x": 0,
+                     "y": 95
+                  },
+                  "id": 93,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-70005",
+                  "title": "We use tenant sharding so each compactor will process a subset of the tenants.",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 96
+                  },
+                  "id": 83,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n  0.99,\n  increase(\n    loki_bloomcompactor_tenants_started_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n  )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p99",
+                        "range": true,
+                        "refId": "C"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n  0.90,\n  increase(\n    loki_bloomcompactor_tenants_started_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n  )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p90",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n  0.50,\n  increase(\n    loki_bloomcompactor_tenants_started_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[30m]\n  )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p50",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "Tenants",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 96
+                  },
+                  "id": 84,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (pod) (\n  increase(\n    loki_bloomcompactor_tenants_started_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n  )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Tenants per pod",
+                  "type": "timeseries"
+               },
+               {
+                  "description": "",
+                  "gridPos": {
+                     "h": 1,
+                     "w": 24,
+                     "x": 0,
+                     "y": 103
+                  },
+                  "id": 86,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-70005",
+                  "title": "Number of tenant tables processed. ",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 104
+                  },
+                  "id": 88,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n  0.99,\n  increase(\n    loki_bloomcompactor_tenant_table_ranges_completed_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n  )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p99",
+                        "range": true,
+                        "refId": "C"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n  0.90,\n  increase(\n    loki_bloomcompactor_tenant_table_ranges_completed_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n  )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p90",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n  0.50,\n  increase(\n    loki_bloomcompactor_tenant_table_ranges_completed_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n  )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p50",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "Tenant Tables",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 104
+                  },
+                  "id": 89,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (pod) (\n  increase(\n    loki_bloomcompactor_tenant_table_ranges_completed_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n  )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Tenant Tables per pod",
+                  "type": "timeseries"
+               },
+               {
+                  "description": "",
+                  "gridPos": {
+                     "h": 1,
+                     "w": 24,
+                     "x": 0,
+                     "y": 111
+                  },
+                  "id": 87,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-70005",
+                  "title": "Series per compaction (includes series copied from other blocks)",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 112
+                  },
+                  "id": 81,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "# series checked per compaction\nhistogram_quantile(\n    0.99, \n    sum by (le) (\n        rate(loki_bloomcompactor_series_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p99",
+                        "range": true,
+                        "refId": "C"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "# series checked per compaction\nhistogram_quantile(\n    0.9, \n    sum by (le) (\n        rate(loki_bloomcompactor_series_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p90",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "# series checked per compaction\nhistogram_quantile(\n    0.5, \n    sum by (le) (\n        rate(loki_bloomcompactor_series_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p50",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Series",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 112
+                  },
+                  "id": 82,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (pod) (\n    rate(loki_bloomcompactor_series_per_compaction_sum{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    /\n    rate(loki_bloomcompactor_series_per_compaction_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "avg series per compaction by pod",
+                  "type": "timeseries"
+               },
+               {
+                  "description": "",
+                  "gridPos": {
+                     "h": 1,
+                     "w": 24,
+                     "x": 0,
+                     "y": 119
+                  },
+                  "id": 90,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-70005",
+                  "title": "Number of bytes from chunks added to blocks during each compaction.",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        },
+                        "unit": "bytes"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 120
+                  },
+                  "id": 91,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "# series checked per compaction\nhistogram_quantile(\n    0.99, \n    sum by (le) (\n        rate(loki_bloomcompactor_bytes_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p99",
+                        "range": true,
+                        "refId": "C"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "# series checked per compaction\nhistogram_quantile(\n    0.9, \n    sum by (le) (\n        rate(loki_bloomcompactor_bytes_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p90",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "# series checked per compaction\nhistogram_quantile(\n    0.5, \n    sum by (le) (\n        rate(loki_bloomcompactor_bytes_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p50",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Bytes",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        },
+                        "unit": "bytes"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 120
+                  },
+                  "id": 92,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (pod) (\n    rate(loki_bloomcompactor_bytes_per_compaction_sum{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    /\n    rate(loki_bloomcompactor_bytes_per_compaction_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p99",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "avg bytes per compaction by pod",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "Data processed",
+            "type": "row"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 29
+            },
+            "id": 58,
+            "panels": [
+               {
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": { },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 3,
+                     "w": 24,
+                     "x": 0,
+                     "y": 82
+                  },
+                  "id": 47,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "\nCompactors write blocks to the attached PVs before flushing them into the object store.\nIt also download chunks and index files.\n\nAfter compacting a given tenant, all the downloaded index files and chunks, as well as the already flushed blocks are deleted.",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-69747",
+                  "title": "",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "area"
+                           }
+                        },
+                        "mappings": [ ],
+                        "max": 1,
+                        "min": 0,
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 0.80000000000000004
+                              }
+                           ]
+                        },
+                        "unit": "percentunit"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 0,
+                     "y": 85
+                  },
+                  "id": 9,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n    0.99,\n    max by(persistentvolumeclaim) (\n        kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} \n        / \n        kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n    ) \n    and \n    count by(persistentvolumeclaim) (\n        kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"bloom-compactor\"}\n    )\n)",
+                        "instant": false,
+                        "legendFormat": "p99",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n    0.90,\n    max by(persistentvolumeclaim) (\n        kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} \n        / \n        kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n    ) \n    and \n    count by(persistentvolumeclaim) (\n        kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"bloom-compactor\"}\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p90",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n    0.50,\n    max by(persistentvolumeclaim) (\n        kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} \n        / \n        kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n    ) \n    and \n    count by(persistentvolumeclaim) (\n        kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"bloom-compactor\"}\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p50",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Disk Utilization",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "area"
+                           }
+                        },
+                        "mappings": [ ],
+                        "max": 1,
+                        "min": 0,
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 0.80000000000000004
+                              }
+                           ]
+                        },
+                        "unit": "percentunit"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 12,
+                     "y": 85
+                  },
+                  "id": 100,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"bloom-compactor\"})",
+                        "instant": false,
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Disk Utilization per pod",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "bytes"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 93
+                  },
+                  "id": 7,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n    0.99,\n    sum by(instance, pod, device) (\n        rate(node_disk_written_bytes_total[$__rate_interval])\n    ) \n    + ignoring(pod) group_right() \n    (\n        label_replace(\n            count by(instance, pod, device) (\n                container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n            ), \n            \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n        ) * 0\n    )\n)",
+                        "instant": false,
+                        "legendFormat": "p99",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n    0.90,\n    sum by(instance, pod, device) (\n        rate(node_disk_written_bytes_total[$__rate_interval])\n    ) \n    + ignoring(pod) group_right() \n    (\n        label_replace(\n            count by(instance, pod, device) (\n                container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n            ), \n            \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n        ) * 0\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p90",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n    0.50,\n    sum by(instance, pod, device) (\n        rate(node_disk_written_bytes_total[$__rate_interval])\n    ) \n    + ignoring(pod) group_right() \n    (\n        label_replace(\n            count by(instance, pod, device) (\n                container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n            ), \n            \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n        ) * 0\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p50",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Disk Writes",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "bytes"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 93
+                  },
+                  "id": 101,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)",
+                        "instant": false,
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Disk Writes per pod",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "bytes"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 0,
+                     "y": 100
+                  },
+                  "id": 8,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n    0.99,\n    sum by(instance, pod, device) (\n        rate(node_disk_read_bytes_total[$__rate_interval])\n    ) + ignoring(pod) group_right()\n    (\n        label_replace(\n            count by(instance, pod, device) (\n                container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n            ), \n            \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n        ) * 0\n    )\n)",
+                        "instant": false,
+                        "legendFormat": "p99",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n    0.90,\n    sum by(instance, pod, device) (\n        rate(node_disk_read_bytes_total[$__rate_interval])\n    ) + ignoring(pod) group_right()\n    (\n        label_replace(\n            count by(instance, pod, device) (\n                container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n            ), \n            \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n        ) * 0\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p90",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "quantile(\n    0.50,\n    sum by(instance, pod, device) (\n        rate(node_disk_read_bytes_total[$__rate_interval])\n    ) + ignoring(pod) group_right()\n    (\n        label_replace(\n            count by(instance, pod, device) (\n                container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n            ), \n            \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n        ) * 0\n    )\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p50",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Disk Reads",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "bytes"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 12,
+                     "y": 100
+                  },
+                  "id": 102,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)",
+                        "instant": false,
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Disk Reads per pod",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "Disk Usage",
+            "type": "row"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 30
+            },
+            "id": 62,
+            "panels": [
+               {
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": { },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 3,
+                     "w": 24,
+                     "x": 0,
+                     "y": 83
+                  },
+                  "id": 71,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "Once all blocks and metas are built locally, the compactor flushes them to the object store.\n\nAfter each iteration, the compactor deletes the metas and blocks marked for deletion in the tombstones.",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-69747",
+                  "title": "",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": { },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 2,
+                     "x": 0,
+                     "y": 86
+                  },
+                  "id": 63,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "---\n#### GCS\n",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-69747",
+                  "title": "",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 25,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 11,
+                     "x": 2,
+                     "y": 86
+                  },
+                  "id": 61,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} {{status_code}}",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "QPS",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 25,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "s"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 11,
+                     "x": 13,
+                     "y": 86
+                  },
+                  "id": 64,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p99",
+                        "range": true,
+                        "refId": "D"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p90",
+                        "range": true,
+                        "refId": "E"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p50",
+                        "range": true,
+                        "refId": "F"
+                     }
+                  ],
+                  "title": "Latency",
+                  "type": "timeseries"
+               },
+               {
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": { },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 2,
+                     "x": 0,
+                     "y": 93
+                  },
+                  "id": 65,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "---\n#### S3\n",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-69747",
+                  "title": "",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 25,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 11,
+                     "x": 2,
+                     "y": 93
+                  },
+                  "id": 67,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} {{status_code}}",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "QPS",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 25,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "s"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 11,
+                     "x": 13,
+                     "y": 93
+                  },
+                  "id": 69,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p99",
+                        "range": true,
+                        "refId": "D"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p90",
+                        "range": true,
+                        "refId": "E"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p50",
+                        "range": true,
+                        "refId": "F"
+                     }
+                  ],
+                  "title": "Latency",
+                  "type": "timeseries"
+               },
+               {
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": { },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 2,
+                     "x": 0,
+                     "y": 100
+                  },
+                  "id": 66,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "---\n#### Azure\nBlob Storage",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-69747",
+                  "title": "",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 25,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 11,
+                     "x": 2,
+                     "y": 100
+                  },
+                  "id": 68,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} {{status_code}}",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "QPS",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 25,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "s"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 11,
+                     "x": 13,
+                     "y": 100
+                  },
+                  "id": 70,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p99",
+                        "range": true,
+                        "refId": "D"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p90",
+                        "range": true,
+                        "refId": "E"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p50",
+                        "range": true,
+                        "refId": "F"
+                     }
+                  ],
+                  "title": "Latency",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "Object Store",
+            "type": "row"
+         }
+      ],
+      "refresh": "10s",
+      "rows": [ ],
+      "schemaVersion": 14,
+      "style": "dark",
+      "tags": [
+         "loki"
+      ],
+      "templating": {
+         "list": [
+            {
+               "current": {
+                  "text": "default",
+                  "value": "default"
+               },
+               "hide": 0,
+               "label": "Data source",
+               "name": "datasource",
+               "options": [ ],
+               "query": "prometheus",
+               "refresh": 1,
+               "regex": "",
+               "type": "datasource"
+            },
+            {
+               "allValue": null,
+               "current": {
+                  "text": "prod",
+                  "value": "prod"
+               },
+               "datasource": "$datasource",
+               "hide": 0,
+               "includeAll": false,
+               "label": "cluster",
+               "multi": false,
+               "name": "cluster",
+               "options": [ ],
+               "query": "label_values(loki_build_info, cluster)",
+               "refresh": 1,
+               "regex": "",
+               "sort": 2,
+               "tagValuesQuery": "",
+               "tags": [ ],
+               "tagsQuery": "",
+               "type": "query",
+               "useTags": false
+            },
+            {
+               "allValue": null,
+               "current": {
+                  "text": "prod",
+                  "value": "prod"
+               },
+               "datasource": "$datasource",
+               "hide": 0,
+               "includeAll": false,
+               "label": "namespace",
+               "multi": false,
+               "name": "namespace",
+               "options": [ ],
+               "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)",
+               "refresh": 1,
+               "regex": "",
+               "sort": 2,
+               "tagValuesQuery": "",
+               "tags": [ ],
+               "tagsQuery": "",
+               "type": "query",
+               "useTags": false
+            },
+            {
+               "hide": 0,
+               "label": null,
+               "name": "loki_datasource",
+               "options": [ ],
+               "query": "loki",
+               "refresh": 1,
+               "regex": "",
+               "type": "datasource"
+            }
+         ]
+      },
+      "time": {
+         "from": "now-1h",
+         "to": "now"
+      },
+      "timeRangeUpdatedDuringEditOrView": false,
+      "timepicker": {
+         "refresh_intervals": [
+            "5s",
+            "10s",
+            "30s",
+            "1m",
+            "5m",
+            "15m",
+            "30m",
+            "1h",
+            "2h",
+            "1d"
+         ],
+         "time_options": [
+            "5m",
+            "15m",
+            "1h",
+            "6h",
+            "12h",
+            "24h",
+            "2d",
+            "7d",
+            "30d"
+         ]
+      },
+      "timezone": "utc",
+      "title": "Loki / Bloom Compactor",
+      "uid": "bloom-compactor",
+      "version": 0,
+      "weekStart": ""
+   }
\ No newline at end of file
diff --git a/production/loki-mixin-compiled/dashboards/loki-bloom-gateway.json b/production/loki-mixin-compiled/dashboards/loki-bloom-gateway.json
new file mode 100644
index 0000000000000..27a058ae800e2
--- /dev/null
+++ b/production/loki-mixin-compiled/dashboards/loki-bloom-gateway.json
@@ -0,0 +1,5035 @@
+{
+      "annotations": {
+         "list": [ ]
+      },
+      "editable": true,
+      "fiscalYearStartMonth": 0,
+      "gnetId": null,
+      "graphTooltip": 0,
+      "hideControls": false,
+      "links": [
+         {
+            "asDropdown": true,
+            "icon": "external link",
+            "includeVars": true,
+            "keepTime": true,
+            "tags": [
+               "loki"
+            ],
+            "targetBlank": false,
+            "title": "Loki Dashboards",
+            "type": "dashboards"
+         }
+      ],
+      "liveNow": false,
+      "panels": [
+         {
+            "collapsed": false,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 0
+            },
+            "id": 73,
+            "panels": [ ],
+            "title": "Overview",
+            "type": "row"
+         },
+         {
+            "datasource": {
+               "type": "prometheus",
+               "uid": "${datasource}"
+            },
+            "description": "Percentage of chunks that are filtered by using bloom filters",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "thresholds",
+                     "seriesBy": "last"
+                  },
+                  "custom": {
+                     "axisBorderShow": false,
+                     "axisCenteredZero": false,
+                     "axisColorMode": "text",
+                     "axisLabel": "",
+                     "axisPlacement": "auto",
+                     "barAlignment": 0,
+                     "drawStyle": "line",
+                     "fillOpacity": 0,
+                     "gradientMode": "none",
+                     "hideFrom": {
+                        "legend": false,
+                        "tooltip": false,
+                        "viz": false
+                     },
+                     "insertNulls": false,
+                     "lineInterpolation": "linear",
+                     "lineStyle": {
+                        "fill": "solid"
+                     },
+                     "lineWidth": 2,
+                     "pointSize": 5,
+                     "scaleDistribution": {
+                        "type": "linear"
+                     },
+                     "showPoints": "auto",
+                     "spanNulls": false,
+                     "stacking": {
+                        "group": "A",
+                        "mode": "none"
+                     },
+                     "thresholdsStyle": {
+                        "mode": "area"
+                     }
+                  },
+                  "mappings": [ ],
+                  "max": 1,
+                  "min": 0,
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [
+                        {
+                           "color": "red",
+                           "value": null
+                        },
+                        {
+                           "color": "orange",
+                           "value": 0.5
+                        },
+                        {
+                           "color": "yellow",
+                           "value": 0.75
+                        },
+                        {
+                           "color": "green",
+                           "value": 0.90000000000000002
+                        }
+                     ]
+                  },
+                  "unit": "percentunit"
+               },
+               "overrides": [ ]
+            },
+            "gridPos": {
+               "h": 6,
+               "w": 6,
+               "x": 0,
+               "y": 1
+            },
+            "id": 23,
+            "options": {
+               "legend": {
+                  "calcs": [ ],
+                  "displayMode": "list",
+                  "placement": "bottom",
+                  "showLegend": true
+               },
+               "tooltip": {
+                  "maxHeight": 600,
+                  "mode": "single",
+                  "sort": "none"
+               }
+            },
+            "targets": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "sum(rate(loki_bloom_gateway_filtered_chunks_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_chunks_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))",
+                  "instant": false,
+                  "legendFormat": "Chunks",
+                  "range": true,
+                  "refId": "A"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "exemplar": false,
+                  "expr": "sum(rate(loki_bloom_gateway_filtered_series_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_series_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))",
+                  "hide": false,
+                  "instant": false,
+                  "legendFormat": "Series",
+                  "range": true,
+                  "refId": "B"
+               }
+            ],
+            "title": "Filter ratio",
+            "type": "timeseries"
+         },
+         {
+            "datasource": {
+               "type": "prometheus",
+               "uid": "${datasource}"
+            },
+            "description": "Percentage of chunks that are filtered by using bloom filters",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "thresholds"
+                  },
+                  "mappings": [ ],
+                  "max": 1,
+                  "min": 0,
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [
+                        {
+                           "color": "red",
+                           "value": null
+                        },
+                        {
+                           "color": "orange",
+                           "value": 0.5
+                        },
+                        {
+                           "color": "yellow",
+                           "value": 0.75
+                        },
+                        {
+                           "color": "green",
+                           "value": 0.90000000000000002
+                        }
+                     ]
+                  },
+                  "unit": "percentunit"
+               },
+               "overrides": [ ]
+            },
+            "gridPos": {
+               "h": 6,
+               "w": 6,
+               "x": 6,
+               "y": 1
+            },
+            "id": 75,
+            "options": {
+               "minVizHeight": 75,
+               "minVizWidth": 75,
+               "orientation": "auto",
+               "reduceOptions": {
+                  "calcs": [
+                     "lastNotNull"
+                  ],
+                  "fields": "",
+                  "values": false
+               },
+               "showThresholdLabels": false,
+               "showThresholdMarkers": true,
+               "sizing": "auto"
+            },
+            "pluginVersion": "11.1.0-70005",
+            "targets": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "exemplar": false,
+                  "expr": "sum(rate(loki_bloom_gateway_filtered_chunks_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_chunks_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))",
+                  "instant": true,
+                  "legendFormat": "Chunks",
+                  "range": false,
+                  "refId": "A"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "exemplar": false,
+                  "expr": "sum(rate(loki_bloom_gateway_filtered_series_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_series_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))",
+                  "hide": false,
+                  "instant": true,
+                  "legendFormat": "Series",
+                  "range": false,
+                  "refId": "B"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "exemplar": false,
+                  "expr": "sum(loki_bloom_gateway_filtered_chunks_sum{job=\"$namespace/bloom-gateway\"})\n/\nsum(loki_bloom_gateway_requested_chunks_sum{job=\"$namespace/bloom-gateway\"})",
+                  "hide": true,
+                  "instant": true,
+                  "legendFormat": "Chunks avg",
+                  "range": false,
+                  "refId": "C"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "exemplar": false,
+                  "expr": "sum(loki_bloom_gateway_filtered_series_sum{job=\"$namespace/bloom-gateway\"})\n/\nsum(loki_bloom_gateway_requested_series_sum{job=\"$namespace/bloom-gateway\"})",
+                  "hide": true,
+                  "instant": true,
+                  "legendFormat": "Series avg",
+                  "range": false,
+                  "refId": "D"
+               }
+            ],
+            "title": "Filter ratio",
+            "type": "gauge"
+         },
+         {
+            "datasource": {
+               "type": "prometheus",
+               "uid": "${datasource}"
+            },
+            "description": "",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "palette-classic"
+                  },
+                  "custom": {
+                     "axisBorderShow": false,
+                     "axisCenteredZero": false,
+                     "axisColorMode": "text",
+                     "axisLabel": "",
+                     "axisPlacement": "auto",
+                     "barAlignment": 0,
+                     "drawStyle": "line",
+                     "fillOpacity": 10,
+                     "gradientMode": "none",
+                     "hideFrom": {
+                        "legend": false,
+                        "tooltip": false,
+                        "viz": false
+                     },
+                     "insertNulls": false,
+                     "lineInterpolation": "linear",
+                     "lineStyle": {
+                        "fill": "solid"
+                     },
+                     "lineWidth": 1,
+                     "pointSize": 5,
+                     "scaleDistribution": {
+                        "type": "linear"
+                     },
+                     "showPoints": "auto",
+                     "spanNulls": false,
+                     "stacking": {
+                        "group": "A",
+                        "mode": "none"
+                     },
+                     "thresholdsStyle": {
+                        "mode": "off"
+                     }
+                  },
+                  "mappings": [ ],
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [
+                        {
+                           "color": "green",
+                           "value": null
+                        },
+                        {
+                           "color": "red",
+                           "value": 80
+                        }
+                     ]
+                  },
+                  "unit": "none"
+               },
+               "overrides": [
+                  {
+                     "matcher": {
+                        "id": "byName",
+                        "options": "Desired"
+                     },
+                     "properties": [
+                        {
+                           "id": "custom.fillOpacity",
+                           "value": 0
+                        },
+                        {
+                           "id": "custom.lineStyle",
+                           "value": {
+                              "dash": [
+                                 10,
+                                 10
+                              ],
+                              "fill": "dash"
+                           }
+                        },
+                        {
+                           "id": "custom.lineWidth",
+                           "value": 2
+                        }
+                     ]
+                  }
+               ]
+            },
+            "gridPos": {
+               "h": 6,
+               "w": 6,
+               "x": 12,
+               "y": 1
+            },
+            "id": 72,
+            "options": {
+               "legend": {
+                  "calcs": [ ],
+                  "displayMode": "list",
+                  "placement": "bottom",
+                  "showLegend": true
+               },
+               "tooltip": {
+                  "maxHeight": 600,
+                  "mode": "single",
+                  "sort": "none"
+               }
+            },
+            "targets": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "kube_statefulset_status_replicas_ready{cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"bloom-gateway\"}",
+                  "hide": false,
+                  "instant": false,
+                  "legendFormat": "Ready",
+                  "range": true,
+                  "refId": "D"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "sum(kube_pod_container_status_ready{container=\"bloom-gateway\", cluster=\"$cluster\", namespace=\"$namespace\"})",
+                  "hide": true,
+                  "instant": false,
+                  "legendFormat": "Running",
+                  "range": true,
+                  "refId": "A"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "kube_statefulset_replicas{cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"bloom-gateway\"}",
+                  "hide": false,
+                  "instant": false,
+                  "legendFormat": "Desired",
+                  "range": true,
+                  "refId": "B"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "kube_statefulset_status_replicas_available{cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"bloom-gateway\"}",
+                  "hide": true,
+                  "instant": false,
+                  "legendFormat": "Available",
+                  "range": true,
+                  "refId": "C"
+               }
+            ],
+            "title": "Readiness",
+            "type": "timeseries"
+         },
+         {
+            "datasource": {
+               "type": "prometheus",
+               "uid": "${datasource}"
+            },
+            "description": "",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "palette-classic"
+                  },
+                  "custom": {
+                     "axisBorderShow": false,
+                     "axisCenteredZero": false,
+                     "axisColorMode": "text",
+                     "axisLabel": "",
+                     "axisPlacement": "auto",
+                     "barAlignment": 0,
+                     "drawStyle": "line",
+                     "fillOpacity": 50,
+                     "gradientMode": "none",
+                     "hideFrom": {
+                        "legend": false,
+                        "tooltip": false,
+                        "viz": false
+                     },
+                     "insertNulls": false,
+                     "lineInterpolation": "linear",
+                     "lineStyle": {
+                        "fill": "solid"
+                     },
+                     "lineWidth": 1,
+                     "pointSize": 5,
+                     "scaleDistribution": {
+                        "type": "linear"
+                     },
+                     "showPoints": "auto",
+                     "spanNulls": false,
+                     "stacking": {
+                        "group": "A",
+                        "mode": "none"
+                     },
+                     "thresholdsStyle": {
+                        "mode": "off"
+                     }
+                  },
+                  "mappings": [ ],
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [
+                        {
+                           "color": "green",
+                           "value": null
+                        },
+                        {
+                           "color": "red",
+                           "value": 80
+                        }
+                     ]
+                  },
+                  "unit": "none"
+               },
+               "overrides": [ ]
+            },
+            "gridPos": {
+               "h": 6,
+               "w": 6,
+               "x": 18,
+               "y": 1
+            },
+            "id": 37,
+            "options": {
+               "legend": {
+                  "calcs": [ ],
+                  "displayMode": "list",
+                  "placement": "bottom",
+                  "showLegend": true
+               },
+               "tooltip": {
+                  "maxHeight": 600,
+                  "mode": "single",
+                  "sort": "none"
+               }
+            },
+            "targets": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "(\n    max by (pod, reason) (kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})\n    * on (pod) group_left\n    sum by (pod) (increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))\n) > 0",
+                  "hide": false,
+                  "instant": false,
+                  "interval": "",
+                  "legendFormat": "{{pod}} ({{reason}})",
+                  "range": true,
+                  "refId": "C"
+               }
+            ],
+            "title": "Container restarts",
+            "type": "timeseries"
+         },
+         {
+            "datasource": {
+               "type": "loki",
+               "uid": "${loki_datasource}"
+            },
+            "gridPos": {
+               "h": 9,
+               "w": 15,
+               "x": 0,
+               "y": 7
+            },
+            "id": 48,
+            "options": {
+               "dedupStrategy": "none",
+               "enableLogDetails": true,
+               "prettifyLogMessage": false,
+               "showCommonLabels": false,
+               "showLabels": false,
+               "showTime": false,
+               "sortOrder": "Descending",
+               "wrapLogMessage": true
+            },
+            "targets": [
+               {
+                  "datasource": {
+                     "type": "loki",
+                     "uid": "${loki_datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"level=error\" or \"panic:\" | logfmt",
+                  "queryType": "range",
+                  "refId": "A"
+               },
+               {
+                  "datasource": {
+                     "type": "loki",
+                     "uid": "${loki_datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"level=warn\" | logfmt",
+                  "hide": true,
+                  "queryType": "range",
+                  "refId": "B"
+               }
+            ],
+            "title": "Errors",
+            "type": "logs"
+         },
+         {
+            "datasource": {
+               "type": "loki",
+               "uid": "${loki_datasource}"
+            },
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "fixedColor": "red",
+                     "mode": "fixed"
+                  },
+                  "custom": {
+                     "axisBorderShow": false,
+                     "axisCenteredZero": false,
+                     "axisColorMode": "text",
+                     "axisLabel": "",
+                     "axisPlacement": "auto",
+                     "barAlignment": 0,
+                     "drawStyle": "bars",
+                     "fillOpacity": 0,
+                     "gradientMode": "none",
+                     "hideFrom": {
+                        "legend": false,
+                        "tooltip": false,
+                        "viz": false
+                     },
+                     "insertNulls": false,
+                     "lineInterpolation": "linear",
+                     "lineWidth": 1,
+                     "pointSize": 5,
+                     "scaleDistribution": {
+                        "log": 2,
+                        "type": "symlog"
+                     },
+                     "showPoints": "auto",
+                     "spanNulls": false,
+                     "stacking": {
+                        "group": "A",
+                        "mode": "none"
+                     },
+                     "thresholdsStyle": {
+                        "mode": "off"
+                     }
+                  },
+                  "mappings": [ ],
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [
+                        {
+                           "color": "green",
+                           "value": null
+                        },
+                        {
+                           "color": "red",
+                           "value": 1
+                        }
+                     ]
+                  }
+               },
+               "overrides": [
+                  {
+                     "matcher": {
+                        "id": "byName",
+                        "options": "warn"
+                     },
+                     "properties": [
+                        {
+                           "id": "color",
+                           "value": {
+                              "fixedColor": "orange",
+                              "mode": "fixed"
+                           }
+                        }
+                     ]
+                  },
+                  {
+                     "matcher": {
+                        "id": "byName",
+                        "options": "error"
+                     },
+                     "properties": [
+                        {
+                           "id": "color",
+                           "value": {
+                              "fixedColor": "red",
+                              "mode": "fixed"
+                           }
+                        }
+                     ]
+                  },
+                  {
+                     "matcher": {
+                        "id": "byName",
+                        "options": "panic"
+                     },
+                     "properties": [
+                        {
+                           "id": "color",
+                           "value": {
+                              "fixedColor": "semi-dark-red",
+                              "mode": "fixed"
+                           }
+                        }
+                     ]
+                  }
+               ]
+            },
+            "gridPos": {
+               "h": 9,
+               "w": 9,
+               "x": 15,
+               "y": 7
+            },
+            "id": 52,
+            "options": {
+               "legend": {
+                  "calcs": [ ],
+                  "displayMode": "list",
+                  "placement": "bottom",
+                  "showLegend": true
+               },
+               "tooltip": {
+                  "maxHeight": 600,
+                  "mode": "single",
+                  "sort": "none"
+               }
+            },
+            "targets": [
+               {
+                  "datasource": {
+                     "type": "loki",
+                     "uid": "${loki_datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "sum by (level) (count_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |~ \"level=(warn|error)\" | logfmt [$__auto]))",
+                  "legendFormat": "{{ level }}",
+                  "queryType": "range",
+                  "refId": "A"
+               },
+               {
+                  "datasource": {
+                     "type": "loki",
+                     "uid": "${loki_datasource}"
+                  },
+                  "editorMode": "code",
+                  "expr": "sum (count_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"panic:\" | logfmt [$__auto]))",
+                  "hide": false,
+                  "legendFormat": "panic",
+                  "queryType": "range",
+                  "refId": "B"
+               }
+            ],
+            "title": "Errors Rate",
+            "type": "timeseries"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 16
+            },
+            "id": 56,
+            "panels": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 14,
+                     "w": 12,
+                     "x": 0,
+                     "y": 17
+                  },
+                  "id": 10,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"}[$__rate_interval]))",
+                        "instant": false,
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\", resource=\"cpu\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Request",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\", resource=\"cpu\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Limit",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "CPU",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "bytes"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 17
+                  },
+                  "id": 11,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"})",
+                        "instant": false,
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\", resource=\"memory\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Request",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Limit",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Memory (workingset)",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "bytes"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 24
+                  },
+                  "id": 81,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"})",
+                        "hide": true,
+                        "instant": false,
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\", resource=\"memory\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Request",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"} > 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Limit",
+                        "range": true,
+                        "refId": "C"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"}) by (pod)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "D"
+                     }
+                  ],
+                  "title": "Memory (heap inuse)",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 8,
+                     "x": 0,
+                     "y": 31
+                  },
+                  "id": 87,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (pod) (rate(go_gc_cycles_total_gc_cycles_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "GC rate",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "s"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 8,
+                     "x": 8,
+                     "y": 31
+                  },
+                  "id": 88,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (pod) (rate(go_gc_duration_seconds_sum{container=\"bloom-gateway\"}[$__rate_interval]))\n/\nsum by (pod) (rate(go_gc_duration_seconds_count{container=\"bloom-gateway\"}[$__rate_interval]))",
+                        "hide": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "GC duration",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "s"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 8,
+                     "x": 16,
+                     "y": 31
+                  },
+                  "id": 89,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99, sum(rate(go_gc_pauses_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))",
+                        "hide": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.90, sum(rate(go_gc_pauses_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))",
+                        "hide": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "GC pauses",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "binBps"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 38
+                  },
+                  "id": 84,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by(instance, pod) (rate(node_disk_read_bytes_total[$__rate_interval]))\n+ ignoring(pod) group_right() \n(count by(instance, pod) (container_fs_reads_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}) * 0)",
+                        "hide": false,
+                        "instant": false,
+                        "interval": "",
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "D"
+                     }
+                  ],
+                  "title": "Disk reads",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "binBps"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 38
+                  },
+                  "id": 85,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by(instance, pod) (rate(node_disk_written_bytes_total[$__rate_interval]))\n+ ignoring(pod) group_right() \n(count by(instance, pod) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}) * 0)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{pod}}",
+                        "range": true,
+                        "refId": "D"
+                     }
+                  ],
+                  "title": "Disk writes",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "Resource usage",
+            "type": "row"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 17
+            },
+            "id": 2,
+            "panels": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 100,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 0,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "normal"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [
+                        {
+                           "matcher": {
+                              "id": "byName",
+                              "options": "cancel"
+                           },
+                           "properties": [
+                              {
+                                 "id": "color",
+                                 "value": {
+                                    "mode": "fixed"
+                                 }
+                              }
+                           ]
+                        },
+                        {
+                           "matcher": {
+                              "id": "byName",
+                              "options": "success"
+                           },
+                           "properties": [
+                              {
+                                 "id": "color",
+                                 "value": {
+                                    "fixedColor": "green",
+                                    "mode": "fixed"
+                                 }
+                              }
+                           ]
+                        },
+                        {
+                           "matcher": {
+                              "id": "byName",
+                              "options": "error"
+                           },
+                           "properties": [
+                              {
+                                 "id": "color",
+                                 "value": {
+                                    "fixedColor": "red",
+                                    "mode": "fixed"
+                                 }
+                              }
+                           ]
+                        }
+                     ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 18
+                  },
+                  "id": 13,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (status_code) (\n    rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/bloom-gateway\", route=\"/logproto.BloomGateway/FilterChunkRefs\"}[$__rate_interval])\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "QPS",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 20,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [
+                        {
+                           "matcher": {
+                              "id": "byName",
+                              "options": "cancel"
+                           },
+                           "properties": [
+                              {
+                                 "id": "color",
+                                 "value": {
+                                    "mode": "fixed"
+                                 }
+                              }
+                           ]
+                        },
+                        {
+                           "matcher": {
+                              "id": "byName",
+                              "options": "success"
+                           },
+                           "properties": [
+                              {
+                                 "id": "color",
+                                 "value": {
+                                    "fixedColor": "green",
+                                    "mode": "fixed"
+                                 }
+                              }
+                           ]
+                        },
+                        {
+                           "matcher": {
+                              "id": "byName",
+                              "options": "error"
+                           },
+                           "properties": [
+                              {
+                                 "id": "color",
+                                 "value": {
+                                    "fixedColor": "red",
+                                    "mode": "fixed"
+                                 }
+                              }
+                           ]
+                        }
+                     ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 18
+                  },
+                  "id": 86,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (pod) (\n    rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/bloom-gateway\", route=\"/logproto.BloomGateway/FilterChunkRefs\"}[$__rate_interval])\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "QPS per Pod",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        },
+                        "unit": "s"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 25
+                  },
+                  "id": 14,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.50, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"/logproto.BloomGateway/FilterChunkRefs\"}))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{ route }} 50th percentile",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(cluster_job_route:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"/logproto.BloomGateway/FilterChunkRefs\"}) by (route)  / sum(cluster_job_route:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"/logproto.BloomGateway/FilterChunkRefs\"}) by (route) ",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{ route }} Average",
+                        "range": true,
+                        "refId": "C"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"/logproto.BloomGateway/FilterChunkRefs\"}))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{ route }} 99th percentile",
+                        "range": true,
+                        "refId": "D"
+                     }
+                  ],
+                  "title": "Latency",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green",
+                                 "value": null
+                              }
+                           ]
+                        },
+                        "unit": "s"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 25
+                  },
+                  "id": 15,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99,\n  sum(\n   rate(loki_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"/logproto.BloomGateway/FilterChunkRefs\"}[$__rate_interval])\n   ) by (pod, le)\n )\n",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "Per Pod Latency (p99)",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "QPS and Latency",
+            "type": "row"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 18
+            },
+            "id": 58,
+            "panels": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 25,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "normal"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 8,
+                     "x": 0,
+                     "y": 11
+                  },
+                  "id": 16,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (pod) (loki_bloom_gateway_queue_duration_seconds_sum{cluster=\"$cluster\", namespace=\"$namespace\"})\n/\nsum by (pod) (loki_bloom_gateway_queue_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"})\n",
+                        "hide": true,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "C"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum (loki_bloom_gateway_queue_length{cluster=\"$cluster\", namespace=\"$namespace\"})",
+                        "hide": true,
+                        "instant": false,
+                        "legendFormat": "Total",
+                        "range": true,
+                        "refId": "D"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (user) (loki_bloom_gateway_queue_length{cluster=\"$cluster\", namespace=\"$namespace\"})",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{user}}",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "Queue Size",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "For how long do pending tasks stay in the queue",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 1,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "s"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 8,
+                     "x": 8,
+                     "y": 11
+                  },
+                  "id": 17,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99, sum by (le) (rate(loki_bloom_gateway_queue_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p99",
+                        "range": true,
+                        "refId": "E"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.90, sum by (le) (rate(loki_bloom_gateway_queue_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))",
+                        "hide": true,
+                        "instant": false,
+                        "legendFormat": "p90",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.50, sum by (le) (rate(loki_bloom_gateway_queue_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p50",
+                        "range": true,
+                        "refId": "C"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum (loki_bloom_gateway_queue_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\"})\n/\nsum (loki_bloom_gateway_queue_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\"})",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "avg",
+                        "range": true,
+                        "refId": "D"
+                     }
+                  ],
+                  "title": "Queue Latency",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "Inflight requests tracks all tasks both queued and in progress",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 1,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 8,
+                     "x": 16,
+                     "y": 11
+                  },
+                  "id": 22,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (quantile) (loki_bloom_gateway_inflight_tasks{cluster=\"$cluster\", namespace=\"$namespace\", quantile=\"0.99\"})",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Inflight tasks",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "Task Queue",
+            "type": "row"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 19
+            },
+            "id": 68,
+            "panels": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "s"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 8,
+                     "x": 0,
+                     "y": 12
+                  },
+                  "id": 69,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99, sum(rate(loki_bloom_gateway_process_duration_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))",
+                        "instant": false,
+                        "legendFormat": "{{status}}-p99",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_gateway_process_duration_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{status}}-p95",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.90, sum(rate(loki_bloom_gateway_process_duration_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{status}}-p90",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Processing time for tasks (per worker iteration)",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        },
+                        "unit": "s"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 8,
+                     "x": 8,
+                     "y": 12
+                  },
+                  "id": 70,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99, sum(rate(loki_bloom_gateway_block_query_latency_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))",
+                        "instant": false,
+                        "legendFormat": "{{status}}-p99",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_gateway_block_query_latency_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{status}}-p95",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.90, sum(rate(loki_bloom_gateway_block_query_latency_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{status}}-p90",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Block query latency (single block)",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 8,
+                     "x": 16,
+                     "y": 12
+                  },
+                  "id": 71,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(rate(loki_bloom_gateway_tasks_dequeued_total{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval]))",
+                        "instant": false,
+                        "legendFormat": "dequeued",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (status) (rate(loki_bloom_gateway_tasks_processed_total{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "processed {{status}}",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "Tasks dequeued/processed",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "Processing",
+            "type": "row"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 20
+            },
+            "id": 59,
+            "panels": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "gridPos": {
+                     "h": 1,
+                     "w": 24,
+                     "x": 0,
+                     "y": 13
+                  },
+                  "id": 19,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-70005",
+                  "title": "We cache bloom blocks in memory to prevent the gateway from hitting the object store too often",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "fieldMinMax": false,
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "bytes"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 14
+                  },
+                  "id": 20,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(loki_embeddedcache_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", cache=\"bloom-blocks-cache\", container=\"bloom-gateway\"}) by (pod)",
+                        "hide": true,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(loki_bloom_blocks_cache_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}) by (pod)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "Cache size (per pod)",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "linearThreshold": 1000,
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "fieldMinMax": false,
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [
+                        {
+                           "matcher": {
+                              "id": "byName",
+                              "options": "Size"
+                           },
+                           "properties": [
+                              {
+                                 "id": "unit",
+                                 "value": "bytes"
+                              },
+                              {
+                                 "id": "custom.fillOpacity",
+                                 "value": 25
+                              },
+                              {
+                                 "id": "custom.lineWidth",
+                                 "value": 0
+                              }
+                           ]
+                        },
+                        {
+                           "matcher": {
+                              "id": "byName",
+                              "options": "Items"
+                           },
+                           "properties": [
+                              {
+                                 "id": "custom.fillOpacity",
+                                 "value": 25
+                              },
+                              {
+                                 "id": "custom.lineWidth",
+                                 "value": 0
+                              },
+                              {
+                                 "id": "unit",
+                                 "value": ""
+                              },
+                              {
+                                 "id": "custom.axisSoftMin",
+                                 "value": 0
+                              }
+                           ]
+                        }
+                     ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 14
+                  },
+                  "id": 83,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(loki_bloom_blocks_cache_entries{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Items",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(rate(loki_bloom_blocks_cache_added_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Added",
+                        "range": true,
+                        "refId": "G"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(rate(loki_bloom_blocks_cache_evicted_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (reason)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Evicted ({{reason}})",
+                        "range": true,
+                        "refId": "F"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(rate(loki_bloom_blocks_cache_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))",
+                        "hide": true,
+                        "instant": false,
+                        "legendFormat": "Size",
+                        "range": true,
+                        "refId": "E"
+                     }
+                  ],
+                  "title": "Cache rate",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 100,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 0,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "linearThreshold": 1000,
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "normal"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "fieldMinMax": false,
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "percentunit"
+                     },
+                     "overrides": [
+                        {
+                           "matcher": {
+                              "id": "byName",
+                              "options": "hit"
+                           },
+                           "properties": [
+                              {
+                                 "id": "color",
+                                 "value": {
+                                    "fixedColor": "green",
+                                    "mode": "fixed"
+                                 }
+                              }
+                           ]
+                        },
+                        {
+                           "matcher": {
+                              "id": "byName",
+                              "options": "miss"
+                           },
+                           "properties": [
+                              {
+                                 "id": "color",
+                                 "value": {
+                                    "fixedColor": "red",
+                                    "mode": "fixed"
+                                 }
+                              }
+                           ]
+                        }
+                     ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 24,
+                     "x": 0,
+                     "y": 21
+                  },
+                  "id": 92,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (status) (\n    rate(loki_bloom_blocks_cache_fetched_total{container=\"bloom-gateway\"}[$__rate_interval])\n)\n/ ignoring(status) group_left\nsum (\n    rate(loki_bloom_blocks_cache_fetched_total{container=\"bloom-gateway\"}[$__rate_interval])\n)",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "Hit/Miss ratio",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "fieldMinMax": false,
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "bytes"
+                     },
+                     "overrides": [
+                        {
+                           "matcher": {
+                              "id": "byRegexp",
+                              "options": "/Size (.*)/"
+                           },
+                           "properties": [
+                              {
+                                 "id": "unit",
+                                 "value": "bytes"
+                              }
+                           ]
+                        }
+                     ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 0,
+                     "y": 28
+                  },
+                  "id": 76,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(loki_embeddedcache_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", cache=\"bloom-blocks-cache\", container=\"bloom-gateway\"})\n/\nsum(loki_embeddedcache_entries{cluster=\"$cluster\", namespace=\"$namespace\", cache=\"bloom-blocks-cache\", container=\"bloom-gateway\"})",
+                        "instant": false,
+                        "legendFormat": "Size",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(loki_bloom_blocks_cache_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})\n/\nsum(loki_bloom_blocks_cache_entries{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "Size",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "Average item size",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "fieldMinMax": false,
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [
+                        {
+                           "matcher": {
+                              "id": "byRegexp",
+                              "options": "/.* (blocks|metas) size/"
+                           },
+                           "properties": [
+                              {
+                                 "id": "unit",
+                                 "value": "bytes"
+                              }
+                           ]
+                        }
+                     ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 12,
+                     "x": 12,
+                     "y": 28
+                  },
+                  "id": 21,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(rate(loki_bloom_store_metas_fetched_sum{cluster=\"$cluster\",namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "metas fetch rate",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum(rate(loki_bloom_store_blocks_fetched_sum{cluster=\"$cluster\",namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "blocks fetch rate",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.9, sum(rate(loki_bloom_store_blocks_fetched_size_bytes_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p90 blocks size",
+                        "range": true,
+                        "refId": "C"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.9, sum(rate(loki_bloom_store_metas_fetched_size_bytes_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "p90 metas size",
+                        "range": true,
+                        "refId": "D"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(1.0, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))",
+                        "hide": true,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "E"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))",
+                        "hide": true,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "F"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.5, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))",
+                        "hide": true,
+                        "instant": false,
+                        "legendFormat": "__auto",
+                        "range": true,
+                        "refId": "G"
+                     }
+                  ],
+                  "title": "Bloom Store",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "Blocks Cache",
+            "type": "row"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 21
+            },
+            "id": 60,
+            "panels": [
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "gridPos": {
+                     "h": 1,
+                     "w": 24,
+                     "x": 0,
+                     "y": 14
+                  },
+                  "id": 61,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-70005",
+                  "title": "The gateway download bloom meta files and blocks from the object store.",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "gridPos": {
+                     "h": 7,
+                     "w": 2,
+                     "x": 0,
+                     "y": 15
+                  },
+                  "id": 24,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "---\n#### GCS\n",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-70005",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 25,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "normal"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 11,
+                     "x": 2,
+                     "y": 15
+                  },
+                  "id": 25,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval]))",
+                        "instant": false,
+                        "legendFormat": "{{operation}} {{status_code}}",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "QPS",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 0,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 11,
+                     "x": 13,
+                     "y": 15
+                  },
+                  "id": 29,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))",
+                        "instant": false,
+                        "legendFormat": "{{operation}} p99",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p90",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p50",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Latency",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "gridPos": {
+                     "h": 7,
+                     "w": 2,
+                     "x": 0,
+                     "y": 22
+                  },
+                  "id": 62,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "---\n#### S3\n",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-70005",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 25,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "normal"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 11,
+                     "x": 2,
+                     "y": 22
+                  },
+                  "id": 63,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval]))",
+                        "instant": false,
+                        "legendFormat": "{{operation}} {{status_code}}",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "QPS",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 25,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "normal"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 11,
+                     "x": 13,
+                     "y": 22
+                  },
+                  "id": 64,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))",
+                        "instant": false,
+                        "legendFormat": "{{operation}} p99",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p90",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p50",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Latency",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "gridPos": {
+                     "h": 7,
+                     "w": 2,
+                     "x": 0,
+                     "y": 29
+                  },
+                  "id": 65,
+                  "options": {
+                     "code": {
+                        "language": "plaintext",
+                        "showLineNumbers": false,
+                        "showMiniMap": false
+                     },
+                     "content": "---\n#### Azure\nBlob Storage\n\n",
+                     "mode": "markdown"
+                  },
+                  "pluginVersion": "11.1.0-70005",
+                  "transparent": true,
+                  "type": "text"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 25,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "normal"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 11,
+                     "x": 2,
+                     "y": 29
+                  },
+                  "id": 66,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval]))",
+                        "instant": false,
+                        "legendFormat": "{{operation}} {{status_code}}",
+                        "range": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "QPS",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 25,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "normal"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              }
+                           ]
+                        },
+                        "unit": "none"
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 7,
+                     "w": 11,
+                     "x": 13,
+                     "y": 29
+                  },
+                  "id": 67,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))",
+                        "instant": false,
+                        "legendFormat": "{{operation}} p99",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p90",
+                        "range": true,
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))",
+                        "hide": false,
+                        "instant": false,
+                        "legendFormat": "{{operation}} p50",
+                        "range": true,
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Latency",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "Object Store",
+            "type": "row"
+         },
+         {
+            "collapsed": true,
+            "gridPos": {
+               "h": 1,
+               "w": 24,
+               "x": 0,
+               "y": 22
+            },
+            "id": 77,
+            "panels": [
+               {
+                  "datasource": {
+                     "type": "loki",
+                     "uid": "${loki_datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 0,
+                     "y": 23
+                  },
+                  "id": 78,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "loki",
+                           "uid": "${loki_datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "topk(3, sum by (tasks) (count_over_time({namespace=\"loki-dev-006\", container=\"bloom-gateway\"} |= \"process tasks with bounds\" | logfmt [5s])))",
+                        "legendFormat": "{{tasks}}",
+                        "queryType": "range",
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Process tasks with bounds",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "loki",
+                     "uid": "${loki_datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [
+                        {
+                           "__systemRef": "hideSeriesFrom",
+                           "matcher": {
+                              "id": "byNames",
+                              "options": {
+                                 "mode": "exclude",
+                                 "names": [
+                                    "max",
+                                    "avg"
+                                 ],
+                                 "prefix": "All except:",
+                                 "readOnly": true
+                              }
+                           },
+                           "properties": [
+                              {
+                                 "id": "custom.hideFrom",
+                                 "value": {
+                                    "legend": false,
+                                    "tooltip": false,
+                                    "viz": true
+                                 }
+                              }
+                           ]
+                        }
+                     ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 12,
+                     "y": 23
+                  },
+                  "id": 79,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "loki",
+                           "uid": "${loki_datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "max(max_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"fetch blocks\" | logfmt | unwrap duration(duration) [$__auto]))",
+                        "hide": false,
+                        "legendFormat": "max",
+                        "queryType": "range",
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "loki",
+                           "uid": "${loki_datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "avg(avg_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"fetch blocks\" | logfmt | unwrap duration(duration) [$__auto]))",
+                        "hide": false,
+                        "legendFormat": "avg",
+                        "queryType": "range",
+                        "refId": "B"
+                     },
+                     {
+                        "datasource": {
+                           "type": "loki",
+                           "uid": "${loki_datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "avg(avg_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"request unavailable blocks in the background\" | logfmt | missing > 0 | unwrap missing [$__auto]))",
+                        "hide": false,
+                        "legendFormat": "avg missing",
+                        "queryType": "range",
+                        "refId": "C"
+                     }
+                  ],
+                  "title": "Download enqueue duration",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": {
+                     "type": "loki",
+                     "uid": "${loki_datasource}"
+                  },
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "fixedColor": "green",
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "fillOpacity": 80,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "lineWidth": 1,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 100
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [ ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 0,
+                     "y": 31
+                  },
+                  "id": 80,
+                  "options": {
+                     "barRadius": 0,
+                     "barWidth": 0.96999999999999997,
+                     "fullHighlight": false,
+                     "groupWidth": 0.69999999999999996,
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "orientation": "horizontal",
+                     "showValue": "auto",
+                     "stacking": "none",
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     },
+                     "xTickLabelRotation": 0,
+                     "xTickLabelSpacing": 0
+                  },
+                  "pluginVersion": "11.0.0-67814",
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "loki",
+                           "uid": "${loki_datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "sort_desc(topk(10, sum by (tasks) (count_over_time({namespace=\"loki-dev-006\", container=\"bloom-gateway\"} |= \"process tasks with bounds\" | logfmt [$__auto]))))",
+                        "legendFormat": "",
+                        "queryType": "instant",
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Tasks multiplexed",
+                  "type": "barchart"
+               },
+               {
+                  "datasource": {
+                     "type": "prometheus",
+                     "uid": "${datasource}"
+                  },
+                  "description": "",
+                  "fieldConfig": {
+                     "defaults": {
+                        "color": {
+                           "mode": "palette-classic"
+                        },
+                        "custom": {
+                           "axisBorderShow": false,
+                           "axisCenteredZero": false,
+                           "axisColorMode": "text",
+                           "axisLabel": "",
+                           "axisPlacement": "auto",
+                           "barAlignment": 0,
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "gradientMode": "none",
+                           "hideFrom": {
+                              "legend": false,
+                              "tooltip": false,
+                              "viz": false
+                           },
+                           "insertNulls": false,
+                           "lineInterpolation": "linear",
+                           "lineStyle": {
+                              "fill": "solid"
+                           },
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "scaleDistribution": {
+                              "type": "linear"
+                           },
+                           "showPoints": "auto",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           },
+                           "thresholdsStyle": {
+                              "mode": "off"
+                           }
+                        },
+                        "mappings": [ ],
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [
+                              {
+                                 "color": "green"
+                              },
+                              {
+                                 "color": "red",
+                                 "value": 80
+                              }
+                           ]
+                        }
+                     },
+                     "overrides": [
+                        {
+                           "matcher": {
+                              "id": "byName",
+                              "options": "Enqueue latency"
+                           },
+                           "properties": [
+                              {
+                                 "id": "unit",
+                                 "value": "s"
+                              }
+                           ]
+                        }
+                     ]
+                  },
+                  "gridPos": {
+                     "h": 8,
+                     "w": 12,
+                     "x": 12,
+                     "y": 31
+                  },
+                  "id": 82,
+                  "options": {
+                     "legend": {
+                        "calcs": [ ],
+                        "displayMode": "list",
+                        "placement": "bottom",
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "maxHeight": 600,
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "targets": [
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99, sum(rate(loki_bloom_store_download_queue_enqueue_time_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))",
+                        "hide": false,
+                        "legendFormat": "Enqueue latency",
+                        "range": true,
+                        "refId": "A"
+                     },
+                     {
+                        "datasource": {
+                           "type": "prometheus",
+                           "uid": "${datasource}"
+                        },
+                        "editorMode": "code",
+                        "expr": "histogram_quantile(0.99, sum(rate(loki_bloom_store_download_queue_size_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))",
+                        "hide": false,
+                        "legendFormat": "Size",
+                        "range": true,
+                        "refId": "B"
+                     }
+                  ],
+                  "title": "Block download queue",
+                  "type": "timeseries"
+               }
+            ],
+            "title": "Misc",
+            "type": "row"
+         }
+      ],
+      "refresh": "10s",
+      "rows": [ ],
+      "schemaVersion": 14,
+      "style": "dark",
+      "tags": [
+         "loki"
+      ],
+      "templating": {
+         "list": [
+            {
+               "current": {
+                  "text": "default",
+                  "value": "default"
+               },
+               "hide": 0,
+               "label": "Data source",
+               "name": "datasource",
+               "options": [ ],
+               "query": "prometheus",
+               "refresh": 1,
+               "regex": "",
+               "type": "datasource"
+            },
+            {
+               "allValue": null,
+               "current": {
+                  "text": "prod",
+                  "value": "prod"
+               },
+               "datasource": "$datasource",
+               "hide": 0,
+               "includeAll": false,
+               "label": "cluster",
+               "multi": false,
+               "name": "cluster",
+               "options": [ ],
+               "query": "label_values(loki_build_info, cluster)",
+               "refresh": 1,
+               "regex": "",
+               "sort": 2,
+               "tagValuesQuery": "",
+               "tags": [ ],
+               "tagsQuery": "",
+               "type": "query",
+               "useTags": false
+            },
+            {
+               "allValue": null,
+               "current": {
+                  "text": "prod",
+                  "value": "prod"
+               },
+               "datasource": "$datasource",
+               "hide": 0,
+               "includeAll": false,
+               "label": "namespace",
+               "multi": false,
+               "name": "namespace",
+               "options": [ ],
+               "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)",
+               "refresh": 1,
+               "regex": "",
+               "sort": 2,
+               "tagValuesQuery": "",
+               "tags": [ ],
+               "tagsQuery": "",
+               "type": "query",
+               "useTags": false
+            },
+            {
+               "hide": 0,
+               "label": null,
+               "name": "loki_datasource",
+               "options": [ ],
+               "query": "loki",
+               "refresh": 1,
+               "regex": "",
+               "type": "datasource"
+            }
+         ]
+      },
+      "time": {
+         "from": "now-1h",
+         "to": "now"
+      },
+      "timeRangeUpdatedDuringEditOrView": false,
+      "timepicker": {
+         "refresh_intervals": [
+            "5s",
+            "10s",
+            "30s",
+            "1m",
+            "5m",
+            "15m",
+            "30m",
+            "1h",
+            "2h",
+            "1d"
+         ],
+         "time_options": [
+            "5m",
+            "15m",
+            "1h",
+            "6h",
+            "12h",
+            "24h",
+            "2d",
+            "7d",
+            "30d"
+         ]
+      },
+      "timezone": "utc",
+      "title": "Loki / Bloom Gateway",
+      "uid": "bloom-gateway",
+      "version": 0,
+      "weekStart": ""
+   }
\ No newline at end of file
diff --git a/production/loki-mixin/.lint b/production/loki-mixin/.lint
index 108cbcc7c8581..4661415c1c7f2 100644
--- a/production/loki-mixin/.lint
+++ b/production/loki-mixin/.lint
@@ -12,6 +12,8 @@ exclusions:
       - dashboard: "Loki / Logs"
       - dashboard: "Loki / Writes Resources"
       - dashboard: "Loki / Writes"
+      - dashboard: "Loki / Bloom Compactor"
+      - dashboard: "Loki / Bloom Gateway"
   template-datasource-rule:
     reason: "Based on new convention we are using variable names prometheus_datasource and loki_datasource where as linter expects 'datasource'"
     entries:
@@ -25,6 +27,8 @@ exclusions:
       - dashboard: "Loki / Logs"
       - dashboard: "Loki / Writes Resources"
       - dashboard: "Loki / Writes"
+      - dashboard: "Loki / Bloom Compactor"
+      - dashboard: "Loki / Bloom Gateway"
   template-instance-rule:
     reason: "These dashboards are cluster overview dashboards, whereas the instance refers to specific pods or nodes"
     entries:
@@ -38,6 +42,8 @@ exclusions:
       - dashboard: "Loki / Reads Resources"
       - dashboard: "Loki / Logs"
       - dashboard: "Loki / Writes Resources"
+      - dashboard: "Loki / Bloom Compactor"
+      - dashboard: "Loki / Bloom Gateway"
   target-instance-rule:
     reason: "These dashboards are cluster overview dashboards, whereas the instance refers to specific pods or nodes"
     entries:
@@ -51,6 +57,8 @@ exclusions:
       - dashboard: "Loki / Logs"
       - dashboard: "Loki / Writes Resources"
       - dashboard: "Loki / Writes"
+      - dashboard: "Loki / Bloom Compactor"
+      - dashboard: "Loki / Bloom Gateway"
   target-job-rule:
     reason: "We don't have/need a job template selector for this dashboard"
     entries:
@@ -64,6 +72,8 @@ exclusions:
       - dashboard: "Loki / Logs"
       - dashboard: "Loki / Writes Resources"
       - dashboard: "Loki / Writes"
+      - dashboard: "Loki / Bloom Compactor"
+      - dashboard: "Loki / Bloom Gateway"
   target-promql-rule:
     reason: "The following are logql queries, not promql"
     entries:
@@ -73,12 +83,22 @@ exclusions:
         panel: "Bad Words"
       - dashboard: "Loki / Logs"
         panel: "Log Rate"
+      - dashboard: "Loki / Bloom Gateway"
+        panel: "Errors Rate"
+      - dashboard: "Loki / Bloom Gateway"
+        panel: "Process tasks with bounds"
+      - dashboard: "Loki / Bloom Gateway"
+        panel: "Download enqueue duration"
+      - dashboard: "Loki / Bloom Compactor"
+        panel: "Errors Rate"
   panel-datasource-rule:
     reason: "Loki datasource variable is being named as loki_datasource now while linter expects 'datasource'"
     entries:
       - dashboard:
       - dashboard: "Loki / Logs"
         panel: "Log Rate"
+      - dashboard: "Loki / Bloom Gateway"
+        panel: "Filter ratio"
   target-rate-interval-rule:
     reason: "This query is using an offset, the lint failure here is not for a rate interval"
     entries:
@@ -162,4 +182,58 @@ exclusions:
       - dashboard: "Loki / Operational"
         panel: "Query Pages"
       - dashboard: "Loki / Operational"
-        panel: "Throttled Rate"
\ No newline at end of file
+        panel: "Throttled Rate"
+      - dashboard: "Loki / Bloom Compactor"
+        panel: "Errors Rate"
+      - dashboard: "Loki / Bloom Compactor"
+        panel: "Required CPUs to not lag behind"
+      - dashboard: "Loki / Bloom Compactor"
+        panel: "Tokens rate (millions)"
+      - dashboard: "Loki / Bloom Compactor"
+        panel: "Created Blocks"
+      - dashboard: "Loki / Bloom Compactor"
+        panel: "Deleted Blocks"
+      - dashboard: "Loki / Bloom Compactor"
+        panel: "Blocks reused"
+      - dashboard: "Loki / Bloom Compactor"
+        panel: "Created Metas"
+      - dashboard: "Loki / Bloom Compactor"
+        panel: "Deleted Metas"
+      - dashboard: "Loki / Bloom Compactor"
+        panel: "Tenants"
+      - dashboard: "Loki / Bloom Compactor"
+        panel: "Tenants per pod"
+      - dashboard: "Loki / Bloom Compactor"
+        panel: "Tenant Tables"
+      - dashboard: "Loki / Bloom Compactor"
+        panel: "Tenant Tables per pod"
+      - dashboard: "Loki / Bloom Compactor"
+        panel: "Series"
+      - dashboard: "Loki / Bloom Compactor"
+        panel: "avg series per compaction by pod"
+      - dashboard: "Loki / Bloom Compactor"
+        panel: "CPU"
+      - dashboard: "Loki / Bloom Compactor"
+        panel: "CPU per pod"
+      - dashboard: "Loki / Bloom Gateway"
+        panel: "Errors Rate"
+      - dashboard: "Loki / Bloom Gateway"
+        panel: "CPU"
+      - dashboard: "Loki / Bloom Gateway"
+        panel: "GC rate"
+      - dashboard: "Loki / Bloom Gateway"
+        panel: "QPS"
+      - dashboard: "Loki / Bloom Gateway"
+        panel: "QPS per Pod"
+      - dashboard: "Loki / Bloom Gateway"
+        panel: "Tasks dequeued/processed"
+      - dashboard: "Loki / Bloom Gateway"
+        panel: "Process tasks with bounds"
+      - dashboard: "Loki / Bloom Gateway"
+        panel: "Download enqueue duration"
+      - dashboard: "Loki / Bloom Gateway"
+        panel: "Block download queue"
+      - dashboard: "Loki / Bloom Gateway"
+        panel: "CPU"
+      - dashboard: "Loki / Bloom Gateway"
+        panel: "CPU per pod"
\ No newline at end of file
diff --git a/production/loki-mixin/dashboards.libsonnet b/production/loki-mixin/dashboards.libsonnet
index 9ea6c2ca82d53..33f3b136d5ad6 100644
--- a/production/loki-mixin/dashboards.libsonnet
+++ b/production/loki-mixin/dashboards.libsonnet
@@ -9,4 +9,6 @@
 (import 'dashboards/loki-reads-resources.libsonnet') +
 (import 'dashboards/loki-deletion.libsonnet') +
 (import 'dashboards/loki-canary-dashboard.libsonnet') +
-(import 'dashboards/recording-rules.libsonnet')
+(import 'dashboards/recording-rules.libsonnet') +
+(import 'dashboards/loki-bloom-compactor.libsonnet') +
+(import 'dashboards/loki-bloom-gateway.libsonnet')
diff --git a/production/loki-mixin/dashboards/dashboard-bloom-compactor.json b/production/loki-mixin/dashboards/dashboard-bloom-compactor.json
new file mode 100644
index 0000000000000..035a7523af8e3
--- /dev/null
+++ b/production/loki-mixin/dashboards/dashboard-bloom-compactor.json
@@ -0,0 +1,5086 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "links": [],
+  "panels": [
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 111,
+      "panels": [],
+      "title": "Overview",
+      "type": "row"
+    },
+    {
+      "gridPos": {
+        "h": 8,
+        "w": 14,
+        "x": 0,
+        "y": 1
+      },
+      "id": 35,
+      "options": {
+        "code": {
+          "language": "plaintext",
+          "showLineNumbers": false,
+          "showMiniMap": false
+        },
+        "content": "## About the Bloom Compactor\nThe compactor iterates through chunks and creates blooms out of them.\nThe size of the resulting blooms depends on the bloom filter settings, the tokenizer settings, the number of ring tokens per compactor and the total number opf compactors.\n\nCompactors are horizontally scalable and uses a ring to:\n- Shard tenants\n- Shard series fingerprints within a tenant subring.\n\nThe blooms for the series are grouped together in blocks which are flushed to object store.",        "mode": "markdown"
+      },
+      "pluginVersion": "11.1.0-70005",
+      "transparent": true,
+      "type": "text"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "description": "Cell-wide compaction progress. Should increase till completion throughout each compaction period.",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 15,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineStyle": {
+              "fill": "solid"
+            },
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 9
+      },
+      "id": 42,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "maxHeight": 600,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.1.0-69868",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum(loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"})\n/\nsum(count(loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}))",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "avg",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "quantile(\n    0.9, \n    sum by (pod) (\n        loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}\n    )\n)",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "p90",
+          "range": true,
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "quantile(\n    0.1, \n    sum by (pod) (\n        loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}\n    )\n)",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "p10",
+          "range": true,
+          "refId": "C"
+        }
+      ],
+      "title": "Progress",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "description": "Uncompressed size of chunks in a series VS the size of the blooms built.",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 15,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineStyle": {
+              "fill": "solid"
+            },
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "bytes"
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Ratio"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "percentunit"
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Ratio over range"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "percentunit"
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 9
+      },
+      "id": 41,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "maxHeight": 600,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.1.0-69868",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(loki_bloom_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "Bloom",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "Chunk",
+          "range": true,
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(loki_bloom_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n/\nsum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "Ratio",
+          "range": true,
+          "refId": "C"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(loki_bloom_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n/\nsum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "Ratio over range",
+          "range": true,
+          "refId": "D"
+        }
+      ],
+      "title": "Chunks and Bloom size",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${loki_datasource}"
+      },
+      "description": "Blooms size vs uncompressed chunk size.",
+      "gridPos": {
+        "h": 7,
+        "w": 17,
+        "x": 0,
+        "y": 16
+      },
+      "id": 51,
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": false,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false
+      },
+      "pluginVersion": "11.1.0-69868",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${loki_datasource}"
+          },
+          "editorMode": "code",
+          "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} |= \"level=error\" |= \"component=bloom-compactor\"",
+          "queryType": "range",
+          "refId": "B"
+        }
+      ],
+      "title": "Errors",
+      "type": "logs"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${loki_datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "red",
+            "mode": "fixed"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "bars",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 3,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 7,
+        "x": 17,
+        "y": 16
+      },
+      "id": 53,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "maxHeight": 600,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.1.0-69868",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${loki_datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum(count_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} |= \"level=error\" |= \"component=bloom-compactor\" [$__auto]))",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Errors Rate",
+      "type": "timeseries"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 23
+      },
+      "id": 112,
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 15,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineStyle": {
+                  "fill": "solid"
+                },
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "percentunit"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 24
+          },
+          "id": 114,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum by (pod) (\n    loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "B"
+            }
+          ],
+          "title": "Progress per pod",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "fieldMinMax": false,
+              "mappings": [],
+              "thresholds": {
+                "mode": "percentage",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "#EAB839",
+                    "value": 0
+                  },
+                  {
+                    "color": "green",
+                    "value": 100
+                  }
+                ]
+              },
+              "unit": "percentunit"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 24
+          },
+          "id": 115,
+          "options": {
+            "minVizHeight": 75,
+            "minVizWidth": 75,
+            "orientation": "auto",
+            "reduceOptions": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "fields": "",
+              "values": false
+            },
+            "showThresholdLabels": false,
+            "showThresholdMarkers": false,
+            "sizing": "auto"
+          },
+          "pluginVersion": "11.0.0-68102",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum by (pod) (\n    loki_bloomcompactor_progress{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "B"
+            }
+          ],
+          "title": "Current Progress per pod",
+          "type": "gauge"
+        }
+      ],
+      "title": "Progress per pod",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 24
+      },
+      "id": 56,
+      "panels": [
+        {
+          "description": "",
+          "gridPos": {
+            "h": 1,
+            "w": 24,
+            "x": 0,
+            "y": 25
+          },
+          "id": 85,
+          "options": {
+            "code": {
+              "language": "plaintext",
+              "showLineNumbers": false,
+              "showMiniMap": false
+            },
+            "content": "",
+            "mode": "markdown"
+          },
+          "pluginVersion": "11.1.0-70005",
+          "title": "We use tenant sharding so each compactor will process a subset of the tenants.",
+          "transparent": true,
+          "type": "text"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "Shows the expected number of cpu cores we need to provision to build blooms as fast as we ingest data so a compaction iteration doesn't take longer than the compaction interval.\n\nWe may decide to have more to speed up compaction.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 26
+          },
+          "id": 94,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "11.1.0-69868",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "# This query shows the expected number of cpu cores we need to not fall behind\n# building blooms for data we're ingesting.\n# conceptually, the formula is:\n# (cell_bytes * space_amplification / bloom_bytes_processed_per_core)\n\n# number of replicas needed\nsum(avg_over_time(loki_cell:bytes:rate1m{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n*\n## Space amplification (how much data do we write compared to what we ingest?)\n(\n  # rep factor\n  3 *\n  sum(\n    # 1 - dedupe_ratio\n    1 - \n    sum(rate(loki_chunk_store_deduped_chunks_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (cluster, namespace)\n    /\n    sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (cluster, namespace)\n  )\n)\n/\n(\nsum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))\n/\nsum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Needed",
+              "range": true,
+              "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Available",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Required CPUs to not lag behind",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 15,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineStyle": {
+                  "fill": "solid"
+                },
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "Bps"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 26
+          },
+          "id": 72,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "11.1.0-69868",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "# MB/s/core chunk data processed\nsum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}[$__rate_interval])) by (pod)\n/\nsum(rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])) by (pod)",
+              "hide": true,
+              "instant": false,
+              "legendFormat": "{{pod}}",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "# MB/s/core chunk data processed\nsum(rate(loki_bloomcompactor_chunk_series_size_sum{cluster=~\"$cluster\", job=~\"$namespace/bloom-compactor\"}[$__rate_interval]))\n/\nsum(rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Total",
+              "range": true,
+              "refId": "B"
+            }
+          ],
+          "title": "MB/s per core",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 33
+          },
+          "id": 1,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "11.1.0-69868",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"cpu\"} > 0)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Request",
+              "range": true,
+              "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "min(kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"cpu\"} > 0)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Limit",
+              "range": true,
+              "refId": "C"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "quantile(\n    0.99,\n    rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}[$__rate_interval])\n)",
+              "instant": false,
+              "legendFormat": "p99",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "quantile(\n    0.50,\n    rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}[$__rate_interval])\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p50",
+              "range": true,
+              "refId": "D"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "avg(\n    rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}[$__rate_interval])\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Avg",
+              "range": true,
+              "refId": "E"
+            }
+          ],
+          "title": "CPU",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 33
+          },
+          "id": 75,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "11.1.0-69868",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}[$__rate_interval]))",
+              "instant": false,
+              "legendFormat": "{{pod}}",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"cpu\"} > 0)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Request",
+              "range": true,
+              "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "min(kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"cpu\"} > 0)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Limit",
+              "range": true,
+              "refId": "C"
+            }
+          ],
+          "title": "CPU per pod",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 40
+          },
+          "id": 76,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "11.1.0-69868",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"memory\"} > 0)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Request",
+              "range": true,
+              "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"} > 0)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Limit",
+              "range": true,
+              "refId": "C"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "quantile (\n  0.99,\n  container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}\n)",
+              "instant": false,
+              "legendFormat": "p99",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "quantile (\n  0.50,\n  container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p50",
+              "range": true,
+              "refId": "D"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "avg (\n  container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"}\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Avg",
+              "range": true,
+              "refId": "E"
+            }
+          ],
+          "title": "Memory (workingset)",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": [
+              {
+                "__systemRef": "hideSeriesFrom",
+                "matcher": {
+                  "id": "byNames",
+                  "options": {
+                    "mode": "exclude",
+                    "names": [
+                      "bloom-compactor-106"
+                    ],
+                    "prefix": "All except:",
+                    "readOnly": true
+                  }
+                },
+                "properties": [
+                  {
+                    "id": "custom.hideFrom",
+                    "value": {
+                      "legend": false,
+                      "tooltip": false,
+                      "viz": true
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 40
+          },
+          "id": 5,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "11.1.0-69868",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"})",
+              "instant": false,
+              "legendFormat": "{{pod}}",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\", resource=\"memory\"} > 0)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Request",
+              "range": true,
+              "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-compactor\"} > 0)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Limit",
+              "range": true,
+              "refId": "C"
+            }
+          ],
+          "title": "Memory per pod (workingset)",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 15,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineStyle": {
+                  "fill": "solid"
+                },
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 47
+          },
+          "id": 27,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "11.1.0-69868",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum (\n    increase(\n        kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[10m]\n    )\n) > 0",
+              "instant": false,
+              "legendFormat": "Restarts",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Container restarts",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 15,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineStyle": {
+                  "fill": "solid"
+                },
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 47
+          },
+          "id": 77,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "11.1.0-69868",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "(\n    sum by (pod) (\n        increase(\n            kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[10m]\n        )\n    )\n    * on (pod) group_right\n    max by (pod, reason) (\n        kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}\n    )\n) > 0",
+              "instant": false,
+              "legendFormat": "{{reason}} / {{pod}}",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Container restarts reason per pod",
+          "type": "timeseries"
+        }
+      ],
+      "title": "Resource Usage",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 25
+      },
+      "id": 95,
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "How many tokens each compactor is appending to blooms. Accounts for tokens that are not actually added to the blooms since they are already there. See the panel on the right for a drill down on the collision.\n",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "log": 2,
+                  "type": "log"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 55
+          },
+          "id": 96,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "11.1.0-69868",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "# tokens checked per pod, millions/s\nsum(rate(loki_bloom_tokens_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))\n/\nsum(count(loki_bloom_tokens_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}))\n/ 1e6",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Per core",
+              "range": true,
+              "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum(rate(loki_bloom_inserts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])) / 1e6",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Total",
+              "range": true,
+              "refId": "C"
+            }
+          ],
+          "title": "Tokens rate (millions)",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "Collision type may be `false` (no collision), `cache` (found in token cache) or true (found in bloom filter).\n\nType may be either `raw` (the original ngram) or `chunk_prefixed` (the ngram with the chunk prefix)",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "percentunit"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 55
+          },
+          "id": 97,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "11.1.0-69868",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "# tokens/s by type+collision\nsum by (collision) (\n    rate(loki_bloom_inserts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n) \n/ on () group_left\nsum (\n    rate(loki_bloom_inserts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "B"
+            }
+          ],
+          "title": "tokens/s by collision type",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "The sizes of the blooms created by the compactor. We build one bloom per series. The more unique ngrams and chunks the series has, the bigger their blooms will be.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 63
+          },
+          "id": 98,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "11.1.0-69868",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(\n    0.99, \n    sum by (le) (\n        rate(loki_bloom_size_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    )\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p99",
+              "range": true,
+              "refId": "D"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(\n    0.90, \n    sum by (le) (\n        rate(loki_bloom_size_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    )\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p90",
+              "range": true,
+              "refId": "E"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(\n    0.50, \n    sum by (le) (\n        rate(loki_bloom_size_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    )\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p50",
+              "range": true,
+              "refId": "F"
+            }
+          ],
+          "title": "Bloom size",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "How many chunks are we indexing in the blooms. Either:\n- `copied` from a pre-existing bloom block, or \n- `iterated` through all its entries if processed for the first time.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 63
+          },
+          "id": 99,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "11.1.0-69868",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "# chunks indexed, by iteration or copied from a pre-existing bloom\nsum(rate(loki_bloom_chunks_indexed_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])) by (type)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "B"
+            }
+          ],
+          "title": "Chunks indexed",
+          "type": "timeseries"
+        }
+      ],
+      "title": "Bloom building",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 26
+      },
+      "id": 103,
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 72
+          },
+          "id": 107,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum(increase(loki_bloomcompactor_blocks_created_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Blocks",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Created Blocks",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "Compactors delete metas and blocks marked for deletion in the metas tombstones.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 72
+          },
+          "id": 106,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum(increase(loki_bloomcompactor_blocks_deleted_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Blocks",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Deleted Blocks",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "Number of overlapping bloom blocks reused when creating new blocks\n",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 79
+          },
+          "id": 109,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum(increase(loki_bloomcompactor_blocks_reused_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Blocks",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Blocks reused",
+          "type": "timeseries"
+        }
+      ],
+      "title": "Blocks building",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 27
+      },
+      "id": 110,
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 87
+          },
+          "id": 108,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "11.1.0-69868",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum(increase(loki_bloomcompactor_metas_created_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Metas",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Created Metas",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "Compactors delete metas and blocks marked for deletion in the metas tombstones.",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 87
+          },
+          "id": 105,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "pluginVersion": "11.1.0-69868",
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum(increase(loki_bloomcompactor_metas_deleted_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Metas",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Deleted Metas",
+          "type": "timeseries"
+        }
+      ],
+      "title": "Metas building",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 28
+      },
+      "id": 80,
+      "panels": [
+        {
+          "description": "",
+          "gridPos": {
+            "h": 1,
+            "w": 24,
+            "x": 0,
+            "y": 95
+          },
+          "id": 93,
+          "options": {
+            "code": {
+              "language": "plaintext",
+              "showLineNumbers": false,
+              "showMiniMap": false
+            },
+            "content": "",
+            "mode": "markdown"
+          },
+          "pluginVersion": "11.1.0-70005",
+          "title": "We use tenant sharding so each compactor will process a subset of the tenants.",
+          "transparent": true,
+          "type": "text"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 96
+          },
+          "id": 83,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "quantile(\n  0.99,\n  increase(\n    loki_bloomcompactor_tenants_started_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n  )\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p99",
+              "range": true,
+              "refId": "C"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "quantile(\n  0.90,\n  increase(\n    loki_bloomcompactor_tenants_started_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n  )\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p90",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "quantile(\n  0.50,\n  increase(\n    loki_bloomcompactor_tenants_started_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[30m]\n  )\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p50",
+              "range": true,
+              "refId": "B"
+            }
+          ],
+          "title": "Tenants",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 96
+          },
+          "id": 84,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum by (pod) (\n  increase(\n    loki_bloomcompactor_tenants_started_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n  )\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{pod}}",
+              "range": true,
+              "refId": "C"
+            }
+          ],
+          "title": "Tenants per pod",
+          "type": "timeseries"
+        },
+        {
+          "description": "",
+          "gridPos": {
+            "h": 1,
+            "w": 24,
+            "x": 0,
+            "y": 103
+          },
+          "id": 86,
+          "options": {
+            "code": {
+              "language": "plaintext",
+              "showLineNumbers": false,
+              "showMiniMap": false
+            },
+            "content": "",
+            "mode": "markdown"
+          },
+          "pluginVersion": "11.1.0-70005",
+          "title": "Number of tenant tables processed. ",
+          "transparent": true,
+          "type": "text"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 104
+          },
+          "id": 88,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "quantile(\n  0.99,\n  increase(\n    loki_bloomcompactor_tenant_table_ranges_completed_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n  )\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p99",
+              "range": true,
+              "refId": "C"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "quantile(\n  0.90,\n  increase(\n    loki_bloomcompactor_tenant_table_ranges_completed_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n  )\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p90",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "quantile(\n  0.50,\n  increase(\n    loki_bloomcompactor_tenant_table_ranges_completed_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n  )\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p50",
+              "range": true,
+              "refId": "B"
+            }
+          ],
+          "title": "Tenant Tables",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 104
+          },
+          "id": 89,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum by (pod) (\n  increase(\n    loki_bloomcompactor_tenant_table_ranges_completed_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval]\n  )\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{pod}}",
+              "range": true,
+              "refId": "C"
+            }
+          ],
+          "title": "Tenant Tables per pod",
+          "type": "timeseries"
+        },
+        {
+          "description": "",
+          "gridPos": {
+            "h": 1,
+            "w": 24,
+            "x": 0,
+            "y": 111
+          },
+          "id": 87,
+          "options": {
+            "code": {
+              "language": "plaintext",
+              "showLineNumbers": false,
+              "showMiniMap": false
+            },
+            "content": "",
+            "mode": "markdown"
+          },
+          "pluginVersion": "11.1.0-70005",
+          "title": "Series per compaction (includes series copied from other blocks)",
+          "transparent": true,
+          "type": "text"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 112
+          },
+          "id": 81,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "# series checked per compaction\nhistogram_quantile(\n    0.99, \n    sum by (le) (\n        rate(loki_bloomcompactor_series_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    )\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p99",
+              "range": true,
+              "refId": "C"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "# series checked per compaction\nhistogram_quantile(\n    0.9, \n    sum by (le) (\n        rate(loki_bloomcompactor_series_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    )\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p90",
+              "range": true,
+              "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "# series checked per compaction\nhistogram_quantile(\n    0.5, \n    sum by (le) (\n        rate(loki_bloomcompactor_series_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    )\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p50",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Series",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 112
+          },
+          "id": 82,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum by (pod) (\n    rate(loki_bloomcompactor_series_per_compaction_sum{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    /\n    rate(loki_bloomcompactor_series_per_compaction_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{pod}}",
+              "range": true,
+              "refId": "C"
+            }
+          ],
+          "title": "avg series per compaction by pod",
+          "type": "timeseries"
+        },
+        {
+          "description": "",
+          "gridPos": {
+            "h": 1,
+            "w": 24,
+            "x": 0,
+            "y": 119
+          },
+          "id": 90,
+          "options": {
+            "code": {
+              "language": "plaintext",
+              "showLineNumbers": false,
+              "showMiniMap": false
+            },
+            "content": "",
+            "mode": "markdown"
+          },
+          "pluginVersion": "11.1.0-70005",
+          "title": "Number of bytes from chunks added to blocks during each compaction.",
+          "transparent": true,
+          "type": "text"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 120
+          },
+          "id": 91,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "# series checked per compaction\nhistogram_quantile(\n    0.99, \n    sum by (le) (\n        rate(loki_bloomcompactor_bytes_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    )\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p99",
+              "range": true,
+              "refId": "C"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "# series checked per compaction\nhistogram_quantile(\n    0.9, \n    sum by (le) (\n        rate(loki_bloomcompactor_bytes_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    )\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p90",
+              "range": true,
+              "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "# series checked per compaction\nhistogram_quantile(\n    0.5, \n    sum by (le) (\n        rate(loki_bloomcompactor_bytes_per_compaction_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    )\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p50",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Bytes",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 120
+          },
+          "id": 92,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum by (pod) (\n    rate(loki_bloomcompactor_bytes_per_compaction_sum{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n    /\n    rate(loki_bloomcompactor_bytes_per_compaction_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"}[$__rate_interval])\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p99",
+              "range": true,
+              "refId": "C"
+            }
+          ],
+          "title": "avg bytes per compaction by pod",
+          "type": "timeseries"
+        }
+      ],
+      "title": "Data processed",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 29
+      },
+      "id": 58,
+      "panels": [
+        {
+          "description": "",
+          "fieldConfig": {
+            "defaults": {},
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 3,
+            "w": 24,
+            "x": 0,
+            "y": 82
+          },
+          "id": 47,
+          "options": {
+            "code": {
+              "language": "plaintext",
+              "showLineNumbers": false,
+              "showMiniMap": false
+            },
+            "content": "\nCompactors write blocks to the attached PVs before flushing them into the object store.\nIt also download chunks and index files.\n\nAfter compacting a given tenant, all the downloaded index files and chunks, as well as the already flushed blocks are deleted.",
+            "mode": "markdown"
+          },
+          "pluginVersion": "11.1.0-69747",
+          "title": "",
+          "transparent": true,
+          "type": "text"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "area"
+                }
+              },
+              "mappings": [],
+              "max": 1,
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 0.8
+                  }
+                ]
+              },
+              "unit": "percentunit"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 85
+          },
+          "id": 9,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "quantile(\n    0.99,\n    max by(persistentvolumeclaim) (\n        kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} \n        / \n        kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n    ) \n    and \n    count by(persistentvolumeclaim) (\n        kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"bloom-compactor\"}\n    )\n)",
+              "instant": false,
+              "legendFormat": "p99",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "quantile(\n    0.90,\n    max by(persistentvolumeclaim) (\n        kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} \n        / \n        kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n    ) \n    and \n    count by(persistentvolumeclaim) (\n        kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"bloom-compactor\"}\n    )\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p90",
+              "range": true,
+              "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "quantile(\n    0.50,\n    max by(persistentvolumeclaim) (\n        kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} \n        / \n        kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n    ) \n    and \n    count by(persistentvolumeclaim) (\n        kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"bloom-compactor\"}\n    )\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p50",
+              "range": true,
+              "refId": "C"
+            }
+          ],
+          "title": "Disk Utilization",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "area"
+                }
+              },
+              "mappings": [],
+              "max": 1,
+              "min": 0,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 0.8
+                  }
+                ]
+              },
+              "unit": "percentunit"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 85
+          },
+          "id": 100,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"bloom-compactor\"})",
+              "instant": false,
+              "legendFormat": "{{pod}}",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Disk Utilization per pod",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 93
+          },
+          "id": 7,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "quantile(\n    0.99,\n    sum by(instance, pod, device) (\n        rate(node_disk_written_bytes_total[$__rate_interval])\n    ) \n    + ignoring(pod) group_right() \n    (\n        label_replace(\n            count by(instance, pod, device) (\n                container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n            ), \n            \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n        ) * 0\n    )\n)",
+              "instant": false,
+              "legendFormat": "p99",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "quantile(\n    0.90,\n    sum by(instance, pod, device) (\n        rate(node_disk_written_bytes_total[$__rate_interval])\n    ) \n    + ignoring(pod) group_right() \n    (\n        label_replace(\n            count by(instance, pod, device) (\n                container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n            ), \n            \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n        ) * 0\n    )\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p90",
+              "range": true,
+              "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "quantile(\n    0.50,\n    sum by(instance, pod, device) (\n        rate(node_disk_written_bytes_total[$__rate_interval])\n    ) \n    + ignoring(pod) group_right() \n    (\n        label_replace(\n            count by(instance, pod, device) (\n                container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n            ), \n            \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n        ) * 0\n    )\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p50",
+              "range": true,
+              "refId": "C"
+            }
+          ],
+          "title": "Disk Writes",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 93
+          },
+          "id": 101,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)",
+              "instant": false,
+              "legendFormat": "{{pod}}",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Disk Writes per pod",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 100
+          },
+          "id": 8,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "quantile(\n    0.99,\n    sum by(instance, pod, device) (\n        rate(node_disk_read_bytes_total[$__rate_interval])\n    ) + ignoring(pod) group_right()\n    (\n        label_replace(\n            count by(instance, pod, device) (\n                container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n            ), \n            \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n        ) * 0\n    )\n)",
+              "instant": false,
+              "legendFormat": "p99",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "quantile(\n    0.90,\n    sum by(instance, pod, device) (\n        rate(node_disk_read_bytes_total[$__rate_interval])\n    ) + ignoring(pod) group_right()\n    (\n        label_replace(\n            count by(instance, pod, device) (\n                container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n            ), \n            \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n        ) * 0\n    )\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p90",
+              "range": true,
+              "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "quantile(\n    0.50,\n    sum by(instance, pod, device) (\n        rate(node_disk_read_bytes_total[$__rate_interval])\n    ) + ignoring(pod) group_right()\n    (\n        label_replace(\n            count by(instance, pod, device) (\n                container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}\n            ), \n            \"device\", \"$1\", \"device\", \"/dev/(.*)\"\n        ) * 0\n    )\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p50",
+              "range": true,
+              "refId": "C"
+            }
+          ],
+          "title": "Disk Reads",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 100
+          },
+          "id": 102,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-compactor\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)",
+              "instant": false,
+              "legendFormat": "{{pod}}",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Disk Reads per pod",
+          "type": "timeseries"
+        }
+      ],
+      "title": "Disk Usage",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 30
+      },
+      "id": 62,
+      "panels": [
+        {
+          "description": "",
+          "fieldConfig": {
+            "defaults": {},
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 3,
+            "w": 24,
+            "x": 0,
+            "y": 83
+          },
+          "id": 71,
+          "options": {
+            "code": {
+              "language": "plaintext",
+              "showLineNumbers": false,
+              "showMiniMap": false
+            },
+            "content": "Once all blocks and metas are built locally, the compactor flushes them to the object store.\n\nAfter each iteration, the compactor deletes the metas and blocks marked for deletion in the tombstones.",
+            "mode": "markdown"
+          },
+          "pluginVersion": "11.1.0-69747",
+          "title": "",
+          "transparent": true,
+          "type": "text"
+        },
+        {
+          "description": "",
+          "fieldConfig": {
+            "defaults": {},
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 2,
+            "x": 0,
+            "y": 86
+          },
+          "id": 63,
+          "options": {
+            "code": {
+              "language": "plaintext",
+              "showLineNumbers": false,
+              "showMiniMap": false
+            },
+            "content": "---\n#### GCS\n",
+            "mode": "markdown"
+          },
+          "pluginVersion": "11.1.0-69747",
+          "title": "",
+          "transparent": true,
+          "type": "text"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 25,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 11,
+            "x": 2,
+            "y": 86
+          },
+          "id": 61,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum by (status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval]))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{operation}} {{status_code}}",
+              "range": true,
+              "refId": "B"
+            }
+          ],
+          "title": "QPS",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 25,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 11,
+            "x": 13,
+            "y": 86
+          },
+          "id": 64,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{operation}} p99",
+              "range": true,
+              "refId": "D"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{operation}} p90",
+              "range": true,
+              "refId": "E"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{operation}} p50",
+              "range": true,
+              "refId": "F"
+            }
+          ],
+          "title": "Latency",
+          "type": "timeseries"
+        },
+        {
+          "description": "",
+          "fieldConfig": {
+            "defaults": {},
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 2,
+            "x": 0,
+            "y": 93
+          },
+          "id": 65,
+          "options": {
+            "code": {
+              "language": "plaintext",
+              "showLineNumbers": false,
+              "showMiniMap": false
+            },
+            "content": "---\n#### S3\n",
+            "mode": "markdown"
+          },
+          "pluginVersion": "11.1.0-69747",
+          "title": "",
+          "transparent": true,
+          "type": "text"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 25,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 11,
+            "x": 2,
+            "y": 93
+          },
+          "id": 67,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval]))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{operation}} {{status_code}}",
+              "range": true,
+              "refId": "B"
+            }
+          ],
+          "title": "QPS",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 25,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 11,
+            "x": 13,
+            "y": 93
+          },
+          "id": 69,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{operation}} p99",
+              "range": true,
+              "refId": "D"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{operation}} p90",
+              "range": true,
+              "refId": "E"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{operation}} p50",
+              "range": true,
+              "refId": "F"
+            }
+          ],
+          "title": "Latency",
+          "type": "timeseries"
+        },
+        {
+          "description": "",
+          "fieldConfig": {
+            "defaults": {},
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 2,
+            "x": 0,
+            "y": 100
+          },
+          "id": 66,
+          "options": {
+            "code": {
+              "language": "plaintext",
+              "showLineNumbers": false,
+              "showMiniMap": false
+            },
+            "content": "---\n#### Azure\nBlob Storage",
+            "mode": "markdown"
+          },
+          "pluginVersion": "11.1.0-69747",
+          "title": "",
+          "transparent": true,
+          "type": "text"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 25,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 11,
+            "x": 2,
+            "y": 100
+          },
+          "id": 68,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval]))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{operation}} {{status_code}}",
+              "range": true,
+              "refId": "B"
+            }
+          ],
+          "title": "QPS",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 25,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 11,
+            "x": 13,
+            "y": 100
+          },
+          "id": 70,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{operation}} p99",
+              "range": true,
+              "refId": "D"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{operation}} p90",
+              "range": true,
+              "refId": "E"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-compactor\"} [$__rate_interval])))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{operation}} p50",
+              "range": true,
+              "refId": "F"
+            }
+          ],
+          "title": "Latency",
+          "type": "timeseries"
+        }
+      ],
+      "title": "Object Store",
+      "type": "row"
+    }
+  ],
+  "refresh": "",
+  "schemaVersion": 39,
+  "tags": [],
+  "time": {
+    "from": "now-2d",
+    "to": "now"
+  },
+  "timeRangeUpdatedDuringEditOrView": false,
+  "timepicker": {},
+  "timezone": "",
+  "title": "Bloom-Compactor New",
+  "uid": "bdeqksjzwxqf4e",
+  "version": 62,
+  "weekStart": ""
+}
\ No newline at end of file
diff --git a/production/loki-mixin/dashboards/dashboard-bloom-gateway.json b/production/loki-mixin/dashboards/dashboard-bloom-gateway.json
new file mode 100644
index 0000000000000..12492f8f00bb5
--- /dev/null
+++ b/production/loki-mixin/dashboards/dashboard-bloom-gateway.json
@@ -0,0 +1,4931 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 73,
+      "panels": [],
+      "title": "Overview",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "description": "Percentage of chunks that are filtered by using bloom filters",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds",
+            "seriesBy": "last"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineStyle": {
+              "fill": "solid"
+            },
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "area"
+            }
+          },
+          "mappings": [],
+          "max": 1,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 0.5
+              },
+              {
+                "color": "yellow",
+                "value": 0.75
+              },
+              {
+                "color": "green",
+                "value": 0.9
+              }
+            ]
+          },
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 6,
+        "x": 0,
+        "y": 1
+      },
+      "id": 23,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "maxHeight": 600,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(loki_bloom_gateway_filtered_chunks_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_chunks_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))",
+          "instant": false,
+          "legendFormat": "Chunks",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "sum(rate(loki_bloom_gateway_filtered_series_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_series_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "Series",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Filter ratio",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "description": "Percentage of chunks that are filtered by using bloom filters",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "max": 1,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 0.5
+              },
+              {
+                "color": "yellow",
+                "value": 0.75
+              },
+              {
+                "color": "green",
+                "value": 0.9
+              }
+            ]
+          },
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 6,
+        "x": 6,
+        "y": 1
+      },
+      "id": 75,
+      "options": {
+        "minVizHeight": 75,
+        "minVizWidth": 75,
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showThresholdLabels": false,
+        "showThresholdMarkers": true,
+        "sizing": "auto"
+      },
+      "pluginVersion": "11.1.0-70005",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "sum(rate(loki_bloom_gateway_filtered_chunks_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_chunks_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))",
+          "instant": true,
+          "legendFormat": "Chunks",
+          "range": false,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "sum(rate(loki_bloom_gateway_filtered_series_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_series_sum{job=\"$namespace/bloom-gateway\"}[$__rate_interval]))",
+          "hide": false,
+          "instant": true,
+          "legendFormat": "Series",
+          "range": false,
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "sum(loki_bloom_gateway_filtered_chunks_sum{job=\"$namespace/bloom-gateway\"})\n/\nsum(loki_bloom_gateway_requested_chunks_sum{job=\"$namespace/bloom-gateway\"})",
+          "hide": true,
+          "instant": true,
+          "legendFormat": "Chunks avg",
+          "range": false,
+          "refId": "C"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "sum(loki_bloom_gateway_filtered_series_sum{job=\"$namespace/bloom-gateway\"})\n/\nsum(loki_bloom_gateway_requested_series_sum{job=\"$namespace/bloom-gateway\"})",
+          "hide": true,
+          "instant": true,
+          "legendFormat": "Series avg",
+          "range": false,
+          "refId": "D"
+        }
+      ],
+      "title": "Filter ratio",
+      "type": "gauge"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineStyle": {
+              "fill": "solid"
+            },
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Desired"
+            },
+            "properties": [
+              {
+                "id": "custom.fillOpacity",
+                "value": 0
+              },
+              {
+                "id": "custom.lineStyle",
+                "value": {
+                  "dash": [
+                    10,
+                    10
+                  ],
+                  "fill": "dash"
+                }
+              },
+              {
+                "id": "custom.lineWidth",
+                "value": 2
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 6,
+        "x": 12,
+        "y": 1
+      },
+      "id": 72,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "maxHeight": 600,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "kube_statefulset_status_replicas_ready{cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"bloom-gateway\"}",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "Ready",
+          "range": true,
+          "refId": "D"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum(kube_pod_container_status_ready{container=\"bloom-gateway\", cluster=\"$cluster\", namespace=\"$namespace\"})",
+          "hide": true,
+          "instant": false,
+          "legendFormat": "Running",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "kube_statefulset_replicas{cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"bloom-gateway\"}",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "Desired",
+          "range": true,
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "kube_statefulset_status_replicas_available{cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"bloom-gateway\"}",
+          "hide": true,
+          "instant": false,
+          "legendFormat": "Available",
+          "range": true,
+          "refId": "C"
+        }
+      ],
+      "title": "Readiness",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${datasource}"
+      },
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 50,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineStyle": {
+              "fill": "solid"
+            },
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 6,
+        "x": 18,
+        "y": 1
+      },
+      "id": 37,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "maxHeight": 600,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "editorMode": "code",
+          "expr": "(\n    max by (pod, reason) (kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})\n    * on (pod) group_left\n    sum by (pod) (increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))\n) > 0",
+          "hide": false,
+          "instant": false,
+          "interval": "",
+          "legendFormat": "{{pod}} ({{reason}})",
+          "range": true,
+          "refId": "C"
+        }
+      ],
+      "title": "Container restarts",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${loki_datasource}"
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 15,
+        "x": 0,
+        "y": 7
+      },
+      "id": 48,
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": false,
+        "sortOrder": "Descending",
+        "wrapLogMessage": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${loki_datasource}"
+          },
+          "editorMode": "code",
+          "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"level=error\" or \"panic:\" | logfmt",
+          "queryType": "range",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${loki_datasource}"
+          },
+          "editorMode": "code",
+          "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"level=warn\" | logfmt",
+          "hide": true,
+          "queryType": "range",
+          "refId": "B"
+        }
+      ],
+      "title": "Errors",
+      "type": "logs"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "${loki_datasource}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "red",
+            "mode": "fixed"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "bars",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "log": 2,
+              "type": "symlog"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "warn"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "orange",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "error"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "red",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "panic"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "semi-dark-red",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 9,
+        "x": 15,
+        "y": 7
+      },
+      "id": 52,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "maxHeight": 600,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${loki_datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum by (level) (count_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |~ \"level=(warn|error)\" | logfmt [$__auto]))",
+          "legendFormat": "{{ level }}",
+          "queryType": "range",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${loki_datasource}"
+          },
+          "editorMode": "code",
+          "expr": "sum (count_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"panic:\" | logfmt [$__auto]))",
+          "hide": false,
+          "legendFormat": "panic",
+          "queryType": "range",
+          "refId": "B"
+        }
+      ],
+      "title": "Errors Rate",
+      "type": "timeseries"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 16
+      },
+      "id": 56,
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 14,
+            "w": 12,
+            "x": 0,
+            "y": 17
+          },
+          "id": 10,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"}[$__rate_interval]))",
+              "instant": false,
+              "legendFormat": "{{pod}}",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\", resource=\"cpu\"} > 0)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Request",
+              "range": true,
+              "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "min(kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\", resource=\"cpu\"} > 0)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Limit",
+              "range": true,
+              "refId": "C"
+            }
+          ],
+          "title": "CPU",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 17
+          },
+          "id": 11,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"})",
+              "instant": false,
+              "legendFormat": "{{pod}}",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\", resource=\"memory\"} > 0)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Request",
+              "range": true,
+              "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"} > 0)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Limit",
+              "range": true,
+              "refId": "C"
+            }
+          ],
+          "title": "Memory (workingset)",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 24
+          },
+          "id": 81,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"})",
+              "hide": true,
+              "instant": false,
+              "legendFormat": "{{pod}}",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\", resource=\"memory\"} > 0)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Request",
+              "range": true,
+              "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"} > 0)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Limit",
+              "range": true,
+              "refId": "C"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum(go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=~\"bloom-gateway\"}) by (pod)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "D"
+            }
+          ],
+          "title": "Memory (heap inuse)",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 10,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineStyle": {
+                  "fill": "solid"
+                },
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 8,
+            "x": 0,
+            "y": 31
+          },
+          "id": 87,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum by (pod) (rate(go_gc_cycles_total_gc_cycles_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "B"
+            }
+          ],
+          "title": "GC rate",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 10,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineStyle": {
+                  "fill": "solid"
+                },
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 8,
+            "x": 8,
+            "y": 31
+          },
+          "id": 88,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum by (pod) (rate(go_gc_duration_seconds_sum{container=\"bloom-gateway\"}[$__rate_interval]))\n/\nsum by (pod) (rate(go_gc_duration_seconds_count{container=\"bloom-gateway\"}[$__rate_interval]))",
+              "hide": false,
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "GC duration",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 10,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineStyle": {
+                  "fill": "solid"
+                },
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 8,
+            "x": 16,
+            "y": 31
+          },
+          "id": 89,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.99, sum(rate(go_gc_pauses_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))",
+              "hide": false,
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.90, sum(rate(go_gc_pauses_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))",
+              "hide": false,
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "B"
+            }
+          ],
+          "title": "GC pauses",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 10,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "binBps"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 38
+          },
+          "id": 84,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum by(instance, pod) (rate(node_disk_read_bytes_total[$__rate_interval]))\n+ ignoring(pod) group_right() \n(count by(instance, pod) (container_fs_reads_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}) * 0)",
+              "hide": false,
+              "instant": false,
+              "interval": "",
+              "legendFormat": "{{pod}}",
+              "range": true,
+              "refId": "D"
+            }
+          ],
+          "title": "Disk reads",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 10,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "binBps"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 38
+          },
+          "id": 85,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum by(instance, pod) (rate(node_disk_written_bytes_total[$__rate_interval]))\n+ ignoring(pod) group_right() \n(count by(instance, pod) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"bloom-gateway\", device!~\".*sda.*\"}) * 0)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{pod}}",
+              "range": true,
+              "refId": "D"
+            }
+          ],
+          "title": "Disk writes",
+          "type": "timeseries"
+        }
+      ],
+      "title": "Resource usage",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 17
+      },
+      "id": 2,
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 100,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 0,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "normal"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "cancel"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "success"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "green",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "error"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "red",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 18
+          },
+          "id": 13,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum by (status_code) (\n    rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/bloom-gateway\", route=\"/logproto.BloomGateway/FilterChunkRefs\"}[$__rate_interval])\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "B"
+            }
+          ],
+          "title": "QPS",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              }
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "cancel"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "success"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "green",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "error"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "red",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 18
+          },
+          "id": 86,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum by (pod) (\n    rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/bloom-gateway\", route=\"/logproto.BloomGateway/FilterChunkRefs\"}[$__rate_interval])\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "B"
+            }
+          ],
+          "title": "QPS per Pod",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 10,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 25
+          },
+          "id": 14,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.50, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"/logproto.BloomGateway/FilterChunkRefs\"}))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{ route }} 50th percentile",
+              "range": true,
+              "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum(cluster_job_route:loki_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"/logproto.BloomGateway/FilterChunkRefs\"}) by (route)  / sum(cluster_job_route:loki_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"/logproto.BloomGateway/FilterChunkRefs\"}) by (route) ",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{ route }} Average",
+              "range": true,
+              "refId": "C"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"/logproto.BloomGateway/FilterChunkRefs\"}))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{ route }} 99th percentile",
+              "range": true,
+              "refId": "D"
+            }
+          ],
+          "title": "Latency",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 10,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 25
+          },
+          "id": 15,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.99,\n  sum(\n   rate(loki_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/bloom-gateway\", route=~\"/logproto.BloomGateway/FilterChunkRefs\"}[$__rate_interval])\n   ) by (pod, le)\n )\n",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "B"
+            }
+          ],
+          "title": "Per Pod Latency (p99)",
+          "type": "timeseries"
+        }
+      ],
+      "title": "QPS and Latency",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 18
+      },
+      "id": 58,
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 25,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "normal"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 8,
+            "x": 0,
+            "y": 11
+          },
+          "id": 16,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum by (pod) (loki_bloom_gateway_queue_duration_seconds_sum{cluster=\"$cluster\", namespace=\"$namespace\"})\n/\nsum by (pod) (loki_bloom_gateway_queue_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"})\n",
+              "hide": true,
+              "instant": false,
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "C"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum (loki_bloom_gateway_queue_length{cluster=\"$cluster\", namespace=\"$namespace\"})",
+              "hide": true,
+              "instant": false,
+              "legendFormat": "Total",
+              "range": true,
+              "refId": "D"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum by (user) (loki_bloom_gateway_queue_length{cluster=\"$cluster\", namespace=\"$namespace\"})",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{user}}",
+              "range": true,
+              "refId": "B"
+            }
+          ],
+          "title": "Queue Size",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "For how long do pending tasks stay in the queue",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 1,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 8,
+            "x": 8,
+            "y": 11
+          },
+          "id": 17,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.99, sum by (le) (rate(loki_bloom_gateway_queue_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p99",
+              "range": true,
+              "refId": "E"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.90, sum by (le) (rate(loki_bloom_gateway_queue_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))",
+              "hide": true,
+              "instant": false,
+              "legendFormat": "p90",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.50, sum by (le) (rate(loki_bloom_gateway_queue_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p50",
+              "range": true,
+              "refId": "C"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum (loki_bloom_gateway_queue_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\"})\n/\nsum (loki_bloom_gateway_queue_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\"})",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "avg",
+              "range": true,
+              "refId": "D"
+            }
+          ],
+          "title": "Queue Latency",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "Inflight requests tracks all tasks both queued and in progress",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 1,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 8,
+            "x": 16,
+            "y": 11
+          },
+          "id": 22,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum by (quantile) (loki_bloom_gateway_inflight_tasks{cluster=\"$cluster\", namespace=\"$namespace\", quantile=\"0.99\"})",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "Inflight tasks",
+          "type": "timeseries"
+        }
+      ],
+      "title": "Task Queue",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 19
+      },
+      "id": 68,
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 8,
+            "x": 0,
+            "y": 12
+          },
+          "id": 69,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.99, sum(rate(loki_bloom_gateway_process_duration_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))",
+              "instant": false,
+              "legendFormat": "{{status}}-p99",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_gateway_process_duration_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{status}}-p95",
+              "range": true,
+              "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.90, sum(rate(loki_bloom_gateway_process_duration_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{status}}-p90",
+              "range": true,
+              "refId": "C"
+            }
+          ],
+          "title": "Processing time for tasks (per worker iteration)",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 8,
+            "x": 8,
+            "y": 12
+          },
+          "id": 70,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.99, sum(rate(loki_bloom_gateway_block_query_latency_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))",
+              "instant": false,
+              "legendFormat": "{{status}}-p99",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_gateway_block_query_latency_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{status}}-p95",
+              "range": true,
+              "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.90, sum(rate(loki_bloom_gateway_block_query_latency_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{status}}-p90",
+              "range": true,
+              "refId": "C"
+            }
+          ],
+          "title": "Block query latency (single block)",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 8,
+            "x": 16,
+            "y": 12
+          },
+          "id": 71,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum(rate(loki_bloom_gateway_tasks_dequeued_total{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval]))",
+              "instant": false,
+              "legendFormat": "dequeued",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum by (status) (rate(loki_bloom_gateway_tasks_processed_total{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval]))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "processed {{status}}",
+              "range": true,
+              "refId": "B"
+            }
+          ],
+          "title": "Tasks dequeued/processed",
+          "type": "timeseries"
+        }
+      ],
+      "title": "Processing",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 20
+      },
+      "id": 59,
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "gridPos": {
+            "h": 1,
+            "w": 24,
+            "x": 0,
+            "y": 13
+          },
+          "id": 19,
+          "options": {
+            "code": {
+              "language": "plaintext",
+              "showLineNumbers": false,
+              "showMiniMap": false
+            },
+            "content": "",
+            "mode": "markdown"
+          },
+          "pluginVersion": "11.1.0-70005",
+          "title": "We cache bloom blocks in memory to prevent the gateway from hitting the object store too often",
+          "transparent": true,
+          "type": "text"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineStyle": {
+                  "fill": "solid"
+                },
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "fieldMinMax": false,
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 14
+          },
+          "id": 20,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum(loki_embeddedcache_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", cache=\"bloom-blocks-cache\", container=\"bloom-gateway\"}) by (pod)",
+              "hide": true,
+              "instant": false,
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum(loki_bloom_blocks_cache_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}) by (pod)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "B"
+            }
+          ],
+          "title": "Cache size (per pod)",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineStyle": {
+                  "fill": "solid"
+                },
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "linearThreshold": 1000,
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "fieldMinMax": false,
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Size"
+                },
+                "properties": [
+                  {
+                    "id": "unit",
+                    "value": "bytes"
+                  },
+                  {
+                    "id": "custom.fillOpacity",
+                    "value": 25
+                  },
+                  {
+                    "id": "custom.lineWidth",
+                    "value": 0
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Items"
+                },
+                "properties": [
+                  {
+                    "id": "custom.fillOpacity",
+                    "value": 25
+                  },
+                  {
+                    "id": "custom.lineWidth",
+                    "value": 0
+                  },
+                  {
+                    "id": "unit",
+                    "value": ""
+                  },
+                  {
+                    "id": "custom.axisSoftMin",
+                    "value": 0
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 14
+          },
+          "id": 83,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum(loki_bloom_blocks_cache_entries{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Items",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum(rate(loki_bloom_blocks_cache_added_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Added",
+              "range": true,
+              "refId": "G"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum(rate(loki_bloom_blocks_cache_evicted_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (reason)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Evicted ({{reason}})",
+              "range": true,
+              "refId": "F"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum(rate(loki_bloom_blocks_cache_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))",
+              "hide": true,
+              "instant": false,
+              "legendFormat": "Size",
+              "range": true,
+              "refId": "E"
+            }
+          ],
+          "title": "Cache rate",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 100,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineStyle": {
+                  "fill": "solid"
+                },
+                "lineWidth": 0,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "linearThreshold": 1000,
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "normal"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "fieldMinMax": false,
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  }
+                ]
+              },
+              "unit": "percentunit"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "hit"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "green",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              },
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "miss"
+                },
+                "properties": [
+                  {
+                    "id": "color",
+                    "value": {
+                      "fixedColor": "red",
+                      "mode": "fixed"
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 24,
+            "x": 0,
+            "y": 21
+          },
+          "id": 92,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum by (status) (\n    rate(loki_bloom_blocks_cache_fetched_total{container=\"bloom-gateway\"}[$__rate_interval])\n)\n/ ignoring(status) group_left\nsum (\n    rate(loki_bloom_blocks_cache_fetched_total{container=\"bloom-gateway\"}[$__rate_interval])\n)",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "B"
+            }
+          ],
+          "title": "Hit/Miss ratio",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineStyle": {
+                  "fill": "solid"
+                },
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "fieldMinMax": false,
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  }
+                ]
+              },
+              "unit": "bytes"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/Size (.*)/"
+                },
+                "properties": [
+                  {
+                    "id": "unit",
+                    "value": "bytes"
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 0,
+            "y": 28
+          },
+          "id": 76,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum(loki_embeddedcache_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", cache=\"bloom-blocks-cache\", container=\"bloom-gateway\"})\n/\nsum(loki_embeddedcache_entries{cluster=\"$cluster\", namespace=\"$namespace\", cache=\"bloom-blocks-cache\", container=\"bloom-gateway\"})",
+              "instant": false,
+              "legendFormat": "Size",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum(loki_bloom_blocks_cache_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})\n/\nsum(loki_bloom_blocks_cache_entries{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "Size",
+              "range": true,
+              "refId": "B"
+            }
+          ],
+          "title": "Average item size",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineStyle": {
+                  "fill": "solid"
+                },
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "fieldMinMax": false,
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byRegexp",
+                  "options": "/.* (blocks|metas) size/"
+                },
+                "properties": [
+                  {
+                    "id": "unit",
+                    "value": "bytes"
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 12,
+            "x": 12,
+            "y": 28
+          },
+          "id": 21,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum(rate(loki_bloom_store_metas_fetched_sum{cluster=\"$cluster\",namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "metas fetch rate",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum(rate(loki_bloom_store_blocks_fetched_sum{cluster=\"$cluster\",namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "blocks fetch rate",
+              "range": true,
+              "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.9, sum(rate(loki_bloom_store_blocks_fetched_size_bytes_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p90 blocks size",
+              "range": true,
+              "refId": "C"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.9, sum(rate(loki_bloom_store_metas_fetched_size_bytes_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "p90 metas size",
+              "range": true,
+              "refId": "D"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(1.0, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))",
+              "hide": true,
+              "instant": false,
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "E"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))",
+              "hide": true,
+              "instant": false,
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "F"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.5, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))",
+              "hide": true,
+              "instant": false,
+              "legendFormat": "__auto",
+              "range": true,
+              "refId": "G"
+            }
+          ],
+          "title": "Bloom Store",
+          "type": "timeseries"
+        }
+      ],
+      "title": "Blocks Cache",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 21
+      },
+      "id": 60,
+      "panels": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "gridPos": {
+            "h": 1,
+            "w": 24,
+            "x": 0,
+            "y": 14
+          },
+          "id": 61,
+          "options": {
+            "code": {
+              "language": "plaintext",
+              "showLineNumbers": false,
+              "showMiniMap": false
+            },
+            "content": "",
+            "mode": "markdown"
+          },
+          "pluginVersion": "11.1.0-70005",
+          "title": "The gateway download bloom meta files and blocks from the object store.",
+          "transparent": true,
+          "type": "text"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "gridPos": {
+            "h": 7,
+            "w": 2,
+            "x": 0,
+            "y": 15
+          },
+          "id": 24,
+          "options": {
+            "code": {
+              "language": "plaintext",
+              "showLineNumbers": false,
+              "showMiniMap": false
+            },
+            "content": "---\n#### GCS\n",
+            "mode": "markdown"
+          },
+          "pluginVersion": "11.1.0-70005",
+          "transparent": true,
+          "type": "text"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 25,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "normal"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 11,
+            "x": 2,
+            "y": 15
+          },
+          "id": 25,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum by (status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval]))",
+              "instant": false,
+              "legendFormat": "{{operation}} {{status_code}}",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "QPS",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 11,
+            "x": 13,
+            "y": 15
+          },
+          "id": 29,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))",
+              "instant": false,
+              "legendFormat": "{{operation}} p99",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{operation}} p90",
+              "range": true,
+              "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{operation}} p50",
+              "range": true,
+              "refId": "C"
+            }
+          ],
+          "title": "Latency",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "gridPos": {
+            "h": 7,
+            "w": 2,
+            "x": 0,
+            "y": 22
+          },
+          "id": 62,
+          "options": {
+            "code": {
+              "language": "plaintext",
+              "showLineNumbers": false,
+              "showMiniMap": false
+            },
+            "content": "---\n#### S3\n",
+            "mode": "markdown"
+          },
+          "pluginVersion": "11.1.0-70005",
+          "transparent": true,
+          "type": "text"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 25,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "normal"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 11,
+            "x": 2,
+            "y": 22
+          },
+          "id": 63,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval]))",
+              "instant": false,
+              "legendFormat": "{{operation}} {{status_code}}",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "QPS",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 25,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "normal"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 11,
+            "x": 13,
+            "y": 22
+          },
+          "id": 64,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))",
+              "instant": false,
+              "legendFormat": "{{operation}} p99",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{operation}} p90",
+              "range": true,
+              "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{operation}} p50",
+              "range": true,
+              "refId": "C"
+            }
+          ],
+          "title": "Latency",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "gridPos": {
+            "h": 7,
+            "w": 2,
+            "x": 0,
+            "y": 29
+          },
+          "id": 65,
+          "options": {
+            "code": {
+              "language": "plaintext",
+              "showLineNumbers": false,
+              "showMiniMap": false
+            },
+            "content": "---\n#### Azure\nBlob Storage\n\n",
+            "mode": "markdown"
+          },
+          "pluginVersion": "11.1.0-70005",
+          "transparent": true,
+          "type": "text"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 25,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "normal"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 11,
+            "x": 2,
+            "y": 29
+          },
+          "id": 66,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval]))",
+              "instant": false,
+              "legendFormat": "{{operation}} {{status_code}}",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "QPS",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 25,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "normal"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  }
+                ]
+              },
+              "unit": "none"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 7,
+            "w": 11,
+            "x": 13,
+            "y": 29
+          },
+          "id": 67,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))",
+              "instant": false,
+              "legendFormat": "{{operation}} p99",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{operation}} p90",
+              "range": true,
+              "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} [$__rate_interval])))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "{{operation}} p50",
+              "range": true,
+              "refId": "C"
+            }
+          ],
+          "title": "Latency",
+          "type": "timeseries"
+        }
+      ],
+      "title": "Object Store",
+      "type": "row"
+    },
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 22
+      },
+      "id": 77,
+      "panels": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${loki_datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 10,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineStyle": {
+                  "fill": "solid"
+                },
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 23
+          },
+          "id": 78,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "loki",
+                "uid": "${loki_datasource}"
+              },
+              "editorMode": "code",
+              "expr": "topk(3, sum by (tasks) (count_over_time({namespace=\"loki-dev-006\", container=\"bloom-gateway\"} |= \"process tasks with bounds\" | logfmt [5s])))",
+              "legendFormat": "{{tasks}}",
+              "queryType": "range",
+              "refId": "A"
+            }
+          ],
+          "title": "Process tasks with bounds",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${loki_datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 10,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineStyle": {
+                  "fill": "solid"
+                },
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": [
+              {
+                "__systemRef": "hideSeriesFrom",
+                "matcher": {
+                  "id": "byNames",
+                  "options": {
+                    "mode": "exclude",
+                    "names": [
+                      "max",
+                      "avg"
+                    ],
+                    "prefix": "All except:",
+                    "readOnly": true
+                  }
+                },
+                "properties": [
+                  {
+                    "id": "custom.hideFrom",
+                    "value": {
+                      "legend": false,
+                      "tooltip": false,
+                      "viz": true
+                    }
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 23
+          },
+          "id": 79,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "loki",
+                "uid": "${loki_datasource}"
+              },
+              "editorMode": "code",
+              "expr": "max(max_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"fetch blocks\" | logfmt | unwrap duration(duration) [$__auto]))",
+              "hide": false,
+              "legendFormat": "max",
+              "queryType": "range",
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "loki",
+                "uid": "${loki_datasource}"
+              },
+              "editorMode": "code",
+              "expr": "avg(avg_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"fetch blocks\" | logfmt | unwrap duration(duration) [$__auto]))",
+              "hide": false,
+              "legendFormat": "avg",
+              "queryType": "range",
+              "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "loki",
+                "uid": "${loki_datasource}"
+              },
+              "editorMode": "code",
+              "expr": "avg(avg_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"request unavailable blocks in the background\" | logfmt | missing > 0 | unwrap missing [$__auto]))",
+              "hide": false,
+              "legendFormat": "avg missing",
+              "queryType": "range",
+              "refId": "C"
+            }
+          ],
+          "title": "Download enqueue duration",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "${loki_datasource}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "fixedColor": "green",
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "fillOpacity": 80,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineWidth": 1,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 100
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 31
+          },
+          "id": 80,
+          "options": {
+            "barRadius": 0,
+            "barWidth": 0.97,
+            "fullHighlight": false,
+            "groupWidth": 0.7,
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "orientation": "horizontal",
+            "showValue": "auto",
+            "stacking": "none",
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            },
+            "xTickLabelRotation": 0,
+            "xTickLabelSpacing": 0
+          },
+          "pluginVersion": "11.0.0-67814",
+          "targets": [
+            {
+              "datasource": {
+                "type": "loki",
+                "uid": "${loki_datasource}"
+              },
+              "editorMode": "code",
+              "expr": "sort_desc(topk(10, sum by (tasks) (count_over_time({namespace=\"loki-dev-006\", container=\"bloom-gateway\"} |= \"process tasks with bounds\" | logfmt [$__auto]))))",
+              "legendFormat": "",
+              "queryType": "instant",
+              "refId": "A"
+            }
+          ],
+          "title": "Tasks multiplexed",
+          "type": "barchart"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${datasource}"
+          },
+          "description": "",
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 10,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineStyle": {
+                  "fill": "solid"
+                },
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": [
+              {
+                "matcher": {
+                  "id": "byName",
+                  "options": "Enqueue latency"
+                },
+                "properties": [
+                  {
+                    "id": "unit",
+                    "value": "s"
+                  }
+                ]
+              }
+            ]
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 31
+          },
+          "id": 82,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "maxHeight": 600,
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.99, sum(rate(loki_bloom_store_download_queue_enqueue_time_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))",
+              "hide": false,
+              "legendFormat": "Enqueue latency",
+              "range": true,
+              "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${datasource}"
+              },
+              "editorMode": "code",
+              "expr": "histogram_quantile(0.99, sum(rate(loki_bloom_store_download_queue_size_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))",
+              "hide": false,
+              "legendFormat": "Size",
+              "range": true,
+              "refId": "B"
+            }
+          ],
+          "title": "Block download queue",
+          "type": "timeseries"
+        }
+      ],
+      "title": "Misc",
+      "type": "row"
+    }
+  ],
+  "refresh": "10s",
+  "schemaVersion": 39,
+  "tags": [],
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timeRangeUpdatedDuringEditOrView": false,
+  "timepicker": {},
+  "timezone": "utc",
+  "title": "Bloom-Gateway",
+  "uid": "c495441a-1639-4ee2-9a32-42488dc5f81d",
+  "version": 208,
+  "weekStart": ""
+}
\ No newline at end of file
diff --git a/production/loki-mixin/dashboards/loki-bloom-compactor.libsonnet b/production/loki-mixin/dashboards/loki-bloom-compactor.libsonnet
new file mode 100644
index 0000000000000..cf93c7c992c53
--- /dev/null
+++ b/production/loki-mixin/dashboards/loki-bloom-compactor.libsonnet
@@ -0,0 +1,19 @@
+local raw = (import './dashboard-bloom-compactor.json');
+
+// !--- HOW TO UPDATE THIS DASHBOARD ---!
+// 1. Export the dashboard from Grafana as JSON
+//    !NOTE: Make sure you collapse all rows but the (first) Overview row.
+// 2. Copy the JSON into `dashboard-bloom-compactor.json`
+// 3. Delete the `id` and `templating` fields from the JSON
+(import 'dashboard-utils.libsonnet') {
+  grafanaDashboards+:
+    {
+      'loki-bloom-compactor.json':
+        raw +
+        $.dashboard('Loki / Bloom Compactor', uid='bloom-compactor')
+        .addCluster()
+        .addNamespace()
+        .addLog()
+        .addTag(),
+    },
+}
diff --git a/production/loki-mixin/dashboards/loki-bloom-gateway.libsonnet b/production/loki-mixin/dashboards/loki-bloom-gateway.libsonnet
new file mode 100644
index 0000000000000..e5ca9f2ff3fd8
--- /dev/null
+++ b/production/loki-mixin/dashboards/loki-bloom-gateway.libsonnet
@@ -0,0 +1,19 @@
+local raw = (import './dashboard-bloom-gateway.json');
+
+// !--- HOW TO UPDATE THIS DASHBOARD ---!
+// 1. Export the dashboard from Grafana as JSON
+//    !NOTE: Make sure you collapse all rows but the (first) Overview row.
+// 2. Copy the JSON into `dashboard-bloom-gateway.json`
+// 3. Delete the `id` and `templating` fields from the JSON
+(import 'dashboard-utils.libsonnet') {
+  grafanaDashboards+:
+    {
+      'loki-bloom-gateway.json':
+        raw +
+        $.dashboard('Loki / Bloom Gateway', uid='bloom-gateway')
+        .addCluster()
+        .addNamespace()
+        .addLog()
+        .addTag(),
+    },
+}
diff --git a/tools/packaging/loki.service b/tools/packaging/loki.service
index b0abbb719103d..2885fac2438ef 100644
--- a/tools/packaging/loki.service
+++ b/tools/packaging/loki.service
@@ -1,6 +1,7 @@
 [Unit]
 Description=Loki service
-After=network.target
+After=network-online.target
+Wants=network-online.target
 
 [Service]
 Type=simple
@@ -12,4 +13,4 @@ Restart = on-failure
 RestartSec = 2
 
 [Install]
-WantedBy=multi-user.target
\ No newline at end of file
+WantedBy=multi-user.target
diff --git a/tools/packaging/promtail.service b/tools/packaging/promtail.service
index a21768d1102d7..258e40a24a05d 100644
--- a/tools/packaging/promtail.service
+++ b/tools/packaging/promtail.service
@@ -1,6 +1,7 @@
 [Unit]
 Description=Promtail service
-After=network.target
+After=network-online.target
+Wants=network-online.target
 
 [Service]
 Type=simple
@@ -12,4 +13,4 @@ Restart = on-failure
 RestartSec = 2
 
 [Install]
-WantedBy=multi-user.target
\ No newline at end of file
+WantedBy=multi-user.target