From 37e38164d50584e5d34e1e3ff41ab4199719c33e Mon Sep 17 00:00:00 2001 From: Andy Asp <90626759+andyasp@users.noreply.github.com> Date: Wed, 26 Jan 2022 14:07:19 -0500 Subject: [PATCH] Change the default of ignore_deletion_mark_delay to 1h (#892) * Change the default of ignore_deletion_mark_delay to 1h * Remove reference to actual default value in docs --- CHANGELOG.md | 1 + docs/sources/blocks-storage/production-tips.md | 2 +- docs/sources/blocks-storage/querier.md | 5 ++--- docs/sources/blocks-storage/store-gateway.md | 5 ++--- docs/sources/configuration/config-file-reference.md | 5 ++--- .../mimir-tests/test-disable-chunk-streaming-generated.yaml | 3 --- operations/mimir-tests/test-query-sharding-generated.yaml | 3 --- operations/mimir-tests/test-shuffle-sharding-generated.yaml | 3 --- operations/mimir-tests/test-storage-azure-generated.yaml | 3 --- operations/mimir-tests/test-storage-gcs-generated.yaml | 3 --- operations/mimir-tests/test-storage-s3-generated.yaml | 3 --- operations/mimir/config.libsonnet | 1 - pkg/storage/tsdb/config.go | 5 ++--- 13 files changed, 10 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e51ef847962..4558216fe39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -219,6 +219,7 @@ * `-ruler.sharding-strategy` option has been removed from ruler. Ruler now uses shuffle-sharding by default, but respects `ruler_tenant_shard_size`, which defaults to 0 (ie. use all rulers for tenant). #889 * `-store-gateway.sharding-strategy` option has been removed store-gateways. Store-gateway now uses shuffle-sharding by default, but respects `store_gateway_tenant_shard_size` for tenant, and this value defaults to 0. #891 * [CHANGE] Server: `-server.http-listen-port` (yaml: `server.http_listen_port`) now defaults to `8080` (previously `80`). #871 +* [CHANGE] Changed the default value of `blocks-storage.bucket-store.ignore-deletion-marks-delay` from 6h to 1h. #892 * [FEATURE] Query Frontend: Add `cortex_query_fetched_chunks_total` per-user counter to expose the number of chunks fetched as part of queries. This metric can be enabled with the `-frontend.query-stats-enabled` flag (or its respective YAML config option `query_stats_enabled`). #31 * [FEATURE] Query Frontend: Add experimental querysharding for the blocks storage (instant and range queries). You can now enable querysharding for blocks storage (`-store.engine=blocks`) by setting `-frontend.parallelize-shardable-queries` to `true`. The following additional config and exported metrics have been added. #79 #80 #100 #124 #140 #148 #150 #151 #153 #154 #155 #156 #157 #158 #159 #160 #163 #169 #172 #196 #205 #225 #226 #227 #228 #230 #235 #240 #239 #246 #244 #319 #330 #371 #385 #400 #458 #586 #630 #660 #707 * New config options: diff --git a/docs/sources/blocks-storage/production-tips.md b/docs/sources/blocks-storage/production-tips.md index 6f828a564d9..30b6dbfe7d9 100644 --- a/docs/sources/blocks-storage/production-tips.md +++ b/docs/sources/blocks-storage/production-tips.md @@ -51,7 +51,7 @@ Because of this, we would suggest to avoid querying non compacted blocks. In ord 3. Configure queriers `-querier.query-store-after` large enough to give compactor enough time to compact newly uploaded blocks (_see below_) 4. Configure queriers `-querier.query-ingesters-within` equal to `-querier.query-store-after` plus 5m (5 minutes is just a delta to query the boundary both from ingesters and queriers) 5. Configure ingesters `-blocks-storage.tsdb.retention-period` at least as `-querier.query-ingesters-within` -6. Lower `-blocks-storage.bucket-store.ignore-deletion-marks-delay` to 1h, otherwise non compacted blocks could be queried anyway, even if their compacted replacement is available +6. Ensure `-blocks-storage.bucket-store.ignore-deletion-marks-delay` is not too large (the default is suggested), otherwise non compacted blocks could be queried anyway, even if their compacted replacement is available #### How to estimate `-querier.query-store-after` diff --git a/docs/sources/blocks-storage/querier.md b/docs/sources/blocks-storage/querier.md index 9ce24e5e7e8..4d55b801d39 100644 --- a/docs/sources/blocks-storage/querier.md +++ b/docs/sources/blocks-storage/querier.md @@ -578,10 +578,9 @@ blocks_storage: # while fetching blocks. The idea of ignore-deletion-marks-delay is to # ignore blocks that are marked for deletion with some delay. This ensures # store can still serve blocks that are meant to be deleted but do not have - # a replacement yet. Default is 6h, half of the default value for - # -compactor.deletion-delay. + # a replacement yet. # CLI flag: -blocks-storage.bucket-store.ignore-deletion-marks-delay - [ignore_deletion_mark_delay: | default = 6h] + [ignore_deletion_mark_delay: | default = 1h] bucket_index: # True to enable querier and store-gateway to discover blocks in the diff --git a/docs/sources/blocks-storage/store-gateway.md b/docs/sources/blocks-storage/store-gateway.md index 3f5899584c8..ca3e0e77a64 100644 --- a/docs/sources/blocks-storage/store-gateway.md +++ b/docs/sources/blocks-storage/store-gateway.md @@ -649,10 +649,9 @@ blocks_storage: # while fetching blocks. The idea of ignore-deletion-marks-delay is to # ignore blocks that are marked for deletion with some delay. This ensures # store can still serve blocks that are meant to be deleted but do not have - # a replacement yet. Default is 6h, half of the default value for - # -compactor.deletion-delay. + # a replacement yet. # CLI flag: -blocks-storage.bucket-store.ignore-deletion-marks-delay - [ignore_deletion_mark_delay: | default = 6h] + [ignore_deletion_mark_delay: | default = 1h] bucket_index: # True to enable querier and store-gateway to discover blocks in the diff --git a/docs/sources/configuration/config-file-reference.md b/docs/sources/configuration/config-file-reference.md index 9d9dfe397bb..158f8d69634 100644 --- a/docs/sources/configuration/config-file-reference.md +++ b/docs/sources/configuration/config-file-reference.md @@ -3047,10 +3047,9 @@ bucket_store: # while fetching blocks. The idea of ignore-deletion-marks-delay is to ignore # blocks that are marked for deletion with some delay. This ensures store can # still serve blocks that are meant to be deleted but do not have a - # replacement yet. Default is 6h, half of the default value for - # -compactor.deletion-delay. + # replacement yet. # CLI flag: -blocks-storage.bucket-store.ignore-deletion-marks-delay - [ignore_deletion_mark_delay: | default = 6h] + [ignore_deletion_mark_delay: | default = 1h] bucket_index: # True to enable querier and store-gateway to discover blocks in the storage diff --git a/operations/mimir-tests/test-disable-chunk-streaming-generated.yaml b/operations/mimir-tests/test-disable-chunk-streaming-generated.yaml index be67c6cc05e..e24104157ce 100644 --- a/operations/mimir-tests/test-disable-chunk-streaming-generated.yaml +++ b/operations/mimir-tests/test-disable-chunk-streaming-generated.yaml @@ -822,7 +822,6 @@ spec: - args: - -blocks-storage.backend=gcs - -blocks-storage.bucket-store.bucket-index.enabled=true - - -blocks-storage.bucket-store.ignore-deletion-marks-delay=1h - -blocks-storage.bucket-store.metadata-cache.backend=memcached - -blocks-storage.bucket-store.metadata-cache.memcached.addresses=dnssrvnoa+memcached-metadata.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.metadata-cache.memcached.max-async-buffer-size=25000 @@ -1062,7 +1061,6 @@ spec: - -api.response-compression-enabled=true - -blocks-storage.backend=gcs - -blocks-storage.bucket-store.bucket-index.enabled=true - - -blocks-storage.bucket-store.ignore-deletion-marks-delay=1h - -blocks-storage.bucket-store.metadata-cache.backend=memcached - -blocks-storage.bucket-store.metadata-cache.memcached.addresses=dnssrvnoa+memcached-metadata.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.metadata-cache.memcached.max-async-buffer-size=25000 @@ -1657,7 +1655,6 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=200ms - -blocks-storage.bucket-store.ignore-blocks-within=10h - - -blocks-storage.bucket-store.ignore-deletion-marks-delay=1h - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-buffer-size=25000 diff --git a/operations/mimir-tests/test-query-sharding-generated.yaml b/operations/mimir-tests/test-query-sharding-generated.yaml index f437c840640..c80218d9365 100644 --- a/operations/mimir-tests/test-query-sharding-generated.yaml +++ b/operations/mimir-tests/test-query-sharding-generated.yaml @@ -821,7 +821,6 @@ spec: - args: - -blocks-storage.backend=gcs - -blocks-storage.bucket-store.bucket-index.enabled=true - - -blocks-storage.bucket-store.ignore-deletion-marks-delay=1h - -blocks-storage.bucket-store.metadata-cache.backend=memcached - -blocks-storage.bucket-store.metadata-cache.memcached.addresses=dnssrvnoa+memcached-metadata.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.metadata-cache.memcached.max-async-buffer-size=25000 @@ -1066,7 +1065,6 @@ spec: - -api.response-compression-enabled=true - -blocks-storage.backend=gcs - -blocks-storage.bucket-store.bucket-index.enabled=true - - -blocks-storage.bucket-store.ignore-deletion-marks-delay=1h - -blocks-storage.bucket-store.metadata-cache.backend=memcached - -blocks-storage.bucket-store.metadata-cache.memcached.addresses=dnssrvnoa+memcached-metadata.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.metadata-cache.memcached.max-async-buffer-size=25000 @@ -1661,7 +1659,6 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=200ms - -blocks-storage.bucket-store.ignore-blocks-within=10h - - -blocks-storage.bucket-store.ignore-deletion-marks-delay=1h - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-buffer-size=25000 diff --git a/operations/mimir-tests/test-shuffle-sharding-generated.yaml b/operations/mimir-tests/test-shuffle-sharding-generated.yaml index ccdf336096f..f2c827b8a64 100644 --- a/operations/mimir-tests/test-shuffle-sharding-generated.yaml +++ b/operations/mimir-tests/test-shuffle-sharding-generated.yaml @@ -822,7 +822,6 @@ spec: - args: - -blocks-storage.backend=gcs - -blocks-storage.bucket-store.bucket-index.enabled=true - - -blocks-storage.bucket-store.ignore-deletion-marks-delay=1h - -blocks-storage.bucket-store.metadata-cache.backend=memcached - -blocks-storage.bucket-store.metadata-cache.memcached.addresses=dnssrvnoa+memcached-metadata.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.metadata-cache.memcached.max-async-buffer-size=25000 @@ -1067,7 +1066,6 @@ spec: - -api.response-compression-enabled=true - -blocks-storage.backend=gcs - -blocks-storage.bucket-store.bucket-index.enabled=true - - -blocks-storage.bucket-store.ignore-deletion-marks-delay=1h - -blocks-storage.bucket-store.metadata-cache.backend=memcached - -blocks-storage.bucket-store.metadata-cache.memcached.addresses=dnssrvnoa+memcached-metadata.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.metadata-cache.memcached.max-async-buffer-size=25000 @@ -1667,7 +1665,6 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=200ms - -blocks-storage.bucket-store.ignore-blocks-within=10h - - -blocks-storage.bucket-store.ignore-deletion-marks-delay=1h - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-buffer-size=25000 diff --git a/operations/mimir-tests/test-storage-azure-generated.yaml b/operations/mimir-tests/test-storage-azure-generated.yaml index 8e7708ac03f..ae7e665920b 100644 --- a/operations/mimir-tests/test-storage-azure-generated.yaml +++ b/operations/mimir-tests/test-storage-azure-generated.yaml @@ -824,7 +824,6 @@ spec: - -blocks-storage.azure.container-name=blocks-bucket - -blocks-storage.backend=azure - -blocks-storage.bucket-store.bucket-index.enabled=true - - -blocks-storage.bucket-store.ignore-deletion-marks-delay=1h - -blocks-storage.bucket-store.metadata-cache.backend=memcached - -blocks-storage.bucket-store.metadata-cache.memcached.addresses=dnssrvnoa+memcached-metadata.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.metadata-cache.memcached.max-async-buffer-size=25000 @@ -1066,7 +1065,6 @@ spec: - -blocks-storage.azure.container-name=blocks-bucket - -blocks-storage.backend=azure - -blocks-storage.bucket-store.bucket-index.enabled=true - - -blocks-storage.bucket-store.ignore-deletion-marks-delay=1h - -blocks-storage.bucket-store.metadata-cache.backend=memcached - -blocks-storage.bucket-store.metadata-cache.memcached.addresses=dnssrvnoa+memcached-metadata.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.metadata-cache.memcached.max-async-buffer-size=25000 @@ -1671,7 +1669,6 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=200ms - -blocks-storage.bucket-store.ignore-blocks-within=10h - - -blocks-storage.bucket-store.ignore-deletion-marks-delay=1h - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-buffer-size=25000 diff --git a/operations/mimir-tests/test-storage-gcs-generated.yaml b/operations/mimir-tests/test-storage-gcs-generated.yaml index 9b512e77669..8bd4055a432 100644 --- a/operations/mimir-tests/test-storage-gcs-generated.yaml +++ b/operations/mimir-tests/test-storage-gcs-generated.yaml @@ -821,7 +821,6 @@ spec: - args: - -blocks-storage.backend=gcs - -blocks-storage.bucket-store.bucket-index.enabled=true - - -blocks-storage.bucket-store.ignore-deletion-marks-delay=1h - -blocks-storage.bucket-store.metadata-cache.backend=memcached - -blocks-storage.bucket-store.metadata-cache.memcached.addresses=dnssrvnoa+memcached-metadata.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.metadata-cache.memcached.max-async-buffer-size=25000 @@ -1061,7 +1060,6 @@ spec: - -api.response-compression-enabled=true - -blocks-storage.backend=gcs - -blocks-storage.bucket-store.bucket-index.enabled=true - - -blocks-storage.bucket-store.ignore-deletion-marks-delay=1h - -blocks-storage.bucket-store.metadata-cache.backend=memcached - -blocks-storage.bucket-store.metadata-cache.memcached.addresses=dnssrvnoa+memcached-metadata.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.metadata-cache.memcached.max-async-buffer-size=25000 @@ -1656,7 +1654,6 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=200ms - -blocks-storage.bucket-store.ignore-blocks-within=10h - - -blocks-storage.bucket-store.ignore-deletion-marks-delay=1h - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-buffer-size=25000 diff --git a/operations/mimir-tests/test-storage-s3-generated.yaml b/operations/mimir-tests/test-storage-s3-generated.yaml index 6ba5090c043..4172785ef2e 100644 --- a/operations/mimir-tests/test-storage-s3-generated.yaml +++ b/operations/mimir-tests/test-storage-s3-generated.yaml @@ -821,7 +821,6 @@ spec: - args: - -blocks-storage.backend=s3 - -blocks-storage.bucket-store.bucket-index.enabled=true - - -blocks-storage.bucket-store.ignore-deletion-marks-delay=1h - -blocks-storage.bucket-store.metadata-cache.backend=memcached - -blocks-storage.bucket-store.metadata-cache.memcached.addresses=dnssrvnoa+memcached-metadata.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.metadata-cache.memcached.max-async-buffer-size=25000 @@ -1062,7 +1061,6 @@ spec: - -api.response-compression-enabled=true - -blocks-storage.backend=s3 - -blocks-storage.bucket-store.bucket-index.enabled=true - - -blocks-storage.bucket-store.ignore-deletion-marks-delay=1h - -blocks-storage.bucket-store.metadata-cache.backend=memcached - -blocks-storage.bucket-store.metadata-cache.memcached.addresses=dnssrvnoa+memcached-metadata.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.metadata-cache.memcached.max-async-buffer-size=25000 @@ -1663,7 +1661,6 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=200ms - -blocks-storage.bucket-store.ignore-blocks-within=10h - - -blocks-storage.bucket-store.ignore-deletion-marks-delay=1h - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-buffer-size=25000 diff --git a/operations/mimir/config.libsonnet b/operations/mimir/config.libsonnet index 58819e2c31b..2ffc05abf84 100644 --- a/operations/mimir/config.libsonnet +++ b/operations/mimir/config.libsonnet @@ -89,7 +89,6 @@ genericBlocksStorageConfig:: {}, queryBlocksStorageConfig:: { 'blocks-storage.bucket-store.sync-dir': '/data/tsdb', - 'blocks-storage.bucket-store.ignore-deletion-marks-delay': '1h', 'store-gateway.sharding-enabled': true, 'store-gateway.sharding-ring.store': 'consul', diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go index f239834c5d5..73b9a948130 100644 --- a/pkg/storage/tsdb/config.go +++ b/pkg/storage/tsdb/config.go @@ -307,9 +307,8 @@ func (cfg *BucketStoreConfig) RegisterFlags(f *flag.FlagSet) { f.IntVar(&cfg.BlockSyncConcurrency, "blocks-storage.bucket-store.block-sync-concurrency", 20, "Maximum number of concurrent blocks synching per tenant.") f.IntVar(&cfg.MetaSyncConcurrency, "blocks-storage.bucket-store.meta-sync-concurrency", 20, "Number of Go routines to use when syncing block meta files from object storage per tenant.") f.DurationVar(&cfg.ConsistencyDelay, "blocks-storage.bucket-store.consistency-delay", 0, "Minimum age of a block before it's being read. Set it to safe value (e.g 30m) if your object storage is eventually consistent. GCS and S3 are (roughly) strongly consistent.") - f.DurationVar(&cfg.IgnoreDeletionMarksDelay, "blocks-storage.bucket-store.ignore-deletion-marks-delay", time.Hour*6, "Duration after which the blocks marked for deletion will be filtered out while fetching blocks. "+ - "The idea of ignore-deletion-marks-delay is to ignore blocks that are marked for deletion with some delay. This ensures store can still serve blocks that are meant to be deleted but do not have a replacement yet. "+ - "Default is 6h, half of the default value for -compactor.deletion-delay.") + f.DurationVar(&cfg.IgnoreDeletionMarksDelay, "blocks-storage.bucket-store.ignore-deletion-marks-delay", time.Hour*1, "Duration after which the blocks marked for deletion will be filtered out while fetching blocks. "+ + "The idea of ignore-deletion-marks-delay is to ignore blocks that are marked for deletion with some delay. This ensures store can still serve blocks that are meant to be deleted but do not have a replacement yet.") f.DurationVar(&cfg.IgnoreBlocksWithin, "blocks-storage.bucket-store.ignore-blocks-within", 0, "Blocks with minimum time within this duration are ignored, and not loaded by store-gateway. Useful when used together with -querier.query-store-after to prevent loading young blocks, because there are usually many of them (depending on number of ingesters) and they are not yet compacted. Negative values or 0 disable the filter.") f.IntVar(&cfg.PostingOffsetsInMemSampling, "blocks-storage.bucket-store.posting-offsets-in-mem-sampling", DefaultPostingOffsetInMemorySampling, "Controls what is the ratio of postings offsets that the store will hold in memory.") f.BoolVar(&cfg.IndexHeaderLazyLoadingEnabled, "blocks-storage.bucket-store.index-header-lazy-loading-enabled", true, "If enabled, store-gateway will lazy load an index-header only once required by a query.")