From 034c9a376c6b14d5ea43c13dead1cd944ccdb9db Mon Sep 17 00:00:00 2001 From: Alan Protasio Date: Wed, 27 Jul 2022 20:14:07 -0700 Subject: [PATCH 01/28] Fix updateCachedShippedBlocks - new thanos Signed-off-by: Alan Protasio --- pkg/ingester/ingester_v2_test.go | 1 - vendor/github.com/vimeo/galaxycache/http/http.go | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/ingester/ingester_v2_test.go b/pkg/ingester/ingester_v2_test.go index e83d5a1099..a6c9644276 100644 --- a/pkg/ingester/ingester_v2_test.go +++ b/pkg/ingester/ingester_v2_test.go @@ -2553,7 +2553,6 @@ func TestIngester_closeAndDeleteUserTSDBIfIdle_shouldNotCloseTSDBIfShippingIsInP defer db.stateMtx.RUnlock() return db.state }) - assert.Equal(t, tsdbNotActive, i.closeAndDeleteUserTSDBIfIdle(userID)) } diff --git a/vendor/github.com/vimeo/galaxycache/http/http.go b/vendor/github.com/vimeo/galaxycache/http/http.go index 3743bb64ba..dd54d0230e 100644 --- a/vendor/github.com/vimeo/galaxycache/http/http.go +++ b/vendor/github.com/vimeo/galaxycache/http/http.go @@ -186,12 +186,12 @@ func (h *HTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } w.Header().Set("Content-Type", "application/octet-stream") - b, expTm, err := value.MarshalBinary() + b, _, err := value.MarshalBinary() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - w.Header().Set(ttlHeader, fmt.Sprintf("%d", expTm.UnixMilli())) + //w.Header().Set(ttlHeader, fmt.Sprintf("%d", expTm.UnixMilli())) w.Write(b) } @@ -233,7 +233,7 @@ func (h *httpFetcher) Fetch(ctx context.Context, galaxy string, key string) ([]b if err != nil { return nil, time.Time{}, fmt.Errorf("parsing TTL header %s: %w", ttlHeader, err) } - return data, time.UnixMilli(expire), nil + return data, time.Unix(expire, 0), nil } // Close here implements the RemoteFetcher interface for closing (does nothing for HTTP) From 7f0fe798545fe2ed4049c06b11504d7e14fb48c2 Mon Sep 17 00:00:00 2001 From: Andrew Bloomgarden Date: Fri, 18 Mar 2022 12:17:54 -0400 Subject: [PATCH 02/28] Remove support for chunks ingestion Signed-off-by: Andrew Bloomgarden --- docs/configuration/config-file-reference.md | 132 +- .../single-process-config-blocks-local.yaml | 95 ++ integration/api_endpoints_test.go | 5 +- integration/asserts.go | 13 +- integration/backward_compatibility_test.go | 180 ++- integration/configs.go | 23 +- integration/e2ecortex/services.go | 11 +- ...tegration_memberlist_single_binary_test.go | 12 +- integration/querier_remote_read_test.go | 20 +- .../querier_streaming_mixed_ingester_test.go | 164 -- integration/querier_test.go | 17 +- integration/ruler_test.go | 28 +- pkg/api/api.go | 18 - pkg/api/handlers.go | 2 +- pkg/api/middlewares.go | 2 +- pkg/chunk/purger/delete_plan.pb.go | 1353 ----------------- pkg/chunk/purger/delete_plan.proto | 34 - pkg/chunk/purger/delete_requests_store.go | 394 ----- pkg/chunk/purger/purger.go | 828 ---------- pkg/chunk/purger/purger_test.go | 532 ------- pkg/chunk/purger/request_handler.go | 183 --- pkg/chunk/purger/table_provisioning.go | 30 - pkg/chunk/purger/tombstones.go | 450 +----- pkg/chunk/purger/tombstones_test.go | 230 --- pkg/chunk/storage/factory.go | 10 +- pkg/cortex/cortex.go | 7 +- pkg/cortex/modules.go | 117 +- pkg/flusher/flusher.go | 19 +- pkg/ingester/client/cortex_mock_test.go | 5 - pkg/ingester/client/ingester.proto | 3 - pkg/ingester/errors.go | 26 - pkg/ingester/flush.go | 417 +---- pkg/ingester/flush_test.go | 253 --- pkg/ingester/index/index.go | 324 ---- pkg/ingester/index/index_test.go | 201 --- pkg/ingester/ingester.go | 742 +-------- pkg/ingester/ingester_test.go | 804 +--------- pkg/ingester/ingester_v2.go | 56 +- pkg/ingester/ingester_v2_test.go | 14 +- pkg/ingester/label_pairs.go | 90 -- pkg/ingester/label_pairs_test.go | 106 -- pkg/ingester/lifecycle_test.go | 265 +--- pkg/ingester/locker.go | 58 - pkg/ingester/locker_test.go | 45 - pkg/ingester/mapper.go | 155 -- pkg/ingester/mapper_test.go | 134 -- pkg/ingester/series.go | 253 --- pkg/ingester/series_map.go | 110 -- pkg/ingester/transfer.go | 377 +---- pkg/ingester/user_state.go | 338 ---- pkg/ingester/user_state_test.go | 147 -- pkg/ingester/wal.go | 1105 -------------- pkg/ingester/wal.pb.go | 607 -------- pkg/ingester/wal.proto | 16 - pkg/ingester/wal_test.go | 313 ---- pkg/querier/querier.go | 6 +- pkg/querier/querier_test.go | 20 +- pkg/querier/series/series_set.go | 4 +- pkg/ring/lifecycler.go | 12 - pkg/ruler/ruler_test.go | 2 +- pkg/util/chunkcompat/compat.go | 14 - tools/doc-generator/main.go | 6 - 62 files changed, 436 insertions(+), 11501 deletions(-) create mode 100644 docs/configuration/single-process-config-blocks-local.yaml delete mode 100644 integration/querier_streaming_mixed_ingester_test.go delete mode 100644 pkg/chunk/purger/delete_plan.pb.go delete mode 100644 pkg/chunk/purger/delete_plan.proto delete mode 100644 pkg/chunk/purger/delete_requests_store.go delete mode 100644 pkg/chunk/purger/purger.go delete mode 100644 pkg/chunk/purger/purger_test.go delete mode 100644 pkg/chunk/purger/request_handler.go delete mode 100644 pkg/chunk/purger/table_provisioning.go delete mode 100644 pkg/chunk/purger/tombstones_test.go delete mode 100644 pkg/ingester/flush_test.go delete mode 100644 pkg/ingester/index/index.go delete mode 100644 pkg/ingester/index/index_test.go delete mode 100644 pkg/ingester/label_pairs.go delete mode 100644 pkg/ingester/label_pairs_test.go delete mode 100644 pkg/ingester/locker.go delete mode 100644 pkg/ingester/locker_test.go delete mode 100644 pkg/ingester/mapper.go delete mode 100644 pkg/ingester/mapper_test.go delete mode 100644 pkg/ingester/series_map.go delete mode 100644 pkg/ingester/wal.pb.go delete mode 100644 pkg/ingester/wal.proto delete mode 100644 pkg/ingester/wal_test.go diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index f15c69e05b..33a8ccfba4 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -140,9 +140,6 @@ api: # blocks storage. [store_gateway: ] -# The purger_config configures the purger which takes care of delete requests. -[purger: ] - tenant_federation: # If enabled on all Cortex services, queries can be federated across multiple # tenants. The tenant IDs involved need to be specified separated by a `|` @@ -604,8 +601,8 @@ instance_limits: The `ingester_config` configures the Cortex ingester. ```yaml -# Configures the Write-Ahead Log (WAL) for the Cortex chunks storage. This -# config is ignored when running the Cortex blocks storage. +# Configures the Write-Ahead Log (WAL) for the removed Cortex chunks storage. +# This config is now always ignored. walconfig: # Enable writing of ingested data into WAL. # CLI flag: -ingester.wal-enabled @@ -2844,9 +2841,9 @@ chunk_tables_provisioning: The `storage_config` configures where Cortex stores the data (chunks storage engine). ```yaml -# The storage engine to use: chunks (deprecated) or blocks. +# The storage engine to use: blocks is the only supported option today. # CLI flag: -store.engine -[engine: | default = "chunks"] +[engine: | default = "blocks"] aws: dynamodb: @@ -3387,93 +3384,6 @@ index_queries_cache_config: # The CLI flags prefix for this block config is: store.index-cache-read [fifocache: ] -delete_store: - # Store for keeping delete request - # CLI flag: -deletes.store - [store: | default = ""] - - # Name of the table which stores delete requests - # CLI flag: -deletes.requests-table-name - [requests_table_name: | default = "delete_requests"] - - table_provisioning: - # Enables on demand throughput provisioning for the storage provider (if - # supported). Applies only to tables which are not autoscaled. Supported by - # DynamoDB - # CLI flag: -deletes.table.enable-ondemand-throughput-mode - [enable_ondemand_throughput_mode: | default = false] - - # Table default write throughput. Supported by DynamoDB - # CLI flag: -deletes.table.write-throughput - [provisioned_write_throughput: | default = 1] - - # Table default read throughput. Supported by DynamoDB - # CLI flag: -deletes.table.read-throughput - [provisioned_read_throughput: | default = 300] - - write_scale: - # Should we enable autoscale for the table. - # CLI flag: -deletes.table.write-throughput.scale.enabled - [enabled: | default = false] - - # AWS AutoScaling role ARN - # CLI flag: -deletes.table.write-throughput.scale.role-arn - [role_arn: | default = ""] - - # DynamoDB minimum provision capacity. - # CLI flag: -deletes.table.write-throughput.scale.min-capacity - [min_capacity: | default = 3000] - - # DynamoDB maximum provision capacity. - # CLI flag: -deletes.table.write-throughput.scale.max-capacity - [max_capacity: | default = 6000] - - # DynamoDB minimum seconds between each autoscale up. - # CLI flag: -deletes.table.write-throughput.scale.out-cooldown - [out_cooldown: | default = 1800] - - # DynamoDB minimum seconds between each autoscale down. - # CLI flag: -deletes.table.write-throughput.scale.in-cooldown - [in_cooldown: | default = 1800] - - # DynamoDB target ratio of consumed capacity to provisioned capacity. - # CLI flag: -deletes.table.write-throughput.scale.target-value - [target: | default = 80] - - read_scale: - # Should we enable autoscale for the table. - # CLI flag: -deletes.table.read-throughput.scale.enabled - [enabled: | default = false] - - # AWS AutoScaling role ARN - # CLI flag: -deletes.table.read-throughput.scale.role-arn - [role_arn: | default = ""] - - # DynamoDB minimum provision capacity. - # CLI flag: -deletes.table.read-throughput.scale.min-capacity - [min_capacity: | default = 3000] - - # DynamoDB maximum provision capacity. - # CLI flag: -deletes.table.read-throughput.scale.max-capacity - [max_capacity: | default = 6000] - - # DynamoDB minimum seconds between each autoscale up. - # CLI flag: -deletes.table.read-throughput.scale.out-cooldown - [out_cooldown: | default = 1800] - - # DynamoDB minimum seconds between each autoscale down. - # CLI flag: -deletes.table.read-throughput.scale.in-cooldown - [in_cooldown: | default = 1800] - - # DynamoDB target ratio of consumed capacity to provisioned capacity. - # CLI flag: -deletes.table.read-throughput.scale.target-value - [target: | default = 80] - - # Tag (of the form key=value) to be added to the tables. Supported by - # DynamoDB - # CLI flag: -deletes.table.tags - [tags: | default = ] - grpc_store: # Hostname or IP of the gRPC store instance. # CLI flag: -grpc-store.server-address @@ -3485,16 +3395,17 @@ grpc_store: The `flusher_config` configures the WAL flusher target, used to manually run one-time flushes when scaling down ingesters. ```yaml -# Directory to read WAL from (chunks storage engine only). +# Has no effect: directory to read WAL from (chunks storage engine only). # CLI flag: -flusher.wal-dir [wal_dir: | default = "wal"] -# Number of concurrent goroutines flushing to storage (chunks storage engine -# only). +# Has no effect: number of concurrent goroutines flushing to storage (chunks +# storage engine only). # CLI flag: -flusher.concurrent-flushes [concurrent_flushes: | default = 50] -# Timeout for individual flush operations (chunks storage engine only). +# Has no effect: timeout for individual flush operations (chunks storage engine +# only). # CLI flag: -flusher.flush-op-timeout [flush_op_timeout: | default = 2m] @@ -5520,31 +5431,6 @@ sharding_ring: [sharding_strategy: | default = "default"] ``` -### `purger_config` - -The `purger_config` configures the purger which takes care of delete requests. - -```yaml -# Enable purger to allow deletion of series. Be aware that Delete series feature -# is still experimental -# CLI flag: -purger.enable -[enable: | default = false] - -# Number of workers executing delete plans in parallel -# CLI flag: -purger.num-workers -[num_workers: | default = 2] - -# Name of the object store to use for storing delete plans -# CLI flag: -purger.object-store-type -[object_store_type: | default = ""] - -# Allow cancellation of delete request until duration after they are created. -# Data would be deleted only after delete requests have been older than this -# duration. Ideally this should be set to at least 24h. -# CLI flag: -purger.delete-request-cancel-period -[delete_request_cancel_period: | default = 24h] -``` - ### `s3_sse_config` The `s3_sse_config` configures the S3 server-side encryption. The supported CLI flags `` used to reference this config block are: diff --git a/docs/configuration/single-process-config-blocks-local.yaml b/docs/configuration/single-process-config-blocks-local.yaml new file mode 100644 index 0000000000..d79a2a6109 --- /dev/null +++ b/docs/configuration/single-process-config-blocks-local.yaml @@ -0,0 +1,95 @@ + +# Configuration for running Cortex in single-process mode. +# This should not be used in production. It is only for getting started +# and development. + +# Disable the requirement that every request to Cortex has a +# X-Scope-OrgID header. `fake` will be substituted in instead. +auth_enabled: false + +server: + http_listen_port: 9009 + + # Configure the server to allow messages up to 100MB. + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + grpc_server_max_concurrent_streams: 1000 + +distributor: + shard_by_all_labels: true + pool: + health_check_ingesters: true + +ingester_client: + grpc_client_config: + # Configure the client to allow messages up to 100MB. + max_recv_msg_size: 104857600 + max_send_msg_size: 104857600 + grpc_compression: gzip + +ingester: + lifecycler: + # The address to advertise for this ingester. Will be autodiscovered by + # looking up address on eth0 or en0; can be specified if this fails. + # address: 127.0.0.1 + + # We want to start immediately and flush on shutdown. + join_after: 0 + min_ready_duration: 0s + final_sleep: 0s + num_tokens: 512 + + # Use an in memory ring store, so we don't need to launch a Consul. + ring: + kvstore: + store: inmemory + replication_factor: 1 + +storage: + engine: blocks + +blocks_storage: + tsdb: + dir: /tmp/cortex/tsdb + + bucket_store: + sync_dir: /tmp/cortex/tsdb-sync + + # You can choose between local storage and Amazon S3, Google GCS and Azure storage. Each option requires additional configuration + # as shown below. All options can be configured via flags as well which might be handy for secret inputs. + backend: filesystem # s3, gcs, azure or filesystem are valid options +# s3: +# bucket_name: cortex +# endpoint: s3.dualstack.us-east-1.amazonaws.com + # Configure your S3 credentials below. + # secret_access_key: "TODO" + # access_key_id: "TODO" +# gcs: +# bucket_name: cortex +# service_account: # if empty or omitted Cortex will use your default service account as per Google's fallback logic +# azure: +# account_name: +# account_key: +# container_name: +# endpoint_suffix: +# max_retries: # Number of retries for recoverable errors (defaults to 20) + filesystem: + dir: ./data/tsdb + +compactor: + data_dir: /tmp/cortex/compactor + sharding_ring: + kvstore: + store: inmemory + +frontend_worker: + match_max_concurrent: true + +ruler: + enable_api: true + enable_sharding: false + +ruler_storage: + backend: local + local: + directory: /tmp/cortex/rules diff --git a/integration/api_endpoints_test.go b/integration/api_endpoints_test.go index 0b65c5b7ed..7ec73a46ad 100644 --- a/integration/api_endpoints_test.go +++ b/integration/api_endpoints_test.go @@ -1,3 +1,4 @@ +//go:build requires_docker // +build requires_docker package integration @@ -22,7 +23,7 @@ func TestIndexAPIEndpoint(t *testing.T) { defer s.Close() // Start Cortex in single binary mode, reading the config from file. - require.NoError(t, copyFileToSharedDir(s, "docs/chunks-storage/single-process-config.yaml", cortexConfigFile)) + require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile)) cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, nil, "", 9009, 9095) require.NoError(t, s.StartAndWaitReady(cortex1)) @@ -44,7 +45,7 @@ func TestConfigAPIEndpoint(t *testing.T) { defer s.Close() // Start Cortex in single binary mode, reading the config from file. - require.NoError(t, copyFileToSharedDir(s, "docs/chunks-storage/single-process-config.yaml", cortexConfigFile)) + require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile)) cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, nil, "", 9009, 9095) require.NoError(t, s.StartAndWaitReady(cortex1)) diff --git a/integration/asserts.go b/integration/asserts.go index ba4206045a..2958133ae9 100644 --- a/integration/asserts.go +++ b/integration/asserts.go @@ -30,16 +30,19 @@ const ( var ( // Service-specific metrics prefixes which shouldn't be used by any other service. serviceMetricsPrefixes = map[ServiceType][]string{ - Distributor: {}, - Ingester: {"!cortex_ingester_client", "cortex_ingester"}, // The metrics prefix cortex_ingester_client may be used by other components so we ignore it. - Querier: {"cortex_querier"}, + Distributor: {}, + // The metrics prefix cortex_ingester_client may be used by other components so we ignore it. + Ingester: {"!cortex_ingester_client", "cortex_ingester"}, + // The metrics prefixes cortex_querier_storegateway and cortex_querier_blocks may be used by other components so we ignore them. + Querier: {"!cortex_querier_storegateway", "!cortex_querier_blocks", "cortex_querier"}, QueryFrontend: {"cortex_frontend", "cortex_query_frontend"}, QueryScheduler: {"cortex_query_scheduler"}, TableManager: {}, AlertManager: {"cortex_alertmanager"}, Ruler: {}, - StoreGateway: {"!cortex_storegateway_client", "cortex_storegateway"}, // The metrics prefix cortex_storegateway_client may be used by other components so we ignore it. - Purger: {"cortex_purger"}, + // The metrics prefix cortex_storegateway_client may be used by other components so we ignore it. + StoreGateway: {"!cortex_storegateway_client", "cortex_storegateway"}, + Purger: {"cortex_purger"}, } // Blacklisted metrics prefixes across any Cortex service. diff --git a/integration/backward_compatibility_test.go b/integration/backward_compatibility_test.go index 96da22c428..79396269da 100644 --- a/integration/backward_compatibility_test.go +++ b/integration/backward_compatibility_test.go @@ -1,3 +1,4 @@ +//go:build requires_docker // +build requires_docker package integration @@ -20,12 +21,6 @@ var ( // If you change the image tag, remember to update it in the preloading done // by GitHub Actions too (see .github/workflows/test-build-deploy.yml). previousVersionImages = map[string]func(map[string]string) map[string]string{ - "quay.io/cortexproject/cortex:v1.0.0": preCortex14Flags, - "quay.io/cortexproject/cortex:v1.1.0": preCortex14Flags, - "quay.io/cortexproject/cortex:v1.2.0": preCortex14Flags, - "quay.io/cortexproject/cortex:v1.3.0": preCortex14Flags, - "quay.io/cortexproject/cortex:v1.4.0": preCortex16Flags, - "quay.io/cortexproject/cortex:v1.5.0": preCortex16Flags, "quay.io/cortexproject/cortex:v1.6.0": preCortex110Flags, "quay.io/cortexproject/cortex:v1.7.0": preCortex110Flags, "quay.io/cortexproject/cortex:v1.8.0": preCortex110Flags, @@ -34,48 +29,37 @@ var ( } ) -func preCortex14Flags(flags map[string]string) map[string]string { +func preCortex110Flags(flags map[string]string) map[string]string { return e2e.MergeFlagsWithoutRemovingEmpty(flags, map[string]string{ - // Blocks storage CLI flags removed the "experimental" prefix in 1.4. - "-store-gateway.sharding-enabled": "", - "-store-gateway.sharding-ring.store": "", - "-store-gateway.sharding-ring.consul.hostname": "", - "-store-gateway.sharding-ring.replication-factor": "", - // Query-scheduler has been introduced in 1.6.0 - "-frontend.scheduler-dns-lookup-period": "", // Store-gateway "wait ring stability" has been introduced in 1.10.0 "-store-gateway.sharding-ring.wait-stability-min-duration": "", "-store-gateway.sharding-ring.wait-stability-max-duration": "", }) } -func preCortex16Flags(flags map[string]string) map[string]string { - return e2e.MergeFlagsWithoutRemovingEmpty(flags, map[string]string{ - // Query-scheduler has been introduced in 1.6.0 - "-frontend.scheduler-dns-lookup-period": "", - // Store-gateway "wait ring stability" has been introduced in 1.10.0 - "-store-gateway.sharding-ring.wait-stability-min-duration": "", - "-store-gateway.sharding-ring.wait-stability-max-duration": "", - }) -} +func TestBackwardCompatibilityWithBlocksStorage(t *testing.T) { + for previousImage, flagsFn := range previousVersionImages { + t.Run(fmt.Sprintf("Backward compatibility upgrading from %s", previousImage), func(t *testing.T) { + flags := blocksStorageFlagsWithFlushOnShutdown() + if flagsFn != nil { + flags = flagsFn(flags) + } -func preCortex110Flags(flags map[string]string) map[string]string { - return e2e.MergeFlagsWithoutRemovingEmpty(flags, map[string]string{ - // Store-gateway "wait ring stability" has been introduced in 1.10.0 - "-store-gateway.sharding-ring.wait-stability-min-duration": "", - "-store-gateway.sharding-ring.wait-stability-max-duration": "", - }) + runBackwardCompatibilityTestWithBlocksStorage(t, previousImage, flags) + }) + } } -func TestBackwardCompatibilityWithChunksStorage(t *testing.T) { +func TestBackwardCompatibilityWithChunksStorageAsSecondStore(t *testing.T) { for previousImage, flagsFn := range previousVersionImages { - t.Run(fmt.Sprintf("Backward compatibility upgrading from %s", previousImage), func(t *testing.T) { + t.Run(fmt.Sprintf("Backward compatibility with data from %s", previousImage), func(t *testing.T) { flags := ChunksStorageFlags() + flags["-ingester.max-transfer-retries"] = "0" if flagsFn != nil { flags = flagsFn(flags) } - runBackwardCompatibilityTestWithChunksStorage(t, previousImage, flags) + runBackwardCompatibilityTestWithChunksStorageAsSecondStore(t, previousImage, flags) }) } } @@ -83,7 +67,7 @@ func TestBackwardCompatibilityWithChunksStorage(t *testing.T) { func TestNewDistributorsCanPushToOldIngestersWithReplication(t *testing.T) { for previousImage, flagsFn := range previousVersionImages { t.Run(fmt.Sprintf("Backward compatibility upgrading from %s", previousImage), func(t *testing.T) { - flags := ChunksStorageFlags() + flags := blocksStorageFlagsWithFlushOnShutdown() if flagsFn != nil { flags = flagsFn(flags) } @@ -93,30 +77,27 @@ func TestNewDistributorsCanPushToOldIngestersWithReplication(t *testing.T) { } } -func runBackwardCompatibilityTestWithChunksStorage(t *testing.T, previousImage string, flagsForOldImage map[string]string) { +func blocksStorageFlagsWithFlushOnShutdown() map[string]string { + return mergeFlags(BlocksStorageFlags(), map[string]string{ + "-blocks-storage.tsdb.flush-blocks-on-shutdown": "true", + }) +} + +func runBackwardCompatibilityTestWithBlocksStorage(t *testing.T, previousImage string, flagsForOldImage map[string]string) { s, err := e2e.NewScenario(networkName) require.NoError(t, err) defer s.Close() // Start dependencies. - dynamo := e2edb.NewDynamoDB() + minio := e2edb.NewMinio(9000, flagsForOldImage["-blocks-storage.s3.bucket-name"]) consul := e2edb.NewConsul() - require.NoError(t, s.StartAndWaitReady(dynamo, consul)) - - require.NoError(t, writeFileToSharedDir(s, cortexSchemaConfigFile, []byte(cortexSchemaConfigYaml))) - - // Start Cortex table-manager (running on current version since the backward compatibility - // test is about testing a rolling update of other services). - tableManager := e2ecortex.NewTableManager("table-manager", ChunksStorageFlags(), "") - require.NoError(t, s.StartAndWaitReady(tableManager)) + require.NoError(t, s.StartAndWaitReady(minio, consul)) - // Wait until the first table-manager sync has completed, so that we're - // sure the tables have been created. - require.NoError(t, tableManager.WaitSumMetrics(e2e.Greater(0), "cortex_table_manager_sync_success_timestamp_seconds")) + flagsForNewImage := blocksStorageFlagsWithFlushOnShutdown() // Start other Cortex components (ingester running on previous version). ingester1 := e2ecortex.NewIngester("ingester-1", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flagsForOldImage, previousImage) - distributor := e2ecortex.NewDistributor("distributor", "consul", consul.NetworkHTTPEndpoint(), ChunksStorageFlags(), "") + distributor := e2ecortex.NewDistributor("distributor", "consul", consul.NetworkHTTPEndpoint(), flagsForNewImage, "") require.NoError(t, s.StartAndWaitReady(distributor, ingester1)) // Wait until the distributor has updated the ring. @@ -133,53 +114,107 @@ func runBackwardCompatibilityTestWithChunksStorage(t *testing.T, previousImage s require.NoError(t, err) require.Equal(t, 200, res.StatusCode) - ingester2 := e2ecortex.NewIngester("ingester-2", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), mergeFlags(ChunksStorageFlags(), map[string]string{ + ingester2 := e2ecortex.NewIngester("ingester-2", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), mergeFlags(flagsForNewImage, map[string]string{ "-ingester.join-after": "10s", }), "") - // Start ingester-2 on new version, to ensure the transfer is backward compatible. require.NoError(t, s.Start(ingester2)) // Stop ingester-1. This function will return once the ingester-1 is successfully - // stopped, which means the transfer to ingester-2 is completed. + // stopped, which means it has uploaded all its data to the object store. require.NoError(t, s.Stop(ingester1)) checkQueries(t, consul, expectedVector, previousImage, flagsForOldImage, - ChunksStorageFlags(), + flagsForNewImage, now, s, 1, ) } -// Check for issues like https://github.com/cortexproject/cortex/issues/2356 -func runNewDistributorsCanPushToOldIngestersWithReplication(t *testing.T, previousImage string, flagsForPreviousImage map[string]string) { +func runBackwardCompatibilityTestWithChunksStorageAsSecondStore(t *testing.T, previousImage string, flagsForOldImage map[string]string) { s, err := e2e.NewScenario(networkName) require.NoError(t, err) defer s.Close() + flagsForNewImage := mergeFlags(blocksStorageFlagsWithFlushOnShutdown(), ChunksStorageFlags()) + flagsForNewImage = mergeFlags(flagsForNewImage, map[string]string{ + "-querier.second-store-engine": chunksStorageEngine, + "-store.engine": blocksStorageEngine, + }) + // Start dependencies. dynamo := e2edb.NewDynamoDB() + minio := e2edb.NewMinio(9000, flagsForNewImage["-blocks-storage.s3.bucket-name"]) consul := e2edb.NewConsul() - require.NoError(t, s.StartAndWaitReady(dynamo, consul)) - - flagsForNewImage := mergeFlags(ChunksStorageFlags(), map[string]string{ - "-distributor.replication-factor": "3", - }) + require.NoError(t, s.StartAndWaitReady(dynamo, minio, consul)) require.NoError(t, writeFileToSharedDir(s, cortexSchemaConfigFile, []byte(cortexSchemaConfigYaml))) - // Start Cortex table-manager (running on current version since the backward compatibility - // test is about testing a rolling update of other services). - tableManager := e2ecortex.NewTableManager("table-manager", ChunksStorageFlags(), "") + // Start Cortex table-manager (on old version since it's gone in current) + tableManager := e2ecortex.NewTableManager("table-manager", flagsForOldImage, previousImage) require.NoError(t, s.StartAndWaitReady(tableManager)) // Wait until the first table-manager sync has completed, so that we're // sure the tables have been created. require.NoError(t, tableManager.WaitSumMetrics(e2e.Greater(0), "cortex_table_manager_sync_success_timestamp_seconds")) + // Start other Cortex components (ingester running on previous version). + ingester1 := e2ecortex.NewIngester("ingester-1", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flagsForOldImage, previousImage) + distributor := e2ecortex.NewDistributor("distributor", "consul", consul.NetworkHTTPEndpoint(), flagsForOldImage, previousImage) + require.NoError(t, s.StartAndWaitReady(distributor, ingester1)) + + // Wait until the distributor has updated the ring. + require.NoError(t, distributor.WaitSumMetrics(e2e.Equals(512), "cortex_ring_tokens_total")) + + // Push some series to Cortex. + now := time.Now() + series, expectedVector := generateSeries("series_1", now) + + c, err := e2ecortex.NewClient(distributor.HTTPEndpoint(), "", "", "", "user-1") + require.NoError(t, err) + + res, err := c.Push(series) + require.NoError(t, err) + require.Equal(t, 200, res.StatusCode) + + // Stop ingester-1. This function will return once the ingester-1 is successfully + // stopped, which has been configured to flush all data to the backing store. + require.NoError(t, s.Stop(ingester1)) + + ingester2 := e2ecortex.NewIngester("ingester-2", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), mergeFlags(flagsForNewImage, map[string]string{ + "-ingester.join-after": "10s", + }), "") + require.NoError(t, s.Start(ingester2)) + + checkQueries(t, consul, + expectedVector, + "", + flagsForNewImage, + flagsForNewImage, + now, + s, + 1, + ) +} + +// Check for issues like https://github.com/cortexproject/cortex/issues/2356 +func runNewDistributorsCanPushToOldIngestersWithReplication(t *testing.T, previousImage string, flagsForPreviousImage map[string]string) { + s, err := e2e.NewScenario(networkName) + require.NoError(t, err) + defer s.Close() + + // Start dependencies. + minio := e2edb.NewMinio(9000, flagsForPreviousImage["-blocks-storage.s3.bucket-name"]) + consul := e2edb.NewConsul() + require.NoError(t, s.StartAndWaitReady(minio, consul)) + + flagsForNewImage := mergeFlags(blocksStorageFlagsWithFlushOnShutdown(), map[string]string{ + "-distributor.replication-factor": "3", + }) + // Start other Cortex components (ingester running on previous version). ingester1 := e2ecortex.NewIngester("ingester-1", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flagsForPreviousImage, previousImage) ingester2 := e2ecortex.NewIngester("ingester-2", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flagsForPreviousImage, previousImage) @@ -227,18 +262,24 @@ func checkQueries( queryFrontendFlags map[string]string querierImage string querierFlags map[string]string + storeGatewayImage string + storeGatewayFlags map[string]string }{ - "old query-frontend, new querier": { + "old query-frontend, new querier, old store-gateway": { queryFrontendImage: previousImage, queryFrontendFlags: flagsForOldImage, querierImage: "", querierFlags: flagsForNewImage, + storeGatewayImage: previousImage, + storeGatewayFlags: flagsForOldImage, }, - "new query-frontend, old querier": { + "new query-frontend, old querier, new store-gateway": { queryFrontendImage: "", queryFrontendFlags: flagsForNewImage, querierImage: previousImage, querierFlags: flagsForOldImage, + storeGatewayImage: "", + storeGatewayFlags: flagsForNewImage, }, } @@ -261,9 +302,18 @@ func checkQueries( require.NoError(t, s.Stop(querier)) }() + // Start store gateway. + storeGateway := e2ecortex.NewStoreGateway("store-gateway", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), c.storeGatewayFlags, c.storeGatewayImage) + + require.NoError(t, s.Start(storeGateway)) + defer func() { + require.NoError(t, s.Stop(storeGateway)) + }() + // Wait until querier and query-frontend are ready, and the querier has updated the ring. - require.NoError(t, s.WaitReady(querier, queryFrontend)) - require.NoError(t, querier.WaitSumMetrics(e2e.Equals(float64(numIngesters*512)), "cortex_ring_tokens_total")) + require.NoError(t, s.WaitReady(querier, queryFrontend, storeGateway)) + expectedTokens := float64((numIngesters + 1) * 512) // Ingesters and Store Gateway. + require.NoError(t, querier.WaitSumMetrics(e2e.Equals(expectedTokens), "cortex_ring_tokens_total")) // Query the series. for _, endpoint := range []string{queryFrontend.HTTPEndpoint(), querier.HTTPEndpoint()} { diff --git a/integration/configs.go b/integration/configs.go index 5d2f8c78c3..bfae33c54d 100644 --- a/integration/configs.go +++ b/integration/configs.go @@ -1,3 +1,4 @@ +//go:build requires_docker // +build requires_docker package integration @@ -27,6 +28,7 @@ const ( cortexConfigFile = "config.yaml" cortexSchemaConfigFile = "schema.yaml" blocksStorageEngine = "blocks" + chunksStorageEngine = "chunks" clientCertFile = "certs/client.crt" clientKeyFile = "certs/client.key" caCertFile = "certs/root.crt" @@ -233,32 +235,13 @@ blocks_storage: ChunksStorageFlags = func() map[string]string { return map[string]string{ + "-store.engine": chunksStorageEngine, "-dynamodb.url": fmt.Sprintf("dynamodb://u:p@%s-dynamodb.:8000", networkName), "-table-manager.poll-interval": "1m", "-schema-config-file": filepath.Join(e2e.ContainerSharedDir, cortexSchemaConfigFile), "-table-manager.retention-period": "168h", } } - - ChunksStorageConfig = buildConfigFromTemplate(` -storage: - aws: - dynamodb: - dynamodb_url: {{.DynamoDBURL}} - -table_manager: - poll_interval: 1m - retention_period: 168h - -schema: -{{.SchemaConfig}} -`, struct { - DynamoDBURL string - SchemaConfig string - }{ - DynamoDBURL: fmt.Sprintf("dynamodb://u:p@%s-dynamodb.:8000", networkName), - SchemaConfig: indentConfig(cortexSchemaConfigYaml, 2), - }) ) func buildConfigFromTemplate(tmpl string, data interface{}) string { diff --git a/integration/e2ecortex/services.go b/integration/e2ecortex/services.go index 1ed6c53e2d..ebda7a2734 100644 --- a/integration/e2ecortex/services.go +++ b/integration/e2ecortex/services.go @@ -424,9 +424,14 @@ func NewRuler(name string, consulAddress string, flags map[string]string, image e2e.NewCommandWithoutEntrypoint("cortex", e2e.BuildArgs(e2e.MergeFlags(map[string]string{ "-target": "ruler", "-log.level": "warn", - // Configure the ingesters ring backend - "-ring.store": "consul", - "-consul.hostname": consulAddress, + // Configure the ring backend + "-ring.store": "consul", + "-store-gateway.sharding-ring.store": "consul", + "-consul.hostname": consulAddress, + "-store-gateway.sharding-ring.consul.hostname": consulAddress, + // Store-gateway ring backend. + "-store-gateway.sharding-enabled": "true", + "-store-gateway.sharding-ring.replication-factor": "1", }, flags))...), e2e.NewHTTPReadinessProbe(httpPort, "/ready", 200, 299), httpPort, diff --git a/integration/integration_memberlist_single_binary_test.go b/integration/integration_memberlist_single_binary_test.go index df1a3cf8a6..6146f477c4 100644 --- a/integration/integration_memberlist_single_binary_test.go +++ b/integration/integration_memberlist_single_binary_test.go @@ -43,11 +43,10 @@ func testSingleBinaryEnv(t *testing.T, tlsEnabled bool, flags map[string]string) defer s.Close() // Start dependencies - dynamo := e2edb.NewDynamoDB() + minio := e2edb.NewMinio(9000, bucketName) // Look ma, no Consul! - require.NoError(t, s.StartAndWaitReady(dynamo)) + require.NoError(t, s.StartAndWaitReady(minio)) - require.NoError(t, writeFileToSharedDir(s, cortexSchemaConfigFile, []byte(cortexSchemaConfigYaml))) var cortex1, cortex2, cortex3 *e2ecortex.CortexService if tlsEnabled { var ( @@ -136,7 +135,7 @@ func newSingleBinary(name string, servername string, join string, testFlags map[ serv := e2ecortex.NewSingleBinary( name, mergeFlags( - ChunksStorageFlags(), + BlocksStorageFlags(), flags, testFlags, getTLSFlagsWithPrefix("memberlist", servername, servername == ""), @@ -160,9 +159,8 @@ func TestSingleBinaryWithMemberlistScaling(t *testing.T) { require.NoError(t, err) defer s.Close() - dynamo := e2edb.NewDynamoDB() - require.NoError(t, s.StartAndWaitReady(dynamo)) - require.NoError(t, writeFileToSharedDir(s, cortexSchemaConfigFile, []byte(cortexSchemaConfigYaml))) + minio := e2edb.NewMinio(9000, bucketName) + require.NoError(t, s.StartAndWaitReady(minio)) // Scale up instances. These numbers seem enough to reliably reproduce some unwanted // consequences of slow propagation, such as missing tombstones. diff --git a/integration/querier_remote_read_test.go b/integration/querier_remote_read_test.go index 217044f3ee..20be210df3 100644 --- a/integration/querier_remote_read_test.go +++ b/integration/querier_remote_read_test.go @@ -29,20 +29,12 @@ func TestQuerierRemoteRead(t *testing.T) { defer s.Close() require.NoError(t, writeFileToSharedDir(s, cortexSchemaConfigFile, []byte(cortexSchemaConfigYaml))) - flags := mergeFlags(ChunksStorageFlags(), map[string]string{}) + flags := mergeFlags(BlocksStorageFlags(), map[string]string{}) // Start dependencies. - dynamo := e2edb.NewDynamoDB() - + minio := e2edb.NewMinio(9000, bucketName) consul := e2edb.NewConsul() - require.NoError(t, s.StartAndWaitReady(consul, dynamo)) - - tableManager := e2ecortex.NewTableManager("table-manager", ChunksStorageFlags(), "") - require.NoError(t, s.StartAndWaitReady(tableManager)) - - // Wait until the first table-manager sync has completed, so that we're - // sure the tables have been created. - require.NoError(t, tableManager.WaitSumMetrics(e2e.Greater(0), "cortex_table_manager_sync_success_timestamp_seconds")) + require.NoError(t, s.StartAndWaitReady(consul, minio)) // Start Cortex components for the write path. distributor := e2ecortex.NewDistributor("distributor", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "") @@ -63,11 +55,13 @@ func TestQuerierRemoteRead(t *testing.T) { require.NoError(t, err) require.Equal(t, 200, res.StatusCode) - querier := e2ecortex.NewQuerier("querier", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), ChunksStorageFlags(), "") + storeGateway := e2ecortex.NewStoreGateway("store-gateway", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "") + require.NoError(t, s.StartAndWaitReady(storeGateway)) + querier := e2ecortex.NewQuerier("querier", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), BlocksStorageFlags(), "") require.NoError(t, s.StartAndWaitReady(querier)) // Wait until the querier has updated the ring. - require.NoError(t, querier.WaitSumMetrics(e2e.Equals(512), "cortex_ring_tokens_total")) + require.NoError(t, querier.WaitSumMetrics(e2e.Equals(2*512), "cortex_ring_tokens_total")) matcher, err := labels.NewMatcher(labels.MatchEqual, "__name__", "series_1") require.NoError(t, err) diff --git a/integration/querier_streaming_mixed_ingester_test.go b/integration/querier_streaming_mixed_ingester_test.go deleted file mode 100644 index 910356b546..0000000000 --- a/integration/querier_streaming_mixed_ingester_test.go +++ /dev/null @@ -1,164 +0,0 @@ -// +build requires_docker - -package integration - -import ( - "context" - "flag" - "fmt" - "strings" - "testing" - "time" - - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - "github.com/stretchr/testify/require" - "github.com/weaveworks/common/user" - - "github.com/cortexproject/cortex/integration/e2e" - e2edb "github.com/cortexproject/cortex/integration/e2e/db" - "github.com/cortexproject/cortex/integration/e2ecortex" - "github.com/cortexproject/cortex/pkg/cortexpb" - ingester_client "github.com/cortexproject/cortex/pkg/ingester/client" -) - -func TestQuerierWithStreamingBlocksAndChunksIngesters(t *testing.T) { - for _, streamChunks := range []bool{false, true} { - t.Run(fmt.Sprintf("%v", streamChunks), func(t *testing.T) { - testQuerierWithStreamingBlocksAndChunksIngesters(t, streamChunks) - }) - } -} - -func testQuerierWithStreamingBlocksAndChunksIngesters(t *testing.T, streamChunks bool) { - s, err := e2e.NewScenario(networkName) - require.NoError(t, err) - defer s.Close() - - require.NoError(t, writeFileToSharedDir(s, cortexSchemaConfigFile, []byte(cortexSchemaConfigYaml))) - chunksFlags := ChunksStorageFlags() - blockFlags := mergeFlags(BlocksStorageFlags(), map[string]string{ - "-blocks-storage.tsdb.block-ranges-period": "1h", - "-blocks-storage.tsdb.head-compaction-interval": "1m", - "-store-gateway.sharding-enabled": "false", - "-querier.ingester-streaming": "true", - }) - blockFlags["-ingester.stream-chunks-when-using-blocks"] = fmt.Sprintf("%v", streamChunks) - - // Start dependencies. - consul := e2edb.NewConsul() - minio := e2edb.NewMinio(9000, blockFlags["-blocks-storage.s3.bucket-name"]) - require.NoError(t, s.StartAndWaitReady(consul, minio)) - - // Start Cortex components. - ingesterBlocks := e2ecortex.NewIngester("ingester-blocks", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), blockFlags, "") - ingesterChunks := e2ecortex.NewIngester("ingester-chunks", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), chunksFlags, "") - storeGateway := e2ecortex.NewStoreGateway("store-gateway", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), blockFlags, "") - require.NoError(t, s.StartAndWaitReady(ingesterBlocks, ingesterChunks, storeGateway)) - - // Sharding is disabled, pass gateway address. - querierFlags := mergeFlags(blockFlags, map[string]string{ - "-querier.store-gateway-addresses": strings.Join([]string{storeGateway.NetworkGRPCEndpoint()}, ","), - "-distributor.shard-by-all-labels": "true", - }) - querier := e2ecortex.NewQuerier("querier", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), querierFlags, "") - require.NoError(t, s.StartAndWaitReady(querier)) - - require.NoError(t, querier.WaitSumMetrics(e2e.Equals(1024), "cortex_ring_tokens_total")) - - s1 := []cortexpb.Sample{ - {Value: 1, TimestampMs: 1000}, - {Value: 2, TimestampMs: 2000}, - {Value: 3, TimestampMs: 3000}, - {Value: 4, TimestampMs: 4000}, - {Value: 5, TimestampMs: 5000}, - } - - s2 := []cortexpb.Sample{ - {Value: 1, TimestampMs: 1000}, - {Value: 2.5, TimestampMs: 2500}, - {Value: 3, TimestampMs: 3000}, - {Value: 5.5, TimestampMs: 5500}, - } - - clientConfig := ingester_client.Config{} - clientConfig.RegisterFlags(flag.NewFlagSet("unused", flag.ContinueOnError)) // registers default values - - // Push data to chunks ingester. - { - ingesterChunksClient, err := ingester_client.MakeIngesterClient(ingesterChunks.GRPCEndpoint(), clientConfig) - require.NoError(t, err) - defer ingesterChunksClient.Close() - - _, err = ingesterChunksClient.Push(user.InjectOrgID(context.Background(), "user"), &cortexpb.WriteRequest{ - Timeseries: []cortexpb.PreallocTimeseries{ - {TimeSeries: &cortexpb.TimeSeries{Labels: []cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "s"}, {Name: "l", Value: "1"}}, Samples: s1}}, - {TimeSeries: &cortexpb.TimeSeries{Labels: []cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "s"}, {Name: "l", Value: "2"}}, Samples: s1}}}, - Source: cortexpb.API, - }) - require.NoError(t, err) - } - - // Push data to blocks ingester. - { - ingesterBlocksClient, err := ingester_client.MakeIngesterClient(ingesterBlocks.GRPCEndpoint(), clientConfig) - require.NoError(t, err) - defer ingesterBlocksClient.Close() - - _, err = ingesterBlocksClient.Push(user.InjectOrgID(context.Background(), "user"), &cortexpb.WriteRequest{ - Timeseries: []cortexpb.PreallocTimeseries{ - {TimeSeries: &cortexpb.TimeSeries{Labels: []cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "s"}, {Name: "l", Value: "2"}}, Samples: s2}}, - {TimeSeries: &cortexpb.TimeSeries{Labels: []cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "s"}, {Name: "l", Value: "3"}}, Samples: s1}}}, - Source: cortexpb.API, - }) - require.NoError(t, err) - } - - c, err := e2ecortex.NewClient("", querier.HTTPEndpoint(), "", "", "user") - require.NoError(t, err) - - // Query back the series (1 only in the storage, 1 only in the ingesters, 1 on both). - result, err := c.Query("s[1m]", time.Unix(10, 0)) - require.NoError(t, err) - - s1Values := []model.SamplePair{ - {Value: 1, Timestamp: 1000}, - {Value: 2, Timestamp: 2000}, - {Value: 3, Timestamp: 3000}, - {Value: 4, Timestamp: 4000}, - {Value: 5, Timestamp: 5000}, - } - - s1AndS2ValuesMerged := []model.SamplePair{ - {Value: 1, Timestamp: 1000}, - {Value: 2, Timestamp: 2000}, - {Value: 2.5, Timestamp: 2500}, - {Value: 3, Timestamp: 3000}, - {Value: 4, Timestamp: 4000}, - {Value: 5, Timestamp: 5000}, - {Value: 5.5, Timestamp: 5500}, - } - - expectedMatrix := model.Matrix{ - // From chunks ingester only. - &model.SampleStream{ - Metric: model.Metric{labels.MetricName: "s", "l": "1"}, - Values: s1Values, - }, - - // From blocks ingester only. - &model.SampleStream{ - Metric: model.Metric{labels.MetricName: "s", "l": "3"}, - Values: s1Values, - }, - - // Merged from both ingesters. - &model.SampleStream{ - Metric: model.Metric{labels.MetricName: "s", "l": "2"}, - Values: s1AndS2ValuesMerged, - }, - } - - require.Equal(t, model.ValMatrix, result.Type()) - require.ElementsMatch(t, expectedMatrix, result.(model.Matrix)) -} diff --git a/integration/querier_test.go b/integration/querier_test.go index 14e4161d60..29f101d954 100644 --- a/integration/querier_test.go +++ b/integration/querier_test.go @@ -854,20 +854,13 @@ func TestHashCollisionHandling(t *testing.T) { defer s.Close() require.NoError(t, writeFileToSharedDir(s, cortexSchemaConfigFile, []byte(cortexSchemaConfigYaml))) - flags := ChunksStorageFlags() + flags := BlocksStorageFlags() // Start dependencies. - dynamo := e2edb.NewDynamoDB() + minio := e2edb.NewMinio(9000, bucketName) consul := e2edb.NewConsul() - require.NoError(t, s.StartAndWaitReady(consul, dynamo)) - - tableManager := e2ecortex.NewTableManager("table-manager", ChunksStorageFlags(), "") - require.NoError(t, s.StartAndWaitReady(tableManager)) - - // Wait until the first table-manager sync has completed, so that we're - // sure the tables have been created. - require.NoError(t, tableManager.WaitSumMetrics(e2e.Greater(0), "cortex_table_manager_sync_success_timestamp_seconds")) + require.NoError(t, s.StartAndWaitReady(consul, minio)) // Start Cortex components for the write path. distributor := e2ecortex.NewDistributor("distributor", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "") @@ -923,11 +916,13 @@ func TestHashCollisionHandling(t *testing.T) { require.NoError(t, err) require.Equal(t, 200, res.StatusCode) + storeGateway := e2ecortex.NewStoreGateway("store-gateway", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "") + require.NoError(t, s.StartAndWaitReady(storeGateway)) querier := e2ecortex.NewQuerier("querier", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "") require.NoError(t, s.StartAndWaitReady(querier)) // Wait until the querier has updated the ring. - require.NoError(t, querier.WaitSumMetrics(e2e.Equals(512), "cortex_ring_tokens_total")) + require.NoError(t, querier.WaitSumMetrics(e2e.Equals(2*512), "cortex_ring_tokens_total")) // Query the series. c, err = e2ecortex.NewClient("", querier.HTTPEndpoint(), "", "", "user-0") diff --git a/integration/ruler_test.go b/integration/ruler_test.go index 15c7d4b4ba..ac8ab0ea40 100644 --- a/integration/ruler_test.go +++ b/integration/ruler_test.go @@ -56,13 +56,11 @@ func TestRulerAPI(t *testing.T) { // Start dependencies. consul := e2edb.NewConsul() - dynamo := e2edb.NewDynamoDB() - minio := e2edb.NewMinio(9000, rulestoreBucketName) - require.NoError(t, s.StartAndWaitReady(consul, minio, dynamo)) + minio := e2edb.NewMinio(9000, rulestoreBucketName, bucketName) + require.NoError(t, s.StartAndWaitReady(consul, minio)) // Start Cortex components. - require.NoError(t, writeFileToSharedDir(s, cortexSchemaConfigFile, []byte(cortexSchemaConfigYaml))) - ruler := e2ecortex.NewRuler("ruler", consul.NetworkHTTPEndpoint(), mergeFlags(ChunksStorageFlags(), RulerFlags(testCfg.legacyRuleStore)), "") + ruler := e2ecortex.NewRuler("ruler", consul.NetworkHTTPEndpoint(), mergeFlags(BlocksStorageFlags(), RulerFlags(testCfg.legacyRuleStore)), "") require.NoError(t, s.StartAndWaitReady(ruler)) // Create a client with the ruler address configured @@ -159,7 +157,7 @@ func TestRulerAPISingleBinary(t *testing.T) { } // Start Cortex components. - require.NoError(t, copyFileToSharedDir(s, "docs/chunks-storage/single-process-config.yaml", cortexConfigFile)) + require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile)) require.NoError(t, writeFileToSharedDir(s, filepath.Join("ruler_configs", user, namespace), []byte(cortexRulerUserConfigYaml))) cortex := e2ecortex.NewSingleBinaryWithConfigFile("cortex", cortexConfigFile, configOverrides, "", 9009, 9095) require.NoError(t, s.StartAndWaitReady(cortex)) @@ -218,7 +216,7 @@ func TestRulerEvaluationDelay(t *testing.T) { } // Start Cortex components. - require.NoError(t, copyFileToSharedDir(s, "docs/chunks-storage/single-process-config.yaml", cortexConfigFile)) + require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile)) require.NoError(t, writeFileToSharedDir(s, filepath.Join("ruler_configs", user, namespace), []byte(cortexRulerEvalStaleNanConfigYaml))) cortex := e2ecortex.NewSingleBinaryWithConfigFile("cortex", cortexConfigFile, configOverrides, "", 9009, 9095) require.NoError(t, s.StartAndWaitReady(cortex)) @@ -404,9 +402,8 @@ func TestRulerAlertmanager(t *testing.T) { // Start dependencies. consul := e2edb.NewConsul() - dynamo := e2edb.NewDynamoDB() - minio := e2edb.NewMinio(9000, rulestoreBucketName) - require.NoError(t, s.StartAndWaitReady(consul, minio, dynamo)) + minio := e2edb.NewMinio(9000, rulestoreBucketName, bucketName) + require.NoError(t, s.StartAndWaitReady(consul, minio)) // Have at least one alertmanager configuration. require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs/user-1.yaml", []byte(cortexAlertmanagerUserConfigYaml))) @@ -426,8 +423,7 @@ func TestRulerAlertmanager(t *testing.T) { } // Start Ruler. - require.NoError(t, writeFileToSharedDir(s, cortexSchemaConfigFile, []byte(cortexSchemaConfigYaml))) - ruler := e2ecortex.NewRuler("ruler", consul.NetworkHTTPEndpoint(), mergeFlags(ChunksStorageFlags(), RulerFlags(false), configOverrides), "") + ruler := e2ecortex.NewRuler("ruler", consul.NetworkHTTPEndpoint(), mergeFlags(BlocksStorageFlags(), RulerFlags(false), configOverrides), "") require.NoError(t, s.StartAndWaitReady(ruler)) // Create a client with the ruler address configured @@ -454,9 +450,8 @@ func TestRulerAlertmanagerTLS(t *testing.T) { // Start dependencies. consul := e2edb.NewConsul() - dynamo := e2edb.NewDynamoDB() - minio := e2edb.NewMinio(9000, rulestoreBucketName) - require.NoError(t, s.StartAndWaitReady(consul, minio, dynamo)) + minio := e2edb.NewMinio(9000, rulestoreBucketName, bucketName) + require.NoError(t, s.StartAndWaitReady(consul, minio)) // set the ca cert := ca.New("Ruler/Alertmanager Test") @@ -506,8 +501,7 @@ func TestRulerAlertmanagerTLS(t *testing.T) { ) // Start Ruler. - require.NoError(t, writeFileToSharedDir(s, cortexSchemaConfigFile, []byte(cortexSchemaConfigYaml))) - ruler := e2ecortex.NewRuler("ruler", consul.NetworkHTTPEndpoint(), mergeFlags(ChunksStorageFlags(), RulerFlags(false), configOverrides), "") + ruler := e2ecortex.NewRuler("ruler", consul.NetworkHTTPEndpoint(), mergeFlags(BlocksStorageFlags(), RulerFlags(false), configOverrides), "") require.NoError(t, s.StartAndWaitReady(ruler)) // Create a client with the ruler address configured diff --git a/pkg/api/api.go b/pkg/api/api.go index ced7685448..4d9e2f5cab 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -6,13 +6,11 @@ import ( "net/http" "path" "strings" - "time" "github.com/NYTimes/gziphandler" "github.com/felixge/fgprof" "github.com/go-kit/log" "github.com/go-kit/log/level" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/storage" "github.com/weaveworks/common/middleware" "github.com/weaveworks/common/server" @@ -263,22 +261,6 @@ func (a *API) RegisterIngester(i Ingester, pushConfig distributor.Config) { a.RegisterRoute("/push", push.Handler(pushConfig.MaxRecvMsgSize, a.sourceIPs, i.Push), true, "POST") // For testing and debugging. } -// RegisterChunksPurger registers the endpoints associated with the Purger/DeleteStore. They do not exactly -// match the Prometheus API but mirror it closely enough to justify their routing under the Prometheus -// component/ -func (a *API) RegisterChunksPurger(store *purger.DeleteStore, deleteRequestCancelPeriod time.Duration) { - deleteRequestHandler := purger.NewDeleteRequestHandler(store, deleteRequestCancelPeriod, prometheus.DefaultRegisterer) - - a.RegisterRoute(path.Join(a.cfg.PrometheusHTTPPrefix, "/api/v1/admin/tsdb/delete_series"), http.HandlerFunc(deleteRequestHandler.AddDeleteRequestHandler), true, "PUT", "POST") - a.RegisterRoute(path.Join(a.cfg.PrometheusHTTPPrefix, "/api/v1/admin/tsdb/delete_series"), http.HandlerFunc(deleteRequestHandler.GetAllDeleteRequestsHandler), true, "GET") - a.RegisterRoute(path.Join(a.cfg.PrometheusHTTPPrefix, "/api/v1/admin/tsdb/cancel_delete_request"), http.HandlerFunc(deleteRequestHandler.CancelDeleteRequestHandler), true, "PUT", "POST") - - // Legacy Routes - a.RegisterRoute(path.Join(a.cfg.LegacyHTTPPrefix, "/api/v1/admin/tsdb/delete_series"), http.HandlerFunc(deleteRequestHandler.AddDeleteRequestHandler), true, "PUT", "POST") - a.RegisterRoute(path.Join(a.cfg.LegacyHTTPPrefix, "/api/v1/admin/tsdb/delete_series"), http.HandlerFunc(deleteRequestHandler.GetAllDeleteRequestsHandler), true, "GET") - a.RegisterRoute(path.Join(a.cfg.LegacyHTTPPrefix, "/api/v1/admin/tsdb/cancel_delete_request"), http.HandlerFunc(deleteRequestHandler.CancelDeleteRequestHandler), true, "PUT", "POST") -} - func (a *API) RegisterTenantDeletion(api *purger.TenantDeletionAPI) { a.RegisterRoute("/purger/delete_tenant", http.HandlerFunc(api.DeleteTenant), true, "POST") a.RegisterRoute("/purger/delete_tenant_status", http.HandlerFunc(api.DeleteTenantStatus), true, "GET") diff --git a/pkg/api/handlers.go b/pkg/api/handlers.go index b775573e83..67cdadfcd9 100644 --- a/pkg/api/handlers.go +++ b/pkg/api/handlers.go @@ -160,7 +160,7 @@ func NewQuerierHandler( exemplarQueryable storage.ExemplarQueryable, engine *promql.Engine, distributor Distributor, - tombstonesLoader *purger.TombstonesLoader, + tombstonesLoader purger.TombstonesLoader, reg prometheus.Registerer, logger log.Logger, ) http.Handler { diff --git a/pkg/api/middlewares.go b/pkg/api/middlewares.go index 7e0e88e803..b60326e51c 100644 --- a/pkg/api/middlewares.go +++ b/pkg/api/middlewares.go @@ -11,7 +11,7 @@ import ( ) // middleware for setting cache gen header to let consumer of response know all previous responses could be invalid due to delete operation -func getHTTPCacheGenNumberHeaderSetterMiddleware(cacheGenNumbersLoader *purger.TombstonesLoader) middleware.Interface { +func getHTTPCacheGenNumberHeaderSetterMiddleware(cacheGenNumbersLoader purger.TombstonesLoader) middleware.Interface { return middleware.Func(func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { tenantIDs, err := tenant.TenantIDs(r.Context()) diff --git a/pkg/chunk/purger/delete_plan.pb.go b/pkg/chunk/purger/delete_plan.pb.go deleted file mode 100644 index 5646b2b4eb..0000000000 --- a/pkg/chunk/purger/delete_plan.pb.go +++ /dev/null @@ -1,1353 +0,0 @@ -// Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: delete_plan.proto - -package purger - -import ( - fmt "fmt" - _ "github.com/cortexproject/cortex/pkg/cortexpb" - github_com_cortexproject_cortex_pkg_cortexpb "github.com/cortexproject/cortex/pkg/cortexpb" - _ "github.com/gogo/protobuf/gogoproto" - proto "github.com/gogo/protobuf/proto" - io "io" - math "math" - math_bits "math/bits" - reflect "reflect" - strings "strings" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package - -// DeletePlan holds all the chunks that are supposed to be deleted within an interval(usually a day) -// This Proto file is used just for storing Delete Plans in proto format. -type DeletePlan struct { - PlanInterval *Interval `protobuf:"bytes,1,opt,name=plan_interval,json=planInterval,proto3" json:"plan_interval,omitempty"` - ChunksGroup []ChunksGroup `protobuf:"bytes,2,rep,name=chunks_group,json=chunksGroup,proto3" json:"chunks_group"` -} - -func (m *DeletePlan) Reset() { *m = DeletePlan{} } -func (*DeletePlan) ProtoMessage() {} -func (*DeletePlan) Descriptor() ([]byte, []int) { - return fileDescriptor_c38868cf63b27372, []int{0} -} -func (m *DeletePlan) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *DeletePlan) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_DeletePlan.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *DeletePlan) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeletePlan.Merge(m, src) -} -func (m *DeletePlan) XXX_Size() int { - return m.Size() -} -func (m *DeletePlan) XXX_DiscardUnknown() { - xxx_messageInfo_DeletePlan.DiscardUnknown(m) -} - -var xxx_messageInfo_DeletePlan proto.InternalMessageInfo - -func (m *DeletePlan) GetPlanInterval() *Interval { - if m != nil { - return m.PlanInterval - } - return nil -} - -func (m *DeletePlan) GetChunksGroup() []ChunksGroup { - if m != nil { - return m.ChunksGroup - } - return nil -} - -// ChunksGroup holds ChunkDetails and Labels for a group of chunks which have same series ID -type ChunksGroup struct { - Labels []github_com_cortexproject_cortex_pkg_cortexpb.LabelAdapter `protobuf:"bytes,1,rep,name=labels,proto3,customtype=github.com/cortexproject/cortex/pkg/cortexpb.LabelAdapter" json:"labels"` - Chunks []ChunkDetails `protobuf:"bytes,2,rep,name=chunks,proto3" json:"chunks"` -} - -func (m *ChunksGroup) Reset() { *m = ChunksGroup{} } -func (*ChunksGroup) ProtoMessage() {} -func (*ChunksGroup) Descriptor() ([]byte, []int) { - return fileDescriptor_c38868cf63b27372, []int{1} -} -func (m *ChunksGroup) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ChunksGroup) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ChunksGroup.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *ChunksGroup) XXX_Merge(src proto.Message) { - xxx_messageInfo_ChunksGroup.Merge(m, src) -} -func (m *ChunksGroup) XXX_Size() int { - return m.Size() -} -func (m *ChunksGroup) XXX_DiscardUnknown() { - xxx_messageInfo_ChunksGroup.DiscardUnknown(m) -} - -var xxx_messageInfo_ChunksGroup proto.InternalMessageInfo - -func (m *ChunksGroup) GetChunks() []ChunkDetails { - if m != nil { - return m.Chunks - } - return nil -} - -type ChunkDetails struct { - ID string `protobuf:"bytes,1,opt,name=ID,proto3" json:"ID,omitempty"` - PartiallyDeletedInterval *Interval `protobuf:"bytes,2,opt,name=partially_deleted_interval,json=partiallyDeletedInterval,proto3" json:"partially_deleted_interval,omitempty"` -} - -func (m *ChunkDetails) Reset() { *m = ChunkDetails{} } -func (*ChunkDetails) ProtoMessage() {} -func (*ChunkDetails) Descriptor() ([]byte, []int) { - return fileDescriptor_c38868cf63b27372, []int{2} -} -func (m *ChunkDetails) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ChunkDetails) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ChunkDetails.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *ChunkDetails) XXX_Merge(src proto.Message) { - xxx_messageInfo_ChunkDetails.Merge(m, src) -} -func (m *ChunkDetails) XXX_Size() int { - return m.Size() -} -func (m *ChunkDetails) XXX_DiscardUnknown() { - xxx_messageInfo_ChunkDetails.DiscardUnknown(m) -} - -var xxx_messageInfo_ChunkDetails proto.InternalMessageInfo - -func (m *ChunkDetails) GetID() string { - if m != nil { - return m.ID - } - return "" -} - -func (m *ChunkDetails) GetPartiallyDeletedInterval() *Interval { - if m != nil { - return m.PartiallyDeletedInterval - } - return nil -} - -type Interval struct { - StartTimestampMs int64 `protobuf:"varint,1,opt,name=start_timestamp_ms,json=startTimestampMs,proto3" json:"start_timestamp_ms,omitempty"` - EndTimestampMs int64 `protobuf:"varint,2,opt,name=end_timestamp_ms,json=endTimestampMs,proto3" json:"end_timestamp_ms,omitempty"` -} - -func (m *Interval) Reset() { *m = Interval{} } -func (*Interval) ProtoMessage() {} -func (*Interval) Descriptor() ([]byte, []int) { - return fileDescriptor_c38868cf63b27372, []int{3} -} -func (m *Interval) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Interval) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Interval.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *Interval) XXX_Merge(src proto.Message) { - xxx_messageInfo_Interval.Merge(m, src) -} -func (m *Interval) XXX_Size() int { - return m.Size() -} -func (m *Interval) XXX_DiscardUnknown() { - xxx_messageInfo_Interval.DiscardUnknown(m) -} - -var xxx_messageInfo_Interval proto.InternalMessageInfo - -func (m *Interval) GetStartTimestampMs() int64 { - if m != nil { - return m.StartTimestampMs - } - return 0 -} - -func (m *Interval) GetEndTimestampMs() int64 { - if m != nil { - return m.EndTimestampMs - } - return 0 -} - -func init() { - proto.RegisterType((*DeletePlan)(nil), "purgeplan.DeletePlan") - proto.RegisterType((*ChunksGroup)(nil), "purgeplan.ChunksGroup") - proto.RegisterType((*ChunkDetails)(nil), "purgeplan.ChunkDetails") - proto.RegisterType((*Interval)(nil), "purgeplan.Interval") -} - -func init() { proto.RegisterFile("delete_plan.proto", fileDescriptor_c38868cf63b27372) } - -var fileDescriptor_c38868cf63b27372 = []byte{ - // 446 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x52, 0x41, 0x8b, 0xd4, 0x30, - 0x18, 0x6d, 0xba, 0x52, 0xdc, 0x74, 0x5c, 0xd6, 0x2c, 0x68, 0x99, 0x43, 0x76, 0xe9, 0x69, 0x0e, - 0xda, 0x81, 0x15, 0x41, 0x41, 0x90, 0x1d, 0x0b, 0x32, 0xa0, 0xb0, 0x16, 0x4f, 0x5e, 0x4a, 0xda, - 0xc6, 0x6e, 0xdd, 0xb4, 0x89, 0x69, 0x2a, 0x7a, 0xf3, 0xe6, 0xd5, 0x9f, 0xe1, 0x0f, 0xf0, 0x47, - 0xec, 0x71, 0x8e, 0x8b, 0x87, 0xc1, 0xe9, 0x5c, 0x3c, 0xce, 0x4f, 0x90, 0xa6, 0xed, 0x4c, 0x15, - 0x3c, 0x78, 0xcb, 0xfb, 0xde, 0x7b, 0xc9, 0xcb, 0x4b, 0xe0, 0xed, 0x84, 0x32, 0xaa, 0x68, 0x28, - 0x18, 0x29, 0x3c, 0x21, 0xb9, 0xe2, 0x68, 0x5f, 0x54, 0x32, 0xa5, 0xcd, 0x60, 0x7c, 0x3f, 0xcd, - 0xd4, 0x45, 0x15, 0x79, 0x31, 0xcf, 0xa7, 0x29, 0x4f, 0xf9, 0x54, 0x2b, 0xa2, 0xea, 0xad, 0x46, - 0x1a, 0xe8, 0x55, 0xeb, 0x1c, 0x3f, 0x1e, 0xc8, 0x63, 0x2e, 0x15, 0xfd, 0x28, 0x24, 0x7f, 0x47, - 0x63, 0xd5, 0xa1, 0xa9, 0xb8, 0x4c, 0x7b, 0x22, 0xea, 0x16, 0xad, 0xd5, 0xfd, 0x02, 0x20, 0xf4, - 0x75, 0x94, 0x73, 0x46, 0x0a, 0xf4, 0x08, 0xde, 0x6a, 0x02, 0x84, 0x59, 0xa1, 0xa8, 0xfc, 0x40, - 0x98, 0x03, 0x4e, 0xc0, 0xc4, 0x3e, 0x3d, 0xf2, 0xb6, 0xd9, 0xbc, 0x79, 0x47, 0x05, 0xa3, 0x06, - 0xf6, 0x08, 0x3d, 0x85, 0xa3, 0xf8, 0xa2, 0x2a, 0x2e, 0xcb, 0x30, 0x95, 0xbc, 0x12, 0x8e, 0x79, - 0xb2, 0x37, 0xb1, 0x4f, 0xef, 0x0c, 0x8c, 0xcf, 0x34, 0xfd, 0xbc, 0x61, 0x67, 0x37, 0xae, 0x96, - 0xc7, 0x46, 0x60, 0xc7, 0xbb, 0x91, 0xfb, 0x1d, 0x40, 0x7b, 0x20, 0x41, 0x05, 0xb4, 0x18, 0x89, - 0x28, 0x2b, 0x1d, 0xa0, 0xb7, 0x3a, 0xf2, 0xfa, 0x1b, 0x78, 0x2f, 0x9a, 0xf9, 0x39, 0xc9, 0xe4, - 0xec, 0xac, 0xd9, 0xe7, 0xc7, 0xf2, 0xf8, 0xbf, 0x1a, 0x68, 0xfd, 0x67, 0x09, 0x11, 0x8a, 0xca, - 0xa0, 0x3b, 0x05, 0x3d, 0x84, 0x56, 0x1b, 0xa7, 0x8b, 0x7e, 0xf7, 0xef, 0xe8, 0x3e, 0x55, 0x24, - 0x63, 0x65, 0x97, 0xbd, 0x13, 0xbb, 0xef, 0xe1, 0x68, 0xc8, 0xa2, 0x03, 0x68, 0xce, 0x7d, 0x5d, - 0xdb, 0x7e, 0x60, 0xce, 0x7d, 0xf4, 0x0a, 0x8e, 0x05, 0x91, 0x2a, 0x23, 0x8c, 0x7d, 0x0a, 0xdb, - 0x47, 0x4f, 0x76, 0xf5, 0x9a, 0xff, 0xae, 0xd7, 0xd9, 0xda, 0xda, 0xf7, 0x49, 0x7a, 0xc6, 0x8d, - 0xe0, 0xcd, 0x6d, 0xed, 0xf7, 0x20, 0x2a, 0x15, 0x91, 0x2a, 0x54, 0x59, 0x4e, 0x4b, 0x45, 0x72, - 0x11, 0xe6, 0xa5, 0x3e, 0x7e, 0x2f, 0x38, 0xd4, 0xcc, 0xeb, 0x9e, 0x78, 0x59, 0xa2, 0x09, 0x3c, - 0xa4, 0x45, 0xf2, 0xa7, 0xd6, 0xd4, 0xda, 0x03, 0x5a, 0x24, 0x03, 0xe5, 0xec, 0xc9, 0x62, 0x85, - 0x8d, 0xeb, 0x15, 0x36, 0x36, 0x2b, 0x0c, 0x3e, 0xd7, 0x18, 0x7c, 0xab, 0x31, 0xb8, 0xaa, 0x31, - 0x58, 0xd4, 0x18, 0xfc, 0xac, 0x31, 0xf8, 0x55, 0x63, 0x63, 0x53, 0x63, 0xf0, 0x75, 0x8d, 0x8d, - 0xc5, 0x1a, 0x1b, 0xd7, 0x6b, 0x6c, 0xbc, 0xb1, 0xf4, 0x3d, 0x64, 0x64, 0xe9, 0xcf, 0xf5, 0xe0, - 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0xf5, 0x46, 0x96, 0xf6, 0xe6, 0x02, 0x00, 0x00, -} - -func (this *DeletePlan) Equal(that interface{}) bool { - if that == nil { - return this == nil - } - - that1, ok := that.(*DeletePlan) - if !ok { - that2, ok := that.(DeletePlan) - if ok { - that1 = &that2 - } else { - return false - } - } - if that1 == nil { - return this == nil - } else if this == nil { - return false - } - if !this.PlanInterval.Equal(that1.PlanInterval) { - return false - } - if len(this.ChunksGroup) != len(that1.ChunksGroup) { - return false - } - for i := range this.ChunksGroup { - if !this.ChunksGroup[i].Equal(&that1.ChunksGroup[i]) { - return false - } - } - return true -} -func (this *ChunksGroup) Equal(that interface{}) bool { - if that == nil { - return this == nil - } - - that1, ok := that.(*ChunksGroup) - if !ok { - that2, ok := that.(ChunksGroup) - if ok { - that1 = &that2 - } else { - return false - } - } - if that1 == nil { - return this == nil - } else if this == nil { - return false - } - if len(this.Labels) != len(that1.Labels) { - return false - } - for i := range this.Labels { - if !this.Labels[i].Equal(that1.Labels[i]) { - return false - } - } - if len(this.Chunks) != len(that1.Chunks) { - return false - } - for i := range this.Chunks { - if !this.Chunks[i].Equal(&that1.Chunks[i]) { - return false - } - } - return true -} -func (this *ChunkDetails) Equal(that interface{}) bool { - if that == nil { - return this == nil - } - - that1, ok := that.(*ChunkDetails) - if !ok { - that2, ok := that.(ChunkDetails) - if ok { - that1 = &that2 - } else { - return false - } - } - if that1 == nil { - return this == nil - } else if this == nil { - return false - } - if this.ID != that1.ID { - return false - } - if !this.PartiallyDeletedInterval.Equal(that1.PartiallyDeletedInterval) { - return false - } - return true -} -func (this *Interval) Equal(that interface{}) bool { - if that == nil { - return this == nil - } - - that1, ok := that.(*Interval) - if !ok { - that2, ok := that.(Interval) - if ok { - that1 = &that2 - } else { - return false - } - } - if that1 == nil { - return this == nil - } else if this == nil { - return false - } - if this.StartTimestampMs != that1.StartTimestampMs { - return false - } - if this.EndTimestampMs != that1.EndTimestampMs { - return false - } - return true -} -func (this *DeletePlan) GoString() string { - if this == nil { - return "nil" - } - s := make([]string, 0, 6) - s = append(s, "&purger.DeletePlan{") - if this.PlanInterval != nil { - s = append(s, "PlanInterval: "+fmt.Sprintf("%#v", this.PlanInterval)+",\n") - } - if this.ChunksGroup != nil { - vs := make([]*ChunksGroup, len(this.ChunksGroup)) - for i := range vs { - vs[i] = &this.ChunksGroup[i] - } - s = append(s, "ChunksGroup: "+fmt.Sprintf("%#v", vs)+",\n") - } - s = append(s, "}") - return strings.Join(s, "") -} -func (this *ChunksGroup) GoString() string { - if this == nil { - return "nil" - } - s := make([]string, 0, 6) - s = append(s, "&purger.ChunksGroup{") - s = append(s, "Labels: "+fmt.Sprintf("%#v", this.Labels)+",\n") - if this.Chunks != nil { - vs := make([]*ChunkDetails, len(this.Chunks)) - for i := range vs { - vs[i] = &this.Chunks[i] - } - s = append(s, "Chunks: "+fmt.Sprintf("%#v", vs)+",\n") - } - s = append(s, "}") - return strings.Join(s, "") -} -func (this *ChunkDetails) GoString() string { - if this == nil { - return "nil" - } - s := make([]string, 0, 6) - s = append(s, "&purger.ChunkDetails{") - s = append(s, "ID: "+fmt.Sprintf("%#v", this.ID)+",\n") - if this.PartiallyDeletedInterval != nil { - s = append(s, "PartiallyDeletedInterval: "+fmt.Sprintf("%#v", this.PartiallyDeletedInterval)+",\n") - } - s = append(s, "}") - return strings.Join(s, "") -} -func (this *Interval) GoString() string { - if this == nil { - return "nil" - } - s := make([]string, 0, 6) - s = append(s, "&purger.Interval{") - s = append(s, "StartTimestampMs: "+fmt.Sprintf("%#v", this.StartTimestampMs)+",\n") - s = append(s, "EndTimestampMs: "+fmt.Sprintf("%#v", this.EndTimestampMs)+",\n") - s = append(s, "}") - return strings.Join(s, "") -} -func valueToGoStringDeletePlan(v interface{}, typ string) string { - rv := reflect.ValueOf(v) - if rv.IsNil() { - return "nil" - } - pv := reflect.Indirect(rv).Interface() - return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) -} -func (m *DeletePlan) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *DeletePlan) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *DeletePlan) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.ChunksGroup) > 0 { - for iNdEx := len(m.ChunksGroup) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.ChunksGroup[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintDeletePlan(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x12 - } - } - if m.PlanInterval != nil { - { - size, err := m.PlanInterval.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintDeletePlan(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - -func (m *ChunksGroup) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *ChunksGroup) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *ChunksGroup) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.Chunks) > 0 { - for iNdEx := len(m.Chunks) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Chunks[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintDeletePlan(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x12 - } - } - if len(m.Labels) > 0 { - for iNdEx := len(m.Labels) - 1; iNdEx >= 0; iNdEx-- { - { - size := m.Labels[iNdEx].Size() - i -= size - if _, err := m.Labels[iNdEx].MarshalTo(dAtA[i:]); err != nil { - return 0, err - } - i = encodeVarintDeletePlan(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0xa - } - } - return len(dAtA) - i, nil -} - -func (m *ChunkDetails) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *ChunkDetails) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *ChunkDetails) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.PartiallyDeletedInterval != nil { - { - size, err := m.PartiallyDeletedInterval.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintDeletePlan(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x12 - } - if len(m.ID) > 0 { - i -= len(m.ID) - copy(dAtA[i:], m.ID) - i = encodeVarintDeletePlan(dAtA, i, uint64(len(m.ID))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - -func (m *Interval) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *Interval) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *Interval) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.EndTimestampMs != 0 { - i = encodeVarintDeletePlan(dAtA, i, uint64(m.EndTimestampMs)) - i-- - dAtA[i] = 0x10 - } - if m.StartTimestampMs != 0 { - i = encodeVarintDeletePlan(dAtA, i, uint64(m.StartTimestampMs)) - i-- - dAtA[i] = 0x8 - } - return len(dAtA) - i, nil -} - -func encodeVarintDeletePlan(dAtA []byte, offset int, v uint64) int { - offset -= sovDeletePlan(v) - base := offset - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ - } - dAtA[offset] = uint8(v) - return base -} -func (m *DeletePlan) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.PlanInterval != nil { - l = m.PlanInterval.Size() - n += 1 + l + sovDeletePlan(uint64(l)) - } - if len(m.ChunksGroup) > 0 { - for _, e := range m.ChunksGroup { - l = e.Size() - n += 1 + l + sovDeletePlan(uint64(l)) - } - } - return n -} - -func (m *ChunksGroup) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if len(m.Labels) > 0 { - for _, e := range m.Labels { - l = e.Size() - n += 1 + l + sovDeletePlan(uint64(l)) - } - } - if len(m.Chunks) > 0 { - for _, e := range m.Chunks { - l = e.Size() - n += 1 + l + sovDeletePlan(uint64(l)) - } - } - return n -} - -func (m *ChunkDetails) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - l = len(m.ID) - if l > 0 { - n += 1 + l + sovDeletePlan(uint64(l)) - } - if m.PartiallyDeletedInterval != nil { - l = m.PartiallyDeletedInterval.Size() - n += 1 + l + sovDeletePlan(uint64(l)) - } - return n -} - -func (m *Interval) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.StartTimestampMs != 0 { - n += 1 + sovDeletePlan(uint64(m.StartTimestampMs)) - } - if m.EndTimestampMs != 0 { - n += 1 + sovDeletePlan(uint64(m.EndTimestampMs)) - } - return n -} - -func sovDeletePlan(x uint64) (n int) { - return (math_bits.Len64(x|1) + 6) / 7 -} -func sozDeletePlan(x uint64) (n int) { - return sovDeletePlan(uint64((x << 1) ^ uint64((int64(x) >> 63)))) -} -func (this *DeletePlan) String() string { - if this == nil { - return "nil" - } - repeatedStringForChunksGroup := "[]ChunksGroup{" - for _, f := range this.ChunksGroup { - repeatedStringForChunksGroup += strings.Replace(strings.Replace(f.String(), "ChunksGroup", "ChunksGroup", 1), `&`, ``, 1) + "," - } - repeatedStringForChunksGroup += "}" - s := strings.Join([]string{`&DeletePlan{`, - `PlanInterval:` + strings.Replace(this.PlanInterval.String(), "Interval", "Interval", 1) + `,`, - `ChunksGroup:` + repeatedStringForChunksGroup + `,`, - `}`, - }, "") - return s -} -func (this *ChunksGroup) String() string { - if this == nil { - return "nil" - } - repeatedStringForChunks := "[]ChunkDetails{" - for _, f := range this.Chunks { - repeatedStringForChunks += strings.Replace(strings.Replace(f.String(), "ChunkDetails", "ChunkDetails", 1), `&`, ``, 1) + "," - } - repeatedStringForChunks += "}" - s := strings.Join([]string{`&ChunksGroup{`, - `Labels:` + fmt.Sprintf("%v", this.Labels) + `,`, - `Chunks:` + repeatedStringForChunks + `,`, - `}`, - }, "") - return s -} -func (this *ChunkDetails) String() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&ChunkDetails{`, - `ID:` + fmt.Sprintf("%v", this.ID) + `,`, - `PartiallyDeletedInterval:` + strings.Replace(this.PartiallyDeletedInterval.String(), "Interval", "Interval", 1) + `,`, - `}`, - }, "") - return s -} -func (this *Interval) String() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&Interval{`, - `StartTimestampMs:` + fmt.Sprintf("%v", this.StartTimestampMs) + `,`, - `EndTimestampMs:` + fmt.Sprintf("%v", this.EndTimestampMs) + `,`, - `}`, - }, "") - return s -} -func valueToStringDeletePlan(v interface{}) string { - rv := reflect.ValueOf(v) - if rv.IsNil() { - return "nil" - } - pv := reflect.Indirect(rv).Interface() - return fmt.Sprintf("*%v", pv) -} -func (m *DeletePlan) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: DeletePlan: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: DeletePlan: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field PlanInterval", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthDeletePlan - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthDeletePlan - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.PlanInterval == nil { - m.PlanInterval = &Interval{} - } - if err := m.PlanInterval.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ChunksGroup", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthDeletePlan - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthDeletePlan - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.ChunksGroup = append(m.ChunksGroup, ChunksGroup{}) - if err := m.ChunksGroup[len(m.ChunksGroup)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipDeletePlan(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthDeletePlan - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthDeletePlan - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *ChunksGroup) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: ChunksGroup: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: ChunksGroup: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthDeletePlan - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthDeletePlan - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Labels = append(m.Labels, github_com_cortexproject_cortex_pkg_cortexpb.LabelAdapter{}) - if err := m.Labels[len(m.Labels)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Chunks", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthDeletePlan - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthDeletePlan - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Chunks = append(m.Chunks, ChunkDetails{}) - if err := m.Chunks[len(m.Chunks)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipDeletePlan(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthDeletePlan - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthDeletePlan - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *ChunkDetails) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: ChunkDetails: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: ChunkDetails: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthDeletePlan - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthDeletePlan - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.ID = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field PartiallyDeletedInterval", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthDeletePlan - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthDeletePlan - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.PartiallyDeletedInterval == nil { - m.PartiallyDeletedInterval = &Interval{} - } - if err := m.PartiallyDeletedInterval.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipDeletePlan(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthDeletePlan - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthDeletePlan - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *Interval) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: Interval: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: Interval: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field StartTimestampMs", wireType) - } - m.StartTimestampMs = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.StartTimestampMs |= int64(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field EndTimestampMs", wireType) - } - m.EndTimestampMs = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.EndTimestampMs |= int64(b&0x7F) << shift - if b < 0x80 { - break - } - } - default: - iNdEx = preIndex - skippy, err := skipDeletePlan(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthDeletePlan - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthDeletePlan - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func skipDeletePlan(dAtA []byte) (n int, err error) { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - wireType := int(wire & 0x7) - switch wireType { - case 0: - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - iNdEx++ - if dAtA[iNdEx-1] < 0x80 { - break - } - } - return iNdEx, nil - case 1: - iNdEx += 8 - return iNdEx, nil - case 2: - var length int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - length |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if length < 0 { - return 0, ErrInvalidLengthDeletePlan - } - iNdEx += length - if iNdEx < 0 { - return 0, ErrInvalidLengthDeletePlan - } - return iNdEx, nil - case 3: - for { - var innerWire uint64 - var start int = iNdEx - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - innerWire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - innerWireType := int(innerWire & 0x7) - if innerWireType == 4 { - break - } - next, err := skipDeletePlan(dAtA[start:]) - if err != nil { - return 0, err - } - iNdEx = start + next - if iNdEx < 0 { - return 0, ErrInvalidLengthDeletePlan - } - } - return iNdEx, nil - case 4: - return iNdEx, nil - case 5: - iNdEx += 4 - return iNdEx, nil - default: - return 0, fmt.Errorf("proto: illegal wireType %d", wireType) - } - } - panic("unreachable") -} - -var ( - ErrInvalidLengthDeletePlan = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowDeletePlan = fmt.Errorf("proto: integer overflow") -) diff --git a/pkg/chunk/purger/delete_plan.proto b/pkg/chunk/purger/delete_plan.proto deleted file mode 100644 index 834fc08749..0000000000 --- a/pkg/chunk/purger/delete_plan.proto +++ /dev/null @@ -1,34 +0,0 @@ -syntax = "proto3"; - -package purgeplan; - -option go_package = "purger"; - -import "github.com/gogo/protobuf/gogoproto/gogo.proto"; -import "github.com/cortexproject/cortex/pkg/cortexpb/cortex.proto"; - -option (gogoproto.marshaler_all) = true; -option (gogoproto.unmarshaler_all) = true; - -// DeletePlan holds all the chunks that are supposed to be deleted within an interval(usually a day) -// This Proto file is used just for storing Delete Plans in proto format. -message DeletePlan { - Interval plan_interval = 1; - repeated ChunksGroup chunks_group = 2 [(gogoproto.nullable) = false]; -} - -// ChunksGroup holds ChunkDetails and Labels for a group of chunks which have same series ID -message ChunksGroup { - repeated cortexpb.LabelPair labels = 1 [(gogoproto.nullable) = false, (gogoproto.customtype) = "github.com/cortexproject/cortex/pkg/cortexpb.LabelAdapter"]; - repeated ChunkDetails chunks = 2 [(gogoproto.nullable) = false]; -} - -message ChunkDetails { - string ID = 1; - Interval partially_deleted_interval = 2; -} - -message Interval { - int64 start_timestamp_ms = 1; - int64 end_timestamp_ms = 2; -} diff --git a/pkg/chunk/purger/delete_requests_store.go b/pkg/chunk/purger/delete_requests_store.go deleted file mode 100644 index f3ec1edbc1..0000000000 --- a/pkg/chunk/purger/delete_requests_store.go +++ /dev/null @@ -1,394 +0,0 @@ -package purger - -import ( - "context" - "encoding/binary" - "encoding/hex" - "errors" - "flag" - "fmt" - "hash/fnv" - "strconv" - "strings" - "time" - - "github.com/cortexproject/cortex/pkg/chunk" - - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" -) - -type ( - DeleteRequestStatus string - CacheKind string - indexType string -) - -const ( - StatusReceived DeleteRequestStatus = "received" - StatusBuildingPlan DeleteRequestStatus = "buildingPlan" - StatusDeleting DeleteRequestStatus = "deleting" - StatusProcessed DeleteRequestStatus = "processed" - - separator = "\000" // separator for series selectors in delete requests - - // CacheKindStore is for cache gen number for store cache - CacheKindStore CacheKind = "store" - // CacheKindResults is for cache gen number for results cache - CacheKindResults CacheKind = "results" - - deleteRequestID indexType = "1" - deleteRequestDetails indexType = "2" - cacheGenNum indexType = "3" -) - -var ( - pendingDeleteRequestStatuses = []DeleteRequestStatus{StatusReceived, StatusBuildingPlan, StatusDeleting} - - ErrDeleteRequestNotFound = errors.New("could not find matching delete request") -) - -// DeleteRequest holds all the details about a delete request. -type DeleteRequest struct { - RequestID string `json:"request_id"` - UserID string `json:"-"` - StartTime model.Time `json:"start_time"` - EndTime model.Time `json:"end_time"` - Selectors []string `json:"selectors"` - Status DeleteRequestStatus `json:"status"` - Matchers [][]*labels.Matcher `json:"-"` - CreatedAt model.Time `json:"created_at"` -} - -// cacheGenNumbers holds store and results cache gen numbers for a user. -type cacheGenNumbers struct { - store, results string -} - -// DeleteStore provides all the methods required to manage lifecycle of delete request and things related to it. -type DeleteStore struct { - cfg DeleteStoreConfig - indexClient chunk.IndexClient -} - -// DeleteStoreConfig holds configuration for delete store. -type DeleteStoreConfig struct { - Store string `yaml:"store"` - RequestsTableName string `yaml:"requests_table_name"` - ProvisionConfig TableProvisioningConfig `yaml:"table_provisioning"` -} - -// RegisterFlags adds the flags required to configure this flag set. -func (cfg *DeleteStoreConfig) RegisterFlags(f *flag.FlagSet) { - cfg.ProvisionConfig.RegisterFlags("deletes.table", f) - f.StringVar(&cfg.Store, "deletes.store", "", "Store for keeping delete request") - f.StringVar(&cfg.RequestsTableName, "deletes.requests-table-name", "delete_requests", "Name of the table which stores delete requests") -} - -// NewDeleteStore creates a store for managing delete requests. -func NewDeleteStore(cfg DeleteStoreConfig, indexClient chunk.IndexClient) (*DeleteStore, error) { - ds := DeleteStore{ - cfg: cfg, - indexClient: indexClient, - } - - return &ds, nil -} - -// Add creates entries for a new delete request. -func (ds *DeleteStore) AddDeleteRequest(ctx context.Context, userID string, startTime, endTime model.Time, selectors []string) error { - return ds.addDeleteRequest(ctx, userID, model.Now(), startTime, endTime, selectors) - -} - -// addDeleteRequest is also used for tests to create delete requests with different createdAt time. -func (ds *DeleteStore) addDeleteRequest(ctx context.Context, userID string, createdAt, startTime, endTime model.Time, selectors []string) error { - requestID := generateUniqueID(userID, selectors) - - for { - _, err := ds.GetDeleteRequest(ctx, userID, string(requestID)) - if err != nil { - if err == ErrDeleteRequestNotFound { - break - } - return err - } - - // we have a collision here, lets recreate a new requestID and check for collision - time.Sleep(time.Millisecond) - requestID = generateUniqueID(userID, selectors) - } - - // userID, requestID - userIDAndRequestID := fmt.Sprintf("%s:%s", userID, requestID) - - // Add an entry with userID, requestID as range key and status as value to make it easy to manage and lookup status - // We don't want to set anything in hash key here since we would want to find delete requests by just status - writeBatch := ds.indexClient.NewWriteBatch() - writeBatch.Add(ds.cfg.RequestsTableName, string(deleteRequestID), []byte(userIDAndRequestID), []byte(StatusReceived)) - - // Add another entry with additional details like creation time, time range of delete request and selectors in value - rangeValue := fmt.Sprintf("%x:%x:%x", int64(createdAt), int64(startTime), int64(endTime)) - writeBatch.Add(ds.cfg.RequestsTableName, fmt.Sprintf("%s:%s", deleteRequestDetails, userIDAndRequestID), - []byte(rangeValue), []byte(strings.Join(selectors, separator))) - - // we update only cache gen number because only query responses are changing at this stage. - // we still have to query data from store for doing query time filtering and we don't want to invalidate its results now. - writeBatch.Add(ds.cfg.RequestsTableName, fmt.Sprintf("%s:%s:%s", cacheGenNum, userID, CacheKindResults), - []byte{}, []byte(strconv.FormatInt(time.Now().Unix(), 10))) - - return ds.indexClient.BatchWrite(ctx, writeBatch) -} - -// GetDeleteRequestsByStatus returns all delete requests for given status. -func (ds *DeleteStore) GetDeleteRequestsByStatus(ctx context.Context, status DeleteRequestStatus) ([]DeleteRequest, error) { - return ds.queryDeleteRequests(ctx, chunk.IndexQuery{ - TableName: ds.cfg.RequestsTableName, - HashValue: string(deleteRequestID), - ValueEqual: []byte(status), - }) -} - -// GetDeleteRequestsForUserByStatus returns all delete requests for a user with given status. -func (ds *DeleteStore) GetDeleteRequestsForUserByStatus(ctx context.Context, userID string, status DeleteRequestStatus) ([]DeleteRequest, error) { - return ds.queryDeleteRequests(ctx, chunk.IndexQuery{ - TableName: ds.cfg.RequestsTableName, - HashValue: string(deleteRequestID), - RangeValuePrefix: []byte(userID), - ValueEqual: []byte(status), - }) -} - -// GetAllDeleteRequestsForUser returns all delete requests for a user. -func (ds *DeleteStore) GetAllDeleteRequestsForUser(ctx context.Context, userID string) ([]DeleteRequest, error) { - return ds.queryDeleteRequests(ctx, chunk.IndexQuery{ - TableName: ds.cfg.RequestsTableName, - HashValue: string(deleteRequestID), - RangeValuePrefix: []byte(userID), - }) -} - -// UpdateStatus updates status of a delete request. -func (ds *DeleteStore) UpdateStatus(ctx context.Context, userID, requestID string, newStatus DeleteRequestStatus) error { - userIDAndRequestID := fmt.Sprintf("%s:%s", userID, requestID) - - writeBatch := ds.indexClient.NewWriteBatch() - writeBatch.Add(ds.cfg.RequestsTableName, string(deleteRequestID), []byte(userIDAndRequestID), []byte(newStatus)) - - if newStatus == StatusProcessed { - // we have deleted data from store so invalidate cache only for store since we don't have to do runtime filtering anymore. - // we don't have to change cache gen number because we were anyways doing runtime filtering - writeBatch.Add(ds.cfg.RequestsTableName, fmt.Sprintf("%s:%s:%s", cacheGenNum, userID, CacheKindStore), []byte{}, []byte(strconv.FormatInt(time.Now().Unix(), 10))) - } - - return ds.indexClient.BatchWrite(ctx, writeBatch) -} - -// GetDeleteRequest returns delete request with given requestID. -func (ds *DeleteStore) GetDeleteRequest(ctx context.Context, userID, requestID string) (*DeleteRequest, error) { - userIDAndRequestID := fmt.Sprintf("%s:%s", userID, requestID) - - deleteRequests, err := ds.queryDeleteRequests(ctx, chunk.IndexQuery{ - TableName: ds.cfg.RequestsTableName, - HashValue: string(deleteRequestID), - RangeValuePrefix: []byte(userIDAndRequestID), - }) - - if err != nil { - return nil, err - } - - if len(deleteRequests) == 0 { - return nil, ErrDeleteRequestNotFound - } - - return &deleteRequests[0], nil -} - -// GetPendingDeleteRequestsForUser returns all delete requests for a user which are not processed. -func (ds *DeleteStore) GetPendingDeleteRequestsForUser(ctx context.Context, userID string) ([]DeleteRequest, error) { - pendingDeleteRequests := []DeleteRequest{} - for _, status := range pendingDeleteRequestStatuses { - deleteRequests, err := ds.GetDeleteRequestsForUserByStatus(ctx, userID, status) - if err != nil { - return nil, err - } - - pendingDeleteRequests = append(pendingDeleteRequests, deleteRequests...) - } - - return pendingDeleteRequests, nil -} - -func (ds *DeleteStore) queryDeleteRequests(ctx context.Context, deleteQuery chunk.IndexQuery) ([]DeleteRequest, error) { - deleteRequests := []DeleteRequest{} - // No need to lock inside the callback since we run a single index query. - err := ds.indexClient.QueryPages(ctx, []chunk.IndexQuery{deleteQuery}, func(query chunk.IndexQuery, batch chunk.ReadBatch) (shouldContinue bool) { - itr := batch.Iterator() - for itr.Next() { - userID, requestID := splitUserIDAndRequestID(string(itr.RangeValue())) - - deleteRequests = append(deleteRequests, DeleteRequest{ - UserID: userID, - RequestID: requestID, - Status: DeleteRequestStatus(itr.Value()), - }) - } - return true - }) - if err != nil { - return nil, err - } - - for i, deleteRequest := range deleteRequests { - deleteRequestQuery := []chunk.IndexQuery{ - { - TableName: ds.cfg.RequestsTableName, - HashValue: fmt.Sprintf("%s:%s:%s", deleteRequestDetails, deleteRequest.UserID, deleteRequest.RequestID), - }, - } - - var parseError error - err := ds.indexClient.QueryPages(ctx, deleteRequestQuery, func(query chunk.IndexQuery, batch chunk.ReadBatch) (shouldContinue bool) { - itr := batch.Iterator() - itr.Next() - - deleteRequest, err = parseDeleteRequestTimestamps(itr.RangeValue(), deleteRequest) - if err != nil { - parseError = err - return false - } - - deleteRequest.Selectors = strings.Split(string(itr.Value()), separator) - deleteRequests[i] = deleteRequest - - return true - }) - - if err != nil { - return nil, err - } - - if parseError != nil { - return nil, parseError - } - } - - return deleteRequests, nil -} - -// getCacheGenerationNumbers returns cache gen numbers for a user. -func (ds *DeleteStore) getCacheGenerationNumbers(ctx context.Context, userID string) (*cacheGenNumbers, error) { - storeCacheGen, err := ds.queryCacheGenerationNumber(ctx, userID, CacheKindStore) - if err != nil { - return nil, err - } - - resultsCacheGen, err := ds.queryCacheGenerationNumber(ctx, userID, CacheKindResults) - if err != nil { - return nil, err - } - - return &cacheGenNumbers{storeCacheGen, resultsCacheGen}, nil -} - -func (ds *DeleteStore) queryCacheGenerationNumber(ctx context.Context, userID string, kind CacheKind) (string, error) { - query := chunk.IndexQuery{TableName: ds.cfg.RequestsTableName, HashValue: fmt.Sprintf("%s:%s:%s", cacheGenNum, userID, kind)} - - genNumber := "" - err := ds.indexClient.QueryPages(ctx, []chunk.IndexQuery{query}, func(query chunk.IndexQuery, batch chunk.ReadBatch) (shouldContinue bool) { - itr := batch.Iterator() - for itr.Next() { - genNumber = string(itr.Value()) - break - } - return false - }) - - if err != nil { - return "", err - } - - return genNumber, nil -} - -// RemoveDeleteRequest removes a delete request and increments cache gen number -func (ds *DeleteStore) RemoveDeleteRequest(ctx context.Context, userID, requestID string, createdAt, startTime, endTime model.Time) error { - userIDAndRequestID := fmt.Sprintf("%s:%s", userID, requestID) - - writeBatch := ds.indexClient.NewWriteBatch() - writeBatch.Delete(ds.cfg.RequestsTableName, string(deleteRequestID), []byte(userIDAndRequestID)) - - // Add another entry with additional details like creation time, time range of delete request and selectors in value - rangeValue := fmt.Sprintf("%x:%x:%x", int64(createdAt), int64(startTime), int64(endTime)) - writeBatch.Delete(ds.cfg.RequestsTableName, fmt.Sprintf("%s:%s", deleteRequestDetails, userIDAndRequestID), - []byte(rangeValue)) - - // we need to invalidate results cache since removal of delete request would cause query results to change - writeBatch.Add(ds.cfg.RequestsTableName, fmt.Sprintf("%s:%s:%s", cacheGenNum, userID, CacheKindResults), - []byte{}, []byte(strconv.FormatInt(time.Now().Unix(), 10))) - - return ds.indexClient.BatchWrite(ctx, writeBatch) -} - -func parseDeleteRequestTimestamps(rangeValue []byte, deleteRequest DeleteRequest) (DeleteRequest, error) { - hexParts := strings.Split(string(rangeValue), ":") - if len(hexParts) != 3 { - return deleteRequest, errors.New("invalid key in parsing delete request lookup response") - } - - createdAt, err := strconv.ParseInt(hexParts[0], 16, 64) - if err != nil { - return deleteRequest, err - } - - from, err := strconv.ParseInt(hexParts[1], 16, 64) - if err != nil { - return deleteRequest, err - - } - through, err := strconv.ParseInt(hexParts[2], 16, 64) - if err != nil { - return deleteRequest, err - - } - - deleteRequest.CreatedAt = model.Time(createdAt) - deleteRequest.StartTime = model.Time(from) - deleteRequest.EndTime = model.Time(through) - - return deleteRequest, nil -} - -// An id is useful in managing delete requests -func generateUniqueID(orgID string, selectors []string) []byte { - uniqueID := fnv.New32() - _, _ = uniqueID.Write([]byte(orgID)) - - timeNow := make([]byte, 8) - binary.LittleEndian.PutUint64(timeNow, uint64(time.Now().UnixNano())) - _, _ = uniqueID.Write(timeNow) - - for _, selector := range selectors { - _, _ = uniqueID.Write([]byte(selector)) - } - - return encodeUniqueID(uniqueID.Sum32()) -} - -func encodeUniqueID(t uint32) []byte { - throughBytes := make([]byte, 4) - binary.BigEndian.PutUint32(throughBytes, t) - encodedThroughBytes := make([]byte, 8) - hex.Encode(encodedThroughBytes, throughBytes) - return encodedThroughBytes -} - -func splitUserIDAndRequestID(rangeValue string) (userID, requestID string) { - lastIndex := strings.LastIndex(rangeValue, ":") - - userID = rangeValue[:lastIndex] - requestID = rangeValue[lastIndex+1:] - - return -} diff --git a/pkg/chunk/purger/purger.go b/pkg/chunk/purger/purger.go deleted file mode 100644 index 7c37c29300..0000000000 --- a/pkg/chunk/purger/purger.go +++ /dev/null @@ -1,828 +0,0 @@ -package purger - -import ( - "bytes" - "context" - "flag" - "fmt" - "io/ioutil" - "sync" - "time" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/gogo/protobuf/proto" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/promql" - "github.com/prometheus/prometheus/promql/parser" - "github.com/weaveworks/common/user" - - "github.com/cortexproject/cortex/pkg/chunk" - "github.com/cortexproject/cortex/pkg/cortexpb" - util_log "github.com/cortexproject/cortex/pkg/util/log" - "github.com/cortexproject/cortex/pkg/util/services" -) - -const ( - millisecondPerDay = int64(24 * time.Hour / time.Millisecond) - statusSuccess = "success" - statusFail = "fail" - loadRequestsInterval = time.Hour - retryFailedRequestsInterval = 15 * time.Minute -) - -type purgerMetrics struct { - deleteRequestsProcessedTotal *prometheus.CounterVec - deleteRequestsChunksSelectedTotal *prometheus.CounterVec - deleteRequestsProcessingFailures *prometheus.CounterVec - loadPendingRequestsAttempsTotal *prometheus.CounterVec - oldestPendingDeleteRequestAgeSeconds prometheus.Gauge - pendingDeleteRequestsCount prometheus.Gauge -} - -func newPurgerMetrics(r prometheus.Registerer) *purgerMetrics { - m := purgerMetrics{} - - m.deleteRequestsProcessedTotal = promauto.With(r).NewCounterVec(prometheus.CounterOpts{ - Namespace: "cortex", - Name: "purger_delete_requests_processed_total", - Help: "Number of delete requests processed per user", - }, []string{"user"}) - m.deleteRequestsChunksSelectedTotal = promauto.With(r).NewCounterVec(prometheus.CounterOpts{ - Namespace: "cortex", - Name: "purger_delete_requests_chunks_selected_total", - Help: "Number of chunks selected while building delete plans per user", - }, []string{"user"}) - m.deleteRequestsProcessingFailures = promauto.With(r).NewCounterVec(prometheus.CounterOpts{ - Namespace: "cortex", - Name: "purger_delete_requests_processing_failures_total", - Help: "Number of delete requests processing failures per user", - }, []string{"user"}) - m.loadPendingRequestsAttempsTotal = promauto.With(r).NewCounterVec(prometheus.CounterOpts{ - Namespace: "cortex", - Name: "purger_load_pending_requests_attempts_total", - Help: "Number of attempts that were made to load pending requests with status", - }, []string{"status"}) - m.oldestPendingDeleteRequestAgeSeconds = promauto.With(r).NewGauge(prometheus.GaugeOpts{ - Namespace: "cortex", - Name: "purger_oldest_pending_delete_request_age_seconds", - Help: "Age of oldest pending delete request in seconds, since they are over their cancellation period", - }) - m.pendingDeleteRequestsCount = promauto.With(r).NewGauge(prometheus.GaugeOpts{ - Namespace: "cortex", - Name: "purger_pending_delete_requests_count", - Help: "Count of delete requests which are over their cancellation period and have not finished processing yet", - }) - - return &m -} - -type deleteRequestWithLogger struct { - DeleteRequest - logger log.Logger // logger is initialized with userID and requestID to add context to every log generated using this -} - -// Config holds config for chunks Purger -type Config struct { - Enable bool `yaml:"enable"` - NumWorkers int `yaml:"num_workers"` - ObjectStoreType string `yaml:"object_store_type"` - DeleteRequestCancelPeriod time.Duration `yaml:"delete_request_cancel_period"` -} - -// RegisterFlags registers CLI flags for Config -func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - f.BoolVar(&cfg.Enable, "purger.enable", false, "Enable purger to allow deletion of series. Be aware that Delete series feature is still experimental") - f.IntVar(&cfg.NumWorkers, "purger.num-workers", 2, "Number of workers executing delete plans in parallel") - f.StringVar(&cfg.ObjectStoreType, "purger.object-store-type", "", "Name of the object store to use for storing delete plans") - f.DurationVar(&cfg.DeleteRequestCancelPeriod, "purger.delete-request-cancel-period", 24*time.Hour, "Allow cancellation of delete request until duration after they are created. Data would be deleted only after delete requests have been older than this duration. Ideally this should be set to at least 24h.") -} - -type workerJob struct { - planNo int - userID string - deleteRequestID string - logger log.Logger -} - -// Purger does the purging of data which is requested to be deleted. Purger only works for chunks. -type Purger struct { - services.Service - - cfg Config - deleteStore *DeleteStore - chunkStore chunk.Store - objectClient chunk.ObjectClient - metrics *purgerMetrics - - executePlansChan chan deleteRequestWithLogger - workerJobChan chan workerJob - - // we would only allow processing of singe delete request at a time since delete requests touching same chunks could change the chunk IDs of partially deleted chunks - // and break the purge plan for other requests - inProcessRequests *inProcessRequestsCollection - - // We do not want to limit pulling new delete requests to a fixed interval which otherwise would limit number of delete requests we process per user. - // While loading delete requests if we find more requests from user pending to be processed, we just set their id in usersWithPendingRequests and - // when a user's delete request gets processed we just check this map to see whether we want to load more requests without waiting for next ticker to load new batch. - usersWithPendingRequests map[string]struct{} - usersWithPendingRequestsMtx sync.Mutex - pullNewRequestsChan chan struct{} - - pendingPlansCount map[string]int // per request pending plan count - pendingPlansCountMtx sync.Mutex - - wg sync.WaitGroup -} - -// NewPurger creates a new Purger -func NewPurger(cfg Config, deleteStore *DeleteStore, chunkStore chunk.Store, storageClient chunk.ObjectClient, registerer prometheus.Registerer) (*Purger, error) { - util_log.WarnExperimentalUse("Delete series API") - - purger := Purger{ - cfg: cfg, - deleteStore: deleteStore, - chunkStore: chunkStore, - objectClient: storageClient, - metrics: newPurgerMetrics(registerer), - pullNewRequestsChan: make(chan struct{}, 1), - executePlansChan: make(chan deleteRequestWithLogger, 50), - workerJobChan: make(chan workerJob, 50), - inProcessRequests: newInProcessRequestsCollection(), - usersWithPendingRequests: map[string]struct{}{}, - pendingPlansCount: map[string]int{}, - } - - purger.Service = services.NewBasicService(purger.init, purger.loop, purger.stop) - return &purger, nil -} - -// init starts workers, scheduler and then loads in process delete requests -func (p *Purger) init(ctx context.Context) error { - for i := 0; i < p.cfg.NumWorkers; i++ { - p.wg.Add(1) - go p.worker() - } - - p.wg.Add(1) - go p.jobScheduler(ctx) - - return p.loadInprocessDeleteRequests() -} - -func (p *Purger) loop(ctx context.Context) error { - loadRequests := func() { - status := statusSuccess - - err := p.pullDeleteRequestsToPlanDeletes() - if err != nil { - status = statusFail - level.Error(util_log.Logger).Log("msg", "error pulling delete requests for building plans", "err", err) - } - - p.metrics.loadPendingRequestsAttempsTotal.WithLabelValues(status).Inc() - } - - // load requests on startup instead of waiting for first ticker - loadRequests() - - loadRequestsTicker := time.NewTicker(loadRequestsInterval) - defer loadRequestsTicker.Stop() - - retryFailedRequestsTicker := time.NewTicker(retryFailedRequestsInterval) - defer retryFailedRequestsTicker.Stop() - - for { - select { - case <-loadRequestsTicker.C: - loadRequests() - case <-p.pullNewRequestsChan: - loadRequests() - case <-retryFailedRequestsTicker.C: - p.retryFailedRequests() - case <-ctx.Done(): - return nil - } - } -} - -// Stop waits until all background tasks stop. -func (p *Purger) stop(_ error) error { - p.wg.Wait() - return nil -} - -func (p *Purger) retryFailedRequests() { - userIDsWithFailedRequest := p.inProcessRequests.listUsersWithFailedRequest() - - for _, userID := range userIDsWithFailedRequest { - deleteRequest := p.inProcessRequests.get(userID) - if deleteRequest == nil { - level.Error(util_log.Logger).Log("msg", "expected an in-process delete request", "user", userID) - continue - } - - p.inProcessRequests.unsetFailedRequestForUser(userID) - err := p.resumeStalledRequest(*deleteRequest) - if err != nil { - reqWithLogger := makeDeleteRequestWithLogger(*deleteRequest, util_log.Logger) - level.Error(reqWithLogger.logger).Log("msg", "failed to resume failed request", "err", err) - } - } -} - -func (p *Purger) workerJobCleanup(job workerJob) { - err := p.removeDeletePlan(context.Background(), job.userID, job.deleteRequestID, job.planNo) - if err != nil { - level.Error(job.logger).Log("msg", "error removing delete plan", - "plan_no", job.planNo, "err", err) - return - } - - p.pendingPlansCountMtx.Lock() - p.pendingPlansCount[job.deleteRequestID]-- - - if p.pendingPlansCount[job.deleteRequestID] == 0 { - level.Info(job.logger).Log("msg", "finished execution of all plans, cleaning up and updating status of request") - - err := p.deleteStore.UpdateStatus(context.Background(), job.userID, job.deleteRequestID, StatusProcessed) - if err != nil { - level.Error(job.logger).Log("msg", "error updating delete request status to process", "err", err) - } - - p.metrics.deleteRequestsProcessedTotal.WithLabelValues(job.userID).Inc() - delete(p.pendingPlansCount, job.deleteRequestID) - p.pendingPlansCountMtx.Unlock() - - p.inProcessRequests.remove(job.userID) - - // request loading of more delete request if - // - user has more pending requests and - // - we do not have a pending request to load more requests - p.usersWithPendingRequestsMtx.Lock() - defer p.usersWithPendingRequestsMtx.Unlock() - if _, ok := p.usersWithPendingRequests[job.userID]; ok { - delete(p.usersWithPendingRequests, job.userID) - select { - case p.pullNewRequestsChan <- struct{}{}: - // sent - default: - // already sent - } - } else if len(p.usersWithPendingRequests) == 0 { - // there are no pending requests from any of the users, set the oldest pending request and number of pending requests to 0 - p.metrics.oldestPendingDeleteRequestAgeSeconds.Set(0) - p.metrics.pendingDeleteRequestsCount.Set(0) - } - } else { - p.pendingPlansCountMtx.Unlock() - } -} - -// we send all the delete plans to workerJobChan -func (p *Purger) jobScheduler(ctx context.Context) { - defer p.wg.Done() - - for { - select { - case req := <-p.executePlansChan: - numPlans := numPlans(req.StartTime, req.EndTime) - level.Info(req.logger).Log("msg", "sending jobs to workers for purging data", "num_jobs", numPlans) - - p.pendingPlansCountMtx.Lock() - p.pendingPlansCount[req.RequestID] = numPlans - p.pendingPlansCountMtx.Unlock() - - for i := 0; i < numPlans; i++ { - p.workerJobChan <- workerJob{planNo: i, userID: req.UserID, - deleteRequestID: req.RequestID, logger: req.logger} - } - case <-ctx.Done(): - close(p.workerJobChan) - return - } - } -} - -func (p *Purger) worker() { - defer p.wg.Done() - - for job := range p.workerJobChan { - err := p.executePlan(job.userID, job.deleteRequestID, job.planNo, job.logger) - if err != nil { - p.metrics.deleteRequestsProcessingFailures.WithLabelValues(job.userID).Inc() - level.Error(job.logger).Log("msg", "error executing delete plan", - "plan_no", job.planNo, "err", err) - continue - } - - p.workerJobCleanup(job) - } -} - -func (p *Purger) executePlan(userID, requestID string, planNo int, logger log.Logger) (err error) { - logger = log.With(logger, "plan_no", planNo) - - defer func() { - if err != nil { - p.inProcessRequests.setFailedRequestForUser(userID) - } - }() - - plan, err := p.getDeletePlan(context.Background(), userID, requestID, planNo) - if err != nil { - if err == chunk.ErrStorageObjectNotFound { - level.Info(logger).Log("msg", "plan not found, must have been executed already") - // this means plan was already executed and got removed. Do nothing. - return nil - } - return err - } - - level.Info(logger).Log("msg", "executing plan") - - ctx := user.InjectOrgID(context.Background(), userID) - - for i := range plan.ChunksGroup { - level.Debug(logger).Log("msg", "deleting chunks", "labels", plan.ChunksGroup[i].Labels) - - for _, chunkDetails := range plan.ChunksGroup[i].Chunks { - chunkRef, err := chunk.ParseExternalKey(userID, chunkDetails.ID) - if err != nil { - return err - } - - var partiallyDeletedInterval *model.Interval = nil - if chunkDetails.PartiallyDeletedInterval != nil { - partiallyDeletedInterval = &model.Interval{ - Start: model.Time(chunkDetails.PartiallyDeletedInterval.StartTimestampMs), - End: model.Time(chunkDetails.PartiallyDeletedInterval.EndTimestampMs), - } - } - - err = p.chunkStore.DeleteChunk(ctx, chunkRef.From, chunkRef.Through, chunkRef.UserID, - chunkDetails.ID, cortexpb.FromLabelAdaptersToLabels(plan.ChunksGroup[i].Labels), partiallyDeletedInterval) - if err != nil { - if isMissingChunkErr(err) { - level.Error(logger).Log("msg", "chunk not found for deletion. We may have already deleted it", - "chunk_id", chunkDetails.ID) - continue - } - return err - } - } - - level.Debug(logger).Log("msg", "deleting series", "labels", plan.ChunksGroup[i].Labels) - - // this is mostly required to clean up series ids from series store - err := p.chunkStore.DeleteSeriesIDs(ctx, model.Time(plan.PlanInterval.StartTimestampMs), model.Time(plan.PlanInterval.EndTimestampMs), - userID, cortexpb.FromLabelAdaptersToLabels(plan.ChunksGroup[i].Labels)) - if err != nil { - return err - } - } - - level.Info(logger).Log("msg", "finished execution of plan") - - return -} - -// we need to load all in process delete requests on startup to finish them first -func (p *Purger) loadInprocessDeleteRequests() error { - inprocessRequests, err := p.deleteStore.GetDeleteRequestsByStatus(context.Background(), StatusBuildingPlan) - if err != nil { - return err - } - - requestsWithDeletingStatus, err := p.deleteStore.GetDeleteRequestsByStatus(context.Background(), StatusDeleting) - if err != nil { - return err - } - - inprocessRequests = append(inprocessRequests, requestsWithDeletingStatus...) - - for i := range inprocessRequests { - deleteRequest := inprocessRequests[i] - p.inProcessRequests.set(deleteRequest.UserID, &deleteRequest) - req := makeDeleteRequestWithLogger(deleteRequest, util_log.Logger) - - level.Info(req.logger).Log("msg", "resuming in process delete requests", "status", deleteRequest.Status) - err = p.resumeStalledRequest(deleteRequest) - if err != nil { - level.Error(req.logger).Log("msg", "failed to resume stalled request", "err", err) - } - - } - - return nil -} - -func (p *Purger) resumeStalledRequest(deleteRequest DeleteRequest) error { - req := makeDeleteRequestWithLogger(deleteRequest, util_log.Logger) - - if deleteRequest.Status == StatusBuildingPlan { - err := p.buildDeletePlan(req) - if err != nil { - p.metrics.deleteRequestsProcessingFailures.WithLabelValues(deleteRequest.UserID).Inc() - return errors.Wrap(err, "failed to build delete plan") - } - - deleteRequest.Status = StatusDeleting - } - - if deleteRequest.Status == StatusDeleting { - level.Info(req.logger).Log("msg", "sending delete request for execution") - p.executePlansChan <- req - } - - return nil -} - -// pullDeleteRequestsToPlanDeletes pulls delete requests which do not have their delete plans built yet and sends them for building delete plans -// after pulling delete requests for building plans, it updates its status to StatusBuildingPlan status to avoid picking this up again next time -func (p *Purger) pullDeleteRequestsToPlanDeletes() error { - deleteRequests, err := p.deleteStore.GetDeleteRequestsByStatus(context.Background(), StatusReceived) - if err != nil { - return err - } - - pendingDeleteRequestsCount := p.inProcessRequests.len() - now := model.Now() - oldestPendingRequestCreatedAt := model.Time(0) - - // requests which are still being processed are also considered pending - if pendingDeleteRequestsCount != 0 { - oldestInProcessRequest := p.inProcessRequests.getOldest() - if oldestInProcessRequest != nil { - oldestPendingRequestCreatedAt = oldestInProcessRequest.CreatedAt - } - } - - for i := range deleteRequests { - deleteRequest := deleteRequests[i] - - // adding an extra minute here to avoid a race between cancellation of request and picking of the request for processing - if deleteRequest.CreatedAt.Add(p.cfg.DeleteRequestCancelPeriod).Add(time.Minute).After(model.Now()) { - continue - } - - pendingDeleteRequestsCount++ - if oldestPendingRequestCreatedAt == 0 || deleteRequest.CreatedAt.Before(oldestPendingRequestCreatedAt) { - oldestPendingRequestCreatedAt = deleteRequest.CreatedAt - } - - if inprocessDeleteRequest := p.inProcessRequests.get(deleteRequest.UserID); inprocessDeleteRequest != nil { - p.usersWithPendingRequestsMtx.Lock() - p.usersWithPendingRequests[deleteRequest.UserID] = struct{}{} - p.usersWithPendingRequestsMtx.Unlock() - - level.Debug(util_log.Logger).Log("msg", "skipping delete request processing for now since another request from same user is already in process", - "inprocess_request_id", inprocessDeleteRequest.RequestID, - "skipped_request_id", deleteRequest.RequestID, "user_id", deleteRequest.UserID) - continue - } - - err = p.deleteStore.UpdateStatus(context.Background(), deleteRequest.UserID, deleteRequest.RequestID, StatusBuildingPlan) - if err != nil { - return err - } - - deleteRequest.Status = StatusBuildingPlan - p.inProcessRequests.set(deleteRequest.UserID, &deleteRequest) - req := makeDeleteRequestWithLogger(deleteRequest, util_log.Logger) - - level.Info(req.logger).Log("msg", "building plan for a new delete request") - - err := p.buildDeletePlan(req) - if err != nil { - p.metrics.deleteRequestsProcessingFailures.WithLabelValues(deleteRequest.UserID).Inc() - - // We do not want to remove this delete request from inProcessRequests to make sure - // we do not move multiple deleting requests in deletion process. - // None of the other delete requests from the user would be considered for processing until then. - level.Error(req.logger).Log("msg", "error building delete plan", "err", err) - return err - } - - level.Info(req.logger).Log("msg", "sending delete request for execution") - p.executePlansChan <- req - } - - // track age of oldest delete request since they are over their cancellation period - oldestPendingRequestAge := time.Duration(0) - if oldestPendingRequestCreatedAt != 0 { - oldestPendingRequestAge = now.Sub(oldestPendingRequestCreatedAt.Add(p.cfg.DeleteRequestCancelPeriod)) - } - p.metrics.oldestPendingDeleteRequestAgeSeconds.Set(float64(oldestPendingRequestAge / time.Second)) - p.metrics.pendingDeleteRequestsCount.Set(float64(pendingDeleteRequestsCount)) - - return nil -} - -// buildDeletePlan builds per day delete plan for given delete requests. -// A days plan will include chunk ids and labels of all the chunks which are supposed to be deleted. -// Chunks are grouped together by labels to avoid storing labels repetitively. -// After building delete plans it updates status of delete request to StatusDeleting and sends it for execution -func (p *Purger) buildDeletePlan(req deleteRequestWithLogger) (err error) { - ctx := context.Background() - ctx = user.InjectOrgID(ctx, req.UserID) - - defer func() { - if err != nil { - p.inProcessRequests.setFailedRequestForUser(req.UserID) - } else { - req.Status = StatusDeleting - p.inProcessRequests.set(req.UserID, &req.DeleteRequest) - } - }() - - perDayTimeRange := splitByDay(req.StartTime, req.EndTime) - level.Info(req.logger).Log("msg", "building delete plan", "num_plans", len(perDayTimeRange)) - - plans := make([][]byte, len(perDayTimeRange)) - includedChunkIDs := map[string]struct{}{} - - for i, planRange := range perDayTimeRange { - chunksGroups := []ChunksGroup{} - - for _, selector := range req.Selectors { - matchers, err := parser.ParseMetricSelector(selector) - if err != nil { - return err - } - - chunks, err := p.chunkStore.Get(ctx, req.UserID, planRange.Start, planRange.End, matchers...) - if err != nil { - return err - } - - var cg []ChunksGroup - cg, includedChunkIDs = groupChunks(chunks, req.StartTime, req.EndTime, includedChunkIDs) - - if len(cg) != 0 { - chunksGroups = append(chunksGroups, cg...) - } - } - - plan := DeletePlan{ - PlanInterval: &Interval{ - StartTimestampMs: int64(planRange.Start), - EndTimestampMs: int64(planRange.End), - }, - ChunksGroup: chunksGroups, - } - - pb, err := proto.Marshal(&plan) - if err != nil { - return err - } - - plans[i] = pb - } - - err = p.putDeletePlans(ctx, req.UserID, req.RequestID, plans) - if err != nil { - return - } - - err = p.deleteStore.UpdateStatus(ctx, req.UserID, req.RequestID, StatusDeleting) - if err != nil { - return - } - - p.metrics.deleteRequestsChunksSelectedTotal.WithLabelValues(req.UserID).Add(float64(len(includedChunkIDs))) - - level.Info(req.logger).Log("msg", "built delete plans", "num_plans", len(perDayTimeRange)) - - return -} - -func (p *Purger) putDeletePlans(ctx context.Context, userID, requestID string, plans [][]byte) error { - for i, plan := range plans { - objectKey := buildObjectKeyForPlan(userID, requestID, i) - - err := p.objectClient.PutObject(ctx, objectKey, bytes.NewReader(plan)) - if err != nil { - return err - } - } - - return nil -} - -func (p *Purger) getDeletePlan(ctx context.Context, userID, requestID string, planNo int) (*DeletePlan, error) { - objectKey := buildObjectKeyForPlan(userID, requestID, planNo) - - readCloser, err := p.objectClient.GetObject(ctx, objectKey) - if err != nil { - return nil, err - } - - defer readCloser.Close() - - buf, err := ioutil.ReadAll(readCloser) - if err != nil { - return nil, err - } - - var plan DeletePlan - err = proto.Unmarshal(buf, &plan) - if err != nil { - return nil, err - } - - return &plan, nil -} - -func (p *Purger) removeDeletePlan(ctx context.Context, userID, requestID string, planNo int) error { - objectKey := buildObjectKeyForPlan(userID, requestID, planNo) - return p.objectClient.DeleteObject(ctx, objectKey) -} - -// returns interval per plan -func splitByDay(start, end model.Time) []model.Interval { - numOfDays := numPlans(start, end) - - perDayTimeRange := make([]model.Interval, numOfDays) - startOfNextDay := model.Time(((int64(start) / millisecondPerDay) + 1) * millisecondPerDay) - perDayTimeRange[0] = model.Interval{Start: start, End: startOfNextDay - 1} - - for i := 1; i < numOfDays; i++ { - interval := model.Interval{Start: startOfNextDay} - startOfNextDay += model.Time(millisecondPerDay) - interval.End = startOfNextDay - 1 - perDayTimeRange[i] = interval - } - - perDayTimeRange[numOfDays-1].End = end - - return perDayTimeRange -} - -func numPlans(start, end model.Time) int { - // rounding down start to start of the day - if start%model.Time(millisecondPerDay) != 0 { - start = model.Time((int64(start) / millisecondPerDay) * millisecondPerDay) - } - - // rounding up end to end of the day - if end%model.Time(millisecondPerDay) != 0 { - end = model.Time((int64(end)/millisecondPerDay)*millisecondPerDay + millisecondPerDay) - } - - return int(int64(end-start) / millisecondPerDay) -} - -// groups chunks together by unique label sets i.e all the chunks with same labels would be stored in a group -// chunk details are stored in groups for each unique label set to avoid storing them repetitively for each chunk -func groupChunks(chunks []chunk.Chunk, deleteFrom, deleteThrough model.Time, includedChunkIDs map[string]struct{}) ([]ChunksGroup, map[string]struct{}) { - metricToChunks := make(map[string]ChunksGroup) - - for _, chk := range chunks { - chunkID := chk.ExternalKey() - - if _, ok := includedChunkIDs[chunkID]; ok { - continue - } - // chunk.Metric are assumed to be sorted which should give same value from String() for same series. - // If they stop being sorted then in the worst case we would lose the benefit of grouping chunks to avoid storing labels repetitively. - metricString := chk.Metric.String() - group, ok := metricToChunks[metricString] - if !ok { - group = ChunksGroup{Labels: cortexpb.FromLabelsToLabelAdapters(chk.Metric)} - } - - chunkDetails := ChunkDetails{ID: chunkID} - - if deleteFrom > chk.From || deleteThrough < chk.Through { - partiallyDeletedInterval := Interval{StartTimestampMs: int64(chk.From), EndTimestampMs: int64(chk.Through)} - - if deleteFrom > chk.From { - partiallyDeletedInterval.StartTimestampMs = int64(deleteFrom) - } - - if deleteThrough < chk.Through { - partiallyDeletedInterval.EndTimestampMs = int64(deleteThrough) - } - chunkDetails.PartiallyDeletedInterval = &partiallyDeletedInterval - } - - group.Chunks = append(group.Chunks, chunkDetails) - includedChunkIDs[chunkID] = struct{}{} - metricToChunks[metricString] = group - } - - chunksGroups := make([]ChunksGroup, 0, len(metricToChunks)) - - for _, group := range metricToChunks { - chunksGroups = append(chunksGroups, group) - } - - return chunksGroups, includedChunkIDs -} - -func isMissingChunkErr(err error) bool { - if err == chunk.ErrStorageObjectNotFound { - return true - } - if promqlStorageErr, ok := err.(promql.ErrStorage); ok && promqlStorageErr.Err == chunk.ErrStorageObjectNotFound { - return true - } - - return false -} - -func buildObjectKeyForPlan(userID, requestID string, planNo int) string { - return fmt.Sprintf("%s:%s/%d", userID, requestID, planNo) -} - -func makeDeleteRequestWithLogger(deleteRequest DeleteRequest, l log.Logger) deleteRequestWithLogger { - logger := log.With(l, "user_id", deleteRequest.UserID, "request_id", deleteRequest.RequestID) - return deleteRequestWithLogger{deleteRequest, logger} -} - -// inProcessRequestsCollection stores DeleteRequests which are in process by each user. -// Currently we only allow processing of one delete request per user so it stores single DeleteRequest per user. -type inProcessRequestsCollection struct { - requests map[string]*DeleteRequest - usersWithFailedRequests map[string]struct{} - mtx sync.RWMutex -} - -func newInProcessRequestsCollection() *inProcessRequestsCollection { - return &inProcessRequestsCollection{ - requests: map[string]*DeleteRequest{}, - usersWithFailedRequests: map[string]struct{}{}, - } -} - -func (i *inProcessRequestsCollection) set(userID string, request *DeleteRequest) { - i.mtx.Lock() - defer i.mtx.Unlock() - - i.requests[userID] = request -} - -func (i *inProcessRequestsCollection) get(userID string) *DeleteRequest { - i.mtx.RLock() - defer i.mtx.RUnlock() - - return i.requests[userID] -} - -func (i *inProcessRequestsCollection) remove(userID string) { - i.mtx.Lock() - defer i.mtx.Unlock() - - delete(i.requests, userID) -} - -func (i *inProcessRequestsCollection) len() int { - i.mtx.RLock() - defer i.mtx.RUnlock() - - return len(i.requests) -} - -func (i *inProcessRequestsCollection) getOldest() *DeleteRequest { - i.mtx.RLock() - defer i.mtx.RUnlock() - - var oldestRequest *DeleteRequest - for _, request := range i.requests { - if oldestRequest == nil || request.CreatedAt.Before(oldestRequest.CreatedAt) { - oldestRequest = request - } - } - - return oldestRequest -} - -func (i *inProcessRequestsCollection) setFailedRequestForUser(userID string) { - i.mtx.Lock() - defer i.mtx.Unlock() - - i.usersWithFailedRequests[userID] = struct{}{} -} - -func (i *inProcessRequestsCollection) unsetFailedRequestForUser(userID string) { - i.mtx.Lock() - defer i.mtx.Unlock() - - delete(i.usersWithFailedRequests, userID) -} - -func (i *inProcessRequestsCollection) listUsersWithFailedRequest() []string { - i.mtx.RLock() - defer i.mtx.RUnlock() - - userIDs := make([]string, 0, len(i.usersWithFailedRequests)) - for userID := range i.usersWithFailedRequests { - userIDs = append(userIDs, userID) - } - - return userIDs -} diff --git a/pkg/chunk/purger/purger_test.go b/pkg/chunk/purger/purger_test.go deleted file mode 100644 index 58453e0e29..0000000000 --- a/pkg/chunk/purger/purger_test.go +++ /dev/null @@ -1,532 +0,0 @@ -package purger - -import ( - "context" - "fmt" - "sort" - "strings" - "testing" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/testutil" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/promql/parser" - "github.com/stretchr/testify/require" - - "github.com/cortexproject/cortex/pkg/chunk" - "github.com/cortexproject/cortex/pkg/chunk/testutils" - "github.com/cortexproject/cortex/pkg/util/flagext" - util_log "github.com/cortexproject/cortex/pkg/util/log" - "github.com/cortexproject/cortex/pkg/util/services" - "github.com/cortexproject/cortex/pkg/util/test" -) - -const ( - userID = "userID" - modelTimeDay = model.Time(millisecondPerDay) - modelTimeHour = model.Time(time.Hour / time.Millisecond) -) - -func setupTestDeleteStore(t *testing.T) *DeleteStore { - var ( - deleteStoreConfig DeleteStoreConfig - tbmConfig chunk.TableManagerConfig - schemaCfg = chunk.DefaultSchemaConfig("", "v10", 0) - ) - flagext.DefaultValues(&deleteStoreConfig) - flagext.DefaultValues(&tbmConfig) - - mockStorage := chunk.NewMockStorage() - - extraTables := []chunk.ExtraTables{{TableClient: mockStorage, Tables: deleteStoreConfig.GetTables()}} - tableManager, err := chunk.NewTableManager(tbmConfig, schemaCfg, 12*time.Hour, mockStorage, nil, extraTables, nil) - require.NoError(t, err) - - require.NoError(t, tableManager.SyncTables(context.Background())) - - deleteStore, err := NewDeleteStore(deleteStoreConfig, mockStorage) - require.NoError(t, err) - - return deleteStore -} - -func setupStoresAndPurger(t *testing.T) (*DeleteStore, chunk.Store, chunk.ObjectClient, *Purger, *prometheus.Registry) { - deleteStore := setupTestDeleteStore(t) - - chunkStore, err := testutils.SetupTestChunkStore() - require.NoError(t, err) - - storageClient, err := testutils.SetupTestObjectStore() - require.NoError(t, err) - - purger, registry := setupPurger(t, deleteStore, chunkStore, storageClient) - - return deleteStore, chunkStore, storageClient, purger, registry -} - -func setupPurger(t *testing.T, deleteStore *DeleteStore, chunkStore chunk.Store, storageClient chunk.ObjectClient) (*Purger, *prometheus.Registry) { - registry := prometheus.NewRegistry() - - var cfg Config - flagext.DefaultValues(&cfg) - - purger, err := NewPurger(cfg, deleteStore, chunkStore, storageClient, registry) - require.NoError(t, err) - - return purger, registry -} - -func buildChunks(from, through model.Time, batchSize int) ([]chunk.Chunk, error) { - var chunks []chunk.Chunk - for ; from < through; from = from.Add(time.Hour) { - // creating batchSize number of chunks chunks per hour - _, testChunks, err := testutils.CreateChunks(0, batchSize, from, from.Add(time.Hour)) - if err != nil { - return nil, err - } - - chunks = append(chunks, testChunks...) - } - - return chunks, nil -} - -var purgePlanTestCases = []struct { - name string - chunkStoreDataInterval model.Interval - deleteRequestInterval model.Interval - expectedNumberOfPlans int - numChunksToDelete int - firstChunkPartialDeletionInterval *Interval - lastChunkPartialDeletionInterval *Interval - batchSize int -}{ - { - name: "deleting whole hour from a one hour data", - chunkStoreDataInterval: model.Interval{End: modelTimeHour}, - deleteRequestInterval: model.Interval{End: modelTimeHour}, - expectedNumberOfPlans: 1, - numChunksToDelete: 1, - }, - { - name: "deleting half a day from a days data", - chunkStoreDataInterval: model.Interval{End: modelTimeDay}, - deleteRequestInterval: model.Interval{End: model.Time(millisecondPerDay / 2)}, - expectedNumberOfPlans: 1, - numChunksToDelete: 12 + 1, // one chunk for each hour + end time touches chunk at boundary - lastChunkPartialDeletionInterval: &Interval{StartTimestampMs: int64(millisecondPerDay / 2), - EndTimestampMs: int64(millisecondPerDay / 2)}, - }, - { - name: "deleting a full day from 2 days data", - chunkStoreDataInterval: model.Interval{End: modelTimeDay * 2}, - deleteRequestInterval: model.Interval{End: modelTimeDay}, - expectedNumberOfPlans: 1, - numChunksToDelete: 24 + 1, // one chunk for each hour + end time touches chunk at boundary - lastChunkPartialDeletionInterval: &Interval{StartTimestampMs: millisecondPerDay, - EndTimestampMs: millisecondPerDay}, - }, - { - name: "deleting 2 days partially from 2 days data", - chunkStoreDataInterval: model.Interval{End: modelTimeDay * 2}, - deleteRequestInterval: model.Interval{Start: model.Time(millisecondPerDay / 2), - End: model.Time(millisecondPerDay + millisecondPerDay/2)}, - expectedNumberOfPlans: 2, - numChunksToDelete: 24 + 2, // one chunk for each hour + start and end time touches chunk at boundary - firstChunkPartialDeletionInterval: &Interval{StartTimestampMs: int64(millisecondPerDay / 2), - EndTimestampMs: int64(millisecondPerDay / 2)}, - lastChunkPartialDeletionInterval: &Interval{StartTimestampMs: millisecondPerDay + millisecondPerDay/2, - EndTimestampMs: millisecondPerDay + millisecondPerDay/2}, - }, - { - name: "deleting 2 days partially, not aligned with hour, from 2 days data", - chunkStoreDataInterval: model.Interval{End: modelTimeDay * 2}, - deleteRequestInterval: model.Interval{Start: model.Time(millisecondPerDay / 2).Add(time.Minute), - End: model.Time(millisecondPerDay + millisecondPerDay/2).Add(-time.Minute)}, - expectedNumberOfPlans: 2, - numChunksToDelete: 24, // one chunk for each hour, no chunks touched at boundary - firstChunkPartialDeletionInterval: &Interval{StartTimestampMs: int64(model.Time(millisecondPerDay / 2).Add(time.Minute)), - EndTimestampMs: int64(model.Time(millisecondPerDay / 2).Add(time.Hour))}, - lastChunkPartialDeletionInterval: &Interval{StartTimestampMs: int64(model.Time(millisecondPerDay + millisecondPerDay/2).Add(-time.Hour)), - EndTimestampMs: int64(model.Time(millisecondPerDay + millisecondPerDay/2).Add(-time.Minute))}, - }, - { - name: "deleting data outside of period of existing data", - chunkStoreDataInterval: model.Interval{End: modelTimeDay}, - deleteRequestInterval: model.Interval{Start: model.Time(millisecondPerDay * 2), End: model.Time(millisecondPerDay * 3)}, - expectedNumberOfPlans: 1, - numChunksToDelete: 0, - }, - { - name: "building multi-day chunk and deleting part of it from first day", - chunkStoreDataInterval: model.Interval{Start: modelTimeDay.Add(-30 * time.Minute), End: modelTimeDay.Add(30 * time.Minute)}, - deleteRequestInterval: model.Interval{Start: modelTimeDay.Add(-30 * time.Minute), End: modelTimeDay.Add(-15 * time.Minute)}, - expectedNumberOfPlans: 1, - numChunksToDelete: 1, - firstChunkPartialDeletionInterval: &Interval{StartTimestampMs: int64(modelTimeDay.Add(-30 * time.Minute)), - EndTimestampMs: int64(modelTimeDay.Add(-15 * time.Minute))}, - }, - { - name: "building multi-day chunk and deleting part of it for each day", - chunkStoreDataInterval: model.Interval{Start: modelTimeDay.Add(-30 * time.Minute), End: modelTimeDay.Add(30 * time.Minute)}, - deleteRequestInterval: model.Interval{Start: modelTimeDay.Add(-15 * time.Minute), End: modelTimeDay.Add(15 * time.Minute)}, - expectedNumberOfPlans: 2, - numChunksToDelete: 1, - firstChunkPartialDeletionInterval: &Interval{StartTimestampMs: int64(modelTimeDay.Add(-15 * time.Minute)), - EndTimestampMs: int64(modelTimeDay.Add(15 * time.Minute))}, - }, -} - -func TestPurger_BuildPlan(t *testing.T) { - for _, tc := range purgePlanTestCases { - for batchSize := 1; batchSize <= 5; batchSize++ { - t.Run(fmt.Sprintf("%s/batch-size=%d", tc.name, batchSize), func(t *testing.T) { - deleteStore, chunkStore, storageClient, purger, _ := setupStoresAndPurger(t) - defer func() { - purger.StopAsync() - chunkStore.Stop() - }() - - chunks, err := buildChunks(tc.chunkStoreDataInterval.Start, tc.chunkStoreDataInterval.End, batchSize) - require.NoError(t, err) - - require.NoError(t, chunkStore.Put(context.Background(), chunks)) - - err = deleteStore.AddDeleteRequest(context.Background(), userID, tc.deleteRequestInterval.Start, - tc.deleteRequestInterval.End, []string{"foo"}) - require.NoError(t, err) - - deleteRequests, err := deleteStore.GetAllDeleteRequestsForUser(context.Background(), userID) - require.NoError(t, err) - - deleteRequest := deleteRequests[0] - requestWithLogger := makeDeleteRequestWithLogger(deleteRequest, util_log.Logger) - - err = purger.buildDeletePlan(requestWithLogger) - require.NoError(t, err) - planPath := fmt.Sprintf("%s:%s/", userID, deleteRequest.RequestID) - - plans, _, err := storageClient.List(context.Background(), planPath, "/") - require.NoError(t, err) - require.Equal(t, tc.expectedNumberOfPlans, len(plans)) - - numPlans := tc.expectedNumberOfPlans - var nilPurgePlanInterval *Interval - numChunks := 0 - - chunkIDs := map[string]struct{}{} - - for i := range plans { - deletePlan, err := purger.getDeletePlan(context.Background(), userID, deleteRequest.RequestID, i) - require.NoError(t, err) - for _, chunksGroup := range deletePlan.ChunksGroup { - numChunksInGroup := len(chunksGroup.Chunks) - chunks := chunksGroup.Chunks - numChunks += numChunksInGroup - - sort.Slice(chunks, func(i, j int) bool { - chunkI, err := chunk.ParseExternalKey(userID, chunks[i].ID) - require.NoError(t, err) - - chunkJ, err := chunk.ParseExternalKey(userID, chunks[j].ID) - require.NoError(t, err) - - return chunkI.From < chunkJ.From - }) - - for j, chunkDetails := range chunksGroup.Chunks { - chunkIDs[chunkDetails.ID] = struct{}{} - if i == 0 && j == 0 && tc.firstChunkPartialDeletionInterval != nil { - require.Equal(t, *tc.firstChunkPartialDeletionInterval, *chunkDetails.PartiallyDeletedInterval) - } else if i == numPlans-1 && j == numChunksInGroup-1 && tc.lastChunkPartialDeletionInterval != nil { - require.Equal(t, *tc.lastChunkPartialDeletionInterval, *chunkDetails.PartiallyDeletedInterval) - } else { - require.Equal(t, nilPurgePlanInterval, chunkDetails.PartiallyDeletedInterval) - } - } - } - } - - require.Equal(t, tc.numChunksToDelete*batchSize, len(chunkIDs)) - require.Equal(t, float64(tc.numChunksToDelete*batchSize), testutil.ToFloat64(purger.metrics.deleteRequestsChunksSelectedTotal)) - }) - } - } -} - -func TestPurger_ExecutePlan(t *testing.T) { - fooMetricNameMatcher, err := parser.ParseMetricSelector(`foo`) - if err != nil { - t.Fatal(err) - } - - for _, tc := range purgePlanTestCases { - for batchSize := 1; batchSize <= 5; batchSize++ { - t.Run(fmt.Sprintf("%s/batch-size=%d", tc.name, batchSize), func(t *testing.T) { - deleteStore, chunkStore, _, purger, _ := setupStoresAndPurger(t) - defer func() { - purger.StopAsync() - chunkStore.Stop() - }() - - chunks, err := buildChunks(tc.chunkStoreDataInterval.Start, tc.chunkStoreDataInterval.End, batchSize) - require.NoError(t, err) - - require.NoError(t, chunkStore.Put(context.Background(), chunks)) - - // calculate the expected number of chunks that should be there in store before deletion - chunkStoreDataIntervalTotal := tc.chunkStoreDataInterval.End - tc.chunkStoreDataInterval.Start - numChunksExpected := int(chunkStoreDataIntervalTotal / model.Time(time.Hour/time.Millisecond)) - - // see if store actually has expected number of chunks - chunks, err = chunkStore.Get(context.Background(), userID, tc.chunkStoreDataInterval.Start, tc.chunkStoreDataInterval.End, fooMetricNameMatcher...) - require.NoError(t, err) - require.Equal(t, numChunksExpected*batchSize, len(chunks)) - - // delete chunks - err = deleteStore.AddDeleteRequest(context.Background(), userID, tc.deleteRequestInterval.Start, - tc.deleteRequestInterval.End, []string{"foo"}) - require.NoError(t, err) - - // get the delete request - deleteRequests, err := deleteStore.GetAllDeleteRequestsForUser(context.Background(), userID) - require.NoError(t, err) - - deleteRequest := deleteRequests[0] - requestWithLogger := makeDeleteRequestWithLogger(deleteRequest, util_log.Logger) - err = purger.buildDeletePlan(requestWithLogger) - require.NoError(t, err) - - // execute all the plans - for i := 0; i < tc.expectedNumberOfPlans; i++ { - err := purger.executePlan(userID, deleteRequest.RequestID, i, requestWithLogger.logger) - require.NoError(t, err) - } - - // calculate the expected number of chunks that should be there in store after deletion - numChunksExpectedAfterDeletion := 0 - for chunkStart := tc.chunkStoreDataInterval.Start; chunkStart < tc.chunkStoreDataInterval.End; chunkStart += modelTimeHour { - numChunksExpectedAfterDeletion += len(getNonDeletedIntervals(model.Interval{Start: chunkStart, End: chunkStart + modelTimeHour}, tc.deleteRequestInterval)) - } - - // see if store actually has expected number of chunks - chunks, err = chunkStore.Get(context.Background(), userID, tc.chunkStoreDataInterval.Start, tc.chunkStoreDataInterval.End, fooMetricNameMatcher...) - require.NoError(t, err) - require.Equal(t, numChunksExpectedAfterDeletion*batchSize, len(chunks)) - }) - } - } -} - -func TestPurger_Restarts(t *testing.T) { - fooMetricNameMatcher, err := parser.ParseMetricSelector(`foo`) - if err != nil { - t.Fatal(err) - } - - deleteStore, chunkStore, storageClient, purger, _ := setupStoresAndPurger(t) - defer func() { - chunkStore.Stop() - }() - - chunks, err := buildChunks(0, model.Time(0).Add(10*24*time.Hour), 1) - require.NoError(t, err) - - require.NoError(t, chunkStore.Put(context.Background(), chunks)) - - // delete chunks - err = deleteStore.AddDeleteRequest(context.Background(), userID, model.Time(0).Add(24*time.Hour), - model.Time(0).Add(8*24*time.Hour), []string{"foo"}) - require.NoError(t, err) - - // get the delete request - deleteRequests, err := deleteStore.GetAllDeleteRequestsForUser(context.Background(), userID) - require.NoError(t, err) - - deleteRequest := deleteRequests[0] - requestWithLogger := makeDeleteRequestWithLogger(deleteRequest, util_log.Logger) - err = purger.buildDeletePlan(requestWithLogger) - require.NoError(t, err) - - // stop the existing purger - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), purger)) - - // create a new purger to check whether it picks up in process delete requests - newPurger, _ := setupPurger(t, deleteStore, chunkStore, storageClient) - - // load in process delete requests by calling Run - require.NoError(t, services.StartAndAwaitRunning(context.Background(), newPurger)) - - defer newPurger.StopAsync() - - test.Poll(t, time.Minute, 0, func() interface{} { - return newPurger.inProcessRequests.len() - }) - - // check whether data got deleted from the store since delete request has been processed - chunks, err = chunkStore.Get(context.Background(), userID, 0, model.Time(0).Add(10*24*time.Hour), fooMetricNameMatcher...) - require.NoError(t, err) - - // we are deleting 7 days out of 10 so there should we 3 days data left in store which means 72 chunks - require.Equal(t, 72, len(chunks)) - - deleteRequests, err = deleteStore.GetAllDeleteRequestsForUser(context.Background(), userID) - require.NoError(t, err) - require.Equal(t, StatusProcessed, deleteRequests[0].Status) - - require.Equal(t, float64(1), testutil.ToFloat64(newPurger.metrics.deleteRequestsProcessedTotal)) - require.PanicsWithError(t, "collected 0 metrics instead of exactly 1", func() { - testutil.ToFloat64(newPurger.metrics.deleteRequestsProcessingFailures) - }) -} - -func TestPurger_Metrics(t *testing.T) { - deleteStore, chunkStore, storageClient, purger, registry := setupStoresAndPurger(t) - defer func() { - purger.StopAsync() - chunkStore.Stop() - }() - - // add delete requests without starting purger loops to load and process delete requests. - // add delete request whose createdAt is now - err := deleteStore.AddDeleteRequest(context.Background(), userID, model.Time(0).Add(24*time.Hour), - model.Time(0).Add(2*24*time.Hour), []string{"foo"}) - require.NoError(t, err) - - // add delete request whose createdAt is 2 days back - err = deleteStore.addDeleteRequest(context.Background(), userID, model.Now().Add(-2*24*time.Hour), model.Time(0).Add(24*time.Hour), - model.Time(0).Add(2*24*time.Hour), []string{"foo"}) - require.NoError(t, err) - - // add delete request whose createdAt is 3 days back - err = deleteStore.addDeleteRequest(context.Background(), userID, model.Now().Add(-3*24*time.Hour), model.Time(0).Add(24*time.Hour), - model.Time(0).Add(8*24*time.Hour), []string{"foo"}) - require.NoError(t, err) - - // load new delete requests for processing - require.NoError(t, purger.pullDeleteRequestsToPlanDeletes()) - - // there must be 2 pending delete requests, oldest being 2 days old since its cancellation time is over - require.InDelta(t, float64(2*86400), testutil.ToFloat64(purger.metrics.oldestPendingDeleteRequestAgeSeconds), 1) - require.Equal(t, float64(2), testutil.ToFloat64(purger.metrics.pendingDeleteRequestsCount)) - - // stop the existing purger - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), purger)) - - // create a new purger - purger, registry = setupPurger(t, deleteStore, chunkStore, storageClient) - - // load in process delete requests by starting the service - require.NoError(t, services.StartAndAwaitRunning(context.Background(), purger)) - - defer purger.StopAsync() - - // wait until purger_delete_requests_processed_total starts to show up. - test.Poll(t, 2*time.Second, 1, func() interface{} { - count, err := testutil.GatherAndCount(registry, "cortex_purger_delete_requests_processed_total") - require.NoError(t, err) - return count - }) - - // wait until both the pending delete requests are processed. - test.Poll(t, 2*time.Second, float64(2), func() interface{} { - return testutil.ToFloat64(purger.metrics.deleteRequestsProcessedTotal) - }) - - // wait until oldest pending request age becomes 0 - test.Poll(t, 2*time.Second, float64(0), func() interface{} { - return testutil.ToFloat64(purger.metrics.oldestPendingDeleteRequestAgeSeconds) - }) - - // wait until pending delete requests count becomes 0 - test.Poll(t, 2*time.Second, float64(0), func() interface{} { - return testutil.ToFloat64(purger.metrics.pendingDeleteRequestsCount) - }) -} - -func TestPurger_retryFailedRequests(t *testing.T) { - // setup chunks store - indexMockStorage := chunk.NewMockStorage() - chunksMockStorage := chunk.NewMockStorage() - - deleteStore := setupTestDeleteStore(t) - chunkStore, err := testutils.SetupTestChunkStoreWithClients(indexMockStorage, chunksMockStorage, indexMockStorage) - require.NoError(t, err) - - // create a purger instance - purgerMockStorage := chunk.NewMockStorage() - purger, _ := setupPurger(t, deleteStore, chunkStore, purgerMockStorage) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), purger)) - - defer func() { - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), purger)) - }() - - // add some chunks - chunks, err := buildChunks(0, model.Time(0).Add(3*24*time.Hour), 1) - require.NoError(t, err) - - require.NoError(t, chunkStore.Put(context.Background(), chunks)) - - // add a request to delete some chunks - err = deleteStore.addDeleteRequest(context.Background(), userID, model.Now().Add(-25*time.Hour), model.Time(0).Add(24*time.Hour), - model.Time(0).Add(2*24*time.Hour), []string{"foo"}) - require.NoError(t, err) - - // change purgerMockStorage to allow only reads. This would fail putting plans to the storage and hence fail build plans operation. - purgerMockStorage.SetMode(chunk.MockStorageModeReadOnly) - - // pull requests to process and ensure that it has failed. - err = purger.pullDeleteRequestsToPlanDeletes() - require.Error(t, err) - require.True(t, strings.Contains(err.Error(), "permission denied")) - - // there must be 1 delete request in process and the userID must be in failed requests list. - require.NotNil(t, purger.inProcessRequests.get(userID)) - require.Len(t, purger.inProcessRequests.listUsersWithFailedRequest(), 1) - - // now allow writes to purgerMockStorage to allow building plans to succeed. - purgerMockStorage.SetMode(chunk.MockStorageModeReadWrite) - - // but change mode of chunksMockStorage to read only which would deny permission to delete any chunks and in turn - // fail to execute delete plans. - chunksMockStorage.SetMode(chunk.MockStorageModeReadOnly) - - // retry processing of failed requests - purger.retryFailedRequests() - - // the delete request status should now change to StatusDeleting since the building of plan should have succeeded. - test.Poll(t, time.Second, StatusDeleting, func() interface{} { - return purger.inProcessRequests.get(userID).Status - }) - // the request should have failed again since we did not give permission to delete chunks. - test.Poll(t, time.Second, 1, func() interface{} { - return len(purger.inProcessRequests.listUsersWithFailedRequest()) - }) - - // now allow writes to chunksMockStorage so the requests do not fail anymore. - chunksMockStorage.SetMode(chunk.MockStorageModeReadWrite) - - // retry processing of failed requests. - purger.retryFailedRequests() - // there must be no in process requests anymore. - test.Poll(t, time.Second, true, func() interface{} { - return purger.inProcessRequests.get(userID) == nil - }) - // there must be no users having failed requests. - require.Len(t, purger.inProcessRequests.listUsersWithFailedRequest(), 0) -} - -func getNonDeletedIntervals(originalInterval, deletedInterval model.Interval) []model.Interval { - nonDeletedIntervals := []model.Interval{} - if deletedInterval.Start > originalInterval.Start { - nonDeletedIntervals = append(nonDeletedIntervals, model.Interval{Start: originalInterval.Start, End: deletedInterval.Start - 1}) - } - - if deletedInterval.End < originalInterval.End { - nonDeletedIntervals = append(nonDeletedIntervals, model.Interval{Start: deletedInterval.End + 1, End: originalInterval.End}) - } - - return nonDeletedIntervals -} diff --git a/pkg/chunk/purger/request_handler.go b/pkg/chunk/purger/request_handler.go deleted file mode 100644 index d9657b3ee7..0000000000 --- a/pkg/chunk/purger/request_handler.go +++ /dev/null @@ -1,183 +0,0 @@ -package purger - -import ( - "encoding/json" - "fmt" - "net/http" - "time" - - "github.com/go-kit/log/level" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/promql/parser" - - "github.com/cortexproject/cortex/pkg/tenant" - "github.com/cortexproject/cortex/pkg/util" - util_log "github.com/cortexproject/cortex/pkg/util/log" -) - -type deleteRequestHandlerMetrics struct { - deleteRequestsReceivedTotal *prometheus.CounterVec -} - -func newDeleteRequestHandlerMetrics(r prometheus.Registerer) *deleteRequestHandlerMetrics { - m := deleteRequestHandlerMetrics{} - - m.deleteRequestsReceivedTotal = promauto.With(r).NewCounterVec(prometheus.CounterOpts{ - Namespace: "cortex", - Name: "purger_delete_requests_received_total", - Help: "Number of delete requests received per user", - }, []string{"user"}) - - return &m -} - -// DeleteRequestHandler provides handlers for delete requests -type DeleteRequestHandler struct { - deleteStore *DeleteStore - metrics *deleteRequestHandlerMetrics - deleteRequestCancelPeriod time.Duration -} - -// NewDeleteRequestHandler creates a DeleteRequestHandler -func NewDeleteRequestHandler(deleteStore *DeleteStore, deleteRequestCancelPeriod time.Duration, registerer prometheus.Registerer) *DeleteRequestHandler { - deleteMgr := DeleteRequestHandler{ - deleteStore: deleteStore, - deleteRequestCancelPeriod: deleteRequestCancelPeriod, - metrics: newDeleteRequestHandlerMetrics(registerer), - } - - return &deleteMgr -} - -// AddDeleteRequestHandler handles addition of new delete request -func (dm *DeleteRequestHandler) AddDeleteRequestHandler(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - userID, err := tenant.TenantID(ctx) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - params := r.URL.Query() - match := params["match[]"] - if len(match) == 0 { - http.Error(w, "selectors not set", http.StatusBadRequest) - return - } - - for i := range match { - _, err := parser.ParseMetricSelector(match[i]) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - } - - startParam := params.Get("start") - startTime := int64(0) - if startParam != "" { - startTime, err = util.ParseTime(startParam) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - } - - endParam := params.Get("end") - endTime := int64(model.Now()) - - if endParam != "" { - endTime, err = util.ParseTime(endParam) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if endTime > int64(model.Now()) { - http.Error(w, "deletes in future not allowed", http.StatusBadRequest) - return - } - } - - if startTime > endTime { - http.Error(w, "start time can't be greater than end time", http.StatusBadRequest) - return - } - - if err := dm.deleteStore.AddDeleteRequest(ctx, userID, model.Time(startTime), model.Time(endTime), match); err != nil { - level.Error(util_log.Logger).Log("msg", "error adding delete request to the store", "err", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - dm.metrics.deleteRequestsReceivedTotal.WithLabelValues(userID).Inc() - w.WriteHeader(http.StatusNoContent) -} - -// GetAllDeleteRequestsHandler handles get all delete requests -func (dm *DeleteRequestHandler) GetAllDeleteRequestsHandler(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - userID, err := tenant.TenantID(ctx) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - deleteRequests, err := dm.deleteStore.GetAllDeleteRequestsForUser(ctx, userID) - if err != nil { - level.Error(util_log.Logger).Log("msg", "error getting delete requests from the store", "err", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - if err := json.NewEncoder(w).Encode(deleteRequests); err != nil { - level.Error(util_log.Logger).Log("msg", "error marshalling response", "err", err) - http.Error(w, fmt.Sprintf("Error marshalling response: %v", err), http.StatusInternalServerError) - } -} - -// CancelDeleteRequestHandler handles delete request cancellation -func (dm *DeleteRequestHandler) CancelDeleteRequestHandler(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - userID, err := tenant.TenantID(ctx) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - params := r.URL.Query() - requestID := params.Get("request_id") - - deleteRequest, err := dm.deleteStore.GetDeleteRequest(ctx, userID, requestID) - if err != nil { - level.Error(util_log.Logger).Log("msg", "error getting delete request from the store", "err", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - if deleteRequest == nil { - http.Error(w, "could not find delete request with given id", http.StatusBadRequest) - return - } - - if deleteRequest.Status != StatusReceived { - http.Error(w, "deletion of request which is in process or already processed is not allowed", http.StatusBadRequest) - return - } - - if deleteRequest.CreatedAt.Add(dm.deleteRequestCancelPeriod).Before(model.Now()) { - http.Error(w, fmt.Sprintf("deletion of request past the deadline of %s since its creation is not allowed", dm.deleteRequestCancelPeriod.String()), http.StatusBadRequest) - return - } - - if err := dm.deleteStore.RemoveDeleteRequest(ctx, userID, requestID, deleteRequest.CreatedAt, deleteRequest.StartTime, deleteRequest.EndTime); err != nil { - level.Error(util_log.Logger).Log("msg", "error cancelling the delete request", "err", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - w.WriteHeader(http.StatusNoContent) -} diff --git a/pkg/chunk/purger/table_provisioning.go b/pkg/chunk/purger/table_provisioning.go deleted file mode 100644 index e8ce5d6364..0000000000 --- a/pkg/chunk/purger/table_provisioning.go +++ /dev/null @@ -1,30 +0,0 @@ -package purger - -import ( - "flag" - - "github.com/cortexproject/cortex/pkg/chunk" -) - -// TableProvisioningConfig holds config for table throuput and autoscaling. Currently only used by DynamoDB. -type TableProvisioningConfig struct { - chunk.ActiveTableProvisionConfig `yaml:",inline"` - TableTags chunk.Tags `yaml:"tags"` -} - -// RegisterFlags adds the flags required to config this to the given FlagSet. -// Adding a separate RegisterFlags here instead of using it from embedded chunk.ActiveTableProvisionConfig to be able to manage defaults separately. -// Defaults for WriteScale and ReadScale are shared for now to avoid adding further complexity since autoscaling is disabled anyways by default. -func (cfg *TableProvisioningConfig) RegisterFlags(argPrefix string, f *flag.FlagSet) { - // default values ActiveTableProvisionConfig - cfg.ProvisionedWriteThroughput = 1 - cfg.ProvisionedReadThroughput = 300 - cfg.ProvisionedThroughputOnDemandMode = false - - cfg.ActiveTableProvisionConfig.RegisterFlags(argPrefix, f) - f.Var(&cfg.TableTags, argPrefix+".tags", "Tag (of the form key=value) to be added to the tables. Supported by DynamoDB") -} - -func (cfg DeleteStoreConfig) GetTables() []chunk.TableDesc { - return []chunk.TableDesc{cfg.ProvisionConfig.BuildTableDesc(cfg.RequestsTableName, cfg.ProvisionConfig.TableTags)} -} diff --git a/pkg/chunk/purger/tombstones.go b/pkg/chunk/purger/tombstones.go index 00eeeee1d6..31084dd529 100644 --- a/pkg/chunk/purger/tombstones.go +++ b/pkg/chunk/purger/tombstones.go @@ -1,450 +1,74 @@ package purger import ( - "context" - "sort" - "strconv" - "sync" - "time" - - "github.com/go-kit/log/level" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/promql/parser" - - util_log "github.com/cortexproject/cortex/pkg/util/log" ) -const tombstonesReloadDuration = 5 * time.Minute - -type tombstonesLoaderMetrics struct { - cacheGenLoadFailures prometheus.Counter - deleteRequestsLoadFailures prometheus.Counter -} - -func newtombstonesLoaderMetrics(r prometheus.Registerer) *tombstonesLoaderMetrics { - m := tombstonesLoaderMetrics{} - - m.cacheGenLoadFailures = promauto.With(r).NewCounter(prometheus.CounterOpts{ - Namespace: "cortex", - Name: "tombstones_loader_cache_gen_load_failures_total", - Help: "Total number of failures while loading cache generation number using tombstones loader", - }) - m.deleteRequestsLoadFailures = promauto.With(r).NewCounter(prometheus.CounterOpts{ - Namespace: "cortex", - Name: "tombstones_loader_cache_delete_requests_load_failures_total", - Help: "Total number of failures while loading delete requests using tombstones loader", - }) +// TombstonesSet holds all the pending delete requests for a user +type TombstonesSet interface { + // GetDeletedIntervals returns non-overlapping, sorted deleted intervals. + GetDeletedIntervals(lbls labels.Labels, from, to model.Time) []model.Interval - return &m -} + // Len returns number of tombstones that are there + Len() int -// TombstonesSet holds all the pending delete requests for a user -type TombstonesSet struct { - tombstones []DeleteRequest - oldestTombstoneStart, newestTombstoneEnd model.Time // Used as optimization to find whether we want to iterate over tombstones or not + // HasTombstonesForInterval tells whether there are any tombstones which overlapping given interval + HasTombstonesForInterval(from, to model.Time) bool } -// Used for easier injection of mocks. -type DeleteStoreAPI interface { - getCacheGenerationNumbers(ctx context.Context, user string) (*cacheGenNumbers, error) - GetPendingDeleteRequestsForUser(ctx context.Context, id string) ([]DeleteRequest, error) +type noopTombstonesSet struct { } // TombstonesLoader loads delete requests and gen numbers from store and keeps checking for updates. // It keeps checking for changes in gen numbers, which also means changes in delete requests and reloads specific users delete requests. -type TombstonesLoader struct { - tombstones map[string]*TombstonesSet - tombstonesMtx sync.RWMutex - - cacheGenNumbers map[string]*cacheGenNumbers - cacheGenNumbersMtx sync.RWMutex - - deleteStore DeleteStoreAPI - metrics *tombstonesLoaderMetrics - quit chan struct{} -} - -// NewTombstonesLoader creates a TombstonesLoader -func NewTombstonesLoader(deleteStore DeleteStoreAPI, registerer prometheus.Registerer) *TombstonesLoader { - tl := TombstonesLoader{ - tombstones: map[string]*TombstonesSet{}, - cacheGenNumbers: map[string]*cacheGenNumbers{}, - deleteStore: deleteStore, - metrics: newtombstonesLoaderMetrics(registerer), - } - go tl.loop() - - return &tl -} - -// Stop stops TombstonesLoader -func (tl *TombstonesLoader) Stop() { - close(tl.quit) -} - -func (tl *TombstonesLoader) loop() { - if tl.deleteStore == nil { - return - } - - tombstonesReloadTimer := time.NewTicker(tombstonesReloadDuration) - for { - select { - case <-tombstonesReloadTimer.C: - err := tl.reloadTombstones() - if err != nil { - level.Error(util_log.Logger).Log("msg", "error reloading tombstones", "err", err) - } - case <-tl.quit: - return - } - } -} - -func (tl *TombstonesLoader) reloadTombstones() error { - updatedGenNumbers := make(map[string]*cacheGenNumbers) - tl.cacheGenNumbersMtx.RLock() - - // check for updates in loaded gen numbers - for userID, oldGenNumbers := range tl.cacheGenNumbers { - newGenNumbers, err := tl.deleteStore.getCacheGenerationNumbers(context.Background(), userID) - if err != nil { - tl.cacheGenNumbersMtx.RUnlock() - return err - } - - if *oldGenNumbers != *newGenNumbers { - updatedGenNumbers[userID] = newGenNumbers - } - } - - tl.cacheGenNumbersMtx.RUnlock() - - // in frontend we load only cache gen numbers so short circuit here if there are no loaded deleted requests - // first call to GetPendingTombstones would avoid doing this. - tl.tombstonesMtx.RLock() - if len(tl.tombstones) == 0 { - tl.tombstonesMtx.RUnlock() - return nil - } - tl.tombstonesMtx.RUnlock() - - // for all the updated gen numbers, reload delete requests - for userID, genNumbers := range updatedGenNumbers { - err := tl.loadPendingTombstones(userID) - if err != nil { - return err - } - - tl.cacheGenNumbersMtx.Lock() - tl.cacheGenNumbers[userID] = genNumbers - tl.cacheGenNumbersMtx.Unlock() - } - - return nil -} - -// GetPendingTombstones returns all pending tombstones -func (tl *TombstonesLoader) GetPendingTombstones(userID string) (*TombstonesSet, error) { - tl.tombstonesMtx.RLock() - - tombstoneSet, isOK := tl.tombstones[userID] - if isOK { - tl.tombstonesMtx.RUnlock() - return tombstoneSet, nil - } - - tl.tombstonesMtx.RUnlock() - err := tl.loadPendingTombstones(userID) - if err != nil { - return nil, err - } - - tl.tombstonesMtx.RLock() - defer tl.tombstonesMtx.RUnlock() - - return tl.tombstones[userID], nil -} - -// GetPendingTombstones returns all pending tombstones -func (tl *TombstonesLoader) GetPendingTombstonesForInterval(userID string, from, to model.Time) (*TombstonesSet, error) { - allTombstones, err := tl.GetPendingTombstones(userID) - if err != nil { - return nil, err - } - - if !allTombstones.HasTombstonesForInterval(from, to) { - return &TombstonesSet{}, nil - } - - filteredSet := TombstonesSet{oldestTombstoneStart: model.Now()} - - for _, tombstone := range allTombstones.tombstones { - if !intervalsOverlap(model.Interval{Start: from, End: to}, model.Interval{Start: tombstone.StartTime, End: tombstone.EndTime}) { - continue - } - - filteredSet.tombstones = append(filteredSet.tombstones, tombstone) - - if tombstone.StartTime < filteredSet.oldestTombstoneStart { - filteredSet.oldestTombstoneStart = tombstone.StartTime - } - - if tombstone.EndTime > filteredSet.newestTombstoneEnd { - filteredSet.newestTombstoneEnd = tombstone.EndTime - } - } - - return &filteredSet, nil -} - -func (tl *TombstonesLoader) loadPendingTombstones(userID string) error { - if tl.deleteStore == nil { - tl.tombstonesMtx.Lock() - defer tl.tombstonesMtx.Unlock() - - tl.tombstones[userID] = &TombstonesSet{oldestTombstoneStart: 0, newestTombstoneEnd: 0} - return nil - } - - pendingDeleteRequests, err := tl.deleteStore.GetPendingDeleteRequestsForUser(context.Background(), userID) - if err != nil { - tl.metrics.deleteRequestsLoadFailures.Inc() - return errors.Wrap(err, "error loading delete requests") - } - - tombstoneSet := TombstonesSet{tombstones: pendingDeleteRequests, oldestTombstoneStart: model.Now()} - for i := range tombstoneSet.tombstones { - tombstoneSet.tombstones[i].Matchers = make([][]*labels.Matcher, len(tombstoneSet.tombstones[i].Selectors)) - - for j, selector := range tombstoneSet.tombstones[i].Selectors { - tombstoneSet.tombstones[i].Matchers[j], err = parser.ParseMetricSelector(selector) +type TombstonesLoader interface { + // GetPendingTombstones returns all pending tombstones + GetPendingTombstones(userID string) (TombstonesSet, error) - if err != nil { - tl.metrics.deleteRequestsLoadFailures.Inc() - return errors.Wrapf(err, "error parsing metric selector") - } - } + // GetPendingTombstonesForInterval returns all pending tombstones between two times + GetPendingTombstonesForInterval(userID string, from, to model.Time) (TombstonesSet, error) - if tombstoneSet.tombstones[i].StartTime < tombstoneSet.oldestTombstoneStart { - tombstoneSet.oldestTombstoneStart = tombstoneSet.tombstones[i].StartTime - } - - if tombstoneSet.tombstones[i].EndTime > tombstoneSet.newestTombstoneEnd { - tombstoneSet.newestTombstoneEnd = tombstoneSet.tombstones[i].EndTime - } - } - - tl.tombstonesMtx.Lock() - defer tl.tombstonesMtx.Unlock() - tl.tombstones[userID] = &tombstoneSet - - return nil -} + // GetStoreCacheGenNumber returns store cache gen number for a user + GetStoreCacheGenNumber(tenantIDs []string) string -// GetStoreCacheGenNumber returns store cache gen number for a user -func (tl *TombstonesLoader) GetStoreCacheGenNumber(tenantIDs []string) string { - return tl.getCacheGenNumbersPerTenants(tenantIDs).store + // GetResultsCacheGenNumber returns results cache gen number for a user + GetResultsCacheGenNumber(tenantIDs []string) string } -// GetResultsCacheGenNumber returns results cache gen number for a user -func (tl *TombstonesLoader) GetResultsCacheGenNumber(tenantIDs []string) string { - return tl.getCacheGenNumbersPerTenants(tenantIDs).results +type noopTombstonesLoader struct { + ts noopTombstonesSet } -func (tl *TombstonesLoader) getCacheGenNumbersPerTenants(tenantIDs []string) *cacheGenNumbers { - var result cacheGenNumbers - - if len(tenantIDs) == 0 { - return &result - } - - // keep the maximum value that's currently in result - var maxResults, maxStore int - - for pos, tenantID := range tenantIDs { - numbers := tl.getCacheGenNumbers(tenantID) - - // handle first tenant in the list - if pos == 0 { - // short cut if there is only one tenant - if len(tenantIDs) == 1 { - return numbers - } - - // set first tenant string whatever happens next - result.results = numbers.results - result.store = numbers.store - } - - // set results number string if it's higher than the ones before - if numbers.results != "" { - results, err := strconv.Atoi(numbers.results) - if err != nil { - level.Error(util_log.Logger).Log("msg", "error parsing resultsCacheGenNumber", "user", tenantID, "err", err) - } else if maxResults < results { - maxResults = results - result.results = numbers.results - } - } - - // set store number string if it's higher than the ones before - if numbers.store != "" { - store, err := strconv.Atoi(numbers.store) - if err != nil { - level.Error(util_log.Logger).Log("msg", "error parsing storeCacheGenNumber", "user", tenantID, "err", err) - } else if maxStore < store { - maxStore = store - result.store = numbers.store - } - } - } - - return &result +// NewNoopTombstonesLoader creates a TombstonesLoader that does nothing +func NewNoopTombstonesLoader() TombstonesLoader { + return &noopTombstonesLoader{} } -func (tl *TombstonesLoader) getCacheGenNumbers(userID string) *cacheGenNumbers { - tl.cacheGenNumbersMtx.RLock() - if genNumbers, isOK := tl.cacheGenNumbers[userID]; isOK { - tl.cacheGenNumbersMtx.RUnlock() - return genNumbers - } - - tl.cacheGenNumbersMtx.RUnlock() - - if tl.deleteStore == nil { - tl.cacheGenNumbersMtx.Lock() - defer tl.cacheGenNumbersMtx.Unlock() - - tl.cacheGenNumbers[userID] = &cacheGenNumbers{} - return tl.cacheGenNumbers[userID] - } - - genNumbers, err := tl.deleteStore.getCacheGenerationNumbers(context.Background(), userID) - if err != nil { - level.Error(util_log.Logger).Log("msg", "error loading cache generation numbers", "err", err) - tl.metrics.cacheGenLoadFailures.Inc() - return &cacheGenNumbers{} - } - - tl.cacheGenNumbersMtx.Lock() - defer tl.cacheGenNumbersMtx.Unlock() - - tl.cacheGenNumbers[userID] = genNumbers - return genNumbers +func (tl *noopTombstonesLoader) GetPendingTombstones(userID string) (TombstonesSet, error) { + return &tl.ts, nil } -// GetDeletedIntervals returns non-overlapping, sorted deleted intervals. -func (ts TombstonesSet) GetDeletedIntervals(lbls labels.Labels, from, to model.Time) []model.Interval { - if len(ts.tombstones) == 0 || to < ts.oldestTombstoneStart || from > ts.newestTombstoneEnd { - return nil - } - - var deletedIntervals []model.Interval - requestedInterval := model.Interval{Start: from, End: to} - - for i := range ts.tombstones { - overlaps, overlappingInterval := getOverlappingInterval(requestedInterval, - model.Interval{Start: ts.tombstones[i].StartTime, End: ts.tombstones[i].EndTime}) - - if !overlaps { - continue - } - - matches := false - for _, matchers := range ts.tombstones[i].Matchers { - if labels.Selector(matchers).Matches(lbls) { - matches = true - break - } - } - - if !matches { - continue - } - - if overlappingInterval == requestedInterval { - // whole interval deleted - return []model.Interval{requestedInterval} - } - - deletedIntervals = append(deletedIntervals, overlappingInterval) - } - - if len(deletedIntervals) == 0 { - return nil - } - - return mergeIntervals(deletedIntervals) +func (tl *noopTombstonesLoader) GetPendingTombstonesForInterval(userID string, from, to model.Time) (TombstonesSet, error) { + return &tl.ts, nil } -// Len returns number of tombstones that are there -func (ts TombstonesSet) Len() int { - return len(ts.tombstones) +func (tl *noopTombstonesLoader) GetStoreCacheGenNumber(tenantIDs []string) string { + return "" } -// HasTombstonesForInterval tells whether there are any tombstones which overlapping given interval -func (ts TombstonesSet) HasTombstonesForInterval(from, to model.Time) bool { - if len(ts.tombstones) == 0 || to < ts.oldestTombstoneStart || from > ts.newestTombstoneEnd { - return false - } - - return true +func (tl *noopTombstonesLoader) GetResultsCacheGenNumber(tenantIDs []string) string { + return "" } -// sorts and merges overlapping intervals -func mergeIntervals(intervals []model.Interval) []model.Interval { - if len(intervals) <= 1 { - return intervals - } - - mergedIntervals := make([]model.Interval, 0, len(intervals)) - sort.Slice(intervals, func(i, j int) bool { - return intervals[i].Start < intervals[j].Start - }) - - ongoingTrFrom, ongoingTrTo := intervals[0].Start, intervals[0].End - for i := 1; i < len(intervals); i++ { - // if there is no overlap add it to mergedIntervals - if intervals[i].Start > ongoingTrTo { - mergedIntervals = append(mergedIntervals, model.Interval{Start: ongoingTrFrom, End: ongoingTrTo}) - ongoingTrFrom = intervals[i].Start - ongoingTrTo = intervals[i].End - continue - } - - // there is an overlap but check whether existing time range is bigger than the current one - if intervals[i].End > ongoingTrTo { - ongoingTrTo = intervals[i].End - } - } - - // add the last time range - mergedIntervals = append(mergedIntervals, model.Interval{Start: ongoingTrFrom, End: ongoingTrTo}) - - return mergedIntervals +func (ts noopTombstonesSet) GetDeletedIntervals(lbls labels.Labels, from, to model.Time) []model.Interval { + return nil } -func getOverlappingInterval(interval1, interval2 model.Interval) (bool, model.Interval) { - if interval2.Start > interval1.Start { - interval1.Start = interval2.Start - } - - if interval2.End < interval1.End { - interval1.End = interval2.End - } - - return interval1.Start < interval1.End, interval1 +func (ts noopTombstonesSet) Len() int { + return 0 } -func intervalsOverlap(interval1, interval2 model.Interval) bool { - if interval1.Start > interval2.End || interval2.Start > interval1.End { - return false - } - - return true +func (ts noopTombstonesSet) HasTombstonesForInterval(from, to model.Time) bool { + return false } diff --git a/pkg/chunk/purger/tombstones_test.go b/pkg/chunk/purger/tombstones_test.go deleted file mode 100644 index e04d6ef02f..0000000000 --- a/pkg/chunk/purger/tombstones_test.go +++ /dev/null @@ -1,230 +0,0 @@ -package purger - -import ( - "context" - "errors" - "testing" - "time" - - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/promql/parser" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestTombstonesLoader(t *testing.T) { - deleteRequestSelectors := []string{"foo"} - metric, err := parser.ParseMetric(deleteRequestSelectors[0]) - require.NoError(t, err) - - for _, tc := range []struct { - name string - deleteRequestIntervals []model.Interval - queryForInterval model.Interval - expectedIntervals []model.Interval - }{ - { - name: "no delete requests", - queryForInterval: model.Interval{End: modelTimeDay}, - }, - { - name: "query out of range of delete requests", - deleteRequestIntervals: []model.Interval{ - {End: modelTimeDay}, - }, - queryForInterval: model.Interval{Start: modelTimeDay.Add(time.Hour), End: modelTimeDay * 2}, - }, - { - name: "no overlap but disjoint deleted intervals", - deleteRequestIntervals: []model.Interval{ - {End: modelTimeDay}, - {Start: modelTimeDay.Add(time.Hour), End: modelTimeDay.Add(2 * time.Hour)}, - }, - queryForInterval: model.Interval{End: modelTimeDay.Add(2 * time.Hour)}, - expectedIntervals: []model.Interval{ - {End: modelTimeDay}, - {Start: modelTimeDay.Add(time.Hour), End: modelTimeDay.Add(2 * time.Hour)}, - }, - }, - { - name: "no overlap but continuous deleted intervals", - deleteRequestIntervals: []model.Interval{ - {End: modelTimeDay}, - {Start: modelTimeDay, End: modelTimeDay.Add(2 * time.Hour)}, - }, - queryForInterval: model.Interval{End: modelTimeDay.Add(2 * time.Hour)}, - expectedIntervals: []model.Interval{ - {End: modelTimeDay.Add(2 * time.Hour)}, - }, - }, - { - name: "some overlap in deleted intervals", - deleteRequestIntervals: []model.Interval{ - {End: modelTimeDay}, - {Start: modelTimeDay.Add(-time.Hour), End: modelTimeDay.Add(2 * time.Hour)}, - }, - queryForInterval: model.Interval{End: modelTimeDay.Add(2 * time.Hour)}, - expectedIntervals: []model.Interval{ - {End: modelTimeDay.Add(2 * time.Hour)}, - }, - }, - { - name: "complete overlap in deleted intervals", - deleteRequestIntervals: []model.Interval{ - {End: modelTimeDay}, - {End: modelTimeDay}, - }, - queryForInterval: model.Interval{End: modelTimeDay.Add(2 * time.Hour)}, - expectedIntervals: []model.Interval{ - {End: modelTimeDay}, - }, - }, - { - name: "mix of overlaps in deleted intervals", - deleteRequestIntervals: []model.Interval{ - {End: modelTimeDay}, - {End: modelTimeDay}, - {Start: modelTimeDay.Add(time.Hour), End: modelTimeDay.Add(2 * time.Hour)}, - {Start: modelTimeDay.Add(2 * time.Hour), End: modelTimeDay.Add(24 * time.Hour)}, - {Start: modelTimeDay.Add(23 * time.Hour), End: modelTimeDay * 3}, - }, - queryForInterval: model.Interval{End: modelTimeDay * 10}, - expectedIntervals: []model.Interval{ - {End: modelTimeDay}, - {Start: modelTimeDay.Add(time.Hour), End: modelTimeDay * 3}, - }, - }, - } { - t.Run(tc.name, func(t *testing.T) { - deleteStore := setupTestDeleteStore(t) - tombstonesLoader := NewTombstonesLoader(deleteStore, nil) - - // add delete requests - for _, interval := range tc.deleteRequestIntervals { - err := deleteStore.AddDeleteRequest(context.Background(), userID, interval.Start, interval.End, deleteRequestSelectors) - require.NoError(t, err) - } - - // get all delete requests for user - tombstonesAnalyzer, err := tombstonesLoader.GetPendingTombstones(userID) - require.NoError(t, err) - - // verify whether number of delete requests is same as what we added - require.Equal(t, len(tc.deleteRequestIntervals), tombstonesAnalyzer.Len()) - - // if we are expecting to get deleted intervals then HasTombstonesForInterval should return true else false - expectedHasTombstonesForInterval := true - if len(tc.expectedIntervals) == 0 { - expectedHasTombstonesForInterval = false - } - - hasTombstonesForInterval := tombstonesAnalyzer.HasTombstonesForInterval(tc.queryForInterval.Start, tc.queryForInterval.End) - require.Equal(t, expectedHasTombstonesForInterval, hasTombstonesForInterval) - - // get deleted intervals - intervals := tombstonesAnalyzer.GetDeletedIntervals(metric, tc.queryForInterval.Start, tc.queryForInterval.End) - require.Equal(t, len(tc.expectedIntervals), len(intervals)) - - // verify whether we got expected intervals back - for i, interval := range intervals { - require.Equal(t, tc.expectedIntervals[i].Start, interval.Start) - require.Equal(t, tc.expectedIntervals[i].End, interval.End) - } - }) - } -} - -func TestTombstonesLoader_GetCacheGenNumber(t *testing.T) { - s := &store{ - numbers: map[string]*cacheGenNumbers{ - "tenant-a": { - results: "1000", - store: "2050", - }, - "tenant-b": { - results: "1050", - store: "2000", - }, - "tenant-c": { - results: "", - store: "", - }, - "tenant-d": { - results: "results-c", - store: "store-c", - }, - }, - } - tombstonesLoader := NewTombstonesLoader(s, nil) - - for _, tc := range []struct { - name string - expectedResultsCacheGenNumber string - expectedStoreCacheGenNumber string - tenantIDs []string - }{ - { - name: "single tenant with numeric values", - tenantIDs: []string{"tenant-a"}, - expectedResultsCacheGenNumber: "1000", - expectedStoreCacheGenNumber: "2050", - }, - { - name: "single tenant with non-numeric values", - tenantIDs: []string{"tenant-d"}, - expectedResultsCacheGenNumber: "results-c", - expectedStoreCacheGenNumber: "store-c", - }, - { - name: "multiple tenants with numeric values", - tenantIDs: []string{"tenant-a", "tenant-b"}, - expectedResultsCacheGenNumber: "1050", - expectedStoreCacheGenNumber: "2050", - }, - { - name: "multiple tenants with numeric and non-numeric values", - tenantIDs: []string{"tenant-d", "tenant-c", "tenant-b", "tenant-a"}, - expectedResultsCacheGenNumber: "1050", - expectedStoreCacheGenNumber: "2050", - }, - { - name: "no tenants", // not really an expected call, edge case check to avoid any panics - }, - } { - t.Run(tc.name, func(t *testing.T) { - assert.Equal(t, tc.expectedResultsCacheGenNumber, tombstonesLoader.GetResultsCacheGenNumber(tc.tenantIDs)) - assert.Equal(t, tc.expectedStoreCacheGenNumber, tombstonesLoader.GetStoreCacheGenNumber(tc.tenantIDs)) - }) - } -} - -func TestTombstonesReloadDoesntDeadlockOnFailure(t *testing.T) { - s := &store{} - tombstonesLoader := NewTombstonesLoader(s, nil) - tombstonesLoader.getCacheGenNumbers("test") - - s.err = errors.New("error") - require.NotNil(t, tombstonesLoader.reloadTombstones()) - - s.err = nil - require.NotNil(t, tombstonesLoader.getCacheGenNumbers("test2")) -} - -type store struct { - numbers map[string]*cacheGenNumbers - err error -} - -func (f *store) getCacheGenerationNumbers(ctx context.Context, user string) (*cacheGenNumbers, error) { - if f.numbers != nil { - number, ok := f.numbers[user] - if ok { - return number, nil - } - } - return &cacheGenNumbers{}, f.err -} - -func (f *store) GetPendingDeleteRequestsForUser(ctx context.Context, id string) ([]DeleteRequest, error) { - return nil, nil -} diff --git a/pkg/chunk/storage/factory.go b/pkg/chunk/storage/factory.go index 8076590d5d..c78cb89d69 100644 --- a/pkg/chunk/storage/factory.go +++ b/pkg/chunk/storage/factory.go @@ -22,7 +22,6 @@ import ( "github.com/cortexproject/cortex/pkg/chunk/local" "github.com/cortexproject/cortex/pkg/chunk/objectclient" "github.com/cortexproject/cortex/pkg/chunk/openstack" - "github.com/cortexproject/cortex/pkg/chunk/purger" util_log "github.com/cortexproject/cortex/pkg/util/log" ) @@ -93,8 +92,6 @@ type Config struct { IndexQueriesCacheConfig cache.Config `yaml:"index_queries_cache_config"` - DeleteStoreConfig purger.DeleteStoreConfig `yaml:"delete_store"` - GrpcConfig grpc.Config `yaml:"grpc_store"` } @@ -107,19 +104,18 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { cfg.CassandraStorageConfig.RegisterFlags(f) cfg.BoltDBConfig.RegisterFlags(f) cfg.FSConfig.RegisterFlags(f) - cfg.DeleteStoreConfig.RegisterFlags(f) cfg.Swift.RegisterFlags(f) cfg.GrpcConfig.RegisterFlags(f) - f.StringVar(&cfg.Engine, "store.engine", "chunks", "The storage engine to use: chunks (deprecated) or blocks.") + f.StringVar(&cfg.Engine, "store.engine", "blocks", "The storage engine to use: blocks is the only supported option today.") cfg.IndexQueriesCacheConfig.RegisterFlagsWithPrefix("store.index-cache-read.", "Cache config for index entry reading. ", f) f.DurationVar(&cfg.IndexCacheValidity, "store.index-cache-validity", 5*time.Minute, "Cache validity for active index entries. Should be no higher than -ingester.max-chunk-idle.") } // Validate config and returns error on failure func (cfg *Config) Validate() error { - if cfg.Engine != StorageEngineChunks && cfg.Engine != StorageEngineBlocks { - return errors.New("unsupported storage engine") + if cfg.Engine != StorageEngineBlocks { + return errors.New("unsupported storage engine (only blocks is supported for ingest)") } if err := cfg.CassandraStorageConfig.Validate(); err != nil { return errors.Wrap(err, "invalid Cassandra Storage config") diff --git a/pkg/cortex/cortex.go b/pkg/cortex/cortex.go index cb7206e205..9a35b4bae9 100644 --- a/pkg/cortex/cortex.go +++ b/pkg/cortex/cortex.go @@ -112,7 +112,6 @@ type Config struct { BlocksStorage tsdb.BlocksStorageConfig `yaml:"blocks_storage"` Compactor compactor.Config `yaml:"compactor"` StoreGateway storegateway.Config `yaml:"store_gateway"` - PurgerConfig purger.Config `yaml:"purger"` TenantFederation tenantfederation.Config `yaml:"tenant_federation"` Ruler ruler.Config `yaml:"ruler"` @@ -161,7 +160,6 @@ func (c *Config) RegisterFlags(f *flag.FlagSet) { c.BlocksStorage.RegisterFlags(f) c.Compactor.RegisterFlags(f) c.StoreGateway.RegisterFlags(f) - c.PurgerConfig.RegisterFlags(f) c.TenantFederation.RegisterFlags(f) c.Ruler.RegisterFlags(f) @@ -319,12 +317,9 @@ type Cortex struct { Ingester *ingester.Ingester Flusher *flusher.Flusher Store chunk.Store - DeletesStore *purger.DeleteStore Frontend *frontendv1.Frontend - TableManager *chunk.TableManager RuntimeConfig *runtimeconfig.Manager - Purger *purger.Purger - TombstonesLoader *purger.TombstonesLoader + TombstonesLoader purger.TombstonesLoader QuerierQueryable prom_storage.SampleAndChunkQueryable ExemplarQueryable prom_storage.ExemplarQueryable QuerierEngine *promql.Engine diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index 20e2ed6250..38b0dc6ca5 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -5,7 +5,6 @@ import ( "flag" "fmt" "net/http" - "os" "time" "github.com/go-kit/log/level" @@ -79,7 +78,6 @@ const ( Compactor string = "compactor" StoreGateway string = "store-gateway" MemberlistKV string = "memberlist-kv" - ChunksPurger string = "chunks-purger" TenantDeletion string = "tenant-deletion" Purger string = "purger" QueryScheduler string = "query-scheduler" @@ -426,7 +424,7 @@ func (t *Cortex) initIngesterService() (serv services.Service, err error) { t.Cfg.Ingester.InstanceLimitsFn = ingesterInstanceLimits(t.RuntimeConfig) t.tsdbIngesterConfig() - t.Ingester, err = ingester.New(t.Cfg.Ingester, t.Cfg.IngesterClient, t.Overrides, t.Store, prometheus.DefaultRegisterer, util_log.Logger) + t.Ingester, err = ingester.New(t.Cfg.Ingester, t.Overrides, prometheus.DefaultRegisterer, util_log.Logger) if err != nil { return } @@ -446,7 +444,6 @@ func (t *Cortex) initFlusher() (serv services.Service, err error) { t.Flusher, err = flusher.New( t.Cfg.Flusher, t.Cfg.Ingester, - t.Store, t.Overrides, prometheus.DefaultRegisterer, util_log.Logger, @@ -459,7 +456,10 @@ func (t *Cortex) initFlusher() (serv services.Service, err error) { } func (t *Cortex) initChunkStore() (serv services.Service, err error) { - if t.Cfg.Storage.Engine != storage.StorageEngineChunks && t.Cfg.Querier.SecondStoreEngine != storage.StorageEngineChunks { + if t.Cfg.Storage.Engine == storage.StorageEngineChunks { + return nil, errors.New("should not get here: ingesting into chunks storage is no longer supported") + } + if t.Cfg.Querier.SecondStoreEngine != storage.StorageEngineChunks { return nil, nil } err = t.Cfg.Schema.Load() @@ -479,28 +479,8 @@ func (t *Cortex) initChunkStore() (serv services.Service, err error) { } func (t *Cortex) initDeleteRequestsStore() (serv services.Service, err error) { - if t.Cfg.Storage.Engine != storage.StorageEngineChunks || !t.Cfg.PurgerConfig.Enable { - // until we need to explicitly enable delete series support we need to do create TombstonesLoader without DeleteStore which acts as noop - t.TombstonesLoader = purger.NewTombstonesLoader(nil, nil) - - return - } - - var indexClient chunk.IndexClient - reg := prometheus.WrapRegistererWith( - prometheus.Labels{"component": DeleteRequestsStore}, prometheus.DefaultRegisterer) - indexClient, err = storage.NewIndexClient(t.Cfg.Storage.DeleteStoreConfig.Store, t.Cfg.Storage, t.Cfg.Schema, reg) - if err != nil { - return - } - - t.DeletesStore, err = purger.NewDeleteStore(t.Cfg.Storage.DeleteStoreConfig, indexClient) - if err != nil { - return - } - - t.TombstonesLoader = purger.NewTombstonesLoader(t.DeletesStore, prometheus.DefaultRegisterer) - + // no-op while blocks store does not support series deletion + t.TombstonesLoader = purger.NewNoopTombstonesLoader() return } @@ -579,61 +559,6 @@ func (t *Cortex) initQueryFrontend() (serv services.Service, err error) { return nil, nil } -func (t *Cortex) initTableManager() (services.Service, error) { - if t.Cfg.Storage.Engine == storage.StorageEngineBlocks { - return nil, nil // table manager isn't used in v2 - } - - err := t.Cfg.Schema.Load() - if err != nil { - return nil, err - } - - // Assume the newest config is the one to use - lastConfig := &t.Cfg.Schema.Configs[len(t.Cfg.Schema.Configs)-1] - - if (t.Cfg.TableManager.ChunkTables.WriteScale.Enabled || - t.Cfg.TableManager.IndexTables.WriteScale.Enabled || - t.Cfg.TableManager.ChunkTables.InactiveWriteScale.Enabled || - t.Cfg.TableManager.IndexTables.InactiveWriteScale.Enabled || - t.Cfg.TableManager.ChunkTables.ReadScale.Enabled || - t.Cfg.TableManager.IndexTables.ReadScale.Enabled || - t.Cfg.TableManager.ChunkTables.InactiveReadScale.Enabled || - t.Cfg.TableManager.IndexTables.InactiveReadScale.Enabled) && - t.Cfg.Storage.AWSStorageConfig.Metrics.URL == "" { - level.Error(util_log.Logger).Log("msg", "WriteScale is enabled but no Metrics URL has been provided") - os.Exit(1) - } - - reg := prometheus.WrapRegistererWith( - prometheus.Labels{"component": "table-manager-store"}, prometheus.DefaultRegisterer) - - tableClient, err := storage.NewTableClient(lastConfig.IndexType, t.Cfg.Storage, reg) - if err != nil { - return nil, err - } - - bucketClient, err := storage.NewBucketClient(t.Cfg.Storage) - util_log.CheckFatal("initializing bucket client", err) - - var extraTables []chunk.ExtraTables - if t.Cfg.PurgerConfig.Enable { - reg := prometheus.WrapRegistererWith( - prometheus.Labels{"component": "table-manager-" + DeleteRequestsStore}, prometheus.DefaultRegisterer) - - deleteStoreTableClient, err := storage.NewTableClient(t.Cfg.Storage.DeleteStoreConfig.Store, t.Cfg.Storage, reg) - if err != nil { - return nil, err - } - - extraTables = append(extraTables, chunk.ExtraTables{TableClient: deleteStoreTableClient, Tables: t.Cfg.Storage.DeleteStoreConfig.GetTables()}) - } - - t.TableManager, err = chunk.NewTableManager(t.Cfg.TableManager, t.Cfg.Schema, t.Cfg.Ingester.MaxChunkAge, tableClient, - bucketClient, extraTables, prometheus.DefaultRegisterer) - return t.TableManager, err -} - func (t *Cortex) initRulerStorage() (serv services.Service, err error) { // if the ruler is not configured and we're in single binary then let's just log an error and continue. // unfortunately there is no way to generate a "default" config and compare default against actual @@ -791,26 +716,6 @@ func (t *Cortex) initMemberlistKV() (services.Service, error) { return t.MemberlistKV, nil } -func (t *Cortex) initChunksPurger() (services.Service, error) { - if t.Cfg.Storage.Engine != storage.StorageEngineChunks || !t.Cfg.PurgerConfig.Enable { - return nil, nil - } - - storageClient, err := storage.NewObjectClient(t.Cfg.PurgerConfig.ObjectStoreType, t.Cfg.Storage) - if err != nil { - return nil, err - } - - t.Purger, err = purger.NewPurger(t.Cfg.PurgerConfig, t.DeletesStore, t.Store, storageClient, prometheus.DefaultRegisterer) - if err != nil { - return nil, err - } - - t.API.RegisterChunksPurger(t.DeletesStore, t.Cfg.PurgerConfig.DeleteRequestCancelPeriod) - - return t.Purger, nil -} - func (t *Cortex) initTenantDeletionAPI() (services.Service, error) { if t.Cfg.Storage.Engine != storage.StorageEngineBlocks { return nil, nil @@ -860,14 +765,12 @@ func (t *Cortex) setupModuleManager() error { mm.RegisterModule(StoreQueryable, t.initStoreQueryables, modules.UserInvisibleModule) mm.RegisterModule(QueryFrontendTripperware, t.initQueryFrontendTripperware, modules.UserInvisibleModule) mm.RegisterModule(QueryFrontend, t.initQueryFrontend) - mm.RegisterModule(TableManager, t.initTableManager) mm.RegisterModule(RulerStorage, t.initRulerStorage, modules.UserInvisibleModule) mm.RegisterModule(Ruler, t.initRuler) mm.RegisterModule(Configs, t.initConfig) mm.RegisterModule(AlertManager, t.initAlertManager) mm.RegisterModule(Compactor, t.initCompactor) mm.RegisterModule(StoreGateway, t.initStoreGateway) - mm.RegisterModule(ChunksPurger, t.initChunksPurger, modules.UserInvisibleModule) mm.RegisterModule(TenantDeletion, t.initTenantDeletionAPI, modules.UserInvisibleModule) mm.RegisterModule(Purger, nil) mm.RegisterModule(QueryScheduler, t.initQueryScheduler) @@ -894,18 +797,16 @@ func (t *Cortex) setupModuleManager() error { QueryFrontendTripperware: {API, Overrides, DeleteRequestsStore}, QueryFrontend: {QueryFrontendTripperware}, QueryScheduler: {API, Overrides}, - TableManager: {API}, Ruler: {DistributorService, Store, StoreQueryable, RulerStorage}, RulerStorage: {Overrides}, Configs: {API}, AlertManager: {API, MemberlistKV, Overrides}, Compactor: {API, MemberlistKV, Overrides}, StoreGateway: {API, Overrides, MemberlistKV}, - ChunksPurger: {Store, DeleteRequestsStore, API}, TenantDeletion: {Store, API, Overrides}, - Purger: {ChunksPurger, TenantDeletion}, + Purger: {TenantDeletion}, TenantFederation: {Queryable}, - All: {QueryFrontend, Querier, Ingester, Distributor, TableManager, Purger, StoreGateway, Ruler}, + All: {QueryFrontend, Querier, Ingester, Distributor, Purger, StoreGateway, Ruler}, } for mod, targets := range deps { if err := mm.AddDependency(mod, targets...); err != nil { diff --git a/pkg/flusher/flusher.go b/pkg/flusher/flusher.go index ee0992a3a2..39bd4e10f4 100644 --- a/pkg/flusher/flusher.go +++ b/pkg/flusher/flusher.go @@ -26,20 +26,18 @@ type Config struct { // RegisterFlags adds the flags required to config this to the given FlagSet func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - f.StringVar(&cfg.WALDir, "flusher.wal-dir", "wal", "Directory to read WAL from (chunks storage engine only).") - f.IntVar(&cfg.ConcurrentFlushes, "flusher.concurrent-flushes", 50, "Number of concurrent goroutines flushing to storage (chunks storage engine only).") - f.DurationVar(&cfg.FlushOpTimeout, "flusher.flush-op-timeout", 2*time.Minute, "Timeout for individual flush operations (chunks storage engine only).") + f.StringVar(&cfg.WALDir, "flusher.wal-dir", "wal", "Has no effect: directory to read WAL from (chunks storage engine only).") + f.IntVar(&cfg.ConcurrentFlushes, "flusher.concurrent-flushes", 50, "Has no effect: number of concurrent goroutines flushing to storage (chunks storage engine only).") + f.DurationVar(&cfg.FlushOpTimeout, "flusher.flush-op-timeout", 2*time.Minute, "Has no effect: timeout for individual flush operations (chunks storage engine only).") f.BoolVar(&cfg.ExitAfterFlush, "flusher.exit-after-flush", true, "Stop Cortex after flush has finished. If false, Cortex process will keep running, doing nothing.") } -// Flusher is designed to be used as a job to flush the data from the WAL on disk. -// Flusher works with both chunks-based and blocks-based ingesters. +// Flusher is designed to be used as a job to flush the data from the TSDB/WALs on disk. type Flusher struct { services.Service cfg Config ingesterConfig ingester.Config - chunkStore ingester.ChunkStore limits *validation.Overrides registerer prometheus.Registerer logger log.Logger @@ -54,21 +52,14 @@ const ( func New( cfg Config, ingesterConfig ingester.Config, - chunkStore ingester.ChunkStore, limits *validation.Overrides, registerer prometheus.Registerer, logger log.Logger, ) (*Flusher, error) { - // These are ignored by blocks-ingester, but that's fine. - ingesterConfig.WALConfig.Dir = cfg.WALDir - ingesterConfig.ConcurrentFlushes = cfg.ConcurrentFlushes - ingesterConfig.FlushOpTimeout = cfg.FlushOpTimeout - f := &Flusher{ cfg: cfg, ingesterConfig: ingesterConfig, - chunkStore: chunkStore, limits: limits, registerer: registerer, logger: logger, @@ -78,7 +69,7 @@ func New( } func (f *Flusher) running(ctx context.Context) error { - ing, err := ingester.NewForFlusher(f.ingesterConfig, f.chunkStore, f.limits, f.registerer, f.logger) + ing, err := ingester.NewForFlusher(f.ingesterConfig, f.limits, f.registerer, f.logger) if err != nil { return errors.Wrap(err, "create ingester") } diff --git a/pkg/ingester/client/cortex_mock_test.go b/pkg/ingester/client/cortex_mock_test.go index a70a0cfc79..fd98c77082 100644 --- a/pkg/ingester/client/cortex_mock_test.go +++ b/pkg/ingester/client/cortex_mock_test.go @@ -76,8 +76,3 @@ func (m *IngesterServerMock) MetricsMetadata(ctx context.Context, r *MetricsMeta args := m.Called(ctx, r) return args.Get(0).(*MetricsMetadataResponse), args.Error(1) } - -func (m *IngesterServerMock) TransferChunks(s Ingester_TransferChunksServer) error { - args := m.Called(s) - return args.Error(0) -} diff --git a/pkg/ingester/client/ingester.proto b/pkg/ingester/client/ingester.proto index 64da64c9b4..1a9c8a96ce 100644 --- a/pkg/ingester/client/ingester.proto +++ b/pkg/ingester/client/ingester.proto @@ -26,9 +26,6 @@ service Ingester { rpc MetricsForLabelMatchers(MetricsForLabelMatchersRequest) returns (MetricsForLabelMatchersResponse) {}; rpc MetricsForLabelMatchersStream(MetricsForLabelMatchersRequest) returns (stream MetricsForLabelMatchersStreamResponse) {}; rpc MetricsMetadata(MetricsMetadataRequest) returns (MetricsMetadataResponse) {}; - - // TransferChunks allows leaving ingester (client) to stream chunks directly to joining ingesters (server). - rpc TransferChunks(stream TimeSeriesChunk) returns (TransferChunksResponse) {}; } message ReadRequest { diff --git a/pkg/ingester/errors.go b/pkg/ingester/errors.go index febdc1b4f0..b982f6ce09 100644 --- a/pkg/ingester/errors.go +++ b/pkg/ingester/errors.go @@ -5,14 +5,12 @@ import ( "net/http" "github.com/prometheus/prometheus/model/labels" - "github.com/weaveworks/common/httpgrpc" ) type validationError struct { err error // underlying error errorType string code int - noReport bool // if true, error will be counted but not reported to caller labels labels.Labels } @@ -24,22 +22,6 @@ func makeLimitError(errorType string, err error) error { } } -func makeNoReportError(errorType string) error { - return &validationError{ - errorType: errorType, - noReport: true, - } -} - -func makeMetricValidationError(errorType string, labels labels.Labels, err error) error { - return &validationError{ - errorType: errorType, - err: err, - code: http.StatusBadRequest, - labels: labels, - } -} - func makeMetricLimitError(errorType string, labels labels.Labels, err error) error { return &validationError{ errorType: errorType, @@ -59,14 +41,6 @@ func (e *validationError) Error() string { return fmt.Sprintf("%s for series %s", e.err.Error(), e.labels.String()) } -// returns a HTTP gRPC error than is correctly forwarded over gRPC, with no reference to `e` retained. -func grpcForwardableError(userID string, code int, e error) error { - return httpgrpc.ErrorFromHTTPResponse(&httpgrpc.HTTPResponse{ - Code: int32(code), - Body: []byte(wrapWithUser(e, userID).Error()), - }) -} - // wrapWithUser prepends the user to the error. It does not retain a reference to err. func wrapWithUser(err error, userID string) error { return fmt.Errorf("user=%s: %s", userID, err) diff --git a/pkg/ingester/flush.go b/pkg/ingester/flush.go index b79e008091..60f793f079 100644 --- a/pkg/ingester/flush.go +++ b/pkg/ingester/flush.go @@ -1,430 +1,17 @@ package ingester import ( - "context" - "fmt" "net/http" - "time" - - "github.com/go-kit/log/level" - ot "github.com/opentracing/opentracing-go" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - "golang.org/x/time/rate" - - "github.com/cortexproject/cortex/pkg/chunk" - "github.com/cortexproject/cortex/pkg/util" - "github.com/cortexproject/cortex/pkg/util/log" -) - -const ( - // Backoff for retrying 'immediate' flushes. Only counts for queue - // position, not wallclock time. - flushBackoff = 1 * time.Second - // Lower bound on flushes per check period for rate-limiter - minFlushes = 100 ) // Flush triggers a flush of all the chunks and closes the flush queues. // Called from the Lifecycler as part of the ingester shutdown. func (i *Ingester) Flush() { - if i.cfg.BlocksStorageEnabled { - i.v2LifecyclerFlush() - return - } - - level.Info(i.logger).Log("msg", "starting to flush all the chunks") - i.sweepUsers(true) - level.Info(i.logger).Log("msg", "chunks queued for flushing") - - // Close the flush queues, to unblock waiting workers. - for _, flushQueue := range i.flushQueues { - flushQueue.Close() - } - - i.flushQueuesDone.Wait() - level.Info(i.logger).Log("msg", "flushing of chunks complete") + i.v2LifecyclerFlush() } // FlushHandler triggers a flush of all in memory chunks. Mainly used for // local testing. func (i *Ingester) FlushHandler(w http.ResponseWriter, r *http.Request) { - if i.cfg.BlocksStorageEnabled { - i.v2FlushHandler(w, r) - return - } - - level.Info(i.logger).Log("msg", "starting to flush all the chunks") - i.sweepUsers(true) - level.Info(i.logger).Log("msg", "chunks queued for flushing") - w.WriteHeader(http.StatusNoContent) -} - -type flushOp struct { - from model.Time - userID string - fp model.Fingerprint - immediate bool -} - -func (o *flushOp) Key() string { - return fmt.Sprintf("%s-%d-%v", o.userID, o.fp, o.immediate) -} - -func (o *flushOp) Priority() int64 { - return -int64(o.from) -} - -// sweepUsers periodically schedules series for flushing and garbage collects users with no series -func (i *Ingester) sweepUsers(immediate bool) { - if i.chunkStore == nil { - return - } - - oldest := model.Time(0) - - for id, state := range i.userStates.cp() { - for pair := range state.fpToSeries.iter() { - state.fpLocker.Lock(pair.fp) - i.sweepSeries(id, pair.fp, pair.series, immediate) - i.removeFlushedChunks(state, pair.fp, pair.series) - first := pair.series.firstUnflushedChunkTime() - state.fpLocker.Unlock(pair.fp) - - if first > 0 && (oldest == 0 || first < oldest) { - oldest = first - } - } - } - - i.metrics.oldestUnflushedChunkTimestamp.Set(float64(oldest.Unix())) - i.setFlushRate() -} - -// Compute a rate such to spread calls to the store over nearly all of the flush period, -// for example if we have 600 items in the queue and period 1 min we will send 10.5 per second. -// Note if the store can't keep up with this rate then it doesn't make any difference. -func (i *Ingester) setFlushRate() { - totalQueueLength := 0 - for _, q := range i.flushQueues { - totalQueueLength += q.Length() - } - const fudge = 1.05 // aim to finish a little bit before the end of the period - flushesPerSecond := float64(totalQueueLength) / i.cfg.FlushCheckPeriod.Seconds() * fudge - // Avoid going very slowly with tiny queues - if flushesPerSecond*i.cfg.FlushCheckPeriod.Seconds() < minFlushes { - flushesPerSecond = minFlushes / i.cfg.FlushCheckPeriod.Seconds() - } - level.Debug(i.logger).Log("msg", "computed flush rate", "rate", flushesPerSecond) - i.flushRateLimiter.SetLimit(rate.Limit(flushesPerSecond)) -} - -type flushReason int8 - -const ( - noFlush = iota - reasonImmediate - reasonMultipleChunksInSeries - reasonAged - reasonIdle - reasonStale - reasonSpreadFlush - // Following are flush outcomes - noUser - noSeries - noChunks - flushError - reasonDropped - maxFlushReason // Used for testing String() method. Should be last. -) - -func (f flushReason) String() string { - switch f { - case noFlush: - return "NoFlush" - case reasonImmediate: - return "Immediate" - case reasonMultipleChunksInSeries: - return "MultipleChunksInSeries" - case reasonAged: - return "Aged" - case reasonIdle: - return "Idle" - case reasonStale: - return "Stale" - case reasonSpreadFlush: - return "Spread" - case noUser: - return "NoUser" - case noSeries: - return "NoSeries" - case noChunks: - return "NoChunksToFlush" - case flushError: - return "FlushError" - case reasonDropped: - return "Dropped" - default: - panic("unrecognised flushReason") - } -} - -// sweepSeries schedules a series for flushing based on a set of criteria -// -// NB we don't close the head chunk here, as the series could wait in the queue -// for some time, and we want to encourage chunks to be as full as possible. -func (i *Ingester) sweepSeries(userID string, fp model.Fingerprint, series *memorySeries, immediate bool) { - if len(series.chunkDescs) <= 0 { - return - } - - firstTime := series.firstTime() - flush := i.shouldFlushSeries(series, fp, immediate) - if flush == noFlush { - return - } - - flushQueueIndex := int(uint64(fp) % uint64(i.cfg.ConcurrentFlushes)) - if i.flushQueues[flushQueueIndex].Enqueue(&flushOp{firstTime, userID, fp, immediate}) { - i.metrics.seriesEnqueuedForFlush.WithLabelValues(flush.String()).Inc() - util.Event().Log("msg", "add to flush queue", "userID", userID, "reason", flush, "firstTime", firstTime, "fp", fp, "series", series.metric, "nlabels", len(series.metric), "queue", flushQueueIndex) - } -} - -func (i *Ingester) shouldFlushSeries(series *memorySeries, fp model.Fingerprint, immediate bool) flushReason { - if len(series.chunkDescs) == 0 { - return noFlush - } - if immediate { - for _, cd := range series.chunkDescs { - if !cd.flushed { - return reasonImmediate - } - } - return noFlush // Everything is flushed. - } - - // Flush if we have more than one chunk, and haven't already flushed the first chunk - if len(series.chunkDescs) > 1 && !series.chunkDescs[0].flushed { - if series.chunkDescs[0].flushReason != noFlush { - return series.chunkDescs[0].flushReason - } - return reasonMultipleChunksInSeries - } - // Otherwise look in more detail at the first chunk - return i.shouldFlushChunk(series.chunkDescs[0], fp, series.isStale()) -} - -func (i *Ingester) shouldFlushChunk(c *desc, fp model.Fingerprint, lastValueIsStale bool) flushReason { - if c.flushed { // don't flush chunks we've already flushed - return noFlush - } - - // Adjust max age slightly to spread flushes out over time - var jitter time.Duration - if i.cfg.ChunkAgeJitter != 0 { - jitter = time.Duration(fp) % i.cfg.ChunkAgeJitter - } - // Chunks should be flushed if they span longer than MaxChunkAge - if c.LastTime.Sub(c.FirstTime) > (i.cfg.MaxChunkAge - jitter) { - return reasonAged - } - - // Chunk should be flushed if their last update is older then MaxChunkIdle. - if model.Now().Sub(c.LastUpdate) > i.cfg.MaxChunkIdle { - return reasonIdle - } - - // A chunk that has a stale marker can be flushed if possible. - if i.cfg.MaxStaleChunkIdle > 0 && - lastValueIsStale && - model.Now().Sub(c.LastUpdate) > i.cfg.MaxStaleChunkIdle { - return reasonStale - } - - return noFlush -} - -func (i *Ingester) flushLoop(j int) { - defer func() { - level.Debug(i.logger).Log("msg", "Ingester.flushLoop() exited") - i.flushQueuesDone.Done() - }() - - for { - o := i.flushQueues[j].Dequeue() - if o == nil { - return - } - op := o.(*flushOp) - - if !op.immediate { - _ = i.flushRateLimiter.Wait(context.Background()) - } - outcome, err := i.flushUserSeries(j, op.userID, op.fp, op.immediate) - i.metrics.seriesDequeuedOutcome.WithLabelValues(outcome.String()).Inc() - if err != nil { - level.Error(log.WithUserID(op.userID, i.logger)).Log("msg", "failed to flush user", "err", err) - } - - // If we're exiting & we failed to flush, put the failed operation - // back in the queue at a later point. - if op.immediate && err != nil { - op.from = op.from.Add(flushBackoff) - i.flushQueues[j].Enqueue(op) - } - } -} - -// Returns flush outcome (either original reason, if series was flushed, noFlush if it doesn't need flushing anymore, or one of the errors) -func (i *Ingester) flushUserSeries(flushQueueIndex int, userID string, fp model.Fingerprint, immediate bool) (flushReason, error) { - i.metrics.flushSeriesInProgress.Inc() - defer i.metrics.flushSeriesInProgress.Dec() - - if i.preFlushUserSeries != nil { - i.preFlushUserSeries() - } - - userState, ok := i.userStates.get(userID) - if !ok { - return noUser, nil - } - - series, ok := userState.fpToSeries.get(fp) - if !ok { - return noSeries, nil - } - - userState.fpLocker.Lock(fp) - reason := i.shouldFlushSeries(series, fp, immediate) - if reason == noFlush { - userState.fpLocker.Unlock(fp) - return noFlush, nil - } - - // shouldFlushSeries() has told us we have at least one chunk. - // Make a copy of chunks descriptors slice, to avoid possible issues related to removing (and niling) elements from chunkDesc. - // This can happen if first chunk is already flushed -- removeFlushedChunks may set such chunk to nil. - // Since elements in the slice are pointers, we can still safely update chunk descriptors after the copy. - chunks := append([]*desc(nil), series.chunkDescs...) - if immediate { - series.closeHead(reasonImmediate) - } else if chunkReason := i.shouldFlushChunk(series.head(), fp, series.isStale()); chunkReason != noFlush { - series.closeHead(chunkReason) - } else { - // The head chunk doesn't need flushing; step back by one. - chunks = chunks[:len(chunks)-1] - } - - if (reason == reasonIdle || reason == reasonStale) && series.headChunkClosed { - if minChunkLength := i.limits.MinChunkLength(userID); minChunkLength > 0 { - chunkLength := 0 - for _, c := range chunks { - chunkLength += c.C.Len() - } - if chunkLength < minChunkLength { - userState.removeSeries(fp, series.metric) - i.metrics.memoryChunks.Sub(float64(len(chunks))) - i.metrics.droppedChunks.Add(float64(len(chunks))) - util.Event().Log( - "msg", "dropped chunks", - "userID", userID, - "numChunks", len(chunks), - "chunkLength", chunkLength, - "fp", fp, - "series", series.metric, - "queue", flushQueueIndex, - ) - chunks = nil - reason = reasonDropped - } - } - } - - userState.fpLocker.Unlock(fp) - - if reason == reasonDropped { - return reason, nil - } - - // No need to flush these chunks again. - for len(chunks) > 0 && chunks[0].flushed { - chunks = chunks[1:] - } - - if len(chunks) == 0 { - return noChunks, nil - } - - // flush the chunks without locking the series, as we don't want to hold the series lock for the duration of the dynamo/s3 rpcs. - ctx, cancel := context.WithTimeout(context.Background(), i.cfg.FlushOpTimeout) - defer cancel() // releases resources if slowOperation completes before timeout elapses - - sp, ctx := ot.StartSpanFromContext(ctx, "flushUserSeries") - defer sp.Finish() - sp.SetTag("organization", userID) - - util.Event().Log("msg", "flush chunks", "userID", userID, "reason", reason, "numChunks", len(chunks), "firstTime", chunks[0].FirstTime, "fp", fp, "series", series.metric, "nlabels", len(series.metric), "queue", flushQueueIndex) - err := i.flushChunks(ctx, userID, fp, series.metric, chunks) - if err != nil { - return flushError, err - } - - userState.fpLocker.Lock(fp) - for i := 0; i < len(chunks); i++ { - // Mark the chunks as flushed, so we can remove them after the retention period. - // We can safely use chunks[i] here, because elements are pointers to chunk descriptors. - chunks[i].flushed = true - chunks[i].LastUpdate = model.Now() - } - userState.fpLocker.Unlock(fp) - return reason, err -} - -// must be called under fpLocker lock -func (i *Ingester) removeFlushedChunks(userState *userState, fp model.Fingerprint, series *memorySeries) { - now := model.Now() - for len(series.chunkDescs) > 0 { - if series.chunkDescs[0].flushed && now.Sub(series.chunkDescs[0].LastUpdate) > i.cfg.RetainPeriod { - series.chunkDescs[0] = nil // erase reference so the chunk can be garbage-collected - series.chunkDescs = series.chunkDescs[1:] - i.metrics.memoryChunks.Dec() - } else { - break - } - } - if len(series.chunkDescs) == 0 { - userState.removeSeries(fp, series.metric) - } -} - -func (i *Ingester) flushChunks(ctx context.Context, userID string, fp model.Fingerprint, metric labels.Labels, chunkDescs []*desc) error { - if i.preFlushChunks != nil { - i.preFlushChunks() - } - - wireChunks := make([]chunk.Chunk, 0, len(chunkDescs)) - for _, chunkDesc := range chunkDescs { - c := chunk.NewChunk(userID, fp, metric, chunkDesc.C, chunkDesc.FirstTime, chunkDesc.LastTime) - if err := c.Encode(); err != nil { - return err - } - wireChunks = append(wireChunks, c) - } - - if err := i.chunkStore.Put(ctx, wireChunks); err != nil { - return err - } - - // Record statistics only when actual put request did not return error. - for _, chunkDesc := range chunkDescs { - utilization, length, size := chunkDesc.C.Utilization(), chunkDesc.C.Len(), chunkDesc.C.Size() - util.Event().Log("msg", "chunk flushed", "userID", userID, "fp", fp, "series", metric, "nlabels", len(metric), "utilization", utilization, "length", length, "size", size, "firstTime", chunkDesc.FirstTime, "lastTime", chunkDesc.LastTime) - i.metrics.chunkUtilization.Observe(utilization) - i.metrics.chunkLength.Observe(float64(length)) - i.metrics.chunkSize.Observe(float64(size)) - i.metrics.chunkAge.Observe(model.Now().Sub(chunkDesc.FirstTime).Seconds()) - } - - return nil + i.v2FlushHandler(w, r) } diff --git a/pkg/ingester/flush_test.go b/pkg/ingester/flush_test.go deleted file mode 100644 index 2b8aed9575..0000000000 --- a/pkg/ingester/flush_test.go +++ /dev/null @@ -1,253 +0,0 @@ -package ingester - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/go-kit/log" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - "github.com/stretchr/testify/require" - "github.com/weaveworks/common/user" - "go.uber.org/atomic" - - "github.com/cortexproject/cortex/pkg/chunk" - "github.com/cortexproject/cortex/pkg/cortexpb" - "github.com/cortexproject/cortex/pkg/ingester/client" - "github.com/cortexproject/cortex/pkg/ring" - "github.com/cortexproject/cortex/pkg/ring/kv" - "github.com/cortexproject/cortex/pkg/util" - "github.com/cortexproject/cortex/pkg/util/services" - "github.com/cortexproject/cortex/pkg/util/validation" -) - -var singleTestLabel = []labels.Labels{[]labels.Label{{Name: "__name__", Value: "test"}}} - -// This test case demonstrates problem with losing incoming samples while chunks are flushed with "immediate" mode. -func TestSweepImmediateDropsSamples(t *testing.T) { - cfg := emptyIngesterConfig() - cfg.FlushCheckPeriod = 1 * time.Minute - cfg.RetainPeriod = 10 * time.Millisecond - - st := &sleepyCountingStore{} - ing := createTestIngester(t, cfg, st) - - samples := newSampleGenerator(t, time.Now(), time.Millisecond) - - // Generates one sample. - pushSample(t, ing, <-samples) - - notify := make(chan struct{}) - ing.preFlushChunks = func() { - if ing.State() == services.Running { - pushSample(t, ing, <-samples) - notify <- struct{}{} - } - } - - // Simulate /flush. Sweeps everything, but also pushes another sample (in preFlushChunks) - ing.sweepUsers(true) - <-notify // Wait for flushing to happen. - - // Stopping ingester should sweep the remaining samples. - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing)) - - require.Equal(t, 2, st.samples) -} - -// There are several factors in this panic: -// Chunk is first flushed normally -// "/flush" is called (sweepUsers(true)), and that causes new flush of already flushed chunks -// During the flush to store (in flushChunks), chunk is actually removed from list of chunks (and its reference is niled) in removeFlushedChunks. -// After flushing to store, reference is nil, causing panic. -func TestFlushPanicIssue2743(t *testing.T) { - cfg := emptyIngesterConfig() - cfg.FlushCheckPeriod = 50 * time.Millisecond // We want to check for flush-able and removable chunks often. - cfg.RetainPeriod = 500 * time.Millisecond // Remove flushed chunks quickly. This triggers nil-ing. To get a panic, it should happen while Store is "writing" chunks. (We use "sleepy store" to enforce that) - cfg.MaxChunkAge = 1 * time.Hour // We don't use max chunk age for this test, as that is jittered. - cfg.MaxChunkIdle = 200 * time.Millisecond // Flush chunk 200ms after adding last sample. - - st := &sleepyCountingStore{d: 1 * time.Second} // Longer than retain period - - ing := createTestIngester(t, cfg, st) - samples := newSampleGenerator(t, time.Now(), 1*time.Second) - - notifyCh := make(chan bool, 10) - ing.preFlushChunks = func() { - select { - case notifyCh <- true: - default: - } - } - - // Generates one sample - pushSample(t, ing, <-samples) - - // Wait until regular flush kicks in (flushing due to chunk being idle) - <-notifyCh - - // Sweep again -- this causes the same chunks to be queued for flushing again. - // We must hit this *before* flushed chunk is removed from list of chunks. (RetainPeriod) - // While store is flushing (simulated by sleep in the store), previously flushed chunk is removed from memory. - ing.sweepUsers(true) - - // Wait a bit for flushing to end. In buggy version, we get panic while waiting. - time.Sleep(2 * time.Second) -} - -func pushSample(t *testing.T, ing *Ingester, sample cortexpb.Sample) { - _, err := ing.Push(user.InjectOrgID(context.Background(), userID), cortexpb.ToWriteRequest(singleTestLabel, []cortexpb.Sample{sample}, nil, cortexpb.API)) - require.NoError(t, err) -} - -func createTestIngester(t *testing.T, cfg Config, store ChunkStore) *Ingester { - l := validation.Limits{} - overrides, err := validation.NewOverrides(l, nil) - require.NoError(t, err) - - ing, err := New(cfg, client.Config{}, overrides, store, nil, log.NewNopLogger()) - require.NoError(t, err) - - require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) - t.Cleanup(func() { - _ = services.StopAndAwaitTerminated(context.Background(), ing) - }) - - return ing -} - -type sleepyCountingStore struct { - d time.Duration - samples int -} - -func (m *sleepyCountingStore) Put(_ context.Context, chunks []chunk.Chunk) error { - if m.d > 0 { - time.Sleep(m.d) - } - - for _, c := range chunks { - m.samples += c.Data.Len() - } - return nil -} - -func emptyIngesterConfig() Config { - return Config{ - WALConfig: WALConfig{}, - LifecyclerConfig: ring.LifecyclerConfig{ - RingConfig: ring.Config{ - KVStore: kv.Config{ - Store: "inmemory", - }, - ReplicationFactor: 1, - }, - InfNames: []string{"en0", "eth0", "lo0", "lo"}, - HeartbeatPeriod: 10 * time.Second, - }, - - ConcurrentFlushes: 1, // Single queue only. Doesn't really matter for this test (same series is always flushed by same worker), but must be positive. - RateUpdatePeriod: 1 * time.Hour, // Must be positive, doesn't matter for this test. - ActiveSeriesMetricsUpdatePeriod: 5 * time.Minute, // Must be positive. - } -} - -func newSampleGenerator(t *testing.T, initTime time.Time, step time.Duration) <-chan cortexpb.Sample { - ts := make(chan cortexpb.Sample) - - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - go func(ctx context.Context) { - c := initTime - for { - select { - case ts <- cortexpb.Sample{Value: 0, TimestampMs: util.TimeToMillis(c)}: - case <-ctx.Done(): - return - } - - c = c.Add(step) - } - }(ctx) - - return ts -} - -func TestFlushReasonString(t *testing.T) { - for fr := flushReason(0); fr < maxFlushReason; fr++ { - require.True(t, len(fr.String()) > 0) - } -} - -// Issue 3139 depends on a timing between immediate flush, and periodic flush, and the fact that "immediate" chunks get behind "idle" chunks. -// Periodic flush may still find "idle" chunks and put them onto queue, because "ingester for flusher" still runs all the loops. -// When flush of "immediate" chunk fails (eg. due to storage error), it is put back onto the queue, but behind Idle chunk. -// When handling Idle chunks, they are then compared against user limit (MinChunkLength), which panics -- because we were not setting limits. -func TestIssue3139(t *testing.T) { - cfg := emptyIngesterConfig() - cfg.WALConfig.FlushOnShutdown = false - cfg.WALConfig.Dir = t.TempDir() - cfg.WALConfig.WALEnabled = true - - cfg.FlushCheckPeriod = 10 * time.Millisecond - cfg.MaxChunkAge = 1 * time.Hour // We don't want to hit "age" check, but idle-ness check. - cfg.MaxChunkIdle = 0 // Everything is idle immediately - - // Sleep long enough for period flush to happen. Also we want to return errors to the first attempts, so that - // series are flushed again. - st := &sleepyStoreWithErrors{d: 500 * time.Millisecond} - st.errorsToGenerate.Store(1) - - ing := createTestIngester(t, cfg, st) - - // Generates a sample. While it is flushed for the first time (which returns error), it will be put on the queue - // again. - pushSample(t, ing, cortexpb.Sample{Value: 100, TimestampMs: int64(model.Now())}) - - // stop ingester -- no flushing should happen yet - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing)) - - // Make sure nothing was flushed yet... sample should be in WAL - require.Equal(t, int64(0), st.samples.Load()) - require.Equal(t, int64(1), st.errorsToGenerate.Load()) // no error was "consumed" - - // Start new ingester, for flushing only - ing, err := NewForFlusher(cfg, st, nil, nil, log.NewNopLogger()) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) - t.Cleanup(func() { - // Just in case test fails earlier, stop ingester anyay. - _ = services.StopAndAwaitTerminated(context.Background(), ing) - }) - - ing.Flush() - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing)) - - // Verify sample was flushed from WAL. - require.Equal(t, int64(1), st.samples.Load()) -} - -type sleepyStoreWithErrors struct { - d time.Duration - errorsToGenerate atomic.Int64 - samples atomic.Int64 -} - -func (m *sleepyStoreWithErrors) Put(_ context.Context, chunks []chunk.Chunk) error { - if m.d > 0 { - time.Sleep(m.d) - } - - if m.errorsToGenerate.Load() > 0 { - m.errorsToGenerate.Dec() - return fmt.Errorf("put error") - } - - for _, c := range chunks { - m.samples.Add(int64(c.Data.Len())) - } - return nil -} diff --git a/pkg/ingester/index/index.go b/pkg/ingester/index/index.go deleted file mode 100644 index 09d9d84eea..0000000000 --- a/pkg/ingester/index/index.go +++ /dev/null @@ -1,324 +0,0 @@ -package index - -import ( - "sort" - "sync" - "unsafe" - - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - - "github.com/cortexproject/cortex/pkg/chunk" - "github.com/cortexproject/cortex/pkg/cortexpb" - "github.com/cortexproject/cortex/pkg/util" -) - -const indexShards = 32 - -// InvertedIndex implements a in-memory inverter index from label pairs to fingerprints. -// It is sharded to reduce lock contention on writes. -type InvertedIndex struct { - shards []indexShard -} - -// New returns a new InvertedIndex. -func New() *InvertedIndex { - shards := make([]indexShard, indexShards) - for i := 0; i < indexShards; i++ { - shards[i].idx = map[string]indexEntry{} - } - return &InvertedIndex{ - shards: shards, - } -} - -// Add a fingerprint under the specified labels. -// NOTE: memory for `labels` is unsafe; anything retained beyond the -// life of this function must be copied -func (ii *InvertedIndex) Add(labels []cortexpb.LabelAdapter, fp model.Fingerprint) labels.Labels { - shard := &ii.shards[util.HashFP(fp)%indexShards] - return shard.add(labels, fp) // add() returns 'interned' values so the original labels are not retained -} - -// Lookup all fingerprints for the provided matchers. -func (ii *InvertedIndex) Lookup(matchers []*labels.Matcher) []model.Fingerprint { - if len(matchers) == 0 { - return nil - } - - result := []model.Fingerprint{} - for i := range ii.shards { - fps := ii.shards[i].lookup(matchers) - result = append(result, fps...) - } - - return result -} - -// LabelNames returns all label names. -func (ii *InvertedIndex) LabelNames() []string { - results := make([][]string, 0, indexShards) - - for i := range ii.shards { - shardResult := ii.shards[i].labelNames() - results = append(results, shardResult) - } - - return mergeStringSlices(results) -} - -// LabelValues returns the values for the given label. -func (ii *InvertedIndex) LabelValues(name string) []string { - results := make([][]string, 0, indexShards) - - for i := range ii.shards { - shardResult := ii.shards[i].labelValues(name) - results = append(results, shardResult) - } - - return mergeStringSlices(results) -} - -// Delete a fingerprint with the given label pairs. -func (ii *InvertedIndex) Delete(labels labels.Labels, fp model.Fingerprint) { - shard := &ii.shards[util.HashFP(fp)%indexShards] - shard.delete(labels, fp) -} - -// NB slice entries are sorted in fp order. -type indexEntry struct { - name string - fps map[string]indexValueEntry -} - -type indexValueEntry struct { - value string - fps []model.Fingerprint -} - -type unlockIndex map[string]indexEntry - -// This is the prevalent value for Intel and AMD CPUs as-at 2018. -const cacheLineSize = 64 - -type indexShard struct { - mtx sync.RWMutex - idx unlockIndex - //nolint:structcheck,unused - pad [cacheLineSize - unsafe.Sizeof(sync.Mutex{}) - unsafe.Sizeof(unlockIndex{})]byte -} - -func copyString(s string) string { - return string([]byte(s)) -} - -// add metric to the index; return all the name/value pairs as a fresh -// sorted slice, referencing 'interned' strings from the index so that -// no references are retained to the memory of `metric`. -func (shard *indexShard) add(metric []cortexpb.LabelAdapter, fp model.Fingerprint) labels.Labels { - shard.mtx.Lock() - defer shard.mtx.Unlock() - - internedLabels := make(labels.Labels, len(metric)) - - for i, pair := range metric { - values, ok := shard.idx[pair.Name] - if !ok { - values = indexEntry{ - name: copyString(pair.Name), - fps: map[string]indexValueEntry{}, - } - shard.idx[values.name] = values - } - fingerprints, ok := values.fps[pair.Value] - if !ok { - fingerprints = indexValueEntry{ - value: copyString(pair.Value), - } - } - // Insert into the right position to keep fingerprints sorted - j := sort.Search(len(fingerprints.fps), func(i int) bool { - return fingerprints.fps[i] >= fp - }) - fingerprints.fps = append(fingerprints.fps, 0) - copy(fingerprints.fps[j+1:], fingerprints.fps[j:]) - fingerprints.fps[j] = fp - values.fps[fingerprints.value] = fingerprints - internedLabels[i] = labels.Label{Name: values.name, Value: fingerprints.value} - } - sort.Sort(internedLabels) - return internedLabels -} - -func (shard *indexShard) lookup(matchers []*labels.Matcher) []model.Fingerprint { - // index slice values must only be accessed under lock, so all - // code paths must take a copy before returning - shard.mtx.RLock() - defer shard.mtx.RUnlock() - - // per-shard intersection is initially nil, which is a special case - // meaning "everything" when passed to intersect() - // loop invariant: result is sorted - var result []model.Fingerprint - for _, matcher := range matchers { - values, ok := shard.idx[matcher.Name] - if !ok { - return nil - } - var toIntersect model.Fingerprints - if matcher.Type == labels.MatchEqual { - fps := values.fps[matcher.Value] - toIntersect = append(toIntersect, fps.fps...) // deliberate copy - } else if matcher.Type == labels.MatchRegexp && len(chunk.FindSetMatches(matcher.Value)) > 0 { - // The lookup is of the form `=~"a|b|c|d"` - set := chunk.FindSetMatches(matcher.Value) - for _, value := range set { - toIntersect = append(toIntersect, values.fps[value].fps...) - } - sort.Sort(toIntersect) - } else { - // accumulate the matching fingerprints (which are all distinct) - // then sort to maintain the invariant - for value, fps := range values.fps { - if matcher.Matches(value) { - toIntersect = append(toIntersect, fps.fps...) - } - } - sort.Sort(toIntersect) - } - result = intersect(result, toIntersect) - if len(result) == 0 { - return nil - } - } - - return result -} - -func (shard *indexShard) labelNames() []string { - shard.mtx.RLock() - defer shard.mtx.RUnlock() - - results := make([]string, 0, len(shard.idx)) - for name := range shard.idx { - results = append(results, name) - } - - sort.Strings(results) - return results -} - -func (shard *indexShard) labelValues(name string) []string { - shard.mtx.RLock() - defer shard.mtx.RUnlock() - - values, ok := shard.idx[name] - if !ok { - return nil - } - - results := make([]string, 0, len(values.fps)) - for val := range values.fps { - results = append(results, val) - } - - sort.Strings(results) - return results -} - -func (shard *indexShard) delete(labels labels.Labels, fp model.Fingerprint) { - shard.mtx.Lock() - defer shard.mtx.Unlock() - - for _, pair := range labels { - name, value := pair.Name, pair.Value - values, ok := shard.idx[name] - if !ok { - continue - } - fingerprints, ok := values.fps[value] - if !ok { - continue - } - - j := sort.Search(len(fingerprints.fps), func(i int) bool { - return fingerprints.fps[i] >= fp - }) - - // see if search didn't find fp which matches the condition which means we don't have to do anything. - if j >= len(fingerprints.fps) || fingerprints.fps[j] != fp { - continue - } - fingerprints.fps = fingerprints.fps[:j+copy(fingerprints.fps[j:], fingerprints.fps[j+1:])] - - if len(fingerprints.fps) == 0 { - delete(values.fps, value) - } else { - values.fps[value] = fingerprints - } - - if len(values.fps) == 0 { - delete(shard.idx, name) - } else { - shard.idx[name] = values - } - } -} - -// intersect two sorted lists of fingerprints. Assumes there are no duplicate -// fingerprints within the input lists. -func intersect(a, b []model.Fingerprint) []model.Fingerprint { - if a == nil { - return b - } - result := []model.Fingerprint{} - for i, j := 0, 0; i < len(a) && j < len(b); { - if a[i] == b[j] { - result = append(result, a[i]) - } - if a[i] < b[j] { - i++ - } else { - j++ - } - } - return result -} - -func mergeStringSlices(ss [][]string) []string { - switch len(ss) { - case 0: - return nil - case 1: - return ss[0] - case 2: - return mergeTwoStringSlices(ss[0], ss[1]) - default: - halfway := len(ss) / 2 - return mergeTwoStringSlices( - mergeStringSlices(ss[:halfway]), - mergeStringSlices(ss[halfway:]), - ) - } -} - -func mergeTwoStringSlices(a, b []string) []string { - result := make([]string, 0, len(a)+len(b)) - i, j := 0, 0 - for i < len(a) && j < len(b) { - if a[i] < b[j] { - result = append(result, a[i]) - i++ - } else if a[i] > b[j] { - result = append(result, b[j]) - j++ - } else { - result = append(result, a[i]) - i++ - j++ - } - } - result = append(result, a[i:]...) - result = append(result, b[j:]...) - return result -} diff --git a/pkg/ingester/index/index_test.go b/pkg/ingester/index/index_test.go deleted file mode 100644 index d9183d950a..0000000000 --- a/pkg/ingester/index/index_test.go +++ /dev/null @@ -1,201 +0,0 @@ -package index - -import ( - "fmt" - "strconv" - "strings" - "testing" - - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/promql/parser" - "github.com/stretchr/testify/assert" - - "github.com/cortexproject/cortex/pkg/cortexpb" -) - -func TestIndex(t *testing.T) { - index := New() - - for _, entry := range []struct { - m model.Metric - fp model.Fingerprint - }{ - {model.Metric{"foo": "bar", "flip": "flop"}, 3}, - {model.Metric{"foo": "bar", "flip": "flap"}, 2}, - {model.Metric{"foo": "baz", "flip": "flop"}, 1}, - {model.Metric{"foo": "baz", "flip": "flap"}, 0}, - } { - index.Add(cortexpb.FromMetricsToLabelAdapters(entry.m), entry.fp) - } - - for _, tc := range []struct { - matchers []*labels.Matcher - fps []model.Fingerprint - }{ - {nil, nil}, - {mustParseMatcher(`{fizz="buzz"}`), []model.Fingerprint{}}, - - {mustParseMatcher(`{foo="bar"}`), []model.Fingerprint{2, 3}}, - {mustParseMatcher(`{foo="baz"}`), []model.Fingerprint{0, 1}}, - {mustParseMatcher(`{flip="flop"}`), []model.Fingerprint{1, 3}}, - {mustParseMatcher(`{flip="flap"}`), []model.Fingerprint{0, 2}}, - - {mustParseMatcher(`{foo="bar", flip="flop"}`), []model.Fingerprint{3}}, - {mustParseMatcher(`{foo="bar", flip="flap"}`), []model.Fingerprint{2}}, - {mustParseMatcher(`{foo="baz", flip="flop"}`), []model.Fingerprint{1}}, - {mustParseMatcher(`{foo="baz", flip="flap"}`), []model.Fingerprint{0}}, - - {mustParseMatcher(`{fizz=~"b.*"}`), []model.Fingerprint{}}, - - {mustParseMatcher(`{foo=~"bar.*"}`), []model.Fingerprint{2, 3}}, - {mustParseMatcher(`{foo=~"ba.*"}`), []model.Fingerprint{0, 1, 2, 3}}, - {mustParseMatcher(`{flip=~"flop|flap"}`), []model.Fingerprint{0, 1, 2, 3}}, - {mustParseMatcher(`{flip=~"flaps"}`), []model.Fingerprint{}}, - - {mustParseMatcher(`{foo=~"bar|bax", flip="flop"}`), []model.Fingerprint{3}}, - {mustParseMatcher(`{foo=~"bar|baz", flip="flap"}`), []model.Fingerprint{0, 2}}, - {mustParseMatcher(`{foo=~"baz.+", flip="flop"}`), []model.Fingerprint{}}, - {mustParseMatcher(`{foo=~"baz", flip="flap"}`), []model.Fingerprint{0}}, - } { - assert.Equal(t, tc.fps, index.Lookup(tc.matchers)) - } - - assert.Equal(t, []string{"flip", "foo"}, index.LabelNames()) - assert.Equal(t, []string{"bar", "baz"}, index.LabelValues("foo")) - assert.Equal(t, []string{"flap", "flop"}, index.LabelValues("flip")) -} - -func BenchmarkSetRegexLookup(b *testing.B) { - // Prepare the benchmark. - seriesLabels := []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"} - seriesPerLabel := 100000 - - idx := New() - for _, l := range seriesLabels { - for i := 0; i < seriesPerLabel; i++ { - lbls := labels.FromStrings("foo", l, "bar", strconv.Itoa(i)) - idx.Add(cortexpb.FromLabelsToLabelAdapters(lbls), model.Fingerprint(lbls.Hash())) - } - } - - selectionLabels := []string{} - for i := 0; i < 100; i++ { - selectionLabels = append(selectionLabels, strconv.Itoa(i)) - } - - tests := []struct { - name string - matcher string - }{ - { - name: "select all", - matcher: fmt.Sprintf(`{bar=~"%s"}`, strings.Join(selectionLabels, "|")), - }, - { - name: "select two", - matcher: fmt.Sprintf(`{bar=~"%s"}`, strings.Join(selectionLabels[:2], "|")), - }, - { - name: "select half", - matcher: fmt.Sprintf(`{bar=~"%s"}`, strings.Join(selectionLabels[:len(selectionLabels)/2], "|")), - }, - { - name: "select none", - matcher: `{bar=~"bleep|bloop"}`, - }, - { - name: "equality matcher", - matcher: `{bar="1"}`, - }, - { - name: "regex (non-set) matcher", - matcher: `{bar=~"1.*"}`, - }, - } - - b.ResetTimer() - - for _, tc := range tests { - b.Run(fmt.Sprintf("%s:%s", tc.name, tc.matcher), func(b *testing.B) { - matcher := mustParseMatcher(tc.matcher) - for n := 0; n < b.N; n++ { - idx.Lookup(matcher) - } - }) - } - -} - -func mustParseMatcher(s string) []*labels.Matcher { - ms, err := parser.ParseMetricSelector(s) - if err != nil { - panic(err) - } - return ms -} - -func TestIndex_Delete(t *testing.T) { - index := New() - - testData := []struct { - m model.Metric - fp model.Fingerprint - }{ - {model.Metric{"common": "label", "foo": "bar", "flip": "flop"}, 0}, - {model.Metric{"common": "label", "foo": "bar", "flip": "flap"}, 1}, - {model.Metric{"common": "label", "foo": "baz", "flip": "flop"}, 2}, - {model.Metric{"common": "label", "foo": "baz", "flip": "flap"}, 3}, - } - for _, entry := range testData { - index.Add(cortexpb.FromMetricsToLabelAdapters(entry.m), entry.fp) - } - - for _, tc := range []struct { - name string - labelsToDelete labels.Labels - fpToDelete model.Fingerprint - expectedFPs []model.Fingerprint - }{ - { - name: "existing labels and fp", - labelsToDelete: metricToLabels(testData[0].m), - fpToDelete: testData[0].fp, - expectedFPs: []model.Fingerprint{1, 2, 3}, - }, - { - name: "non-existing labels", - labelsToDelete: metricToLabels(model.Metric{"app": "fizz"}), - fpToDelete: testData[1].fp, - expectedFPs: []model.Fingerprint{1, 2, 3}, - }, - { - name: "non-existing fp", - labelsToDelete: metricToLabels(testData[1].m), - fpToDelete: 99, - expectedFPs: []model.Fingerprint{1, 2, 3}, - }, - } { - t.Run(tc.name, func(t *testing.T) { - index.Delete(tc.labelsToDelete, tc.fpToDelete) - assert.Equal(t, tc.expectedFPs, index.Lookup(mustParseMatcher(`{common="label"}`))) - }) - } - - assert.Equal(t, []string{"common", "flip", "foo"}, index.LabelNames()) - assert.Equal(t, []string{"label"}, index.LabelValues("common")) - assert.Equal(t, []string{"bar", "baz"}, index.LabelValues("foo")) - assert.Equal(t, []string{"flap", "flop"}, index.LabelValues("flip")) -} - -func metricToLabels(m model.Metric) labels.Labels { - ls := make(labels.Labels, 0, len(m)) - for k, v := range m { - ls = append(ls, labels.Label{ - Name: string(k), - Value: string(v), - }) - } - - return ls -} diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index b72c369f31..544f01d4bc 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -5,7 +5,6 @@ import ( "flag" "fmt" "net/http" - "os" "strings" "sync" "time" @@ -15,26 +14,17 @@ import ( "github.com/gogo/status" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/tsdb/chunks" - tsdb_record "github.com/prometheus/prometheus/tsdb/record" - "github.com/weaveworks/common/httpgrpc" "go.uber.org/atomic" - "golang.org/x/time/rate" "google.golang.org/grpc/codes" - cortex_chunk "github.com/cortexproject/cortex/pkg/chunk" "github.com/cortexproject/cortex/pkg/cortexpb" "github.com/cortexproject/cortex/pkg/ingester/client" "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/storage/tsdb" "github.com/cortexproject/cortex/pkg/tenant" - "github.com/cortexproject/cortex/pkg/util" logutil "github.com/cortexproject/cortex/pkg/util/log" util_math "github.com/cortexproject/cortex/pkg/util/math" "github.com/cortexproject/cortex/pkg/util/services" - "github.com/cortexproject/cortex/pkg/util/spanlogger" "github.com/cortexproject/cortex/pkg/util/validation" ) @@ -52,15 +42,12 @@ const ( ) var ( - // This is initialised if the WAL is enabled and the records are fetched from this pool. - recordPool sync.Pool - errIngesterStopping = errors.New("ingester stopping") ) // Config for an Ingester. type Config struct { - WALConfig WALConfig `yaml:"walconfig" doc:"description=Configures the Write-Ahead Log (WAL) for the Cortex chunks storage. This config is ignored when running the Cortex blocks storage."` + WALConfig WALConfig `yaml:"walconfig" doc:"description=Configures the Write-Ahead Log (WAL) for the removed Cortex chunks storage. This config is now always ignored."` LifecyclerConfig ring.LifecyclerConfig `yaml:"lifecycler"` // Config for transferring chunks. Zero or negative = no retries. @@ -166,43 +153,23 @@ func (cfg *Config) getIgnoreSeriesLimitForMetricNamesMap() map[string]struct{} { type Ingester struct { *services.BasicService - cfg Config - clientConfig client.Config + cfg Config metrics *ingesterMetrics logger log.Logger - chunkStore ChunkStore lifecycler *ring.Lifecycler limits *validation.Overrides limiter *Limiter subservicesWatcher *services.FailureWatcher - userStatesMtx sync.RWMutex // protects userStates and stopped - userStates *userStates - stopped bool // protected by userStatesMtx + stoppedMtx sync.RWMutex // protects stopped + stopped bool // protected by stoppedMtx // For storing metadata ingested. usersMetadataMtx sync.RWMutex usersMetadata map[string]*userMetricsMetadata - // One queue per flush thread. Fingerprint is used to - // pick a queue. - flushQueues []*util.PriorityQueue - flushQueuesDone sync.WaitGroup - - // Spread out calls to the chunk store over the flush period - flushRateLimiter *rate.Limiter - - // This should never be nil. - wal WAL - // To be passed to the WAL. - registerer prometheus.Registerer - - // Hooks for injecting behaviour from tests. - preFlushUserSeries func() - preFlushChunks func() - // Prometheus block storage TSDBState TSDBState @@ -211,237 +178,28 @@ type Ingester struct { inflightPushRequests atomic.Int64 } -// ChunkStore is the interface we need to store chunks -type ChunkStore interface { - Put(ctx context.Context, chunks []cortex_chunk.Chunk) error -} - // New constructs a new Ingester. -func New(cfg Config, clientConfig client.Config, limits *validation.Overrides, chunkStore ChunkStore, registerer prometheus.Registerer, logger log.Logger) (*Ingester, error) { +func New(cfg Config, limits *validation.Overrides, registerer prometheus.Registerer, logger log.Logger) (*Ingester, error) { + if !cfg.BlocksStorageEnabled { + // TODO FIXME error message + return nil, fmt.Errorf("chunks storage is no longer supported") + } + defaultInstanceLimits = &cfg.DefaultLimits if cfg.ingesterClientFactory == nil { cfg.ingesterClientFactory = client.MakeIngesterClient } - if cfg.BlocksStorageEnabled { - return NewV2(cfg, clientConfig, limits, registerer, logger) - } - - if cfg.WALConfig.WALEnabled { - // If WAL is enabled, we don't transfer out the data to any ingester. - // Either the next ingester which takes it's place should recover from WAL - // or the data has to be flushed during scaledown. - cfg.MaxTransferRetries = 0 - - // Transfers are disabled with WAL, hence no need to wait for transfers. - cfg.LifecyclerConfig.JoinAfter = 0 - - recordPool = sync.Pool{ - New: func() interface{} { - return &WALRecord{} - }, - } - } - - if cfg.WALConfig.WALEnabled || cfg.WALConfig.Recover { - if err := os.MkdirAll(cfg.WALConfig.Dir, os.ModePerm); err != nil { - return nil, err - } - } - - i := &Ingester{ - cfg: cfg, - clientConfig: clientConfig, - - limits: limits, - chunkStore: chunkStore, - flushQueues: make([]*util.PriorityQueue, cfg.ConcurrentFlushes), - flushRateLimiter: rate.NewLimiter(rate.Inf, 1), - usersMetadata: map[string]*userMetricsMetadata{}, - registerer: registerer, - logger: logger, - } - i.metrics = newIngesterMetrics(registerer, true, cfg.ActiveSeriesMetricsEnabled, i.getInstanceLimits, nil, &i.inflightPushRequests) - - var err error - // During WAL recovery, it will create new user states which requires the limiter. - // Hence initialise the limiter before creating the WAL. - // The '!cfg.WALConfig.WALEnabled' argument says don't flush on shutdown if the WAL is enabled. - i.lifecycler, err = ring.NewLifecycler(cfg.LifecyclerConfig, i, "ingester", RingKey, !cfg.WALConfig.WALEnabled || cfg.WALConfig.FlushOnShutdown, logger, prometheus.WrapRegistererWithPrefix("cortex_", registerer)) - if err != nil { - return nil, err - } - - i.limiter = NewLimiter( - limits, - i.lifecycler, - cfg.DistributorShardingStrategy, - cfg.DistributorShardByAllLabels, - cfg.LifecyclerConfig.RingConfig.ReplicationFactor, - cfg.LifecyclerConfig.RingConfig.ZoneAwarenessEnabled) - - i.subservicesWatcher = services.NewFailureWatcher() - i.subservicesWatcher.WatchService(i.lifecycler) - - i.BasicService = services.NewBasicService(i.starting, i.loop, i.stopping) - return i, nil -} - -func (i *Ingester) starting(ctx context.Context) error { - if i.cfg.WALConfig.Recover { - level.Info(i.logger).Log("msg", "recovering from WAL") - start := time.Now() - if err := recoverFromWAL(i); err != nil { - level.Error(i.logger).Log("msg", "failed to recover from WAL", "time", time.Since(start).String()) - return errors.Wrap(err, "failed to recover from WAL") - } - elapsed := time.Since(start) - level.Info(i.logger).Log("msg", "recovery from WAL completed", "time", elapsed.String()) - i.metrics.walReplayDuration.Set(elapsed.Seconds()) - } - - // If the WAL recover happened, then the userStates would already be set. - if i.userStates == nil { - i.userStates = newUserStates(i.limiter, i.cfg, i.metrics, i.logger) - } - - var err error - i.wal, err = newWAL(i.cfg.WALConfig, i.userStates.cp, i.registerer, i.logger) - if err != nil { - return errors.Wrap(err, "starting WAL") - } - - // Now that user states have been created, we can start the lifecycler. - // Important: we want to keep lifecycler running until we ask it to stop, so we need to give it independent context - if err := i.lifecycler.StartAsync(context.Background()); err != nil { - return errors.Wrap(err, "failed to start lifecycler") - } - if err := i.lifecycler.AwaitRunning(ctx); err != nil { - return errors.Wrap(err, "failed to start lifecycler") - } - - i.startFlushLoops() - - return nil -} - -func (i *Ingester) startFlushLoops() { - i.flushQueuesDone.Add(i.cfg.ConcurrentFlushes) - for j := 0; j < i.cfg.ConcurrentFlushes; j++ { - i.flushQueues[j] = util.NewPriorityQueue(i.metrics.flushQueueLength) - go i.flushLoop(j) - } + return NewV2(cfg, limits, registerer, logger) } // NewForFlusher constructs a new Ingester to be used by flusher target. // Compared to the 'New' method: // * Always replays the WAL. // * Does not start the lifecycler. -func NewForFlusher(cfg Config, chunkStore ChunkStore, limits *validation.Overrides, registerer prometheus.Registerer, logger log.Logger) (*Ingester, error) { - if cfg.BlocksStorageEnabled { - return NewV2ForFlusher(cfg, limits, registerer, logger) - } - - i := &Ingester{ - cfg: cfg, - chunkStore: chunkStore, - flushQueues: make([]*util.PriorityQueue, cfg.ConcurrentFlushes), - flushRateLimiter: rate.NewLimiter(rate.Inf, 1), - wal: &noopWAL{}, - limits: limits, - logger: logger, - } - i.metrics = newIngesterMetrics(registerer, true, false, i.getInstanceLimits, nil, &i.inflightPushRequests) - - i.BasicService = services.NewBasicService(i.startingForFlusher, i.loopForFlusher, i.stopping) - return i, nil -} - -func (i *Ingester) startingForFlusher(ctx context.Context) error { - level.Info(i.logger).Log("msg", "recovering from WAL") - - // We recover from WAL always. - start := time.Now() - if err := recoverFromWAL(i); err != nil { - level.Error(i.logger).Log("msg", "failed to recover from WAL", "time", time.Since(start).String()) - return err - } - elapsed := time.Since(start) - - level.Info(i.logger).Log("msg", "recovery from WAL completed", "time", elapsed.String()) - i.metrics.walReplayDuration.Set(elapsed.Seconds()) - - i.startFlushLoops() - return nil -} - -func (i *Ingester) loopForFlusher(ctx context.Context) error { - for { - select { - case <-ctx.Done(): - return nil - - case err := <-i.subservicesWatcher.Chan(): - return errors.Wrap(err, "ingester subservice failed") - } - } -} - -func (i *Ingester) loop(ctx context.Context) error { - flushTicker := time.NewTicker(i.cfg.FlushCheckPeriod) - defer flushTicker.Stop() - - rateUpdateTicker := time.NewTicker(i.cfg.RateUpdatePeriod) - defer rateUpdateTicker.Stop() - - metadataPurgeTicker := time.NewTicker(metadataPurgePeriod) - defer metadataPurgeTicker.Stop() - - var activeSeriesTickerChan <-chan time.Time - if i.cfg.ActiveSeriesMetricsEnabled { - t := time.NewTicker(i.cfg.ActiveSeriesMetricsUpdatePeriod) - activeSeriesTickerChan = t.C - defer t.Stop() - } - - for { - select { - case <-metadataPurgeTicker.C: - i.purgeUserMetricsMetadata() - - case <-flushTicker.C: - i.sweepUsers(false) - - case <-rateUpdateTicker.C: - i.userStates.updateRates() - - case <-activeSeriesTickerChan: - i.userStates.purgeAndUpdateActiveSeries(time.Now().Add(-i.cfg.ActiveSeriesMetricsIdleTimeout)) - - case <-ctx.Done(): - return nil - - case err := <-i.subservicesWatcher.Chan(): - return errors.Wrap(err, "ingester subservice failed") - } - } -} - -// stopping is run when ingester is asked to stop -func (i *Ingester) stopping(_ error) error { - i.wal.Stop() - - // This will prevent us accepting any more samples - i.stopIncomingRequests() - - // Lifecycler can be nil if the ingester is for a flusher. - if i.lifecycler != nil { - // Next initiate our graceful exit from the ring. - return services.StopAndAwaitTerminated(context.Background(), i.lifecycler) - } - - return nil +func NewForFlusher(cfg Config, limits *validation.Overrides, registerer prometheus.Registerer, logger log.Logger) (*Ingester, error) { + return NewV2ForFlusher(cfg, limits, registerer, logger) } // ShutdownHandler triggers the following set of operations in order: @@ -464,13 +222,6 @@ func (i *Ingester) ShutdownHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) } -// stopIncomingRequests is called during the shutdown process. -func (i *Ingester) stopIncomingRequests() { - i.userStatesMtx.Lock() - defer i.userStatesMtx.Unlock() - i.stopped = true -} - // check that ingester has finished starting, i.e. it is in Running or Stopping state. // Why Stopping? Because ingester still runs, even when it is transferring data out in Stopping state. // Ingester handles this state on its own (via `stopped` flag). @@ -509,159 +260,7 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte } } - if i.cfg.BlocksStorageEnabled { - return i.v2Push(ctx, req) - } - - // NOTE: because we use `unsafe` in deserialisation, we must not - // retain anything from `req` past the call to ReuseSlice - defer cortexpb.ReuseSlice(req.Timeseries) - - userID, err := tenant.TenantID(ctx) - if err != nil { - return nil, err - } - - // Given metadata is a best-effort approach, and we don't halt on errors - // process it before samples. Otherwise, we risk returning an error before ingestion. - i.pushMetadata(ctx, userID, req.GetMetadata()) - - var firstPartialErr *validationError - var record *WALRecord - if i.cfg.WALConfig.WALEnabled { - record = recordPool.Get().(*WALRecord) - record.UserID = userID - // Assuming there is not much churn in most cases, there is no use - // keeping the record.Labels slice hanging around. - record.Series = nil - if cap(record.Samples) < len(req.Timeseries) { - record.Samples = make([]tsdb_record.RefSample, 0, len(req.Timeseries)) - } else { - record.Samples = record.Samples[:0] - } - } - - for _, ts := range req.Timeseries { - seriesSamplesIngested := 0 - for _, s := range ts.Samples { - // append() copies the memory in `ts.Labels` except on the error path - err := i.append(ctx, userID, ts.Labels, model.Time(s.TimestampMs), model.SampleValue(s.Value), req.Source, record) - if err == nil { - seriesSamplesIngested++ - continue - } - - i.metrics.ingestedSamplesFail.Inc() - if ve, ok := err.(*validationError); ok { - if firstPartialErr == nil { - firstPartialErr = ve - } - continue - } - - // non-validation error: abandon this request - return nil, grpcForwardableError(userID, http.StatusInternalServerError, err) - } - - if i.cfg.ActiveSeriesMetricsEnabled && seriesSamplesIngested > 0 { - // updateActiveSeries will copy labels if necessary. - i.updateActiveSeries(userID, time.Now(), ts.Labels) - } - } - - if record != nil { - // Log the record only if there was no error in ingestion. - if err := i.wal.Log(record); err != nil { - return nil, err - } - recordPool.Put(record) - } - - if firstPartialErr != nil { - // grpcForwardableError turns the error into a string so it no longer references `req` - return &cortexpb.WriteResponse{}, grpcForwardableError(userID, firstPartialErr.code, firstPartialErr) - } - - return &cortexpb.WriteResponse{}, nil -} - -// NOTE: memory for `labels` is unsafe; anything retained beyond the -// life of this function must be copied -func (i *Ingester) append(ctx context.Context, userID string, labels labelPairs, timestamp model.Time, value model.SampleValue, source cortexpb.WriteRequest_SourceEnum, record *WALRecord) error { - labels.removeBlanks() - - var ( - state *userState - fp model.Fingerprint - ) - i.userStatesMtx.RLock() - defer func() { - i.userStatesMtx.RUnlock() - if state != nil { - state.fpLocker.Unlock(fp) - } - }() - if i.stopped { - return errIngesterStopping - } - - // getOrCreateSeries copies the memory for `labels`, except on the error path. - state, fp, series, err := i.userStates.getOrCreateSeries(ctx, userID, labels, record) - if err != nil { - if ve, ok := err.(*validationError); ok { - state.discardedSamples.WithLabelValues(ve.errorType).Inc() - } - - // Reset the state so that the defer will not try to unlock the fpLocker - // in case of error, because that lock has already been released on error. - state = nil - return err - } - - prevNumChunks := len(series.chunkDescs) - if i.cfg.SpreadFlushes && prevNumChunks > 0 { - // Map from the fingerprint hash to a point in the cycle of period MaxChunkAge - startOfCycle := timestamp.Add(-(timestamp.Sub(model.Time(0)) % i.cfg.MaxChunkAge)) - slot := startOfCycle.Add(time.Duration(uint64(fp) % uint64(i.cfg.MaxChunkAge))) - // If adding this sample means the head chunk will span that point in time, close so it will get flushed - if series.head().FirstTime < slot && timestamp >= slot { - series.closeHead(reasonSpreadFlush) - } - } - - if err := series.add(model.SamplePair{ - Value: value, - Timestamp: timestamp, - }); err != nil { - if ve, ok := err.(*validationError); ok { - state.discardedSamples.WithLabelValues(ve.errorType).Inc() - if ve.noReport { - return nil - } - } - return err - } - - if record != nil { - record.Samples = append(record.Samples, tsdb_record.RefSample{ - Ref: chunks.HeadSeriesRef(fp), - T: int64(timestamp), - V: float64(value), - }) - } - - i.metrics.memoryChunks.Add(float64(len(series.chunkDescs) - prevNumChunks)) - i.metrics.ingestedSamples.Inc() - switch source { - case cortexpb.RULE: - state.ingestedRuleSamples.Inc() - case cortexpb.API: - fallthrough - default: - state.ingestedAPISamples.Inc() - } - - return err + return i.v2Push(ctx, req) } // pushMetadata returns number of ingested metadata. @@ -697,12 +296,12 @@ func (i *Ingester) pushMetadata(ctx context.Context, userID string, metadata []* } func (i *Ingester) appendMetadata(userID string, m *cortexpb.MetricMetadata) error { - i.userStatesMtx.RLock() + i.stoppedMtx.RLock() if i.stopped { - i.userStatesMtx.RUnlock() + i.stoppedMtx.RUnlock() return errIngesterStopping } - i.userStatesMtx.RUnlock() + i.stoppedMtx.RUnlock() userMetadata := i.getOrCreateUserMetadata(userID) @@ -774,187 +373,22 @@ func (i *Ingester) purgeUserMetricsMetadata() { // Query implements service.IngesterServer func (i *Ingester) Query(ctx context.Context, req *client.QueryRequest) (*client.QueryResponse, error) { - if i.cfg.BlocksStorageEnabled { - return i.v2Query(ctx, req) - } - - if err := i.checkRunningOrStopping(); err != nil { - return nil, err - } - - userID, err := tenant.TenantID(ctx) - if err != nil { - return nil, err - } - - from, through, matchers, err := client.FromQueryRequest(req) - if err != nil { - return nil, err - } - - i.metrics.queries.Inc() - - i.userStatesMtx.RLock() - state, ok, err := i.userStates.getViaContext(ctx) - i.userStatesMtx.RUnlock() - if err != nil { - return nil, err - } else if !ok { - return &client.QueryResponse{}, nil - } - - result := &client.QueryResponse{} - numSeries, numSamples := 0, 0 - maxSamplesPerQuery := i.limits.MaxSamplesPerQuery(userID) - err = state.forSeriesMatching(ctx, matchers, func(ctx context.Context, _ model.Fingerprint, series *memorySeries) error { - values, err := series.samplesForRange(from, through) - if err != nil { - return err - } - if len(values) == 0 { - return nil - } - numSeries++ - - numSamples += len(values) - if numSamples > maxSamplesPerQuery { - return httpgrpc.Errorf(http.StatusRequestEntityTooLarge, "exceeded maximum number of samples in a query (%d)", maxSamplesPerQuery) - } - - ts := cortexpb.TimeSeries{ - Labels: cortexpb.FromLabelsToLabelAdapters(series.metric), - Samples: make([]cortexpb.Sample, 0, len(values)), - } - for _, s := range values { - ts.Samples = append(ts.Samples, cortexpb.Sample{ - Value: float64(s.Value), - TimestampMs: int64(s.Timestamp), - }) - } - result.Timeseries = append(result.Timeseries, ts) - return nil - }, nil, 0) - i.metrics.queriedSeries.Observe(float64(numSeries)) - i.metrics.queriedSamples.Observe(float64(numSamples)) - return result, err + return i.v2Query(ctx, req) } // QueryStream implements service.IngesterServer func (i *Ingester) QueryStream(req *client.QueryRequest, stream client.Ingester_QueryStreamServer) error { - if i.cfg.BlocksStorageEnabled { - return i.v2QueryStream(req, stream) - } - - if err := i.checkRunningOrStopping(); err != nil { - return err - } - - spanLog, ctx := spanlogger.New(stream.Context(), "QueryStream") - defer spanLog.Finish() - - from, through, matchers, err := client.FromQueryRequest(req) - if err != nil { - return err - } - - i.metrics.queries.Inc() - - i.userStatesMtx.RLock() - state, ok, err := i.userStates.getViaContext(ctx) - i.userStatesMtx.RUnlock() - if err != nil { - return err - } else if !ok { - return nil - } - - numSeries, numChunks := 0, 0 - reuseWireChunks := [queryStreamBatchSize][]client.Chunk{} - batch := make([]client.TimeSeriesChunk, 0, queryStreamBatchSize) - // We'd really like to have series in label order, not FP order, so we - // can iteratively merge them with entries coming from the chunk store. But - // that would involve locking all the series & sorting, so until we have - // a better solution in the ingesters I'd rather take the hit in the queriers. - err = state.forSeriesMatching(stream.Context(), matchers, func(ctx context.Context, _ model.Fingerprint, series *memorySeries) error { - chunks := make([]*desc, 0, len(series.chunkDescs)) - for _, chunk := range series.chunkDescs { - if !(chunk.FirstTime.After(through) || chunk.LastTime.Before(from)) { - chunks = append(chunks, chunk.slice(from, through)) - } - } - - if len(chunks) == 0 { - return nil - } - - numSeries++ - reusePos := len(batch) - wireChunks, err := toWireChunks(chunks, reuseWireChunks[reusePos]) - if err != nil { - return err - } - reuseWireChunks[reusePos] = wireChunks - - numChunks += len(wireChunks) - batch = append(batch, client.TimeSeriesChunk{ - Labels: cortexpb.FromLabelsToLabelAdapters(series.metric), - Chunks: wireChunks, - }) - - return nil - }, func(ctx context.Context) error { - if len(batch) == 0 { - return nil - } - err = client.SendQueryStream(stream, &client.QueryStreamResponse{ - Chunkseries: batch, - }) - batch = batch[:0] - return err - }, queryStreamBatchSize) - if err != nil { - return err - } - - i.metrics.queriedSeries.Observe(float64(numSeries)) - i.metrics.queriedChunks.Observe(float64(numChunks)) - level.Debug(spanLog).Log("streams", numSeries) - level.Debug(spanLog).Log("chunks", numChunks) - return err + return i.v2QueryStream(req, stream) } // Query implements service.IngesterServer func (i *Ingester) QueryExemplars(ctx context.Context, req *client.ExemplarQueryRequest) (*client.ExemplarQueryResponse, error) { - if !i.cfg.BlocksStorageEnabled { - return nil, errors.New("not supported") - } - return i.v2QueryExemplars(ctx, req) } // LabelValues returns all label values that are associated with a given label name. func (i *Ingester) LabelValues(ctx context.Context, req *client.LabelValuesRequest) (*client.LabelValuesResponse, error) { - if i.cfg.BlocksStorageEnabled { - return i.v2LabelValues(ctx, req) - } - - if err := i.checkRunningOrStopping(); err != nil { - return nil, err - } - - i.userStatesMtx.RLock() - defer i.userStatesMtx.RUnlock() - state, ok, err := i.userStates.getViaContext(ctx) - if err != nil { - return nil, err - } else if !ok { - return &client.LabelValuesResponse{}, nil - } - - resp := &client.LabelValuesResponse{} - resp.LabelValues = append(resp.LabelValues, state.index.LabelValues(req.LabelName)...) - - return resp, nil + return i.v2LabelValues(ctx, req) } func (i *Ingester) LabelValuesStream(req *client.LabelValuesRequest, stream client.Ingester_LabelValuesStreamServer) error { @@ -977,27 +411,7 @@ func (i *Ingester) LabelValuesStream(req *client.LabelValuesRequest, stream clie // LabelNames return all the label names. func (i *Ingester) LabelNames(ctx context.Context, req *client.LabelNamesRequest) (*client.LabelNamesResponse, error) { - if i.cfg.BlocksStorageEnabled { - return i.v2LabelNames(ctx, req) - } - - if err := i.checkRunningOrStopping(); err != nil { - return nil, err - } - - i.userStatesMtx.RLock() - defer i.userStatesMtx.RUnlock() - state, ok, err := i.userStates.getViaContext(ctx) - if err != nil { - return nil, err - } else if !ok { - return &client.LabelNamesResponse{}, nil - } - - resp := &client.LabelNamesResponse{} - resp.LabelNames = append(resp.LabelNames, state.index.LabelNames()...) - - return resp, nil + return i.v2LabelNames(ctx, req) } // LabelNames return all the label names. @@ -1021,49 +435,7 @@ func (i *Ingester) LabelNamesStream(req *client.LabelNamesRequest, stream client // MetricsForLabelMatchers returns all the metrics which match a set of matchers. func (i *Ingester) MetricsForLabelMatchers(ctx context.Context, req *client.MetricsForLabelMatchersRequest) (*client.MetricsForLabelMatchersResponse, error) { - if i.cfg.BlocksStorageEnabled { - return i.v2MetricsForLabelMatchers(ctx, req) - } - - if err := i.checkRunningOrStopping(); err != nil { - return nil, err - } - - i.userStatesMtx.RLock() - defer i.userStatesMtx.RUnlock() - state, ok, err := i.userStates.getViaContext(ctx) - if err != nil { - return nil, err - } else if !ok { - return &client.MetricsForLabelMatchersResponse{}, nil - } - - // TODO Right now we ignore start and end. - _, _, matchersSet, err := client.FromMetricsForLabelMatchersRequest(req) - if err != nil { - return nil, err - } - - lss := map[model.Fingerprint]labels.Labels{} - for _, matchers := range matchersSet { - if err := state.forSeriesMatching(ctx, matchers, func(ctx context.Context, fp model.Fingerprint, series *memorySeries) error { - if _, ok := lss[fp]; !ok { - lss[fp] = series.metric - } - return nil - }, nil, 0); err != nil { - return nil, err - } - } - - result := &client.MetricsForLabelMatchersResponse{ - Metric: make([]*cortexpb.Metric, 0, len(lss)), - } - for _, ls := range lss { - result.Metric = append(result.Metric, &cortexpb.Metric{Labels: cortexpb.FromLabelsToLabelAdapters(ls)}) - } - - return result, nil + return i.v2MetricsForLabelMatchers(ctx, req) } func (i *Ingester) MetricsForLabelMatchersStream(req *client.MetricsForLabelMatchersRequest, stream client.Ingester_MetricsForLabelMatchersStreamServer) error { @@ -1087,12 +459,12 @@ func (i *Ingester) MetricsForLabelMatchersStream(req *client.MetricsForLabelMatc // MetricsMetadata returns all the metric metadata of a user. func (i *Ingester) MetricsMetadata(ctx context.Context, req *client.MetricsMetadataRequest) (*client.MetricsMetadataResponse, error) { - i.userStatesMtx.RLock() + i.stoppedMtx.RLock() if err := i.checkRunningOrStopping(); err != nil { - i.userStatesMtx.RUnlock() + i.stoppedMtx.RUnlock() return nil, err } - i.userStatesMtx.RUnlock() + i.stoppedMtx.RUnlock() userID, err := tenant.TenantID(ctx) if err != nil { @@ -1110,64 +482,12 @@ func (i *Ingester) MetricsMetadata(ctx context.Context, req *client.MetricsMetad // UserStats returns ingestion statistics for the current user. func (i *Ingester) UserStats(ctx context.Context, req *client.UserStatsRequest) (*client.UserStatsResponse, error) { - if i.cfg.BlocksStorageEnabled { - return i.v2UserStats(ctx, req) - } - - if err := i.checkRunningOrStopping(); err != nil { - return nil, err - } - - i.userStatesMtx.RLock() - defer i.userStatesMtx.RUnlock() - state, ok, err := i.userStates.getViaContext(ctx) - if err != nil { - return nil, err - } else if !ok { - return &client.UserStatsResponse{}, nil - } - - apiRate := state.ingestedAPISamples.Rate() - ruleRate := state.ingestedRuleSamples.Rate() - return &client.UserStatsResponse{ - IngestionRate: apiRate + ruleRate, - ApiIngestionRate: apiRate, - RuleIngestionRate: ruleRate, - NumSeries: uint64(state.fpToSeries.length()), - }, nil + return i.v2UserStats(ctx, req) } // AllUserStats returns ingestion statistics for all users known to this ingester. func (i *Ingester) AllUserStats(ctx context.Context, req *client.UserStatsRequest) (*client.UsersStatsResponse, error) { - if i.cfg.BlocksStorageEnabled { - return i.v2AllUserStats(ctx, req) - } - - if err := i.checkRunningOrStopping(); err != nil { - return nil, err - } - - i.userStatesMtx.RLock() - defer i.userStatesMtx.RUnlock() - users := i.userStates.cp() - - response := &client.UsersStatsResponse{ - Stats: make([]*client.UserIDStatsResponse, 0, len(users)), - } - for userID, state := range users { - apiRate := state.ingestedAPISamples.Rate() - ruleRate := state.ingestedRuleSamples.Rate() - response.Stats = append(response.Stats, &client.UserIDStatsResponse{ - UserId: userID, - Data: &client.UserStatsResponse{ - IngestionRate: apiRate + ruleRate, - ApiIngestionRate: apiRate, - RuleIngestionRate: ruleRate, - NumSeries: uint64(state.fpToSeries.length()), - }, - }) - } - return response, nil + return i.v2AllUserStats(ctx, req) } // CheckReady is the readiness handler used to indicate to k8s when the ingesters @@ -1178,11 +498,3 @@ func (i *Ingester) CheckReady(ctx context.Context) error { } return i.lifecycler.CheckReady(ctx) } - -// labels will be copied if needed. -func (i *Ingester) updateActiveSeries(userID string, now time.Time, labels []cortexpb.LabelAdapter) { - i.userStatesMtx.RLock() - defer i.userStatesMtx.RUnlock() - - i.userStates.updateActiveSeriesForUser(userID, now, cortexpb.FromLabelAdaptersToLabels(labels)) -} diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 023a894069..629a65775e 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -3,148 +3,29 @@ package ingester import ( "context" "fmt" - "math" - "math/rand" "net/http" "os" "path/filepath" "sort" - "strconv" - "strings" - "sync" "testing" "time" - "github.com/go-kit/log" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/cortexproject/cortex/pkg/chunk" + "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/weaveworks/common/httpgrpc" "github.com/weaveworks/common/user" - "google.golang.org/grpc" - "github.com/cortexproject/cortex/pkg/chunk" - promchunk "github.com/cortexproject/cortex/pkg/chunk/encoding" "github.com/cortexproject/cortex/pkg/cortexpb" "github.com/cortexproject/cortex/pkg/ingester/client" "github.com/cortexproject/cortex/pkg/ring" - "github.com/cortexproject/cortex/pkg/util/chunkcompat" "github.com/cortexproject/cortex/pkg/util/services" "github.com/cortexproject/cortex/pkg/util/test" - "github.com/cortexproject/cortex/pkg/util/validation" ) -type testStore struct { - mtx sync.Mutex - // Chunks keyed by userID. - chunks map[string][]chunk.Chunk -} - -func newTestStore(t require.TestingT, cfg Config, clientConfig client.Config, limits validation.Limits, reg prometheus.Registerer) (*testStore, *Ingester) { - store := &testStore{ - chunks: map[string][]chunk.Chunk{}, - } - overrides, err := validation.NewOverrides(limits, nil) - require.NoError(t, err) - - ing, err := New(cfg, clientConfig, overrides, store, reg, log.NewNopLogger()) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) - - return store, ing -} - -func newDefaultTestStore(t testing.TB) (*testStore, *Ingester) { - t.Helper() - - return newTestStore(t, - defaultIngesterTestConfig(t), - defaultClientTestConfig(), - defaultLimitsTestConfig(), nil) -} - -func (s *testStore) Put(ctx context.Context, chunks []chunk.Chunk) error { - if len(chunks) == 0 { - return nil - } - s.mtx.Lock() - defer s.mtx.Unlock() - - for _, chunk := range chunks { - for _, v := range chunk.Metric { - if v.Value == "" { - return fmt.Errorf("Chunk has blank label %q", v.Name) - } - } - } - userID := chunks[0].UserID - s.chunks[userID] = append(s.chunks[userID], chunks...) - return nil -} - -func (s *testStore) Stop() {} - -// check that the store is holding data equivalent to what we expect -func (s *testStore) checkData(t *testing.T, userIDs []string, testData map[string]model.Matrix) { - s.mtx.Lock() - defer s.mtx.Unlock() - for _, userID := range userIDs { - res, err := chunk.ChunksToMatrix(context.Background(), s.chunks[userID], model.Time(0), model.Time(math.MaxInt64)) - require.NoError(t, err) - sort.Sort(res) - assert.Equal(t, testData[userID], res, "userID %s", userID) - } -} - -func buildTestMatrix(numSeries int, samplesPerSeries int, offset int) model.Matrix { - m := make(model.Matrix, 0, numSeries) - for i := 0; i < numSeries; i++ { - ss := model.SampleStream{ - Metric: model.Metric{ - model.MetricNameLabel: model.LabelValue(fmt.Sprintf("testmetric_%d", i)), - model.JobLabel: model.LabelValue(fmt.Sprintf("testjob%d", i%2)), - }, - Values: make([]model.SamplePair, 0, samplesPerSeries), - } - for j := 0; j < samplesPerSeries; j++ { - ss.Values = append(ss.Values, model.SamplePair{ - Timestamp: model.Time(i + j + offset), - Value: model.SampleValue(i + j + offset), - }) - } - m = append(m, &ss) - } - sort.Sort(m) - return m -} - -func matrixToSamples(m model.Matrix) []cortexpb.Sample { - var samples []cortexpb.Sample - for _, ss := range m { - for _, sp := range ss.Values { - samples = append(samples, cortexpb.Sample{ - TimestampMs: int64(sp.Timestamp), - Value: float64(sp.Value), - }) - } - } - return samples -} - -// Return one copy of the labels per sample -func matrixToLables(m model.Matrix) []labels.Labels { - var labels []labels.Labels - for _, ss := range m { - for range ss.Values { - labels = append(labels, cortexpb.FromLabelAdaptersToLabels(cortexpb.FromMetricsToLabelAdapters(ss.Metric))) - } - } - return labels -} - func runTestQuery(ctx context.Context, t *testing.T, ing *Ingester, ty labels.MatchType, n, v string) (model.Matrix, *client.QueryRequest, error) { return runTestQueryTimes(ctx, t, ing, ty, n, v, model.Earliest, model.Latest) } @@ -167,351 +48,6 @@ func runTestQueryTimes(ctx context.Context, t *testing.T, ing *Ingester, ty labe return res, req, nil } -func pushTestMetadata(t *testing.T, ing *Ingester, numMetadata, metadataPerMetric int) ([]string, map[string][]*cortexpb.MetricMetadata) { - userIDs := []string{"1", "2", "3"} - - // Create test metadata. - // Map of userIDs, to map of metric => metadataSet - testData := map[string][]*cortexpb.MetricMetadata{} - for _, userID := range userIDs { - metadata := make([]*cortexpb.MetricMetadata, 0, metadataPerMetric) - for i := 0; i < numMetadata; i++ { - metricName := fmt.Sprintf("testmetric_%d", i) - for j := 0; j < metadataPerMetric; j++ { - m := &cortexpb.MetricMetadata{MetricFamilyName: metricName, Help: fmt.Sprintf("a help for %d", j), Unit: "", Type: cortexpb.COUNTER} - metadata = append(metadata, m) - } - } - testData[userID] = metadata - } - - // Append metadata. - for _, userID := range userIDs { - ctx := user.InjectOrgID(context.Background(), userID) - _, err := ing.Push(ctx, cortexpb.ToWriteRequest(nil, nil, testData[userID], cortexpb.API)) - require.NoError(t, err) - } - - return userIDs, testData -} - -func pushTestSamples(t testing.TB, ing *Ingester, numSeries, samplesPerSeries, offset int) ([]string, map[string]model.Matrix) { - userIDs := []string{"1", "2", "3"} - - // Create test samples. - testData := map[string]model.Matrix{} - for i, userID := range userIDs { - testData[userID] = buildTestMatrix(numSeries, samplesPerSeries, i+offset) - } - - // Append samples. - for _, userID := range userIDs { - ctx := user.InjectOrgID(context.Background(), userID) - _, err := ing.Push(ctx, cortexpb.ToWriteRequest(matrixToLables(testData[userID]), matrixToSamples(testData[userID]), nil, cortexpb.API)) - require.NoError(t, err) - } - - return userIDs, testData -} - -func retrieveTestSamples(t *testing.T, ing *Ingester, userIDs []string, testData map[string]model.Matrix) { - // Read samples back via ingester queries. - for _, userID := range userIDs { - ctx := user.InjectOrgID(context.Background(), userID) - res, req, err := runTestQuery(ctx, t, ing, labels.MatchRegexp, model.JobLabel, ".+") - require.NoError(t, err) - assert.Equal(t, testData[userID], res) - - s := stream{ - ctx: ctx, - } - err = ing.QueryStream(req, &s) - require.NoError(t, err) - - res, err = chunkcompat.StreamsToMatrix(model.Earliest, model.Latest, s.responses) - require.NoError(t, err) - assert.Equal(t, testData[userID].String(), res.String()) - } -} - -func TestIngesterAppend(t *testing.T) { - store, ing := newDefaultTestStore(t) - userIDs, testData := pushTestSamples(t, ing, 10, 1000, 0) - retrieveTestSamples(t, ing, userIDs, testData) - - // Read samples back via chunk store. - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing)) - store.checkData(t, userIDs, testData) -} - -func TestIngesterMetadataAppend(t *testing.T) { - for _, tc := range []struct { - desc string - numMetadata int - metadataPerMetric int - expectedMetrics int - expectedMetadata int - err error - }{ - {"with no metadata", 0, 0, 0, 0, nil}, - {"with one metadata per metric", 10, 1, 10, 10, nil}, - {"with multiple metadata per metric", 10, 3, 10, 30, nil}, - } { - t.Run(tc.desc, func(t *testing.T) { - limits := defaultLimitsTestConfig() - limits.MaxLocalMetadataPerMetric = 50 - _, ing := newTestStore(t, defaultIngesterTestConfig(t), defaultClientTestConfig(), limits, nil) - userIDs, _ := pushTestMetadata(t, ing, tc.numMetadata, tc.metadataPerMetric) - - for _, userID := range userIDs { - ctx := user.InjectOrgID(context.Background(), userID) - resp, err := ing.MetricsMetadata(ctx, nil) - - if tc.err != nil { - require.Equal(t, tc.err, err) - } else { - require.NoError(t, err) - require.NotNil(t, resp) - - metricTracker := map[string]bool{} - for _, m := range resp.Metadata { - _, ok := metricTracker[m.GetMetricFamilyName()] - if !ok { - metricTracker[m.GetMetricFamilyName()] = true - } - } - - require.Equal(t, tc.expectedMetrics, len(metricTracker)) - require.Equal(t, tc.expectedMetadata, len(resp.Metadata)) - } - } - }) - } -} - -func TestIngesterPurgeMetadata(t *testing.T) { - cfg := defaultIngesterTestConfig(t) - cfg.MetadataRetainPeriod = 20 * time.Millisecond - _, ing := newTestStore(t, cfg, defaultClientTestConfig(), defaultLimitsTestConfig(), nil) - userIDs, _ := pushTestMetadata(t, ing, 10, 3) - - time.Sleep(40 * time.Millisecond) - for _, userID := range userIDs { - ctx := user.InjectOrgID(context.Background(), userID) - ing.purgeUserMetricsMetadata() - - resp, err := ing.MetricsMetadata(ctx, nil) - require.NoError(t, err) - assert.Equal(t, 0, len(resp.GetMetadata())) - } -} - -func TestIngesterMetadataMetrics(t *testing.T) { - reg := prometheus.NewPedanticRegistry() - cfg := defaultIngesterTestConfig(t) - cfg.MetadataRetainPeriod = 20 * time.Millisecond - _, ing := newTestStore(t, cfg, defaultClientTestConfig(), defaultLimitsTestConfig(), reg) - _, _ = pushTestMetadata(t, ing, 10, 3) - - pushTestMetadata(t, ing, 10, 3) - pushTestMetadata(t, ing, 10, 3) // We push the _exact_ same metrics again to ensure idempotency. Metadata is kept as a set so there shouldn't be a change of metrics. - - metricNames := []string{ - "cortex_ingester_memory_metadata_created_total", - "cortex_ingester_memory_metadata_removed_total", - "cortex_ingester_memory_metadata", - } - - assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` - # HELP cortex_ingester_memory_metadata The current number of metadata in memory. - # TYPE cortex_ingester_memory_metadata gauge - cortex_ingester_memory_metadata 90 - # HELP cortex_ingester_memory_metadata_created_total The total number of metadata that were created per user - # TYPE cortex_ingester_memory_metadata_created_total counter - cortex_ingester_memory_metadata_created_total{user="1"} 30 - cortex_ingester_memory_metadata_created_total{user="2"} 30 - cortex_ingester_memory_metadata_created_total{user="3"} 30 - `), metricNames...)) - - time.Sleep(40 * time.Millisecond) - ing.purgeUserMetricsMetadata() - assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` - # HELP cortex_ingester_memory_metadata The current number of metadata in memory. - # TYPE cortex_ingester_memory_metadata gauge - cortex_ingester_memory_metadata 0 - # HELP cortex_ingester_memory_metadata_created_total The total number of metadata that were created per user - # TYPE cortex_ingester_memory_metadata_created_total counter - cortex_ingester_memory_metadata_created_total{user="1"} 30 - cortex_ingester_memory_metadata_created_total{user="2"} 30 - cortex_ingester_memory_metadata_created_total{user="3"} 30 - # HELP cortex_ingester_memory_metadata_removed_total The total number of metadata that were removed per user. - # TYPE cortex_ingester_memory_metadata_removed_total counter - cortex_ingester_memory_metadata_removed_total{user="1"} 30 - cortex_ingester_memory_metadata_removed_total{user="2"} 30 - cortex_ingester_memory_metadata_removed_total{user="3"} 30 - `), metricNames...)) - -} - -func TestIngesterSendsOnlySeriesWithData(t *testing.T) { - _, ing := newDefaultTestStore(t) - - userIDs, _ := pushTestSamples(t, ing, 10, 1000, 0) - - // Read samples back via ingester queries. - for _, userID := range userIDs { - ctx := user.InjectOrgID(context.Background(), userID) - _, req, err := runTestQueryTimes(ctx, t, ing, labels.MatchRegexp, model.JobLabel, ".+", model.Latest.Add(-15*time.Second), model.Latest) - require.NoError(t, err) - - s := stream{ - ctx: ctx, - } - err = ing.QueryStream(req, &s) - require.NoError(t, err) - - // Nothing should be selected. - require.Equal(t, 0, len(s.responses)) - } - - // Read samples back via chunk store. - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing)) -} - -func TestIngesterIdleFlush(t *testing.T) { - // Create test ingester with short flush cycle - cfg := defaultIngesterTestConfig(t) - cfg.FlushCheckPeriod = 20 * time.Millisecond - cfg.MaxChunkIdle = 100 * time.Millisecond - cfg.RetainPeriod = 500 * time.Millisecond - store, ing := newTestStore(t, cfg, defaultClientTestConfig(), defaultLimitsTestConfig(), nil) - - userIDs, testData := pushTestSamples(t, ing, 4, 100, 0) - - // wait beyond idle time so samples flush - time.Sleep(cfg.MaxChunkIdle * 3) - - store.checkData(t, userIDs, testData) - - // Check data is still retained by ingester - for _, userID := range userIDs { - ctx := user.InjectOrgID(context.Background(), userID) - res, _, err := runTestQuery(ctx, t, ing, labels.MatchRegexp, model.JobLabel, ".+") - require.NoError(t, err) - assert.Equal(t, testData[userID], res) - } - - // now wait beyond retain time so chunks are removed from memory - time.Sleep(cfg.RetainPeriod) - - // Check data has gone from ingester - for _, userID := range userIDs { - ctx := user.InjectOrgID(context.Background(), userID) - res, _, err := runTestQuery(ctx, t, ing, labels.MatchRegexp, model.JobLabel, ".+") - require.NoError(t, err) - assert.Equal(t, model.Matrix{}, res) - } - - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing)) -} - -func TestIngesterSpreadFlush(t *testing.T) { - // Create test ingester with short flush cycle - cfg := defaultIngesterTestConfig(t) - cfg.SpreadFlushes = true - cfg.FlushCheckPeriod = 20 * time.Millisecond - store, ing := newTestStore(t, cfg, defaultClientTestConfig(), defaultLimitsTestConfig(), nil) - - userIDs, testData := pushTestSamples(t, ing, 4, 100, 0) - - // add another sample with timestamp at the end of the cycle to trigger - // head closes and get an extra chunk so we will flush the first one - _, _ = pushTestSamples(t, ing, 4, 1, int(cfg.MaxChunkAge.Seconds()-1)*1000) - - // wait beyond flush time so first set of samples should be sent to store - // (you'd think a shorter wait, like period*2, would work, but Go timers are not reliable enough for that) - time.Sleep(cfg.FlushCheckPeriod * 10) - - // check the first set of samples has been sent to the store - store.checkData(t, userIDs, testData) - - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing)) -} - -type stream struct { - grpc.ServerStream - ctx context.Context - responses []*client.QueryStreamResponse -} - -func (s *stream) Context() context.Context { - return s.ctx -} - -func (s *stream) Send(response *client.QueryStreamResponse) error { - s.responses = append(s.responses, response) - return nil -} - -func TestIngesterAppendOutOfOrderAndDuplicate(t *testing.T) { - _, ing := newDefaultTestStore(t) - defer services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck - - m := labelPairs{ - {Name: model.MetricNameLabel, Value: "testmetric"}, - } - ctx := context.Background() - err := ing.append(ctx, userID, m, 1, 0, cortexpb.API, nil) - require.NoError(t, err) - - // Two times exactly the same sample (noop). - err = ing.append(ctx, userID, m, 1, 0, cortexpb.API, nil) - require.NoError(t, err) - - // Earlier sample than previous one. - err = ing.append(ctx, userID, m, 0, 0, cortexpb.API, nil) - require.Contains(t, err.Error(), "sample timestamp out of order") - errResp, ok := err.(*validationError) - require.True(t, ok) - require.Equal(t, errResp.code, 400) - - // Same timestamp as previous sample, but different value. - err = ing.append(ctx, userID, m, 1, 1, cortexpb.API, nil) - require.Contains(t, err.Error(), "sample with repeated timestamp but different value") - errResp, ok = err.(*validationError) - require.True(t, ok) - require.Equal(t, errResp.code, 400) -} - -// Test that blank labels are removed by the ingester -func TestIngesterAppendBlankLabel(t *testing.T) { - _, ing := newDefaultTestStore(t) - defer services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck - - lp := labelPairs{ - {Name: model.MetricNameLabel, Value: "testmetric"}, - {Name: "foo", Value: ""}, - {Name: "bar", Value: ""}, - } - ctx := user.InjectOrgID(context.Background(), userID) - err := ing.append(ctx, userID, lp, 1, 0, cortexpb.API, nil) - require.NoError(t, err) - - res, _, err := runTestQuery(ctx, t, ing, labels.MatchEqual, labels.MetricName, "testmetric") - require.NoError(t, err) - - expected := model.Matrix{ - { - Metric: model.Metric{labels.MetricName: "testmetric"}, - Values: []model.SamplePair{ - {Timestamp: 1, Value: 0}, - }, - }, - } - - assert.Equal(t, expected, res) -} - func TestIngesterUserLimitExceeded(t *testing.T) { limits := defaultLimitsTestConfig() limits.MaxLocalSeriesPerUser = 1 @@ -524,17 +60,6 @@ func TestIngesterUserLimitExceeded(t *testing.T) { require.NoError(t, os.Mkdir(chunksDir, os.ModePerm)) require.NoError(t, os.Mkdir(blocksDir, os.ModePerm)) - chunksIngesterGenerator := func() *Ingester { - cfg := defaultIngesterTestConfig(t) - cfg.WALConfig.WALEnabled = true - cfg.WALConfig.Recover = true - cfg.WALConfig.Dir = chunksDir - cfg.WALConfig.CheckpointDuration = 100 * time.Minute - - _, ing := newTestStore(t, cfg, defaultClientTestConfig(), limits, nil) - return ing - } - blocksIngesterGenerator := func() *Ingester { ing, err := prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, blocksDir, nil) require.NoError(t, err) @@ -547,8 +72,8 @@ func TestIngesterUserLimitExceeded(t *testing.T) { return ing } - tests := []string{"chunks", "blocks"} - for i, ingGenerator := range []func() *Ingester{chunksIngesterGenerator, blocksIngesterGenerator} { + tests := []string{"blocks"} + for i, ingGenerator := range []func() *Ingester{blocksIngesterGenerator} { t.Run(tests[i], func(t *testing.T) { ing := ingGenerator() @@ -631,6 +156,20 @@ func TestIngesterUserLimitExceeded(t *testing.T) { } +func benchmarkData(nSeries int) (allLabels []labels.Labels, allSamples []cortexpb.Sample) { + for j := 0; j < nSeries; j++ { + labels := chunk.BenchmarkLabels.Copy() + for i := range labels { + if labels[i].Name == "cpu" { + labels[i].Value = fmt.Sprintf("cpu%02d", j) + } + } + allLabels = append(allLabels, labels) + allSamples = append(allSamples, cortexpb.Sample{TimestampMs: 0, Value: float64(j)}) + } + return +} + func TestIngesterMetricLimitExceeded(t *testing.T) { limits := defaultLimitsTestConfig() limits.MaxLocalSeriesPerMetric = 1 @@ -643,17 +182,6 @@ func TestIngesterMetricLimitExceeded(t *testing.T) { require.NoError(t, os.Mkdir(chunksDir, os.ModePerm)) require.NoError(t, os.Mkdir(blocksDir, os.ModePerm)) - chunksIngesterGenerator := func() *Ingester { - cfg := defaultIngesterTestConfig(t) - cfg.WALConfig.WALEnabled = true - cfg.WALConfig.Recover = true - cfg.WALConfig.Dir = chunksDir - cfg.WALConfig.CheckpointDuration = 100 * time.Minute - - _, ing := newTestStore(t, cfg, defaultClientTestConfig(), limits, nil) - return ing - } - blocksIngesterGenerator := func() *Ingester { ing, err := prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, blocksDir, nil) require.NoError(t, err) @@ -667,7 +195,7 @@ func TestIngesterMetricLimitExceeded(t *testing.T) { } tests := []string{"chunks", "blocks"} - for i, ingGenerator := range []func() *Ingester{chunksIngesterGenerator, blocksIngesterGenerator} { + for i, ingGenerator := range []func() *Ingester{blocksIngesterGenerator} { t.Run(tests[i], func(t *testing.T) { ing := ingGenerator() @@ -749,300 +277,6 @@ func TestIngesterMetricLimitExceeded(t *testing.T) { } } -func TestIngesterValidation(t *testing.T) { - _, ing := newDefaultTestStore(t) - defer services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck - userID := "1" - ctx := user.InjectOrgID(context.Background(), userID) - m := labelPairs{{Name: labels.MetricName, Value: "testmetric"}} - - // As a setup, let's append samples. - err := ing.append(context.Background(), userID, m, 1, 0, cortexpb.API, nil) - require.NoError(t, err) - - for _, tc := range []struct { - desc string - lbls []labels.Labels - samples []cortexpb.Sample - err error - }{ - { - desc: "With multiple append failures, only return the first error.", - lbls: []labels.Labels{ - {{Name: labels.MetricName, Value: "testmetric"}}, - {{Name: labels.MetricName, Value: "testmetric"}}, - }, - samples: []cortexpb.Sample{ - {TimestampMs: 0, Value: 0}, // earlier timestamp, out of order. - {TimestampMs: 1, Value: 2}, // same timestamp different value. - }, - err: httpgrpc.Errorf(http.StatusBadRequest, `user=1: sample timestamp out of order; last timestamp: 0.001, incoming timestamp: 0 for series {__name__="testmetric"}`), - }, - } { - t.Run(tc.desc, func(t *testing.T) { - _, err := ing.Push(ctx, cortexpb.ToWriteRequest(tc.lbls, tc.samples, nil, cortexpb.API)) - require.Equal(t, tc.err, err) - }) - } -} - -func BenchmarkIngesterSeriesCreationLocking(b *testing.B) { - for i := 1; i <= 32; i++ { - b.Run(strconv.Itoa(i), func(b *testing.B) { - for n := 0; n < b.N; n++ { - benchmarkIngesterSeriesCreationLocking(b, i) - } - }) - } -} - -func benchmarkIngesterSeriesCreationLocking(b *testing.B, parallelism int) { - _, ing := newDefaultTestStore(b) - defer services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck - - var ( - wg sync.WaitGroup - series = int(1e4) - ctx = context.Background() - ) - wg.Add(parallelism) - ctx = user.InjectOrgID(ctx, "1") - for i := 0; i < parallelism; i++ { - seriesPerGoroutine := series / parallelism - go func(from, through int) { - defer wg.Done() - - for j := from; j < through; j++ { - _, err := ing.Push(ctx, &cortexpb.WriteRequest{ - Timeseries: []cortexpb.PreallocTimeseries{ - { - TimeSeries: &cortexpb.TimeSeries{ - Labels: []cortexpb.LabelAdapter{ - {Name: model.MetricNameLabel, Value: fmt.Sprintf("metric_%d", j)}, - }, - Samples: []cortexpb.Sample{ - {TimestampMs: int64(j), Value: float64(j)}, - }, - }, - }, - }, - }) - require.NoError(b, err) - } - - }(i*seriesPerGoroutine, (i+1)*seriesPerGoroutine) - } - - wg.Wait() -} - -func BenchmarkIngesterPush(b *testing.B) { - limits := defaultLimitsTestConfig() - benchmarkIngesterPush(b, limits, false) -} - -func BenchmarkIngesterPushErrors(b *testing.B) { - limits := defaultLimitsTestConfig() - limits.MaxLocalSeriesPerMetric = 1 - benchmarkIngesterPush(b, limits, true) -} - -// Construct a set of realistic-looking samples, all with slightly different label sets -func benchmarkData(nSeries int) (allLabels []labels.Labels, allSamples []cortexpb.Sample) { - for j := 0; j < nSeries; j++ { - labels := chunk.BenchmarkLabels.Copy() - for i := range labels { - if labels[i].Name == "cpu" { - labels[i].Value = fmt.Sprintf("cpu%02d", j) - } - } - allLabels = append(allLabels, labels) - allSamples = append(allSamples, cortexpb.Sample{TimestampMs: 0, Value: float64(j)}) - } - return -} - -func benchmarkIngesterPush(b *testing.B, limits validation.Limits, errorsExpected bool) { - cfg := defaultIngesterTestConfig(b) - clientCfg := defaultClientTestConfig() - - const ( - series = 100 - samples = 100 - ) - - allLabels, allSamples := benchmarkData(series) - ctx := user.InjectOrgID(context.Background(), "1") - - encodings := []struct { - name string - e promchunk.Encoding - }{ - {"DoubleDelta", promchunk.DoubleDelta}, - {"Varbit", promchunk.Varbit}, - {"Bigchunk", promchunk.Bigchunk}, - } - - for _, enc := range encodings { - b.Run(fmt.Sprintf("encoding=%s", enc.name), func(b *testing.B) { - b.ResetTimer() - for iter := 0; iter < b.N; iter++ { - _, ing := newTestStore(b, cfg, clientCfg, limits, nil) - // Bump the timestamp on each of our test samples each time round the loop - for j := 0; j < samples; j++ { - for i := range allSamples { - allSamples[i].TimestampMs = int64(j + 1) - } - _, err := ing.Push(ctx, cortexpb.ToWriteRequest(allLabels, allSamples, nil, cortexpb.API)) - if !errorsExpected { - require.NoError(b, err) - } - } - _ = services.StopAndAwaitTerminated(context.Background(), ing) - } - }) - } - -} - -func BenchmarkIngester_QueryStream(b *testing.B) { - cfg := defaultIngesterTestConfig(b) - clientCfg := defaultClientTestConfig() - limits := defaultLimitsTestConfig() - _, ing := newTestStore(b, cfg, clientCfg, limits, nil) - ctx := user.InjectOrgID(context.Background(), "1") - - const ( - series = 2000 - samples = 1000 - ) - - allLabels, allSamples := benchmarkData(series) - - // Bump the timestamp and set a random value on each of our test samples each time round the loop - for j := 0; j < samples; j++ { - for i := range allSamples { - allSamples[i].TimestampMs = int64(j + 1) - allSamples[i].Value = rand.Float64() - } - _, err := ing.Push(ctx, cortexpb.ToWriteRequest(allLabels, allSamples, nil, cortexpb.API)) - require.NoError(b, err) - } - - req := &client.QueryRequest{ - StartTimestampMs: 0, - EndTimestampMs: samples + 1, - - Matchers: []*client.LabelMatcher{{ - Type: client.EQUAL, - Name: model.MetricNameLabel, - Value: "container_cpu_usage_seconds_total", - }}, - } - - mockStream := &mockQueryStreamServer{ctx: ctx} - - b.ResetTimer() - - for ix := 0; ix < b.N; ix++ { - err := ing.QueryStream(req, mockStream) - require.NoError(b, err) - } -} - -func TestIngesterActiveSeries(t *testing.T) { - metricLabelAdapters := []cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test"}} - metricLabels := cortexpb.FromLabelAdaptersToLabels(metricLabelAdapters) - metricNames := []string{ - "cortex_ingester_active_series", - } - userID := "test" - - tests := map[string]struct { - reqs []*cortexpb.WriteRequest - expectedMetrics string - disableActiveSeries bool - }{ - "should succeed on valid series and metadata": { - reqs: []*cortexpb.WriteRequest{ - cortexpb.ToWriteRequest( - []labels.Labels{metricLabels}, - []cortexpb.Sample{{Value: 1, TimestampMs: 9}}, - nil, - cortexpb.API), - cortexpb.ToWriteRequest( - []labels.Labels{metricLabels}, - []cortexpb.Sample{{Value: 2, TimestampMs: 10}}, - nil, - cortexpb.API), - }, - expectedMetrics: ` - # HELP cortex_ingester_active_series Number of currently active series per user. - # TYPE cortex_ingester_active_series gauge - cortex_ingester_active_series{user="test"} 1 - `, - }, - "successful push, active series disabled": { - disableActiveSeries: true, - reqs: []*cortexpb.WriteRequest{ - cortexpb.ToWriteRequest( - []labels.Labels{metricLabels}, - []cortexpb.Sample{{Value: 1, TimestampMs: 9}}, - nil, - cortexpb.API), - cortexpb.ToWriteRequest( - []labels.Labels{metricLabels}, - []cortexpb.Sample{{Value: 2, TimestampMs: 10}}, - nil, - cortexpb.API), - }, - expectedMetrics: ``, - }, - } - - for testName, testData := range tests { - t.Run(testName, func(t *testing.T) { - registry := prometheus.NewRegistry() - - // Create a mocked ingester - cfg := defaultIngesterTestConfig(t) - cfg.LifecyclerConfig.JoinAfter = 0 - cfg.ActiveSeriesMetricsEnabled = !testData.disableActiveSeries - - _, i := newTestStore(t, - cfg, - defaultClientTestConfig(), - defaultLimitsTestConfig(), registry) - - defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck - - ctx := user.InjectOrgID(context.Background(), userID) - - // Wait until the ingester is ACTIVE - test.Poll(t, 100*time.Millisecond, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - // Push timeseries - for _, req := range testData.reqs { - _, err := i.Push(ctx, req) - assert.NoError(t, err) - } - - // Update active series for metrics check. - if !testData.disableActiveSeries { - i.userStatesMtx.Lock() - i.userStates.purgeAndUpdateActiveSeries(time.Now().Add(-i.cfg.ActiveSeriesMetricsIdleTimeout)) - i.userStatesMtx.Unlock() - } - - // Check tracked Prometheus metrics - err := testutil.GatherAndCompare(registry, strings.NewReader(testData.expectedMetrics), metricNames...) - assert.NoError(t, err) - }) - } -} - func TestGetIgnoreSeriesLimitForMetricNamesMap(t *testing.T) { cfg := Config{} diff --git a/pkg/ingester/ingester_v2.go b/pkg/ingester/ingester_v2.go index b35411b72d..a2e08803a1 100644 --- a/pkg/ingester/ingester_v2.go +++ b/pkg/ingester/ingester_v2.go @@ -481,7 +481,7 @@ func newTSDBState(bucketClient objstore.Bucket, registerer prometheus.Registerer } // NewV2 returns a new Ingester that uses Cortex block storage instead of chunks storage. -func NewV2(cfg Config, clientConfig client.Config, limits *validation.Overrides, registerer prometheus.Registerer, logger log.Logger) (*Ingester, error) { +func NewV2(cfg Config, limits *validation.Overrides, registerer prometheus.Registerer, logger log.Logger) (*Ingester, error) { bucketClient, err := bucket.NewClient(context.Background(), cfg.BlocksStorageConfig.Bucket, "ingester", logger, registerer) if err != nil { return nil, errors.Wrap(err, "failed to create the bucket client") @@ -489,11 +489,8 @@ func NewV2(cfg Config, clientConfig client.Config, limits *validation.Overrides, i := &Ingester{ cfg: cfg, - clientConfig: clientConfig, limits: limits, - chunkStore: nil, usersMetadata: map[string]*userMetricsMetadata{}, - wal: &noopWAL{}, TSDBState: newTSDBState(bucketClient, registerer), logger: logger, ingestionRate: util_math.NewEWMARate(0.2, instanceIngestionRateTickInterval), @@ -553,7 +550,6 @@ func NewV2ForFlusher(cfg Config, limits *validation.Overrides, registerer promet i := &Ingester{ cfg: cfg, limits: limits, - wal: &noopWAL{}, TSDBState: newTSDBState(bucketClient, registerer), logger: logger, } @@ -681,12 +677,12 @@ func (i *Ingester) updateLoop(ctx context.Context) error { case <-ingestionRateTicker.C: i.ingestionRate.Tick() case <-rateUpdateTicker.C: - i.userStatesMtx.RLock() + i.stoppedMtx.RLock() for _, db := range i.TSDBState.dbs { db.ingestedAPISamples.Tick() db.ingestedRuleSamples.Tick() } - i.userStatesMtx.RUnlock() + i.stoppedMtx.RUnlock() case <-activeSeriesTickerChan: i.v2UpdateActiveSeries() @@ -745,12 +741,12 @@ func (i *Ingester) v2Push(ctx context.Context, req *cortexpb.WriteRequest) (*cor } // Ensure the ingester shutdown procedure hasn't started - i.userStatesMtx.RLock() + i.stoppedMtx.RLock() if i.stopped { - i.userStatesMtx.RUnlock() + i.stoppedMtx.RUnlock() return nil, errIngesterStopping } - i.userStatesMtx.RUnlock() + i.stoppedMtx.RUnlock() if err := db.acquireAppendLock(); err != nil { return &cortexpb.WriteResponse{}, httpgrpc.Errorf(http.StatusServiceUnavailable, wrapWithUser(err, userID).Error()) @@ -1337,8 +1333,8 @@ func (i *Ingester) v2AllUserStats(ctx context.Context, req *client.UserStatsRequ return nil, err } - i.userStatesMtx.RLock() - defer i.userStatesMtx.RUnlock() + i.stoppedMtx.RLock() + defer i.stoppedMtx.RUnlock() users := i.TSDBState.dbs @@ -1589,8 +1585,8 @@ func (i *Ingester) v2QueryStreamChunks(ctx context.Context, db *userTSDB, from, } func (i *Ingester) getTSDB(userID string) *userTSDB { - i.userStatesMtx.RLock() - defer i.userStatesMtx.RUnlock() + i.stoppedMtx.RLock() + defer i.stoppedMtx.RUnlock() db := i.TSDBState.dbs[userID] return db } @@ -1598,8 +1594,8 @@ func (i *Ingester) getTSDB(userID string) *userTSDB { // List all users for which we have a TSDB. We do it here in order // to keep the mutex locked for the shortest time possible. func (i *Ingester) getTSDBUsers() []string { - i.userStatesMtx.RLock() - defer i.userStatesMtx.RUnlock() + i.stoppedMtx.RLock() + defer i.stoppedMtx.RUnlock() ids := make([]string, 0, len(i.TSDBState.dbs)) for userID := range i.TSDBState.dbs { @@ -1615,8 +1611,8 @@ func (i *Ingester) getOrCreateTSDB(userID string, force bool) (*userTSDB, error) return db, nil } - i.userStatesMtx.Lock() - defer i.userStatesMtx.Unlock() + i.stoppedMtx.Lock() + defer i.stoppedMtx.Unlock() // Check again for DB in the event it was created in-between locks var ok bool @@ -1759,7 +1755,7 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { } func (i *Ingester) closeAllTSDB() { - i.userStatesMtx.Lock() + i.stoppedMtx.Lock() wg := &sync.WaitGroup{} wg.Add(len(i.TSDBState.dbs)) @@ -1780,9 +1776,9 @@ func (i *Ingester) closeAllTSDB() { // set of open ones. This lock acquisition doesn't deadlock with the // outer one, because the outer one is released as soon as all go // routines are started. - i.userStatesMtx.Lock() + i.stoppedMtx.Lock() delete(i.TSDBState.dbs, userID) - i.userStatesMtx.Unlock() + i.stoppedMtx.Unlock() i.metrics.memUsers.Dec() i.metrics.activeSeriesPerUser.DeleteLabelValues(userID) @@ -1790,7 +1786,7 @@ func (i *Ingester) closeAllTSDB() { } // Wait until all Close() completed - i.userStatesMtx.Unlock() + i.stoppedMtx.Unlock() wg.Wait() } @@ -1815,9 +1811,9 @@ func (i *Ingester) openExistingTSDB(ctx context.Context) error { } // Add the database to the map of user databases - i.userStatesMtx.Lock() + i.stoppedMtx.Lock() i.TSDBState.dbs[userID] = db - i.userStatesMtx.Unlock() + i.stoppedMtx.Unlock() i.metrics.memUsers.Inc() i.TSDBState.walReplayTime.Observe(time.Since(startTime).Seconds()) @@ -1900,8 +1896,8 @@ func (i *Ingester) getMemorySeriesMetric() float64 { return 0 } - i.userStatesMtx.RLock() - defer i.userStatesMtx.RUnlock() + i.stoppedMtx.RLock() + defer i.stoppedMtx.RUnlock() count := uint64(0) for _, db := range i.TSDBState.dbs { @@ -1914,8 +1910,8 @@ func (i *Ingester) getMemorySeriesMetric() float64 { // getOldestUnshippedBlockMetric returns the unix timestamp of the oldest unshipped block or // 0 if all blocks have been shipped. func (i *Ingester) getOldestUnshippedBlockMetric() float64 { - i.userStatesMtx.RLock() - defer i.userStatesMtx.RUnlock() + i.stoppedMtx.RLock() + defer i.stoppedMtx.RUnlock() oldest := uint64(0) for _, db := range i.TSDBState.dbs { @@ -2172,9 +2168,9 @@ func (i *Ingester) closeAndDeleteUserTSDBIfIdle(userID string) tsdbCloseCheckRes // If this happens now, the request will get reject as the push will not be able to acquire the lock as the tsdb will be // in closed state defer func() { - i.userStatesMtx.Lock() + i.stoppedMtx.Lock() delete(i.TSDBState.dbs, userID) - i.userStatesMtx.Unlock() + i.stoppedMtx.Unlock() }() i.metrics.memUsers.Dec() diff --git a/pkg/ingester/ingester_v2_test.go b/pkg/ingester/ingester_v2_test.go index a6c9644276..a561bca83a 100644 --- a/pkg/ingester/ingester_v2_test.go +++ b/pkg/ingester/ingester_v2_test.go @@ -2173,8 +2173,6 @@ func prepareIngesterWithBlocksStorageAndLimits(t testing.TB, ingesterCfg Config, bucketDir := t.TempDir() - clientCfg := defaultClientTestConfig() - overrides, err := validation.NewOverrides(limits, nil) if err != nil { return nil, err @@ -2185,7 +2183,7 @@ func prepareIngesterWithBlocksStorageAndLimits(t testing.TB, ingesterCfg Config, ingesterCfg.BlocksStorageConfig.Bucket.Backend = "filesystem" ingesterCfg.BlocksStorageConfig.Bucket.Filesystem.Directory = bucketDir - ingester, err := NewV2(ingesterCfg, clientCfg, overrides, registerer, log.NewNopLogger()) + ingester, err := NewV2(ingesterCfg, overrides, registerer, log.NewNopLogger()) if err != nil { return nil, err } @@ -2309,7 +2307,6 @@ func TestIngester_v2OpenExistingTSDBOnStartup(t *testing.T) { testName := name testData := test t.Run(testName, func(t *testing.T) { - clientCfg := defaultClientTestConfig() limits := defaultLimitsTestConfig() overrides, err := validation.NewOverrides(limits, nil) @@ -2328,7 +2325,7 @@ func TestIngester_v2OpenExistingTSDBOnStartup(t *testing.T) { // setup the tsdbs dir testData.setup(t, tempDir) - ingester, err := NewV2(ingesterCfg, clientCfg, overrides, nil, log.NewNopLogger()) + ingester, err := NewV2(ingesterCfg, overrides, nil, log.NewNopLogger()) require.NoError(t, err) startErr := services.StartAndAwaitRunning(context.Background(), ingester) @@ -3233,8 +3230,8 @@ func TestIngesterCompactAndCloseIdleTSDB(t *testing.T) { // Wait until TSDB has been closed and removed. test.Poll(t, 10*time.Second, 0, func() interface{} { - i.userStatesMtx.Lock() - defer i.userStatesMtx.Unlock() + i.stoppedMtx.Lock() + defer i.stoppedMtx.Unlock() return len(i.TSDBState.dbs) }) @@ -3358,7 +3355,6 @@ func TestHeadCompactionOnStartup(t *testing.T) { require.NoError(t, db.Close()) } - clientCfg := defaultClientTestConfig() limits := defaultLimitsTestConfig() overrides, err := validation.NewOverrides(limits, nil) @@ -3371,7 +3367,7 @@ func TestHeadCompactionOnStartup(t *testing.T) { ingesterCfg.BlocksStorageConfig.Bucket.S3.Endpoint = "localhost" ingesterCfg.BlocksStorageConfig.TSDB.Retention = 2 * 24 * time.Hour // Make sure that no newly created blocks are deleted. - ingester, err := NewV2(ingesterCfg, clientCfg, overrides, nil, log.NewNopLogger()) + ingester, err := NewV2(ingesterCfg, overrides, nil, log.NewNopLogger()) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), ingester)) diff --git a/pkg/ingester/label_pairs.go b/pkg/ingester/label_pairs.go deleted file mode 100644 index bd0e8af632..0000000000 --- a/pkg/ingester/label_pairs.go +++ /dev/null @@ -1,90 +0,0 @@ -package ingester - -import ( - "sort" - "strings" - - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - - "github.com/cortexproject/cortex/pkg/cortexpb" - "github.com/cortexproject/cortex/pkg/util/extract" -) - -// A series is uniquely identified by its set of label name/value -// pairs, which may arrive in any order over the wire -type labelPairs []cortexpb.LabelAdapter - -func (a labelPairs) String() string { - var b strings.Builder - - metricName, err := extract.MetricNameFromLabelAdapters(a) - numLabels := len(a) - 1 - if err != nil { - numLabels = len(a) - } - b.WriteString(metricName) - b.WriteByte('{') - count := 0 - for _, pair := range a { - if pair.Name != model.MetricNameLabel { - b.WriteString(pair.Name) - b.WriteString("=\"") - b.WriteString(pair.Value) - b.WriteByte('"') - count++ - if count < numLabels { - b.WriteByte(',') - } - } - } - b.WriteByte('}') - return b.String() -} - -// Remove any label where the value is "" - Prometheus 2+ will remove these -// before sending, but other clients such as Prometheus 1.x might send us blanks. -func (a *labelPairs) removeBlanks() { - for i := 0; i < len(*a); { - if len((*a)[i].Value) == 0 { - // Delete by swap with the value at the end of the slice - (*a)[i] = (*a)[len(*a)-1] - (*a) = (*a)[:len(*a)-1] - continue // go round and check the data that is now at position i - } - i++ - } -} - -func valueForName(s labels.Labels, name string) (string, bool) { - pos := sort.Search(len(s), func(i int) bool { return s[i].Name >= name }) - if pos == len(s) || s[pos].Name != name { - return "", false - } - return s[pos].Value, true -} - -// Check if a and b contain the same name/value pairs -func (a labelPairs) equal(b labels.Labels) bool { - if len(a) != len(b) { - return false - } - // Check as many as we can where the two sets are in the same order - i := 0 - for ; i < len(a); i++ { - if b[i].Name != string(a[i].Name) { - break - } - if b[i].Value != string(a[i].Value) { - return false - } - } - // Now check remaining values using binary search - for ; i < len(a); i++ { - v, found := valueForName(b, a[i].Name) - if !found || v != a[i].Value { - return false - } - } - return true -} diff --git a/pkg/ingester/label_pairs_test.go b/pkg/ingester/label_pairs_test.go deleted file mode 100644 index bb2a8641a7..0000000000 --- a/pkg/ingester/label_pairs_test.go +++ /dev/null @@ -1,106 +0,0 @@ -package ingester - -import ( - "testing" - - "github.com/prometheus/prometheus/model/labels" -) - -func TestLabelPairsEqual(t *testing.T) { - for _, test := range []struct { - name string - a labelPairs - b labels.Labels - equal bool - }{ - { - name: "both blank", - a: labelPairs{}, - b: labels.Labels{}, - equal: true, - }, - { - name: "labelPairs nonblank; labels blank", - a: labelPairs{ - {Name: "foo", Value: "a"}, - }, - b: labels.Labels{}, - equal: false, - }, - { - name: "labelPairs blank; labels nonblank", - a: labelPairs{}, - b: labels.Labels{ - {Name: "foo", Value: "a"}, - }, - equal: false, - }, - { - name: "same contents; labelPairs not sorted", - a: labelPairs{ - {Name: "foo", Value: "a"}, - {Name: "bar", Value: "b"}, - }, - b: labels.Labels{ - {Name: "bar", Value: "b"}, - {Name: "foo", Value: "a"}, - }, - equal: true, - }, - { - name: "same contents", - a: labelPairs{ - {Name: "bar", Value: "b"}, - {Name: "foo", Value: "a"}, - }, - b: labels.Labels{ - {Name: "bar", Value: "b"}, - {Name: "foo", Value: "a"}, - }, - equal: true, - }, - { - name: "same names, different value", - a: labelPairs{ - {Name: "bar", Value: "b"}, - {Name: "foo", Value: "c"}, - }, - b: labels.Labels{ - {Name: "bar", Value: "b"}, - {Name: "foo", Value: "a"}, - }, - equal: false, - }, - { - name: "labels has one extra value", - a: labelPairs{ - {Name: "bar", Value: "b"}, - {Name: "foo", Value: "a"}, - }, - b: labels.Labels{ - {Name: "bar", Value: "b"}, - {Name: "foo", Value: "a"}, - {Name: "firble", Value: "c"}, - }, - equal: false, - }, - { - name: "labelPairs has one extra value", - a: labelPairs{ - {Name: "bar", Value: "b"}, - {Name: "foo", Value: "a"}, - {Name: "firble", Value: "c"}, - }, - b: labels.Labels{ - {Name: "bar", Value: "b"}, - {Name: "foo", Value: "a"}, - {Name: "firble", Value: "a"}, - }, - equal: false, - }, - } { - if test.a.equal(test.b) != test.equal { - t.Errorf("%s: expected equal=%t", test.name, test.equal) - } - } -} diff --git a/pkg/ingester/lifecycle_test.go b/pkg/ingester/lifecycle_test.go index 5298fa75c7..a688cb7707 100644 --- a/pkg/ingester/lifecycle_test.go +++ b/pkg/ingester/lifecycle_test.go @@ -3,24 +3,17 @@ package ingester import ( "context" "fmt" - "io" - "math" "net/http" "net/http/httptest" "testing" "time" + "github.com/prometheus/client_golang/prometheus" + "github.com/go-kit/log" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/weaveworks/common/user" - "google.golang.org/grpc" - "google.golang.org/grpc/health/grpc_health_v1" - "github.com/cortexproject/cortex/pkg/chunk" - "github.com/cortexproject/cortex/pkg/cortexpb" "github.com/cortexproject/cortex/pkg/ingester/client" "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/ring/kv" @@ -71,12 +64,12 @@ func defaultLimitsTestConfig() validation.Limits { // TestIngesterRestart tests a restarting ingester doesn't keep adding more tokens. func TestIngesterRestart(t *testing.T) { config := defaultIngesterTestConfig(t) - clientConfig := defaultClientTestConfig() - limits := defaultLimitsTestConfig() config.LifecyclerConfig.UnregisterOnShutdown = false { - _, ingester := newTestStore(t, config, clientConfig, limits, nil) + ingester, err := prepareIngesterWithBlocksStorage(t, config, prometheus.NewRegistry()) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), ingester)) time.Sleep(100 * time.Millisecond) // Doesn't actually unregister due to UnregisterFromRing: false. require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ingester)) @@ -87,7 +80,9 @@ func TestIngesterRestart(t *testing.T) { }) { - _, ingester := newTestStore(t, config, clientConfig, limits, nil) + ingester, err := prepareIngesterWithBlocksStorage(t, config, prometheus.NewRegistry()) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), ingester)) time.Sleep(100 * time.Millisecond) // Doesn't actually unregister due to UnregisterFromRing: false. require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ingester)) @@ -103,11 +98,12 @@ func TestIngesterRestart(t *testing.T) { func TestIngester_ShutdownHandler(t *testing.T) { for _, unregister := range []bool{false, true} { t.Run(fmt.Sprintf("unregister=%t", unregister), func(t *testing.T) { + registry := prometheus.NewRegistry() config := defaultIngesterTestConfig(t) - clientConfig := defaultClientTestConfig() - limits := defaultLimitsTestConfig() config.LifecyclerConfig.UnregisterOnShutdown = unregister - _, ingester := newTestStore(t, config, clientConfig, limits, nil) + ingester, err := prepareIngesterWithBlocksStorage(t, config, registry) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), ingester)) // Make sure the ingester has been added to the ring. test.Poll(t, 100*time.Millisecond, 1, func() interface{} { @@ -126,243 +122,6 @@ func TestIngester_ShutdownHandler(t *testing.T) { } } -func TestIngesterChunksTransfer(t *testing.T) { - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) - require.NoError(t, err) - - // Start the first ingester, and get it into ACTIVE state. - cfg1 := defaultIngesterTestConfig(t) - cfg1.LifecyclerConfig.ID = "ingester1" - cfg1.LifecyclerConfig.Addr = "ingester1" - cfg1.LifecyclerConfig.JoinAfter = 0 * time.Second - cfg1.MaxTransferRetries = 10 - ing1, err := New(cfg1, defaultClientTestConfig(), limits, nil, nil, log.NewNopLogger()) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing1)) - - test.Poll(t, 100*time.Millisecond, ring.ACTIVE, func() interface{} { - return ing1.lifecycler.GetState() - }) - - // Now write a sample to this ingester - req, expectedResponse, _, _ := mockWriteRequest(t, labels.Labels{{Name: labels.MetricName, Value: "foo"}}, 456, 123000) - ctx := user.InjectOrgID(context.Background(), userID) - _, err = ing1.Push(ctx, req) - require.NoError(t, err) - - // Start a second ingester, but let it go into PENDING - cfg2 := defaultIngesterTestConfig(t) - cfg2.LifecyclerConfig.RingConfig.KVStore.Mock = cfg1.LifecyclerConfig.RingConfig.KVStore.Mock - cfg2.LifecyclerConfig.ID = "ingester2" - cfg2.LifecyclerConfig.Addr = "ingester2" - cfg2.LifecyclerConfig.JoinAfter = 100 * time.Second - ing2, err := New(cfg2, defaultClientTestConfig(), limits, nil, nil, log.NewNopLogger()) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing2)) - - // Let ing2 send chunks to ing1 - ing1.cfg.ingesterClientFactory = func(addr string, _ client.Config) (client.HealthAndIngesterClient, error) { - return ingesterClientAdapater{ - ingester: ing2, - }, nil - } - - // Now stop the first ingester, and wait for the second ingester to become ACTIVE. - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing1)) - - test.Poll(t, 10*time.Second, ring.ACTIVE, func() interface{} { - return ing2.lifecycler.GetState() - }) - - // And check the second ingester has the sample - matcher, err := labels.NewMatcher(labels.MatchEqual, model.MetricNameLabel, "foo") - require.NoError(t, err) - - request, err := client.ToQueryRequest(model.TimeFromUnix(0), model.TimeFromUnix(200), []*labels.Matcher{matcher}) - require.NoError(t, err) - - response, err := ing2.Query(ctx, request) - require.NoError(t, err) - assert.Equal(t, expectedResponse, response) - - // Check we can send the same sample again to the new ingester and get the same result - req, _, _, _ = mockWriteRequest(t, labels.Labels{{Name: labels.MetricName, Value: "foo"}}, 456, 123000) - _, err = ing2.Push(ctx, req) - require.NoError(t, err) - response, err = ing2.Query(ctx, request) - require.NoError(t, err) - assert.Equal(t, expectedResponse, response) -} - -func TestIngesterBadTransfer(t *testing.T) { - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) - require.NoError(t, err) - - // Start ingester in PENDING. - cfg := defaultIngesterTestConfig(t) - cfg.LifecyclerConfig.ID = "ingester1" - cfg.LifecyclerConfig.Addr = "ingester1" - cfg.LifecyclerConfig.JoinAfter = 100 * time.Second - ing, err := New(cfg, defaultClientTestConfig(), limits, nil, nil, log.NewNopLogger()) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) - - test.Poll(t, 100*time.Millisecond, ring.PENDING, func() interface{} { - return ing.lifecycler.GetState() - }) - - // Now transfer 0 series to this ingester, ensure it errors. - client := ingesterClientAdapater{ingester: ing} - stream, err := client.TransferChunks(context.Background()) - require.NoError(t, err) - _, err = stream.CloseAndRecv() - require.Error(t, err) - - // Check the ingester is still waiting. - require.Equal(t, ring.PENDING, ing.lifecycler.GetState()) -} - -type ingesterTransferChunkStreamMock struct { - ctx context.Context - reqs chan *client.TimeSeriesChunk - resp chan *client.TransferChunksResponse - err chan error - - grpc.ServerStream - grpc.ClientStream -} - -func (s *ingesterTransferChunkStreamMock) Send(tsc *client.TimeSeriesChunk) error { - s.reqs <- tsc - return nil -} - -func (s *ingesterTransferChunkStreamMock) CloseAndRecv() (*client.TransferChunksResponse, error) { - close(s.reqs) - select { - case resp := <-s.resp: - return resp, nil - case err := <-s.err: - return nil, err - } -} - -func (s *ingesterTransferChunkStreamMock) SendAndClose(resp *client.TransferChunksResponse) error { - s.resp <- resp - return nil -} - -func (s *ingesterTransferChunkStreamMock) ErrorAndClose(err error) { - s.err <- err -} - -func (s *ingesterTransferChunkStreamMock) Recv() (*client.TimeSeriesChunk, error) { - req, ok := <-s.reqs - if !ok { - return nil, io.EOF - } - return req, nil -} - -func (s *ingesterTransferChunkStreamMock) Context() context.Context { - return s.ctx -} - -func (*ingesterTransferChunkStreamMock) SendMsg(m interface{}) error { - return nil -} - -func (*ingesterTransferChunkStreamMock) RecvMsg(m interface{}) error { - return nil -} - -type ingesterClientAdapater struct { - client.IngesterClient - grpc_health_v1.HealthClient - ingester client.IngesterServer -} - -func (i ingesterClientAdapater) TransferChunks(ctx context.Context, _ ...grpc.CallOption) (client.Ingester_TransferChunksClient, error) { - stream := &ingesterTransferChunkStreamMock{ - ctx: ctx, - reqs: make(chan *client.TimeSeriesChunk), - resp: make(chan *client.TransferChunksResponse), - err: make(chan error), - } - go func() { - err := i.ingester.TransferChunks(stream) - if err != nil { - stream.ErrorAndClose(err) - } - }() - return stream, nil -} - -func (i ingesterClientAdapater) Close() error { - return nil -} - -func (i ingesterClientAdapater) Check(ctx context.Context, in *grpc_health_v1.HealthCheckRequest, opts ...grpc.CallOption) (*grpc_health_v1.HealthCheckResponse, error) { - return nil, nil -} - -// TestIngesterFlush tries to test that the ingester flushes chunks before -// removing itself from the ring. -func TestIngesterFlush(t *testing.T) { - // Start the ingester, and get it into ACTIVE state. - store, ing := newDefaultTestStore(t) - - test.Poll(t, 100*time.Millisecond, ring.ACTIVE, func() interface{} { - return ing.lifecycler.GetState() - }) - - // Now write a sample to this ingester - var ( - lbls = []labels.Labels{{{Name: labels.MetricName, Value: "foo"}}} - sampleData = []cortexpb.Sample{ - { - TimestampMs: 123000, - Value: 456, - }, - } - ) - ctx := user.InjectOrgID(context.Background(), userID) - _, err := ing.Push(ctx, cortexpb.ToWriteRequest(lbls, sampleData, nil, cortexpb.API)) - require.NoError(t, err) - - // We add a 100ms sleep into the flush loop, such that we can reliably detect - // if the ingester is removing its token from Consul before flushing chunks. - ing.preFlushUserSeries = func() { - time.Sleep(100 * time.Millisecond) - } - - // Now stop the ingester. Don't call shutdown, as it waits for all goroutines - // to exit. We just want to check that by the time the token is removed from - // the ring, the data is in the chunk store. - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing.lifecycler)) - test.Poll(t, 200*time.Millisecond, 0, func() interface{} { - r, err := ing.lifecycler.KVStore.Get(context.Background(), RingKey) - if err != nil { - return -1 - } - return len(r.(*ring.Desc).Ingesters) - }) - - // And check the store has the chunk - res, err := chunk.ChunksToMatrix(context.Background(), store.chunks[userID], model.Time(0), model.Time(math.MaxInt64)) - require.NoError(t, err) - assert.Equal(t, model.Matrix{ - &model.SampleStream{ - Metric: model.Metric{ - model.MetricNameLabel: "foo", - }, - Values: []model.SamplePair{ - {Timestamp: model.TimeFromUnix(123), Value: model.SampleValue(456)}, - }, - }, - }, res) -} - // numTokens determines the number of tokens owned by the specified // address func numTokens(c kv.Client, name, ringKey string) int { diff --git a/pkg/ingester/locker.go b/pkg/ingester/locker.go deleted file mode 100644 index 3c97f38ba1..0000000000 --- a/pkg/ingester/locker.go +++ /dev/null @@ -1,58 +0,0 @@ -package ingester - -import ( - "sync" - "unsafe" - - "github.com/prometheus/common/model" - - "github.com/cortexproject/cortex/pkg/util" -) - -const ( - cacheLineSize = 64 -) - -// Avoid false sharing when using array of mutexes. -type paddedMutex struct { - sync.Mutex - //nolint:structcheck,unused - pad [cacheLineSize - unsafe.Sizeof(sync.Mutex{})]byte -} - -// fingerprintLocker allows locking individual fingerprints. To limit the number -// of mutexes needed for that, only a fixed number of mutexes are -// allocated. Fingerprints to be locked are assigned to those pre-allocated -// mutexes by their value. Collisions are not detected. If two fingerprints get -// assigned to the same mutex, only one of them can be locked at the same -// time. As long as the number of pre-allocated mutexes is much larger than the -// number of goroutines requiring a fingerprint lock concurrently, the loss in -// efficiency is small. However, a goroutine must never lock more than one -// fingerprint at the same time. (In that case a collision would try to acquire -// the same mutex twice). -type fingerprintLocker struct { - fpMtxs []paddedMutex - numFpMtxs uint32 -} - -// newFingerprintLocker returns a new fingerprintLocker ready for use. At least -// 1024 preallocated mutexes are used, even if preallocatedMutexes is lower. -func newFingerprintLocker(preallocatedMutexes int) *fingerprintLocker { - if preallocatedMutexes < 1024 { - preallocatedMutexes = 1024 - } - return &fingerprintLocker{ - make([]paddedMutex, preallocatedMutexes), - uint32(preallocatedMutexes), - } -} - -// Lock locks the given fingerprint. -func (l *fingerprintLocker) Lock(fp model.Fingerprint) { - l.fpMtxs[util.HashFP(fp)%l.numFpMtxs].Lock() -} - -// Unlock unlocks the given fingerprint. -func (l *fingerprintLocker) Unlock(fp model.Fingerprint) { - l.fpMtxs[util.HashFP(fp)%l.numFpMtxs].Unlock() -} diff --git a/pkg/ingester/locker_test.go b/pkg/ingester/locker_test.go deleted file mode 100644 index a150397393..0000000000 --- a/pkg/ingester/locker_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package ingester - -import ( - "sync" - "testing" - - "github.com/prometheus/common/model" -) - -func BenchmarkFingerprintLockerParallel(b *testing.B) { - numGoroutines := 10 - numFingerprints := 10 - numLockOps := b.N - locker := newFingerprintLocker(100) - - wg := sync.WaitGroup{} - b.ResetTimer() - for i := 0; i < numGoroutines; i++ { - wg.Add(1) - go func(i int) { - for j := 0; j < numLockOps; j++ { - fp1 := model.Fingerprint(j % numFingerprints) - fp2 := model.Fingerprint(j%numFingerprints + numFingerprints) - locker.Lock(fp1) - locker.Lock(fp2) - locker.Unlock(fp2) - locker.Unlock(fp1) - } - wg.Done() - }(i) - } - wg.Wait() -} - -func BenchmarkFingerprintLockerSerial(b *testing.B) { - numFingerprints := 10 - locker := newFingerprintLocker(100) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - fp := model.Fingerprint(i % numFingerprints) - locker.Lock(fp) - locker.Unlock(fp) - } -} diff --git a/pkg/ingester/mapper.go b/pkg/ingester/mapper.go deleted file mode 100644 index 835f6253ab..0000000000 --- a/pkg/ingester/mapper.go +++ /dev/null @@ -1,155 +0,0 @@ -package ingester - -import ( - "fmt" - "sort" - "strings" - "sync" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/prometheus/common/model" - "go.uber.org/atomic" -) - -const maxMappedFP = 1 << 20 // About 1M fingerprints reserved for mapping. - -var separatorString = string([]byte{model.SeparatorByte}) - -// fpMappings maps original fingerprints to a map of string representations of -// metrics to the truly unique fingerprint. -type fpMappings map[model.Fingerprint]map[string]model.Fingerprint - -// fpMapper is used to map fingerprints in order to work around fingerprint -// collisions. -type fpMapper struct { - highestMappedFP atomic.Uint64 - - mtx sync.RWMutex // Protects mappings. - mappings fpMappings - - fpToSeries *seriesMap - - logger log.Logger -} - -// newFPMapper loads the collision map from the persistence and -// returns an fpMapper ready to use. -func newFPMapper(fpToSeries *seriesMap, logger log.Logger) *fpMapper { - return &fpMapper{ - fpToSeries: fpToSeries, - mappings: map[model.Fingerprint]map[string]model.Fingerprint{}, - logger: logger, - } -} - -// mapFP takes a raw fingerprint (as returned by Metrics.FastFingerprint) and -// returns a truly unique fingerprint. The caller must have locked the raw -// fingerprint. -// -// If an error is encountered, it is returned together with the unchanged raw -// fingerprint. -func (m *fpMapper) mapFP(fp model.Fingerprint, metric labelPairs) model.Fingerprint { - // First check if we are in the reserved FP space, in which case this is - // automatically a collision that has to be mapped. - if fp <= maxMappedFP { - return m.maybeAddMapping(fp, metric) - } - - // Then check the most likely case: This fp belongs to a series that is - // already in memory. - s, ok := m.fpToSeries.get(fp) - if ok { - // FP exists in memory, but is it for the same metric? - if metric.equal(s.metric) { - // Yup. We are done. - return fp - } - // Collision detected! - return m.maybeAddMapping(fp, metric) - } - // Metric is not in memory. Before doing the expensive archive lookup, - // check if we have a mapping for this metric in place already. - m.mtx.RLock() - mappedFPs, fpAlreadyMapped := m.mappings[fp] - m.mtx.RUnlock() - if fpAlreadyMapped { - // We indeed have mapped fp historically. - ms := metricToUniqueString(metric) - // fp is locked by the caller, so no further locking of - // 'collisions' required (it is specific to fp). - mappedFP, ok := mappedFPs[ms] - if ok { - // Historical mapping found, return the mapped FP. - return mappedFP - } - } - return fp -} - -// maybeAddMapping is only used internally. It takes a detected collision and -// adds it to the collisions map if not yet there. In any case, it returns the -// truly unique fingerprint for the colliding metric. -func (m *fpMapper) maybeAddMapping( - fp model.Fingerprint, - collidingMetric labelPairs, -) model.Fingerprint { - ms := metricToUniqueString(collidingMetric) - m.mtx.RLock() - mappedFPs, ok := m.mappings[fp] - m.mtx.RUnlock() - if ok { - // fp is locked by the caller, so no further locking required. - mappedFP, ok := mappedFPs[ms] - if ok { - return mappedFP // Existing mapping. - } - // A new mapping has to be created. - mappedFP = m.nextMappedFP() - mappedFPs[ms] = mappedFP - level.Debug(m.logger).Log( - "msg", "fingerprint collision detected, mapping to new fingerprint", - "old_fp", fp, - "new_fp", mappedFP, - "metric", collidingMetric, - ) - return mappedFP - } - // This is the first collision for fp. - mappedFP := m.nextMappedFP() - mappedFPs = map[string]model.Fingerprint{ms: mappedFP} - m.mtx.Lock() - m.mappings[fp] = mappedFPs - m.mtx.Unlock() - level.Debug(m.logger).Log( - "msg", "fingerprint collision detected, mapping to new fingerprint", - "old_fp", fp, - "new_fp", mappedFP, - "metric", collidingMetric, - ) - return mappedFP -} - -func (m *fpMapper) nextMappedFP() model.Fingerprint { - mappedFP := model.Fingerprint(m.highestMappedFP.Inc()) - if mappedFP > maxMappedFP { - panic(fmt.Errorf("more than %v fingerprints mapped in collision detection", maxMappedFP)) - } - return mappedFP -} - -// metricToUniqueString turns a metric into a string in a reproducible and -// unique way, i.e. the same metric will always create the same string, and -// different metrics will always create different strings. In a way, it is the -// "ideal" fingerprint function, only that it is more expensive than the -// FastFingerprint function, and its result is not suitable as a key for maps -// and indexes as it might become really large, causing a lot of hashing effort -// in maps and a lot of storage overhead in indexes. -func metricToUniqueString(m labelPairs) string { - parts := make([]string, 0, len(m)) - for _, pair := range m { - parts = append(parts, string(pair.Name)+separatorString+string(pair.Value)) - } - sort.Strings(parts) - return strings.Join(parts, separatorString) -} diff --git a/pkg/ingester/mapper_test.go b/pkg/ingester/mapper_test.go deleted file mode 100644 index d18c6cb3c3..0000000000 --- a/pkg/ingester/mapper_test.go +++ /dev/null @@ -1,134 +0,0 @@ -package ingester - -import ( - "sort" - "testing" - - "github.com/go-kit/log" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" -) - -var ( - // cm11, cm12, cm13 are colliding with fp1. - // cm21, cm22 are colliding with fp2. - // cm31, cm32 are colliding with fp3, which is below maxMappedFP. - // Note that fingerprints are set and not actually calculated. - // The collision detection is independent from the actually used - // fingerprinting algorithm. - fp1 = model.Fingerprint(maxMappedFP + 1) - fp2 = model.Fingerprint(maxMappedFP + 2) - fp3 = model.Fingerprint(1) - cm11 = labelPairs{ - {Name: "foo", Value: "bar"}, - {Name: "dings", Value: "bumms"}, - } - cm12 = labelPairs{ - {Name: "bar", Value: "foo"}, - } - cm13 = labelPairs{ - {Name: "foo", Value: "bar"}, - } - cm21 = labelPairs{ - {Name: "foo", Value: "bumms"}, - {Name: "dings", Value: "bar"}, - } - cm22 = labelPairs{ - {Name: "dings", Value: "foo"}, - {Name: "bar", Value: "bumms"}, - } - cm31 = labelPairs{ - {Name: "bumms", Value: "dings"}, - } - cm32 = labelPairs{ - {Name: "bumms", Value: "dings"}, - {Name: "bar", Value: "foo"}, - } -) - -func (a labelPairs) copyValuesAndSort() labels.Labels { - c := make(labels.Labels, len(a)) - for i, pair := range a { - c[i].Name = pair.Name - c[i].Value = pair.Value - } - sort.Sort(c) - return c -} - -func TestFPMapper(t *testing.T) { - sm := newSeriesMap() - - mapper := newFPMapper(sm, log.NewNopLogger()) - - // Everything is empty, resolving a FP should do nothing. - assertFingerprintEqual(t, mapper.mapFP(fp1, cm11), fp1) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm12), fp1) - - // cm11 is in sm. Adding cm11 should do nothing. Mapping cm12 should resolve - // the collision. - sm.put(fp1, &memorySeries{metric: cm11.copyValuesAndSort()}) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm11), fp1) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm12), model.Fingerprint(1)) - - // The mapped cm12 is added to sm, too. That should not change the outcome. - sm.put(model.Fingerprint(1), &memorySeries{metric: cm12.copyValuesAndSort()}) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm11), fp1) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm12), model.Fingerprint(1)) - - // Now map cm13, should reproducibly result in the next mapped FP. - assertFingerprintEqual(t, mapper.mapFP(fp1, cm13), model.Fingerprint(2)) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm13), model.Fingerprint(2)) - - // Add cm13 to sm. Should not change anything. - sm.put(model.Fingerprint(2), &memorySeries{metric: cm13.copyValuesAndSort()}) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm11), fp1) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm12), model.Fingerprint(1)) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm13), model.Fingerprint(2)) - - // Now add cm21 and cm22 in the same way, checking the mapped FPs. - assertFingerprintEqual(t, mapper.mapFP(fp2, cm21), fp2) - sm.put(fp2, &memorySeries{metric: cm21.copyValuesAndSort()}) - assertFingerprintEqual(t, mapper.mapFP(fp2, cm21), fp2) - assertFingerprintEqual(t, mapper.mapFP(fp2, cm22), model.Fingerprint(3)) - sm.put(model.Fingerprint(3), &memorySeries{metric: cm22.copyValuesAndSort()}) - assertFingerprintEqual(t, mapper.mapFP(fp2, cm21), fp2) - assertFingerprintEqual(t, mapper.mapFP(fp2, cm22), model.Fingerprint(3)) - - // Map cm31, resulting in a mapping straight away. - assertFingerprintEqual(t, mapper.mapFP(fp3, cm31), model.Fingerprint(4)) - sm.put(model.Fingerprint(4), &memorySeries{metric: cm31.copyValuesAndSort()}) - - // Map cm32, which is now mapped for two reasons... - assertFingerprintEqual(t, mapper.mapFP(fp3, cm32), model.Fingerprint(5)) - sm.put(model.Fingerprint(5), &memorySeries{metric: cm32.copyValuesAndSort()}) - - // Now check ALL the mappings, just to be sure. - assertFingerprintEqual(t, mapper.mapFP(fp1, cm11), fp1) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm12), model.Fingerprint(1)) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm13), model.Fingerprint(2)) - assertFingerprintEqual(t, mapper.mapFP(fp2, cm21), fp2) - assertFingerprintEqual(t, mapper.mapFP(fp2, cm22), model.Fingerprint(3)) - assertFingerprintEqual(t, mapper.mapFP(fp3, cm31), model.Fingerprint(4)) - assertFingerprintEqual(t, mapper.mapFP(fp3, cm32), model.Fingerprint(5)) - - // Remove all the fingerprints from sm, which should change nothing, as - // the existing mappings stay and should be detected. - sm.del(fp1) - sm.del(fp2) - sm.del(fp3) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm11), fp1) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm12), model.Fingerprint(1)) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm13), model.Fingerprint(2)) - assertFingerprintEqual(t, mapper.mapFP(fp2, cm21), fp2) - assertFingerprintEqual(t, mapper.mapFP(fp2, cm22), model.Fingerprint(3)) - assertFingerprintEqual(t, mapper.mapFP(fp3, cm31), model.Fingerprint(4)) - assertFingerprintEqual(t, mapper.mapFP(fp3, cm32), model.Fingerprint(5)) -} - -// assertFingerprintEqual asserts that two fingerprints are equal. -func assertFingerprintEqual(t *testing.T, gotFP, wantFP model.Fingerprint) { - if gotFP != wantFP { - t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP) - } -} diff --git a/pkg/ingester/series.go b/pkg/ingester/series.go index a5dfcacde4..727cd1ce93 100644 --- a/pkg/ingester/series.go +++ b/pkg/ingester/series.go @@ -1,260 +1,7 @@ package ingester -import ( - "fmt" - "sort" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/model/value" - - "github.com/cortexproject/cortex/pkg/chunk/encoding" - "github.com/cortexproject/cortex/pkg/prom1/storage/metric" -) - const ( sampleOutOfOrder = "sample-out-of-order" newValueForTimestamp = "new-value-for-timestamp" sampleOutOfBounds = "sample-out-of-bounds" - duplicateSample = "duplicate-sample" - duplicateTimestamp = "duplicate-timestamp" ) - -type memorySeries struct { - metric labels.Labels - - // Sorted by start time, overlapping chunk ranges are forbidden. - chunkDescs []*desc - - // Whether the current head chunk has already been finished. If true, - // the current head chunk must not be modified anymore. - headChunkClosed bool - - // The timestamp & value of the last sample in this series. Needed to - // ensure timestamp monotonicity during ingestion. - lastSampleValueSet bool - lastTime model.Time - lastSampleValue model.SampleValue - - // Prometheus metrics. - createdChunks prometheus.Counter -} - -// newMemorySeries returns a pointer to a newly allocated memorySeries for the -// given metric. -func newMemorySeries(m labels.Labels, createdChunks prometheus.Counter) *memorySeries { - return &memorySeries{ - metric: m, - lastTime: model.Earliest, - createdChunks: createdChunks, - } -} - -// add adds a sample pair to the series, possibly creating a new chunk. -// The caller must have locked the fingerprint of the series. -func (s *memorySeries) add(v model.SamplePair) error { - // If sender has repeated the same timestamp, check more closely and perhaps return error. - if v.Timestamp == s.lastTime { - // If we don't know what the last sample value is, silently discard. - // This will mask some errors but better than complaining when we don't really know. - if !s.lastSampleValueSet { - return makeNoReportError(duplicateTimestamp) - } - // If both timestamp and sample value are the same as for the last append, - // ignore as they are a common occurrence when using client-side timestamps - // (e.g. Pushgateway or federation). - if v.Value.Equal(s.lastSampleValue) { - return makeNoReportError(duplicateSample) - } - return makeMetricValidationError(newValueForTimestamp, s.metric, - fmt.Errorf("sample with repeated timestamp but different value; last value: %v, incoming value: %v", s.lastSampleValue, v.Value)) - } - if v.Timestamp < s.lastTime { - return makeMetricValidationError(sampleOutOfOrder, s.metric, - fmt.Errorf("sample timestamp out of order; last timestamp: %v, incoming timestamp: %v", s.lastTime, v.Timestamp)) - } - - if len(s.chunkDescs) == 0 || s.headChunkClosed { - newHead := newDesc(encoding.New(), v.Timestamp, v.Timestamp) - s.chunkDescs = append(s.chunkDescs, newHead) - s.headChunkClosed = false - s.createdChunks.Inc() - } - - newChunk, err := s.head().add(v) - if err != nil { - return err - } - - // If we get a single chunk result, then just replace the head chunk with it - // (no need to update first/last time). Otherwise, we'll need to update first - // and last time. - if newChunk != nil { - first, last, err := firstAndLastTimes(newChunk) - if err != nil { - return err - } - s.chunkDescs = append(s.chunkDescs, newDesc(newChunk, first, last)) - s.createdChunks.Inc() - } - - s.lastTime = v.Timestamp - s.lastSampleValue = v.Value - s.lastSampleValueSet = true - - return nil -} - -func firstAndLastTimes(c encoding.Chunk) (model.Time, model.Time, error) { - var ( - first model.Time - last model.Time - firstSet bool - iter = c.NewIterator(nil) - ) - for iter.Scan() { - sample := iter.Value() - if !firstSet { - first = sample.Timestamp - firstSet = true - } - last = sample.Timestamp - } - return first, last, iter.Err() -} - -// closeHead marks the head chunk closed. The caller must have locked -// the fingerprint of the memorySeries. This method will panic if this -// series has no chunk descriptors. -func (s *memorySeries) closeHead(reason flushReason) { - s.chunkDescs[0].flushReason = reason - s.headChunkClosed = true -} - -// firstTime returns the earliest known time for the series. The caller must have -// locked the fingerprint of the memorySeries. This method will panic if this -// series has no chunk descriptors. -func (s *memorySeries) firstTime() model.Time { - return s.chunkDescs[0].FirstTime -} - -// Returns time of oldest chunk in the series, that isn't flushed. If there are -// no chunks, or all chunks are flushed, returns 0. -// The caller must have locked the fingerprint of the memorySeries. -func (s *memorySeries) firstUnflushedChunkTime() model.Time { - for _, c := range s.chunkDescs { - if !c.flushed { - return c.FirstTime - } - } - - return 0 -} - -// head returns a pointer to the head chunk descriptor. The caller must have -// locked the fingerprint of the memorySeries. This method will panic if this -// series has no chunk descriptors. -func (s *memorySeries) head() *desc { - return s.chunkDescs[len(s.chunkDescs)-1] -} - -func (s *memorySeries) samplesForRange(from, through model.Time) ([]model.SamplePair, error) { - // Find first chunk with start time after "from". - fromIdx := sort.Search(len(s.chunkDescs), func(i int) bool { - return s.chunkDescs[i].FirstTime.After(from) - }) - // Find first chunk with start time after "through". - throughIdx := sort.Search(len(s.chunkDescs), func(i int) bool { - return s.chunkDescs[i].FirstTime.After(through) - }) - if fromIdx == len(s.chunkDescs) { - // Even the last chunk starts before "from". Find out if the - // series ends before "from" and we don't need to do anything. - lt := s.chunkDescs[len(s.chunkDescs)-1].LastTime - if lt.Before(from) { - return nil, nil - } - } - if fromIdx > 0 { - fromIdx-- - } - if throughIdx == len(s.chunkDescs) { - throughIdx-- - } - var values []model.SamplePair - in := metric.Interval{ - OldestInclusive: from, - NewestInclusive: through, - } - var reuseIter encoding.Iterator - for idx := fromIdx; idx <= throughIdx; idx++ { - cd := s.chunkDescs[idx] - reuseIter = cd.C.NewIterator(reuseIter) - chValues, err := encoding.RangeValues(reuseIter, in) - if err != nil { - return nil, err - } - values = append(values, chValues...) - } - return values, nil -} - -func (s *memorySeries) setChunks(descs []*desc) error { - if len(s.chunkDescs) != 0 { - return fmt.Errorf("series already has chunks") - } - - s.chunkDescs = descs - if len(descs) > 0 { - s.lastTime = descs[len(descs)-1].LastTime - } - return nil -} - -func (s *memorySeries) isStale() bool { - return s.lastSampleValueSet && value.IsStaleNaN(float64(s.lastSampleValue)) -} - -type desc struct { - C encoding.Chunk // nil if chunk is evicted. - FirstTime model.Time // Timestamp of first sample. Populated at creation. Immutable. - LastTime model.Time // Timestamp of last sample. Populated at creation & on append. - LastUpdate model.Time // This server's local time on last change - flushReason flushReason // If chunk is closed, holds the reason why. - flushed bool // set to true when flush succeeds -} - -func newDesc(c encoding.Chunk, firstTime model.Time, lastTime model.Time) *desc { - return &desc{ - C: c, - FirstTime: firstTime, - LastTime: lastTime, - LastUpdate: model.Now(), - } -} - -// Add adds a sample pair to the underlying chunk. For safe concurrent access, -// The chunk must be pinned, and the caller must have locked the fingerprint of -// the series. -func (d *desc) add(s model.SamplePair) (encoding.Chunk, error) { - cs, err := d.C.Add(s) - if err != nil { - return nil, err - } - - if cs == nil { - d.LastTime = s.Timestamp // sample was added to this chunk - d.LastUpdate = model.Now() - } - - return cs, nil -} - -func (d *desc) slice(start, end model.Time) *desc { - return &desc{ - C: d.C.Slice(start, end), - FirstTime: start, - LastTime: end, - } -} diff --git a/pkg/ingester/series_map.go b/pkg/ingester/series_map.go deleted file mode 100644 index 4d4a9a5b66..0000000000 --- a/pkg/ingester/series_map.go +++ /dev/null @@ -1,110 +0,0 @@ -package ingester - -import ( - "sync" - "unsafe" - - "github.com/prometheus/common/model" - "go.uber.org/atomic" - - "github.com/cortexproject/cortex/pkg/util" -) - -const seriesMapShards = 128 - -// seriesMap maps fingerprints to memory series. All its methods are -// goroutine-safe. A seriesMap is effectively a goroutine-safe version of -// map[model.Fingerprint]*memorySeries. -type seriesMap struct { - size atomic.Int32 - shards []shard -} - -type shard struct { - mtx sync.Mutex - m map[model.Fingerprint]*memorySeries - - // Align this struct. - _ [cacheLineSize - unsafe.Sizeof(sync.Mutex{}) - unsafe.Sizeof(map[model.Fingerprint]*memorySeries{})]byte -} - -// fingerprintSeriesPair pairs a fingerprint with a memorySeries pointer. -type fingerprintSeriesPair struct { - fp model.Fingerprint - series *memorySeries -} - -// newSeriesMap returns a newly allocated empty seriesMap. To create a seriesMap -// based on a prefilled map, use an explicit initializer. -func newSeriesMap() *seriesMap { - shards := make([]shard, seriesMapShards) - for i := 0; i < seriesMapShards; i++ { - shards[i].m = map[model.Fingerprint]*memorySeries{} - } - return &seriesMap{ - shards: shards, - } -} - -// get returns a memorySeries for a fingerprint. Return values have the same -// semantics as the native Go map. -func (sm *seriesMap) get(fp model.Fingerprint) (*memorySeries, bool) { - shard := &sm.shards[util.HashFP(fp)%seriesMapShards] - shard.mtx.Lock() - ms, ok := shard.m[fp] - shard.mtx.Unlock() - return ms, ok -} - -// put adds a mapping to the seriesMap. -func (sm *seriesMap) put(fp model.Fingerprint, s *memorySeries) { - shard := &sm.shards[util.HashFP(fp)%seriesMapShards] - shard.mtx.Lock() - _, ok := shard.m[fp] - shard.m[fp] = s - shard.mtx.Unlock() - - if !ok { - sm.size.Inc() - } -} - -// del removes a mapping from the series Map. -func (sm *seriesMap) del(fp model.Fingerprint) { - shard := &sm.shards[util.HashFP(fp)%seriesMapShards] - shard.mtx.Lock() - _, ok := shard.m[fp] - delete(shard.m, fp) - shard.mtx.Unlock() - if ok { - sm.size.Dec() - } -} - -// iter returns a channel that produces all mappings in the seriesMap. The -// channel will be closed once all fingerprints have been received. Not -// consuming all fingerprints from the channel will leak a goroutine. The -// semantics of concurrent modification of seriesMap is the similar as the one -// for iterating over a map with a 'range' clause. However, if the next element -// in iteration order is removed after the current element has been received -// from the channel, it will still be produced by the channel. -func (sm *seriesMap) iter() <-chan fingerprintSeriesPair { - ch := make(chan fingerprintSeriesPair) - go func() { - for i := range sm.shards { - sm.shards[i].mtx.Lock() - for fp, ms := range sm.shards[i].m { - sm.shards[i].mtx.Unlock() - ch <- fingerprintSeriesPair{fp, ms} - sm.shards[i].mtx.Lock() - } - sm.shards[i].mtx.Unlock() - } - close(ch) - }() - return ch -} - -func (sm *seriesMap) length() int { - return int(sm.size.Load()) -} diff --git a/pkg/ingester/transfer.go b/pkg/ingester/transfer.go index ce31e9f42a..e20f91d847 100644 --- a/pkg/ingester/transfer.go +++ b/pkg/ingester/transfer.go @@ -1,390 +1,17 @@ package ingester import ( - "bytes" "context" - "fmt" - "io" - "os" - "time" "github.com/go-kit/log/level" - "github.com/pkg/errors" - "github.com/prometheus/common/model" - "github.com/weaveworks/common/user" - "github.com/cortexproject/cortex/pkg/chunk/encoding" - "github.com/cortexproject/cortex/pkg/cortexpb" - "github.com/cortexproject/cortex/pkg/ingester/client" "github.com/cortexproject/cortex/pkg/ring" - "github.com/cortexproject/cortex/pkg/util/backoff" ) -var ( - errTransferNoPendingIngesters = errors.New("no pending ingesters") -) - -// returns source ingesterID, number of received series, added chunks and error -func (i *Ingester) fillUserStatesFromStream(userStates *userStates, stream client.Ingester_TransferChunksServer) (fromIngesterID string, seriesReceived int, retErr error) { - chunksAdded := 0.0 - - defer func() { - if retErr != nil { - // Ensure the in memory chunks are updated to reflect the number of dropped chunks from the transfer - i.metrics.memoryChunks.Sub(chunksAdded) - - // If an error occurs during the transfer and the user state is to be discarded, - // ensure the metrics it exports reflect this. - userStates.teardown() - } - }() - - for { - wireSeries, err := stream.Recv() - if err == io.EOF { - break - } - if err != nil { - retErr = errors.Wrap(err, "TransferChunks: Recv") - return - } - - // We can't send "extra" fields with a streaming call, so we repeat - // wireSeries.FromIngesterId and assume it is the same every time - // round this loop. - if fromIngesterID == "" { - fromIngesterID = wireSeries.FromIngesterId - level.Info(i.logger).Log("msg", "processing TransferChunks request", "from_ingester", fromIngesterID) - - // Before transfer, make sure 'from' ingester is in correct state to call ClaimTokensFor later - err := i.checkFromIngesterIsInLeavingState(stream.Context(), fromIngesterID) - if err != nil { - retErr = errors.Wrap(err, "TransferChunks: checkFromIngesterIsInLeavingState") - return - } - } - descs, err := fromWireChunks(wireSeries.Chunks) - if err != nil { - retErr = errors.Wrap(err, "TransferChunks: fromWireChunks") - return - } - - state, fp, series, err := userStates.getOrCreateSeries(stream.Context(), wireSeries.UserId, wireSeries.Labels, nil) - if err != nil { - retErr = errors.Wrapf(err, "TransferChunks: getOrCreateSeries: user %s series %s", wireSeries.UserId, wireSeries.Labels) - return - } - prevNumChunks := len(series.chunkDescs) - - err = series.setChunks(descs) - state.fpLocker.Unlock(fp) // acquired in getOrCreateSeries - if err != nil { - retErr = errors.Wrapf(err, "TransferChunks: setChunks: user %s series %s", wireSeries.UserId, wireSeries.Labels) - return - } - - seriesReceived++ - chunksDelta := float64(len(series.chunkDescs) - prevNumChunks) - chunksAdded += chunksDelta - i.metrics.memoryChunks.Add(chunksDelta) - i.metrics.receivedChunks.Add(float64(len(descs))) - } - - if seriesReceived == 0 { - level.Error(i.logger).Log("msg", "received TransferChunks request with no series", "from_ingester", fromIngesterID) - retErr = fmt.Errorf("TransferChunks: no series") - return - } - - if fromIngesterID == "" { - level.Error(i.logger).Log("msg", "received TransferChunks request with no ID from ingester") - retErr = fmt.Errorf("no ingester id") - return - } - - if err := i.lifecycler.ClaimTokensFor(stream.Context(), fromIngesterID); err != nil { - retErr = errors.Wrap(err, "TransferChunks: ClaimTokensFor") - return - } - - return -} - -// TransferChunks receives all the chunks from another ingester. -func (i *Ingester) TransferChunks(stream client.Ingester_TransferChunksServer) error { - fromIngesterID := "" - seriesReceived := 0 - - xfer := func() error { - userStates := newUserStates(i.limiter, i.cfg, i.metrics, i.logger) - - var err error - fromIngesterID, seriesReceived, err = i.fillUserStatesFromStream(userStates, stream) - - if err != nil { - return err - } - - i.userStatesMtx.Lock() - defer i.userStatesMtx.Unlock() - - i.userStates = userStates - - return nil - } - - if err := i.transfer(stream.Context(), xfer); err != nil { - return err - } - - // Close the stream last, as this is what tells the "from" ingester that - // it's OK to shut down. - if err := stream.SendAndClose(&client.TransferChunksResponse{}); err != nil { - level.Error(i.logger).Log("msg", "Error closing TransferChunks stream", "from_ingester", fromIngesterID, "err", err) - return err - } - level.Info(i.logger).Log("msg", "Successfully transferred chunks", "from_ingester", fromIngesterID, "series_received", seriesReceived) - - return nil -} - -// Ring gossiping: check if "from" ingester is in LEAVING state. It should be, but we may not see that yet -// when using gossip ring. If we cannot see ingester is the LEAVING state yet, we don't accept this -// transfer, as claiming tokens would possibly end up with this ingester owning no tokens, due to conflict -// resolution in ring merge function. Hopefully the leaving ingester will retry transfer again. -func (i *Ingester) checkFromIngesterIsInLeavingState(ctx context.Context, fromIngesterID string) error { - v, err := i.lifecycler.KVStore.Get(ctx, i.lifecycler.RingKey) - if err != nil { - return errors.Wrap(err, "get ring") - } - if v == nil { - return fmt.Errorf("ring not found when checking state of source ingester") - } - r, ok := v.(*ring.Desc) - if !ok || r == nil { - return fmt.Errorf("ring not found, got %T", v) - } - - if r.Ingesters == nil || r.Ingesters[fromIngesterID].State != ring.LEAVING { - return fmt.Errorf("source ingester is not in a LEAVING state, found state=%v", r.Ingesters[fromIngesterID].State) - } - - // all fine - return nil -} - -func (i *Ingester) transfer(ctx context.Context, xfer func() error) error { - // Enter JOINING state (only valid from PENDING) - if err := i.lifecycler.ChangeState(ctx, ring.JOINING); err != nil { - return err - } - - // The ingesters state effectively works as a giant mutex around this whole - // method, and as such we have to ensure we unlock the mutex. - defer func() { - state := i.lifecycler.GetState() - if i.lifecycler.GetState() == ring.ACTIVE { - return - } - - level.Error(i.logger).Log("msg", "TransferChunks failed, not in ACTIVE state.", "state", state) - - // Enter PENDING state (only valid from JOINING) - if i.lifecycler.GetState() == ring.JOINING { - if err := i.lifecycler.ChangeState(ctx, ring.PENDING); err != nil { - level.Error(i.logger).Log("msg", "error rolling back failed TransferChunks", "err", err) - os.Exit(1) - } - } - }() - - if err := xfer(); err != nil { - return err - } - - if err := i.lifecycler.ChangeState(ctx, ring.ACTIVE); err != nil { - return errors.Wrap(err, "Transfer: ChangeState") - } - - return nil -} - -// The passed wireChunks slice is for re-use. -func toWireChunks(descs []*desc, wireChunks []client.Chunk) ([]client.Chunk, error) { - if cap(wireChunks) < len(descs) { - wireChunks = make([]client.Chunk, len(descs)) - } else { - wireChunks = wireChunks[:len(descs)] - } - for i, d := range descs { - wireChunk := client.Chunk{ - StartTimestampMs: int64(d.FirstTime), - EndTimestampMs: int64(d.LastTime), - Encoding: int32(d.C.Encoding()), - } - - slice := wireChunks[i].Data[:0] // try to re-use the memory from last time - if cap(slice) < d.C.Size() { - slice = make([]byte, 0, d.C.Size()) - } - buf := bytes.NewBuffer(slice) - - if err := d.C.Marshal(buf); err != nil { - return nil, err - } - - wireChunk.Data = buf.Bytes() - wireChunks[i] = wireChunk - } - return wireChunks, nil -} - -func fromWireChunks(wireChunks []client.Chunk) ([]*desc, error) { - descs := make([]*desc, 0, len(wireChunks)) - for _, c := range wireChunks { - desc := &desc{ - FirstTime: model.Time(c.StartTimestampMs), - LastTime: model.Time(c.EndTimestampMs), - LastUpdate: model.Now(), - } - - var err error - desc.C, err = encoding.NewForEncoding(encoding.Encoding(byte(c.Encoding))) - if err != nil { - return nil, err - } - - if err := desc.C.UnmarshalFromBuf(c.Data); err != nil { - return nil, err - } - - descs = append(descs, desc) - } - return descs, nil -} - // TransferOut finds an ingester in PENDING state and transfers our chunks to it. // Called as part of the ingester shutdown process. func (i *Ingester) TransferOut(ctx context.Context) error { // The blocks storage doesn't support blocks transferring. - if i.cfg.BlocksStorageEnabled { - level.Info(i.logger).Log("msg", "transfer between a LEAVING ingester and a PENDING one is not supported for the blocks storage") - return ring.ErrTransferDisabled - } - - if i.cfg.MaxTransferRetries <= 0 { - return ring.ErrTransferDisabled - } - backoff := backoff.New(ctx, backoff.Config{ - MinBackoff: 100 * time.Millisecond, - MaxBackoff: 5 * time.Second, - MaxRetries: i.cfg.MaxTransferRetries, - }) - - // Keep track of the last error so that we can log it with the highest level - // once all retries have completed - var err error - - for backoff.Ongoing() { - err = i.transferOut(ctx) - if err == nil { - level.Info(i.logger).Log("msg", "transfer successfully completed") - return nil - } - - level.Warn(i.logger).Log("msg", "transfer attempt failed", "err", err, "attempt", backoff.NumRetries()+1, "max_retries", i.cfg.MaxTransferRetries) - - backoff.Wait() - } - - level.Error(i.logger).Log("msg", "all transfer attempts failed", "err", err) - return backoff.Err() -} - -func (i *Ingester) transferOut(ctx context.Context) error { - userStatesCopy := i.userStates.cp() - if len(userStatesCopy) == 0 { - level.Info(i.logger).Log("msg", "nothing to transfer") - return nil - } - - targetIngester, err := i.findTargetIngester(ctx) - if err != nil { - return fmt.Errorf("cannot find ingester to transfer chunks to: %w", err) - } - - level.Info(i.logger).Log("msg", "sending chunks", "to_ingester", targetIngester.Addr) - c, err := i.cfg.ingesterClientFactory(targetIngester.Addr, i.clientConfig) - if err != nil { - return err - } - defer c.Close() - - ctx = user.InjectOrgID(ctx, "-1") - stream, err := c.TransferChunks(ctx) - if err != nil { - return errors.Wrap(err, "TransferChunks") - } - - var chunks []client.Chunk - for userID, state := range userStatesCopy { - for pair := range state.fpToSeries.iter() { - state.fpLocker.Lock(pair.fp) - - if len(pair.series.chunkDescs) == 0 { // Nothing to send? - state.fpLocker.Unlock(pair.fp) - continue - } - - chunks, err = toWireChunks(pair.series.chunkDescs, chunks) - if err != nil { - state.fpLocker.Unlock(pair.fp) - return errors.Wrap(err, "toWireChunks") - } - - err = client.SendTimeSeriesChunk(stream, &client.TimeSeriesChunk{ - FromIngesterId: i.lifecycler.ID, - UserId: userID, - Labels: cortexpb.FromLabelsToLabelAdapters(pair.series.metric), - Chunks: chunks, - }) - state.fpLocker.Unlock(pair.fp) - if err != nil { - return errors.Wrap(err, "Send") - } - - i.metrics.sentChunks.Add(float64(len(chunks))) - } - } - - _, err = stream.CloseAndRecv() - if err != nil { - return errors.Wrap(err, "CloseAndRecv") - } - - // Close & empty all the flush queues, to unblock waiting workers. - for _, flushQueue := range i.flushQueues { - flushQueue.DiscardAndClose() - } - i.flushQueuesDone.Wait() - - level.Info(i.logger).Log("msg", "successfully sent chunks", "to_ingester", targetIngester.Addr) - return nil -} - -// findTargetIngester finds an ingester in PENDING state. -func (i *Ingester) findTargetIngester(ctx context.Context) (*ring.InstanceDesc, error) { - ringDesc, err := i.lifecycler.KVStore.Get(ctx, i.lifecycler.RingKey) - if err != nil { - return nil, err - } else if ringDesc == nil { - return nil, errTransferNoPendingIngesters - } - - ingesters := ringDesc.(*ring.Desc).FindIngestersByState(ring.PENDING) - if len(ingesters) <= 0 { - return nil, errTransferNoPendingIngesters - } - - return &ingesters[0], nil + level.Info(i.logger).Log("msg", "transfer between a LEAVING ingester and a PENDING one is not supported for the blocks storage") + return ring.ErrTransferDisabled } diff --git a/pkg/ingester/user_state.go b/pkg/ingester/user_state.go index 685dd54f38..57973e7568 100644 --- a/pkg/ingester/user_state.go +++ b/pkg/ingester/user_state.go @@ -1,358 +1,20 @@ package ingester import ( - "context" - "net/http" "sync" - "time" - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/tsdb/chunks" - tsdb_record "github.com/prometheus/prometheus/tsdb/record" "github.com/segmentio/fasthash/fnv1a" - "github.com/weaveworks/common/httpgrpc" - "github.com/cortexproject/cortex/pkg/cortexpb" - "github.com/cortexproject/cortex/pkg/ingester/client" - "github.com/cortexproject/cortex/pkg/ingester/index" - "github.com/cortexproject/cortex/pkg/tenant" "github.com/cortexproject/cortex/pkg/util" - "github.com/cortexproject/cortex/pkg/util/extract" - util_math "github.com/cortexproject/cortex/pkg/util/math" - "github.com/cortexproject/cortex/pkg/util/spanlogger" - "github.com/cortexproject/cortex/pkg/util/validation" ) -// userStates holds the userState object for all users (tenants), -// each one containing all the in-memory series for a given user. -type userStates struct { - states sync.Map - limiter *Limiter - cfg Config - metrics *ingesterMetrics - logger log.Logger -} - -type userState struct { - limiter *Limiter - userID string - fpLocker *fingerprintLocker - fpToSeries *seriesMap - mapper *fpMapper - index *index.InvertedIndex - ingestedAPISamples *util_math.EwmaRate - ingestedRuleSamples *util_math.EwmaRate - activeSeries *ActiveSeries - logger log.Logger - - seriesInMetric *metricCounter - - // Series metrics. - memSeries prometheus.Gauge - memSeriesCreatedTotal prometheus.Counter - memSeriesRemovedTotal prometheus.Counter - discardedSamples *prometheus.CounterVec - createdChunks prometheus.Counter - activeSeriesGauge prometheus.Gauge -} - // DiscardedSamples metric labels const ( perUserSeriesLimit = "per_user_series_limit" perMetricSeriesLimit = "per_metric_series_limit" ) -func newUserStates(limiter *Limiter, cfg Config, metrics *ingesterMetrics, logger log.Logger) *userStates { - return &userStates{ - limiter: limiter, - cfg: cfg, - metrics: metrics, - logger: logger, - } -} - -func (us *userStates) cp() map[string]*userState { - states := map[string]*userState{} - us.states.Range(func(key, value interface{}) bool { - states[key.(string)] = value.(*userState) - return true - }) - return states -} - -//nolint:unused -func (us *userStates) gc() { - us.states.Range(func(key, value interface{}) bool { - state := value.(*userState) - if state.fpToSeries.length() == 0 { - us.states.Delete(key) - state.activeSeries.clear() - state.activeSeriesGauge.Set(0) - } - return true - }) -} - -func (us *userStates) updateRates() { - us.states.Range(func(key, value interface{}) bool { - state := value.(*userState) - state.ingestedAPISamples.Tick() - state.ingestedRuleSamples.Tick() - return true - }) -} - -// Labels will be copied if they are kept. -func (us *userStates) updateActiveSeriesForUser(userID string, now time.Time, lbls []labels.Label) { - if s, ok := us.get(userID); ok { - s.activeSeries.UpdateSeries(lbls, now, func(l labels.Labels) labels.Labels { return cortexpb.CopyLabels(l) }) - } -} - -func (us *userStates) purgeAndUpdateActiveSeries(purgeTime time.Time) { - us.states.Range(func(key, value interface{}) bool { - state := value.(*userState) - state.activeSeries.Purge(purgeTime) - state.activeSeriesGauge.Set(float64(state.activeSeries.Active())) - return true - }) -} - -func (us *userStates) get(userID string) (*userState, bool) { - state, ok := us.states.Load(userID) - if !ok { - return nil, ok - } - return state.(*userState), ok -} - -func (us *userStates) getOrCreate(userID string) *userState { - state, ok := us.get(userID) - if !ok { - - logger := log.With(us.logger, "user", userID) - // Speculatively create a userState object and try to store it - // in the map. Another goroutine may have got there before - // us, in which case this userState will be discarded - state = &userState{ - userID: userID, - limiter: us.limiter, - fpToSeries: newSeriesMap(), - fpLocker: newFingerprintLocker(16 * 1024), - index: index.New(), - ingestedAPISamples: util_math.NewEWMARate(0.2, us.cfg.RateUpdatePeriod), - ingestedRuleSamples: util_math.NewEWMARate(0.2, us.cfg.RateUpdatePeriod), - seriesInMetric: newMetricCounter(us.limiter, us.cfg.getIgnoreSeriesLimitForMetricNamesMap()), - logger: logger, - - memSeries: us.metrics.memSeries, - memSeriesCreatedTotal: us.metrics.memSeriesCreatedTotal.WithLabelValues(userID), - memSeriesRemovedTotal: us.metrics.memSeriesRemovedTotal.WithLabelValues(userID), - discardedSamples: validation.DiscardedSamples.MustCurryWith(prometheus.Labels{"user": userID}), - createdChunks: us.metrics.createdChunks, - - activeSeries: NewActiveSeries(), - activeSeriesGauge: us.metrics.activeSeriesPerUser.WithLabelValues(userID), - } - state.mapper = newFPMapper(state.fpToSeries, logger) - stored, ok := us.states.LoadOrStore(userID, state) - if !ok { - us.metrics.memUsers.Inc() - } - state = stored.(*userState) - } - - return state -} - -// teardown ensures metrics are accurately updated if a userStates struct is discarded -func (us *userStates) teardown() { - for _, u := range us.cp() { - u.memSeriesRemovedTotal.Add(float64(u.fpToSeries.length())) - u.memSeries.Sub(float64(u.fpToSeries.length())) - u.activeSeriesGauge.Set(0) - us.metrics.memUsers.Dec() - } -} - -func (us *userStates) getViaContext(ctx context.Context) (*userState, bool, error) { - userID, err := tenant.TenantID(ctx) - if err != nil { - return nil, false, err - } - state, ok := us.get(userID) - return state, ok, nil -} - -// NOTE: memory for `labels` is unsafe; anything retained beyond the -// life of this function must be copied -func (us *userStates) getOrCreateSeries(ctx context.Context, userID string, labels []cortexpb.LabelAdapter, record *WALRecord) (*userState, model.Fingerprint, *memorySeries, error) { - state := us.getOrCreate(userID) - // WARNING: `err` may have a reference to unsafe memory in `labels` - fp, series, err := state.getSeries(labels, record) - return state, fp, series, err -} - -// NOTE: memory for `metric` is unsafe; anything retained beyond the -// life of this function must be copied -func (u *userState) getSeries(metric labelPairs, record *WALRecord) (model.Fingerprint, *memorySeries, error) { - rawFP := client.FastFingerprint(metric) - u.fpLocker.Lock(rawFP) - fp := u.mapper.mapFP(rawFP, metric) - if fp != rawFP { - u.fpLocker.Unlock(rawFP) - u.fpLocker.Lock(fp) - } - - series, ok := u.fpToSeries.get(fp) - if ok { - return fp, series, nil - } - - series, err := u.createSeriesWithFingerprint(fp, metric, record, false) - if err != nil { - u.fpLocker.Unlock(fp) - return 0, nil, err - } - - return fp, series, nil -} - -func (u *userState) createSeriesWithFingerprint(fp model.Fingerprint, metric labelPairs, record *WALRecord, recovery bool) (*memorySeries, error) { - // There's theoretically a relatively harmless race here if multiple - // goroutines get the length of the series map at the same time, then - // all proceed to add a new series. This is likely not worth addressing, - // as this should happen rarely (all samples from one push are added - // serially), and the overshoot in allowed series would be minimal. - - if !recovery { - if err := u.limiter.AssertMaxSeriesPerUser(u.userID, u.fpToSeries.length()); err != nil { - return nil, makeLimitError(perUserSeriesLimit, u.limiter.FormatError(u.userID, err)) - } - } - - // MetricNameFromLabelAdapters returns a copy of the string in `metric` - metricName, err := extract.MetricNameFromLabelAdapters(metric) - if err != nil { - return nil, err - } - - if !recovery { - // Check if the per-metric limit has been exceeded - if err = u.seriesInMetric.canAddSeriesFor(u.userID, metricName); err != nil { - // WARNING: returns a reference to `metric` - return nil, makeMetricLimitError(perMetricSeriesLimit, cortexpb.FromLabelAdaptersToLabels(metric), u.limiter.FormatError(u.userID, err)) - } - } - - u.memSeriesCreatedTotal.Inc() - u.memSeries.Inc() - u.seriesInMetric.increaseSeriesForMetric(metricName) - - if record != nil { - lbls := make(labels.Labels, 0, len(metric)) - for _, m := range metric { - lbls = append(lbls, labels.Label(m)) - } - record.Series = append(record.Series, tsdb_record.RefSeries{ - Ref: chunks.HeadSeriesRef(fp), - Labels: lbls, - }) - } - - labels := u.index.Add(metric, fp) // Add() returns 'interned' values so the original labels are not retained - series := newMemorySeries(labels, u.createdChunks) - u.fpToSeries.put(fp, series) - - return series, nil -} - -func (u *userState) removeSeries(fp model.Fingerprint, metric labels.Labels) { - u.fpToSeries.del(fp) - u.index.Delete(metric, fp) - - metricName := metric.Get(model.MetricNameLabel) - if metricName == "" { - // Series without a metric name should never be able to make it into - // the ingester's memory storage. - panic("No metric name label") - } - - u.seriesInMetric.decreaseSeriesForMetric(metricName) - - u.memSeriesRemovedTotal.Inc() - u.memSeries.Dec() -} - -// forSeriesMatching passes all series matching the given matchers to the -// provided callback. Deals with locking and the quirks of zero-length matcher -// values. There are 2 callbacks: -// - The `add` callback is called for each series while the lock is held, and -// is intend to be used by the caller to build a batch. -// - The `send` callback is called at certain intervals specified by batchSize -// with no locks held, and is intended to be used by the caller to send the -// built batches. -func (u *userState) forSeriesMatching(ctx context.Context, allMatchers []*labels.Matcher, - add func(context.Context, model.Fingerprint, *memorySeries) error, - send func(context.Context) error, batchSize int, -) error { - log, ctx := spanlogger.New(ctx, "forSeriesMatching") - defer log.Finish() - - filters, matchers := util.SplitFiltersAndMatchers(allMatchers) - fps := u.index.Lookup(matchers) - if len(fps) > u.limiter.MaxSeriesPerQuery(u.userID) { - return httpgrpc.Errorf(http.StatusRequestEntityTooLarge, "exceeded maximum number of series in a query") - } - - level.Debug(log).Log("series", len(fps)) - - // We only hold one FP lock at once here, so no opportunity to deadlock. - sent := 0 -outer: - for _, fp := range fps { - if err := ctx.Err(); err != nil { - return err - } - - u.fpLocker.Lock(fp) - series, ok := u.fpToSeries.get(fp) - if !ok { - u.fpLocker.Unlock(fp) - continue - } - - for _, filter := range filters { - if !filter.Matches(series.metric.Get(filter.Name)) { - u.fpLocker.Unlock(fp) - continue outer - } - } - - err := add(ctx, fp, series) - u.fpLocker.Unlock(fp) - if err != nil { - return err - } - - sent++ - if batchSize > 0 && sent%batchSize == 0 && send != nil { - if err = send(ctx); err != nil { - return nil - } - } - } - - if batchSize > 0 && sent%batchSize > 0 && send != nil { - return send(ctx) - } - return nil -} - const numMetricCounterShards = 128 type metricCounterShard struct { diff --git a/pkg/ingester/user_state_test.go b/pkg/ingester/user_state_test.go index 1d5ec1238d..ec6234e25a 100644 --- a/pkg/ingester/user_state_test.go +++ b/pkg/ingester/user_state_test.go @@ -1,162 +1,15 @@ package ingester import ( - "context" - "fmt" - "math" - "strings" "testing" - "time" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/testutil" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/weaveworks/common/user" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/validation" ) -// Test forSeriesMatching correctly batches up series. -func TestForSeriesMatchingBatching(t *testing.T) { - matchAllNames, err := labels.NewMatcher(labels.MatchRegexp, model.MetricNameLabel, ".+") - require.NoError(t, err) - // We rely on pushTestSamples() creating jobs "testjob0" and "testjob1" in equal parts - matchNotJob0, err := labels.NewMatcher(labels.MatchNotEqual, model.JobLabel, "testjob0") - require.NoError(t, err) - matchNotJob1, err := labels.NewMatcher(labels.MatchNotEqual, model.JobLabel, "testjob1") - require.NoError(t, err) - - for _, tc := range []struct { - numSeries, batchSize int - matchers []*labels.Matcher - expected int - }{ - {100, 10, []*labels.Matcher{matchAllNames}, 100}, - {99, 10, []*labels.Matcher{matchAllNames}, 99}, - {98, 10, []*labels.Matcher{matchAllNames}, 98}, - {5, 10, []*labels.Matcher{matchAllNames}, 5}, - {10, 1, []*labels.Matcher{matchAllNames}, 10}, - {1, 1, []*labels.Matcher{matchAllNames}, 1}, - {10, 10, []*labels.Matcher{matchAllNames, matchNotJob0}, 5}, - {10, 10, []*labels.Matcher{matchAllNames, matchNotJob1}, 5}, - {100, 10, []*labels.Matcher{matchAllNames, matchNotJob0}, 50}, - {100, 10, []*labels.Matcher{matchAllNames, matchNotJob1}, 50}, - {99, 10, []*labels.Matcher{matchAllNames, matchNotJob0}, 49}, - {99, 10, []*labels.Matcher{matchAllNames, matchNotJob1}, 50}, - } { - t.Run(fmt.Sprintf("numSeries=%d,batchSize=%d,matchers=%s", tc.numSeries, tc.batchSize, tc.matchers), func(t *testing.T) { - _, ing := newDefaultTestStore(t) - userIDs, _ := pushTestSamples(t, ing, tc.numSeries, 100, 0) - - for _, userID := range userIDs { - ctx := user.InjectOrgID(context.Background(), userID) - instance, ok, err := ing.userStates.getViaContext(ctx) - require.NoError(t, err) - require.True(t, ok) - - total, batch, batches := 0, 0, 0 - err = instance.forSeriesMatching(ctx, tc.matchers, - func(_ context.Context, _ model.Fingerprint, s *memorySeries) error { - batch++ - return nil - }, - func(context.Context) error { - require.True(t, batch <= tc.batchSize) - total += batch - batch = 0 - batches++ - return nil - }, - tc.batchSize) - require.NoError(t, err) - require.Equal(t, tc.expected, total) - require.Equal(t, int(math.Ceil(float64(tc.expected)/float64(tc.batchSize))), batches) - } - }) - } -} - -// TestTeardown ensures metrics are updated correctly if the userState is discarded -func TestTeardown(t *testing.T) { - reg := prometheus.NewPedanticRegistry() - _, ing := newTestStore(t, - defaultIngesterTestConfig(t), - defaultClientTestConfig(), - defaultLimitsTestConfig(), - reg) - - pushTestSamples(t, ing, 100, 100, 0) - - // Assert exported metrics (3 blocks, 5 files per block, 2 files WAL). - metricNames := []string{ - "cortex_ingester_memory_series_created_total", - "cortex_ingester_memory_series_removed_total", - "cortex_ingester_memory_series", - "cortex_ingester_memory_users", - "cortex_ingester_active_series", - } - - ing.userStatesMtx.Lock() - ing.userStates.purgeAndUpdateActiveSeries(time.Now().Add(-5 * time.Minute)) - ing.userStatesMtx.Unlock() - - assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` - # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. - # TYPE cortex_ingester_memory_series_removed_total counter - cortex_ingester_memory_series_removed_total{user="1"} 0 - cortex_ingester_memory_series_removed_total{user="2"} 0 - cortex_ingester_memory_series_removed_total{user="3"} 0 - # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. - # TYPE cortex_ingester_memory_series_created_total counter - cortex_ingester_memory_series_created_total{user="1"} 100 - cortex_ingester_memory_series_created_total{user="2"} 100 - cortex_ingester_memory_series_created_total{user="3"} 100 - # HELP cortex_ingester_memory_series The current number of series in memory. - # TYPE cortex_ingester_memory_series gauge - cortex_ingester_memory_series 300 - # HELP cortex_ingester_memory_users The current number of users in memory. - # TYPE cortex_ingester_memory_users gauge - cortex_ingester_memory_users 3 - # HELP cortex_ingester_active_series Number of currently active series per user. - # TYPE cortex_ingester_active_series gauge - cortex_ingester_active_series{user="1"} 100 - cortex_ingester_active_series{user="2"} 100 - cortex_ingester_active_series{user="3"} 100 - `), metricNames...)) - - ing.userStatesMtx.Lock() - defer ing.userStatesMtx.Unlock() - ing.userStates.teardown() - - assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` - # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. - # TYPE cortex_ingester_memory_series_removed_total counter - cortex_ingester_memory_series_removed_total{user="1"} 100 - cortex_ingester_memory_series_removed_total{user="2"} 100 - cortex_ingester_memory_series_removed_total{user="3"} 100 - # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. - # TYPE cortex_ingester_memory_series_created_total counter - cortex_ingester_memory_series_created_total{user="1"} 100 - cortex_ingester_memory_series_created_total{user="2"} 100 - cortex_ingester_memory_series_created_total{user="3"} 100 - # HELP cortex_ingester_memory_series The current number of series in memory. - # TYPE cortex_ingester_memory_series gauge - cortex_ingester_memory_series 0 - # HELP cortex_ingester_memory_users The current number of users in memory. - # TYPE cortex_ingester_memory_users gauge - cortex_ingester_memory_users 0 - # HELP cortex_ingester_active_series Number of currently active series per user. - # TYPE cortex_ingester_active_series gauge - cortex_ingester_active_series{user="1"} 0 - cortex_ingester_active_series{user="2"} 0 - cortex_ingester_active_series{user="3"} 0 - `), metricNames...)) -} - func TestMetricCounter(t *testing.T) { const metric = "metric" diff --git a/pkg/ingester/wal.go b/pkg/ingester/wal.go index d714294be7..704179c391 100644 --- a/pkg/ingester/wal.go +++ b/pkg/ingester/wal.go @@ -2,32 +2,7 @@ package ingester import ( "flag" - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "regexp" - "runtime" - "strconv" - "sync" "time" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/gogo/protobuf/proto" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/tsdb/encoding" - tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" - "github.com/prometheus/prometheus/tsdb/fileutil" - tsdb_record "github.com/prometheus/prometheus/tsdb/record" - "github.com/prometheus/prometheus/tsdb/wal" - - "github.com/cortexproject/cortex/pkg/cortexpb" - "github.com/cortexproject/cortex/pkg/ingester/client" ) // WALConfig is config for the Write Ahead Log. @@ -52,1083 +27,3 @@ func (cfg *WALConfig) RegisterFlags(f *flag.FlagSet) { f.BoolVar(&cfg.FlushOnShutdown, "ingester.flush-on-shutdown-with-wal-enabled", false, "When WAL is enabled, should chunks be flushed to long-term storage on shutdown. Useful eg. for migration to blocks engine.") cfg.checkpointDuringShutdown = true } - -// WAL interface allows us to have a no-op WAL when the WAL is disabled. -type WAL interface { - // Log marshalls the records and writes it into the WAL. - Log(*WALRecord) error - // Stop stops all the WAL operations. - Stop() -} - -// RecordType represents the type of the WAL/Checkpoint record. -type RecordType byte - -const ( - // WALRecordSeries is the type for the WAL record on Prometheus TSDB record for series. - WALRecordSeries RecordType = 1 - // WALRecordSamples is the type for the WAL record based on Prometheus TSDB record for samples. - WALRecordSamples RecordType = 2 - - // CheckpointRecord is the type for the Checkpoint record based on protos. - CheckpointRecord RecordType = 3 -) - -type noopWAL struct{} - -func (noopWAL) Log(*WALRecord) error { return nil } -func (noopWAL) Stop() {} - -type walWrapper struct { - cfg WALConfig - quit chan struct{} - wait sync.WaitGroup - - wal *wal.WAL - getUserStates func() map[string]*userState - checkpointMtx sync.Mutex - bytesPool sync.Pool - - logger log.Logger - - // Metrics. - checkpointDeleteFail prometheus.Counter - checkpointDeleteTotal prometheus.Counter - checkpointCreationFail prometheus.Counter - checkpointCreationTotal prometheus.Counter - checkpointDuration prometheus.Summary - checkpointLoggedBytesTotal prometheus.Counter - walLoggedBytesTotal prometheus.Counter - walRecordsLogged prometheus.Counter -} - -// newWAL creates a WAL object. If the WAL is disabled, then the returned WAL is a no-op WAL. -func newWAL(cfg WALConfig, userStatesFunc func() map[string]*userState, registerer prometheus.Registerer, logger log.Logger) (WAL, error) { - if !cfg.WALEnabled { - return &noopWAL{}, nil - } - - var walRegistry prometheus.Registerer - if registerer != nil { - walRegistry = prometheus.WrapRegistererWith(prometheus.Labels{"kind": "wal"}, registerer) - } - tsdbWAL, err := wal.NewSize(logger, walRegistry, cfg.Dir, wal.DefaultSegmentSize/4, false) - if err != nil { - return nil, err - } - - w := &walWrapper{ - cfg: cfg, - quit: make(chan struct{}), - wal: tsdbWAL, - getUserStates: userStatesFunc, - bytesPool: sync.Pool{ - New: func() interface{} { - return make([]byte, 0, 512) - }, - }, - logger: logger, - } - - w.checkpointDeleteFail = promauto.With(registerer).NewCounter(prometheus.CounterOpts{ - Name: "cortex_ingester_checkpoint_deletions_failed_total", - Help: "Total number of checkpoint deletions that failed.", - }) - w.checkpointDeleteTotal = promauto.With(registerer).NewCounter(prometheus.CounterOpts{ - Name: "cortex_ingester_checkpoint_deletions_total", - Help: "Total number of checkpoint deletions attempted.", - }) - w.checkpointCreationFail = promauto.With(registerer).NewCounter(prometheus.CounterOpts{ - Name: "cortex_ingester_checkpoint_creations_failed_total", - Help: "Total number of checkpoint creations that failed.", - }) - w.checkpointCreationTotal = promauto.With(registerer).NewCounter(prometheus.CounterOpts{ - Name: "cortex_ingester_checkpoint_creations_total", - Help: "Total number of checkpoint creations attempted.", - }) - w.checkpointDuration = promauto.With(registerer).NewSummary(prometheus.SummaryOpts{ - Name: "cortex_ingester_checkpoint_duration_seconds", - Help: "Time taken to create a checkpoint.", - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, - }) - w.walRecordsLogged = promauto.With(registerer).NewCounter(prometheus.CounterOpts{ - Name: "cortex_ingester_wal_records_logged_total", - Help: "Total number of WAL records logged.", - }) - w.checkpointLoggedBytesTotal = promauto.With(registerer).NewCounter(prometheus.CounterOpts{ - Name: "cortex_ingester_checkpoint_logged_bytes_total", - Help: "Total number of bytes written to disk for checkpointing.", - }) - w.walLoggedBytesTotal = promauto.With(registerer).NewCounter(prometheus.CounterOpts{ - Name: "cortex_ingester_wal_logged_bytes_total", - Help: "Total number of bytes written to disk for WAL records.", - }) - - w.wait.Add(1) - go w.run() - return w, nil -} - -func (w *walWrapper) Stop() { - close(w.quit) - w.wait.Wait() - w.wal.Close() -} - -func (w *walWrapper) Log(record *WALRecord) error { - if record == nil || (len(record.Series) == 0 && len(record.Samples) == 0) { - return nil - } - select { - case <-w.quit: - return nil - default: - buf := w.bytesPool.Get().([]byte)[:0] - defer func() { - w.bytesPool.Put(buf) // nolint:staticcheck - }() - - if len(record.Series) > 0 { - buf = record.encodeSeries(buf) - if err := w.wal.Log(buf); err != nil { - return err - } - w.walRecordsLogged.Inc() - w.walLoggedBytesTotal.Add(float64(len(buf))) - buf = buf[:0] - } - if len(record.Samples) > 0 { - buf = record.encodeSamples(buf) - if err := w.wal.Log(buf); err != nil { - return err - } - w.walRecordsLogged.Inc() - w.walLoggedBytesTotal.Add(float64(len(buf))) - } - return nil - } -} - -func (w *walWrapper) run() { - defer w.wait.Done() - - if !w.cfg.CheckpointEnabled { - return - } - - ticker := time.NewTicker(w.cfg.CheckpointDuration) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - start := time.Now() - level.Info(w.logger).Log("msg", "starting checkpoint") - if err := w.performCheckpoint(false); err != nil { - level.Error(w.logger).Log("msg", "error checkpointing series", "err", err) - continue - } - elapsed := time.Since(start) - level.Info(w.logger).Log("msg", "checkpoint done", "time", elapsed.String()) - w.checkpointDuration.Observe(elapsed.Seconds()) - case <-w.quit: - if w.cfg.checkpointDuringShutdown { - level.Info(w.logger).Log("msg", "creating checkpoint before shutdown") - if err := w.performCheckpoint(true); err != nil { - level.Error(w.logger).Log("msg", "error checkpointing series during shutdown", "err", err) - } - } - return - } - } -} - -const checkpointPrefix = "checkpoint." - -func (w *walWrapper) performCheckpoint(immediate bool) (err error) { - if !w.cfg.CheckpointEnabled { - return nil - } - - // This method is called during shutdown which can interfere with ongoing checkpointing. - // Hence to avoid any race between file creation and WAL truncation, we hold this lock here. - w.checkpointMtx.Lock() - defer w.checkpointMtx.Unlock() - - w.checkpointCreationTotal.Inc() - defer func() { - if err != nil { - w.checkpointCreationFail.Inc() - } - }() - - if w.getUserStates == nil { - return errors.New("function to get user states not initialised") - } - - _, lastSegment, err := wal.Segments(w.wal.Dir()) - if err != nil { - return err - } - if lastSegment < 0 { - // There are no WAL segments. No need of checkpoint yet. - return nil - } - - _, lastCh, err := lastCheckpoint(w.wal.Dir()) - if err != nil { - return err - } - - if lastCh == lastSegment { - // As the checkpoint name is taken from last WAL segment, we need to ensure - // a new segment for every checkpoint so that the old checkpoint is not overwritten. - if err := w.wal.NextSegment(); err != nil { - return err - } - - _, lastSegment, err = wal.Segments(w.wal.Dir()) - if err != nil { - return err - } - } - - // Checkpoint is named after the last WAL segment present so that when replaying the WAL - // we can start from that particular WAL segment. - checkpointDir := filepath.Join(w.wal.Dir(), fmt.Sprintf(checkpointPrefix+"%06d", lastSegment)) - level.Info(w.logger).Log("msg", "attempting checkpoint for", "dir", checkpointDir) - checkpointDirTemp := checkpointDir + ".tmp" - - if err := os.MkdirAll(checkpointDirTemp, 0777); err != nil { - return errors.Wrap(err, "create checkpoint dir") - } - checkpoint, err := wal.New(nil, nil, checkpointDirTemp, false) - if err != nil { - return errors.Wrap(err, "open checkpoint") - } - defer func() { - checkpoint.Close() - os.RemoveAll(checkpointDirTemp) - }() - - // Count number of series - we'll use this to rate limit checkpoints. - numSeries := 0 - us := w.getUserStates() - for _, state := range us { - numSeries += state.fpToSeries.length() - } - if numSeries == 0 { - return nil - } - - perSeriesDuration := (95 * w.cfg.CheckpointDuration) / (100 * time.Duration(numSeries)) - - var wireChunkBuf []client.Chunk - var b []byte - bytePool := sync.Pool{ - New: func() interface{} { - return make([]byte, 0, 1024) - }, - } - records := [][]byte{} - totalSize := 0 - ticker := time.NewTicker(perSeriesDuration) - defer ticker.Stop() - start := time.Now() - for userID, state := range us { - for pair := range state.fpToSeries.iter() { - state.fpLocker.Lock(pair.fp) - wireChunkBuf, b, err = w.checkpointSeries(userID, pair.fp, pair.series, wireChunkBuf, bytePool.Get().([]byte)) - state.fpLocker.Unlock(pair.fp) - if err != nil { - return err - } - - records = append(records, b) - totalSize += len(b) - if totalSize >= 1*1024*1024 { // 1 MiB. - if err := checkpoint.Log(records...); err != nil { - return err - } - w.checkpointLoggedBytesTotal.Add(float64(totalSize)) - totalSize = 0 - for i := range records { - bytePool.Put(records[i]) // nolint:staticcheck - } - records = records[:0] - } - - if !immediate { - if time.Since(start) > 2*w.cfg.CheckpointDuration { - // This could indicate a surge in number of series and continuing with - // the old estimation of ticker can make checkpointing run indefinitely in worst case - // and disk running out of space. Re-adjust the ticker might not solve the problem - // as there can be another surge again. Hence let's checkpoint this one immediately. - immediate = true - continue - } - - select { - case <-ticker.C: - case <-w.quit: // When we're trying to shutdown, finish the checkpoint as fast as possible. - } - } - } - } - - if err := checkpoint.Log(records...); err != nil { - return err - } - - if err := checkpoint.Close(); err != nil { - return errors.Wrap(err, "close checkpoint") - } - if err := fileutil.Replace(checkpointDirTemp, checkpointDir); err != nil { - return errors.Wrap(err, "rename checkpoint directory") - } - - // We delete the WAL segments which are before the previous checkpoint and not before the - // current checkpoint created. This is because if the latest checkpoint is corrupted for any reason, we - // should be able to recover from the older checkpoint which would need the older WAL segments. - if err := w.wal.Truncate(lastCh); err != nil { - // It is fine to have old WAL segments hanging around if deletion failed. - // We can try again next time. - level.Error(w.logger).Log("msg", "error deleting old WAL segments", "err", err) - } - - if lastCh >= 0 { - if err := w.deleteCheckpoints(lastCh); err != nil { - // It is fine to have old checkpoints hanging around if deletion failed. - // We can try again next time. - level.Error(w.logger).Log("msg", "error deleting old checkpoint", "err", err) - } - } - - return nil -} - -// lastCheckpoint returns the directory name and index of the most recent checkpoint. -// If dir does not contain any checkpoints, -1 is returned as index. -func lastCheckpoint(dir string) (string, int, error) { - dirs, err := ioutil.ReadDir(dir) - if err != nil { - return "", -1, err - } - var ( - maxIdx = -1 - checkpointDir string - ) - // There may be multiple checkpoints left, so select the one with max index. - for i := 0; i < len(dirs); i++ { - di := dirs[i] - - idx, err := checkpointIndex(di.Name(), false) - if err != nil { - continue - } - if !di.IsDir() { - return "", -1, fmt.Errorf("checkpoint %s is not a directory", di.Name()) - } - if idx > maxIdx { - checkpointDir = di.Name() - maxIdx = idx - } - } - if maxIdx >= 0 { - return filepath.Join(dir, checkpointDir), maxIdx, nil - } - return "", -1, nil -} - -// deleteCheckpoints deletes all checkpoints in a directory which is < maxIndex. -func (w *walWrapper) deleteCheckpoints(maxIndex int) (err error) { - w.checkpointDeleteTotal.Inc() - defer func() { - if err != nil { - w.checkpointDeleteFail.Inc() - } - }() - - errs := tsdb_errors.NewMulti() - - files, err := ioutil.ReadDir(w.wal.Dir()) - if err != nil { - return err - } - for _, fi := range files { - index, err := checkpointIndex(fi.Name(), true) - if err != nil || index >= maxIndex { - continue - } - if err := os.RemoveAll(filepath.Join(w.wal.Dir(), fi.Name())); err != nil { - errs.Add(err) - } - } - return errs.Err() -} - -var checkpointRe = regexp.MustCompile("^" + regexp.QuoteMeta(checkpointPrefix) + "(\\d+)(\\.tmp)?$") - -// checkpointIndex returns the index of a given checkpoint file. It handles -// both regular and temporary checkpoints according to the includeTmp flag. If -// the file is not a checkpoint it returns an error. -func checkpointIndex(filename string, includeTmp bool) (int, error) { - result := checkpointRe.FindStringSubmatch(filename) - if len(result) < 2 { - return 0, errors.New("file is not a checkpoint") - } - // Filter out temporary checkpoints if desired. - if !includeTmp && len(result) == 3 && result[2] != "" { - return 0, errors.New("temporary checkpoint") - } - return strconv.Atoi(result[1]) -} - -// checkpointSeries write the chunks of the series to the checkpoint. -func (w *walWrapper) checkpointSeries(userID string, fp model.Fingerprint, series *memorySeries, wireChunks []client.Chunk, b []byte) ([]client.Chunk, []byte, error) { - var err error - wireChunks, err = toWireChunks(series.chunkDescs, wireChunks) - if err != nil { - return wireChunks, b, err - } - - b, err = encodeWithTypeHeader(&Series{ - UserId: userID, - Fingerprint: uint64(fp), - Labels: cortexpb.FromLabelsToLabelAdapters(series.metric), - Chunks: wireChunks, - }, CheckpointRecord, b) - - return wireChunks, b, err -} - -type walRecoveryParameters struct { - walDir string - ingester *Ingester - numWorkers int - stateCache []map[string]*userState - seriesCache []map[string]map[uint64]*memorySeries -} - -func recoverFromWAL(ingester *Ingester) error { - params := walRecoveryParameters{ - walDir: ingester.cfg.WALConfig.Dir, - numWorkers: runtime.GOMAXPROCS(0), - ingester: ingester, - } - - params.stateCache = make([]map[string]*userState, params.numWorkers) - params.seriesCache = make([]map[string]map[uint64]*memorySeries, params.numWorkers) - for i := 0; i < params.numWorkers; i++ { - params.stateCache[i] = make(map[string]*userState) - params.seriesCache[i] = make(map[string]map[uint64]*memorySeries) - } - - level.Info(ingester.logger).Log("msg", "recovering from checkpoint") - start := time.Now() - userStates, idx, err := processCheckpointWithRepair(params) - if err != nil { - return err - } - elapsed := time.Since(start) - level.Info(ingester.logger).Log("msg", "recovered from checkpoint", "time", elapsed.String()) - - if segExists, err := segmentsExist(params.walDir); err == nil && !segExists { - level.Info(ingester.logger).Log("msg", "no segments found, skipping recover from segments") - ingester.userStatesMtx.Lock() - ingester.userStates = userStates - ingester.userStatesMtx.Unlock() - return nil - } else if err != nil { - return err - } - - level.Info(ingester.logger).Log("msg", "recovering from WAL", "dir", params.walDir, "start_segment", idx) - start = time.Now() - if err := processWALWithRepair(idx, userStates, params); err != nil { - return err - } - elapsed = time.Since(start) - level.Info(ingester.logger).Log("msg", "recovered from WAL", "time", elapsed.String()) - - ingester.userStatesMtx.Lock() - ingester.userStates = userStates - ingester.userStatesMtx.Unlock() - return nil -} - -func processCheckpointWithRepair(params walRecoveryParameters) (*userStates, int, error) { - logger := params.ingester.logger - - // Use a local userStates, so we don't need to worry about locking. - userStates := newUserStates(params.ingester.limiter, params.ingester.cfg, params.ingester.metrics, params.ingester.logger) - - lastCheckpointDir, idx, err := lastCheckpoint(params.walDir) - if err != nil { - return nil, -1, err - } - if idx < 0 { - level.Info(logger).Log("msg", "no checkpoint found") - return userStates, -1, nil - } - - level.Info(logger).Log("msg", fmt.Sprintf("recovering from %s", lastCheckpointDir)) - - err = processCheckpoint(lastCheckpointDir, userStates, params) - if err == nil { - return userStates, idx, nil - } - - // We don't call repair on checkpoint as losing even a single record is like losing the entire data of a series. - // We try recovering from the older checkpoint instead. - params.ingester.metrics.walCorruptionsTotal.Inc() - level.Error(logger).Log("msg", "checkpoint recovery failed, deleting this checkpoint and trying to recover from old checkpoint", "err", err) - - // Deleting this checkpoint to try the previous checkpoint. - if err := os.RemoveAll(lastCheckpointDir); err != nil { - return nil, -1, errors.Wrapf(err, "unable to delete checkpoint directory %s", lastCheckpointDir) - } - - // If we have reached this point, it means the last checkpoint was deleted. - // Now the last checkpoint will be the one before the deleted checkpoint. - lastCheckpointDir, idx, err = lastCheckpoint(params.walDir) - if err != nil { - return nil, -1, err - } - - // Creating new userStates to discard the old chunks. - userStates = newUserStates(params.ingester.limiter, params.ingester.cfg, params.ingester.metrics, params.ingester.logger) - if idx < 0 { - // There was only 1 checkpoint. We don't error in this case - // as for the first checkpoint entire WAL will/should be present. - return userStates, -1, nil - } - - level.Info(logger).Log("msg", fmt.Sprintf("attempting recovery from %s", lastCheckpointDir)) - if err := processCheckpoint(lastCheckpointDir, userStates, params); err != nil { - // We won't attempt the repair again even if its the old checkpoint. - params.ingester.metrics.walCorruptionsTotal.Inc() - return nil, -1, err - } - - return userStates, idx, nil -} - -// segmentsExist is a stripped down version of -// https://github.com/prometheus/prometheus/blob/4c648eddf47d7e07fbc74d0b18244402200dca9e/tsdb/wal/wal.go#L739-L760. -func segmentsExist(dir string) (bool, error) { - files, err := ioutil.ReadDir(dir) - if err != nil { - return false, err - } - for _, f := range files { - if _, err := strconv.Atoi(f.Name()); err == nil { - // First filename which is a number. - // This is how Prometheus stores and this - // is how it checks too. - return true, nil - } - } - return false, nil -} - -// processCheckpoint loads the chunks of the series present in the last checkpoint. -func processCheckpoint(name string, userStates *userStates, params walRecoveryParameters) error { - - reader, closer, err := newWalReader(name, -1) - if err != nil { - return err - } - defer closer.Close() - - var ( - inputs = make([]chan *Series, params.numWorkers) - // errChan is to capture the errors from goroutine. - // The channel size is nWorkers+1 to not block any worker if all of them error out. - errChan = make(chan error, params.numWorkers) - wg = sync.WaitGroup{} - seriesPool = &sync.Pool{ - New: func() interface{} { - return &Series{} - }, - } - ) - - wg.Add(params.numWorkers) - for i := 0; i < params.numWorkers; i++ { - inputs[i] = make(chan *Series, 300) - go func(input <-chan *Series, stateCache map[string]*userState, seriesCache map[string]map[uint64]*memorySeries) { - processCheckpointRecord(userStates, seriesPool, stateCache, seriesCache, input, errChan, params.ingester.metrics.memoryChunks) - wg.Done() - }(inputs[i], params.stateCache[i], params.seriesCache[i]) - } - - var capturedErr error -Loop: - for reader.Next() { - s := seriesPool.Get().(*Series) - m, err := decodeCheckpointRecord(reader.Record(), s) - if err != nil { - // We don't return here in order to close/drain all the channels and - // make sure all goroutines exit. - capturedErr = err - break Loop - } - s = m.(*Series) - - // The yoloString from the unmarshal of LabelAdapter gets corrupted - // when travelling through the channel. Hence making a copy of that. - // This extra alloc during the read path is fine as it's only 1 time - // and saves extra allocs during write path by having LabelAdapter. - s.Labels = copyLabelAdapters(s.Labels) - - select { - case capturedErr = <-errChan: - // Exit early on an error. - // Only acts upon the first error received. - break Loop - default: - mod := s.Fingerprint % uint64(params.numWorkers) - inputs[mod] <- s - } - } - - for i := 0; i < params.numWorkers; i++ { - close(inputs[i]) - } - wg.Wait() - // If any worker errored out, some input channels might not be empty. - // Hence drain them. - for i := 0; i < params.numWorkers; i++ { - for range inputs[i] { - } - } - - if capturedErr != nil { - return capturedErr - } - select { - case capturedErr = <-errChan: - return capturedErr - default: - return reader.Err() - } -} - -func copyLabelAdapters(las []cortexpb.LabelAdapter) []cortexpb.LabelAdapter { - for i := range las { - n, v := make([]byte, len(las[i].Name)), make([]byte, len(las[i].Value)) - copy(n, las[i].Name) - copy(v, las[i].Value) - las[i].Name = string(n) - las[i].Value = string(v) - } - return las -} - -func processCheckpointRecord( - userStates *userStates, - seriesPool *sync.Pool, - stateCache map[string]*userState, - seriesCache map[string]map[uint64]*memorySeries, - seriesChan <-chan *Series, - errChan chan error, - memoryChunks prometheus.Counter, -) { - var la []cortexpb.LabelAdapter - for s := range seriesChan { - state, ok := stateCache[s.UserId] - if !ok { - state = userStates.getOrCreate(s.UserId) - stateCache[s.UserId] = state - seriesCache[s.UserId] = make(map[uint64]*memorySeries) - } - - la = la[:0] - for _, l := range s.Labels { - la = append(la, cortexpb.LabelAdapter{ - Name: string(l.Name), - Value: string(l.Value), - }) - } - series, err := state.createSeriesWithFingerprint(model.Fingerprint(s.Fingerprint), la, nil, true) - if err != nil { - errChan <- err - return - } - - descs, err := fromWireChunks(s.Chunks) - if err != nil { - errChan <- err - return - } - - if err := series.setChunks(descs); err != nil { - errChan <- err - return - } - memoryChunks.Add(float64(len(descs))) - - seriesCache[s.UserId][s.Fingerprint] = series - seriesPool.Put(s) - } -} - -type samplesWithUserID struct { - samples []tsdb_record.RefSample - userID string -} - -func processWALWithRepair(startSegment int, userStates *userStates, params walRecoveryParameters) error { - logger := params.ingester.logger - - corruptErr := processWAL(startSegment, userStates, params) - if corruptErr == nil { - return nil - } - - params.ingester.metrics.walCorruptionsTotal.Inc() - level.Error(logger).Log("msg", "error in replaying from WAL", "err", corruptErr) - - // Attempt repair. - level.Info(logger).Log("msg", "attempting repair of the WAL") - w, err := wal.New(logger, nil, params.walDir, true) - if err != nil { - return err - } - - err = w.Repair(corruptErr) - if err != nil { - level.Error(logger).Log("msg", "error in repairing WAL", "err", err) - } - - return tsdb_errors.NewMulti(err, w.Close()).Err() -} - -// processWAL processes the records in the WAL concurrently. -func processWAL(startSegment int, userStates *userStates, params walRecoveryParameters) error { - - reader, closer, err := newWalReader(params.walDir, startSegment) - if err != nil { - return err - } - defer closer.Close() - - var ( - wg sync.WaitGroup - inputs = make([]chan *samplesWithUserID, params.numWorkers) - outputs = make([]chan *samplesWithUserID, params.numWorkers) - // errChan is to capture the errors from goroutine. - // The channel size is nWorkers to not block any worker if all of them error out. - errChan = make(chan error, params.numWorkers) - shards = make([]*samplesWithUserID, params.numWorkers) - ) - - wg.Add(params.numWorkers) - for i := 0; i < params.numWorkers; i++ { - outputs[i] = make(chan *samplesWithUserID, 300) - inputs[i] = make(chan *samplesWithUserID, 300) - shards[i] = &samplesWithUserID{} - - go func(input <-chan *samplesWithUserID, output chan<- *samplesWithUserID, - stateCache map[string]*userState, seriesCache map[string]map[uint64]*memorySeries) { - processWALSamples(userStates, stateCache, seriesCache, input, output, errChan, params.ingester.logger) - wg.Done() - }(inputs[i], outputs[i], params.stateCache[i], params.seriesCache[i]) - } - - var ( - capturedErr error - walRecord = &WALRecord{} - lp labelPairs - ) -Loop: - for reader.Next() { - select { - case capturedErr = <-errChan: - // Exit early on an error. - // Only acts upon the first error received. - break Loop - default: - } - - if err := decodeWALRecord(reader.Record(), walRecord); err != nil { - // We don't return here in order to close/drain all the channels and - // make sure all goroutines exit. - capturedErr = err - break Loop - } - - if len(walRecord.Series) > 0 { - userID := walRecord.UserID - - state := userStates.getOrCreate(userID) - - for _, s := range walRecord.Series { - fp := model.Fingerprint(s.Ref) - _, ok := state.fpToSeries.get(fp) - if ok { - continue - } - - lp = lp[:0] - for _, l := range s.Labels { - lp = append(lp, cortexpb.LabelAdapter(l)) - } - if _, err := state.createSeriesWithFingerprint(fp, lp, nil, true); err != nil { - // We don't return here in order to close/drain all the channels and - // make sure all goroutines exit. - capturedErr = err - break Loop - } - } - } - - // We split up the samples into chunks of 5000 samples or less. - // With O(300 * #cores) in-flight sample batches, large scrapes could otherwise - // cause thousands of very large in flight buffers occupying large amounts - // of unused memory. - walRecordSamples := walRecord.Samples - for len(walRecordSamples) > 0 { - m := 5000 - userID := walRecord.UserID - if len(walRecordSamples) < m { - m = len(walRecordSamples) - } - - for i := 0; i < params.numWorkers; i++ { - if len(shards[i].samples) == 0 { - // It is possible that the previous iteration did not put - // anything in this shard. In that case no need to get a new buffer. - shards[i].userID = userID - continue - } - select { - case buf := <-outputs[i]: - buf.samples = buf.samples[:0] - buf.userID = userID - shards[i] = buf - default: - shards[i] = &samplesWithUserID{ - userID: userID, - } - } - } - - for _, sam := range walRecordSamples[:m] { - mod := uint64(sam.Ref) % uint64(params.numWorkers) - shards[mod].samples = append(shards[mod].samples, sam) - } - - for i := 0; i < params.numWorkers; i++ { - if len(shards[i].samples) > 0 { - inputs[i] <- shards[i] - } - } - - walRecordSamples = walRecordSamples[m:] - } - } - - for i := 0; i < params.numWorkers; i++ { - close(inputs[i]) - for range outputs[i] { - } - } - wg.Wait() - // If any worker errored out, some input channels might not be empty. - // Hence drain them. - for i := 0; i < params.numWorkers; i++ { - for range inputs[i] { - } - } - - if capturedErr != nil { - return capturedErr - } - select { - case capturedErr = <-errChan: - return capturedErr - default: - return reader.Err() - } -} - -func processWALSamples(userStates *userStates, stateCache map[string]*userState, seriesCache map[string]map[uint64]*memorySeries, - input <-chan *samplesWithUserID, output chan<- *samplesWithUserID, errChan chan error, logger log.Logger) { - defer close(output) - - sp := model.SamplePair{} - for samples := range input { - state, ok := stateCache[samples.userID] - if !ok { - state = userStates.getOrCreate(samples.userID) - stateCache[samples.userID] = state - seriesCache[samples.userID] = make(map[uint64]*memorySeries) - } - sc := seriesCache[samples.userID] - for i := range samples.samples { - series, ok := sc[uint64(samples.samples[i].Ref)] - if !ok { - series, ok = state.fpToSeries.get(model.Fingerprint(samples.samples[i].Ref)) - if !ok { - // This should ideally not happen. - // If the series was not created in recovering checkpoint or - // from the labels of any records previous to this, there - // is no way to get the labels for this fingerprint. - level.Warn(logger).Log("msg", "series not found for sample during wal recovery", "userid", samples.userID, "fingerprint", model.Fingerprint(samples.samples[i].Ref).String()) - continue - } - } - sp.Timestamp = model.Time(samples.samples[i].T) - sp.Value = model.SampleValue(samples.samples[i].V) - // There can be many out of order samples because of checkpoint and WAL overlap. - // Checking this beforehand avoids the allocation of lots of error messages. - if sp.Timestamp.After(series.lastTime) { - if err := series.add(sp); err != nil { - errChan <- err - return - } - } - } - output <- samples - } -} - -// If startSegment is <0, it means all the segments. -func newWalReader(name string, startSegment int) (*wal.Reader, io.Closer, error) { - var ( - segmentReader io.ReadCloser - err error - ) - if startSegment < 0 { - segmentReader, err = wal.NewSegmentsReader(name) - if err != nil { - return nil, nil, err - } - } else { - first, last, err := wal.Segments(name) - if err != nil { - return nil, nil, err - } - if startSegment > last { - return nil, nil, errors.New("start segment is beyond the last WAL segment") - } - if first > startSegment { - startSegment = first - } - segmentReader, err = wal.NewSegmentsRangeReader(wal.SegmentRange{ - Dir: name, - First: startSegment, - Last: -1, // Till the end. - }) - if err != nil { - return nil, nil, err - } - } - return wal.NewReader(segmentReader), segmentReader, nil -} - -func decodeCheckpointRecord(rec []byte, m proto.Message) (_ proto.Message, err error) { - switch RecordType(rec[0]) { - case CheckpointRecord: - if err := proto.Unmarshal(rec[1:], m); err != nil { - return m, err - } - default: - // The legacy proto record will have it's first byte >7. - // Hence it does not match any of the existing record types. - err := proto.Unmarshal(rec, m) - if err != nil { - return m, err - } - } - - return m, err -} - -func encodeWithTypeHeader(m proto.Message, typ RecordType, b []byte) ([]byte, error) { - buf, err := proto.Marshal(m) - if err != nil { - return b, err - } - - b = append(b[:0], byte(typ)) - b = append(b, buf...) - return b, nil -} - -// WALRecord is a struct combining the series and samples record. -type WALRecord struct { - UserID string - Series []tsdb_record.RefSeries - Samples []tsdb_record.RefSample -} - -func (record *WALRecord) encodeSeries(b []byte) []byte { - buf := encoding.Encbuf{B: b} - buf.PutByte(byte(WALRecordSeries)) - buf.PutUvarintStr(record.UserID) - - var enc tsdb_record.Encoder - // The 'encoded' already has the type header and userID here, hence re-using - // the remaining part of the slice (i.e. encoded[len(encoded):])) to encode the series. - encoded := buf.Get() - encoded = append(encoded, enc.Series(record.Series, encoded[len(encoded):])...) - - return encoded -} - -func (record *WALRecord) encodeSamples(b []byte) []byte { - buf := encoding.Encbuf{B: b} - buf.PutByte(byte(WALRecordSamples)) - buf.PutUvarintStr(record.UserID) - - var enc tsdb_record.Encoder - // The 'encoded' already has the type header and userID here, hence re-using - // the remaining part of the slice (i.e. encoded[len(encoded):]))to encode the samples. - encoded := buf.Get() - encoded = append(encoded, enc.Samples(record.Samples, encoded[len(encoded):])...) - - return encoded -} - -func decodeWALRecord(b []byte, walRec *WALRecord) (err error) { - var ( - userID string - dec tsdb_record.Decoder - rseries []tsdb_record.RefSeries - rsamples []tsdb_record.RefSample - - decbuf = encoding.Decbuf{B: b} - t = RecordType(decbuf.Byte()) - ) - - walRec.Series = walRec.Series[:0] - walRec.Samples = walRec.Samples[:0] - switch t { - case WALRecordSamples: - userID = decbuf.UvarintStr() - rsamples, err = dec.Samples(decbuf.B, walRec.Samples) - case WALRecordSeries: - userID = decbuf.UvarintStr() - rseries, err = dec.Series(decbuf.B, walRec.Series) - default: - return errors.New("unknown record type") - } - - // We reach here only if its a record with type header. - if decbuf.Err() != nil { - return decbuf.Err() - } - - if err != nil { - return err - } - - walRec.UserID = userID - walRec.Samples = rsamples - walRec.Series = rseries - - return nil -} diff --git a/pkg/ingester/wal.pb.go b/pkg/ingester/wal.pb.go deleted file mode 100644 index 65f0421470..0000000000 --- a/pkg/ingester/wal.pb.go +++ /dev/null @@ -1,607 +0,0 @@ -// Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: wal.proto - -package ingester - -import ( - fmt "fmt" - _ "github.com/cortexproject/cortex/pkg/cortexpb" - github_com_cortexproject_cortex_pkg_cortexpb "github.com/cortexproject/cortex/pkg/cortexpb" - client "github.com/cortexproject/cortex/pkg/ingester/client" - _ "github.com/gogo/protobuf/gogoproto" - proto "github.com/gogo/protobuf/proto" - io "io" - math "math" - math_bits "math/bits" - reflect "reflect" - strings "strings" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package - -type Series struct { - UserId string `protobuf:"bytes,1,opt,name=user_id,json=userId,proto3" json:"user_id,omitempty"` - Fingerprint uint64 `protobuf:"varint,2,opt,name=fingerprint,proto3" json:"fingerprint,omitempty"` - Labels []github_com_cortexproject_cortex_pkg_cortexpb.LabelAdapter `protobuf:"bytes,3,rep,name=labels,proto3,customtype=github.com/cortexproject/cortex/pkg/cortexpb.LabelAdapter" json:"labels"` - Chunks []client.Chunk `protobuf:"bytes,4,rep,name=chunks,proto3" json:"chunks"` -} - -func (m *Series) Reset() { *m = Series{} } -func (*Series) ProtoMessage() {} -func (*Series) Descriptor() ([]byte, []int) { - return fileDescriptor_ae6364fc8077884f, []int{0} -} -func (m *Series) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Series) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Series.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *Series) XXX_Merge(src proto.Message) { - xxx_messageInfo_Series.Merge(m, src) -} -func (m *Series) XXX_Size() int { - return m.Size() -} -func (m *Series) XXX_DiscardUnknown() { - xxx_messageInfo_Series.DiscardUnknown(m) -} - -var xxx_messageInfo_Series proto.InternalMessageInfo - -func (m *Series) GetUserId() string { - if m != nil { - return m.UserId - } - return "" -} - -func (m *Series) GetFingerprint() uint64 { - if m != nil { - return m.Fingerprint - } - return 0 -} - -func (m *Series) GetChunks() []client.Chunk { - if m != nil { - return m.Chunks - } - return nil -} - -func init() { - proto.RegisterType((*Series)(nil), "ingester.Series") -} - -func init() { proto.RegisterFile("wal.proto", fileDescriptor_ae6364fc8077884f) } - -var fileDescriptor_ae6364fc8077884f = []byte{ - // 323 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x91, 0x31, 0x4e, 0xc3, 0x30, - 0x18, 0x85, 0x6d, 0x5a, 0x05, 0xea, 0x8a, 0x25, 0x0c, 0x44, 0x1d, 0xfe, 0x46, 0x4c, 0x95, 0x10, - 0x89, 0x04, 0x13, 0x0b, 0x52, 0xc3, 0x84, 0xc4, 0x80, 0xc2, 0xc6, 0x82, 0x92, 0xd4, 0x4d, 0x4d, - 0x43, 0x1c, 0x39, 0x8e, 0x60, 0xe4, 0x08, 0x1c, 0x83, 0xa3, 0x74, 0xec, 0x58, 0x31, 0x54, 0xad, - 0xbb, 0x30, 0xf6, 0x08, 0x28, 0xae, 0x5b, 0x75, 0x84, 0xed, 0x7f, 0x2f, 0xef, 0xcb, 0xfb, 0x6d, - 0x93, 0xd6, 0x5b, 0x94, 0x79, 0x85, 0xe0, 0x92, 0xdb, 0x47, 0x2c, 0x4f, 0x69, 0x29, 0xa9, 0xe8, - 0x5c, 0xa4, 0x4c, 0x8e, 0xaa, 0xd8, 0x4b, 0xf8, 0xab, 0x9f, 0xf2, 0x94, 0xfb, 0x3a, 0x10, 0x57, - 0x43, 0xad, 0xb4, 0xd0, 0xd3, 0x06, 0xec, 0x5c, 0xef, 0xc5, 0x13, 0x2e, 0x24, 0x7d, 0x2f, 0x04, - 0x7f, 0xa1, 0x89, 0x34, 0xca, 0x2f, 0xc6, 0xe9, 0xf6, 0x43, 0x6c, 0x06, 0x83, 0x06, 0x7f, 0x41, - 0xb7, 0x7b, 0xf9, 0x49, 0xc6, 0x68, 0x2e, 0x77, 0x7a, 0xf3, 0x8f, 0xb3, 0x05, 0x26, 0xd6, 0x23, - 0x15, 0x8c, 0x96, 0xf6, 0x29, 0x39, 0xac, 0x4a, 0x2a, 0x9e, 0xd9, 0xc0, 0xc1, 0x2e, 0xee, 0xb5, - 0x42, 0xab, 0x96, 0x77, 0x03, 0xdb, 0x25, 0xed, 0x61, 0x8d, 0x89, 0x42, 0xb0, 0x5c, 0x3a, 0x07, - 0x2e, 0xee, 0x35, 0xc3, 0x7d, 0xcb, 0xce, 0x89, 0x95, 0x45, 0x31, 0xcd, 0x4a, 0xa7, 0xe1, 0x36, - 0x7a, 0xed, 0xcb, 0x13, 0x6f, 0xbb, 0xb1, 0x77, 0x5f, 0xfb, 0x0f, 0x11, 0x13, 0x41, 0x7f, 0x32, - 0xef, 0xa2, 0xef, 0x79, 0xf7, 0x5f, 0x27, 0xde, 0xf0, 0xfd, 0x41, 0x54, 0x48, 0x2a, 0x42, 0xd3, - 0x62, 0x9f, 0x13, 0x2b, 0x19, 0x55, 0xf9, 0xb8, 0x74, 0x9a, 0xba, 0xef, 0xd8, 0xf4, 0x79, 0xb7, - 0xb5, 0x1b, 0x34, 0xeb, 0xa6, 0xd0, 0x44, 0x82, 0x9b, 0xe9, 0x12, 0xd0, 0x6c, 0x09, 0x68, 0xbd, - 0x04, 0xfc, 0xa1, 0x00, 0x7f, 0x29, 0xc0, 0x13, 0x05, 0x78, 0xaa, 0x00, 0x2f, 0x14, 0xe0, 0x1f, - 0x05, 0x68, 0xad, 0x00, 0x7f, 0xae, 0x00, 0x4d, 0x57, 0x80, 0x66, 0x2b, 0x40, 0x4f, 0xbb, 0x07, - 0x8d, 0x2d, 0x7d, 0x53, 0x57, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xd2, 0x67, 0x44, 0x9d, 0xee, - 0x01, 0x00, 0x00, -} - -func (this *Series) Equal(that interface{}) bool { - if that == nil { - return this == nil - } - - that1, ok := that.(*Series) - if !ok { - that2, ok := that.(Series) - if ok { - that1 = &that2 - } else { - return false - } - } - if that1 == nil { - return this == nil - } else if this == nil { - return false - } - if this.UserId != that1.UserId { - return false - } - if this.Fingerprint != that1.Fingerprint { - return false - } - if len(this.Labels) != len(that1.Labels) { - return false - } - for i := range this.Labels { - if !this.Labels[i].Equal(that1.Labels[i]) { - return false - } - } - if len(this.Chunks) != len(that1.Chunks) { - return false - } - for i := range this.Chunks { - if !this.Chunks[i].Equal(&that1.Chunks[i]) { - return false - } - } - return true -} -func (this *Series) GoString() string { - if this == nil { - return "nil" - } - s := make([]string, 0, 8) - s = append(s, "&ingester.Series{") - s = append(s, "UserId: "+fmt.Sprintf("%#v", this.UserId)+",\n") - s = append(s, "Fingerprint: "+fmt.Sprintf("%#v", this.Fingerprint)+",\n") - s = append(s, "Labels: "+fmt.Sprintf("%#v", this.Labels)+",\n") - if this.Chunks != nil { - vs := make([]*client.Chunk, len(this.Chunks)) - for i := range vs { - vs[i] = &this.Chunks[i] - } - s = append(s, "Chunks: "+fmt.Sprintf("%#v", vs)+",\n") - } - s = append(s, "}") - return strings.Join(s, "") -} -func valueToGoStringWal(v interface{}, typ string) string { - rv := reflect.ValueOf(v) - if rv.IsNil() { - return "nil" - } - pv := reflect.Indirect(rv).Interface() - return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) -} -func (m *Series) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *Series) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *Series) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.Chunks) > 0 { - for iNdEx := len(m.Chunks) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Chunks[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintWal(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x22 - } - } - if len(m.Labels) > 0 { - for iNdEx := len(m.Labels) - 1; iNdEx >= 0; iNdEx-- { - { - size := m.Labels[iNdEx].Size() - i -= size - if _, err := m.Labels[iNdEx].MarshalTo(dAtA[i:]); err != nil { - return 0, err - } - i = encodeVarintWal(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x1a - } - } - if m.Fingerprint != 0 { - i = encodeVarintWal(dAtA, i, uint64(m.Fingerprint)) - i-- - dAtA[i] = 0x10 - } - if len(m.UserId) > 0 { - i -= len(m.UserId) - copy(dAtA[i:], m.UserId) - i = encodeVarintWal(dAtA, i, uint64(len(m.UserId))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - -func encodeVarintWal(dAtA []byte, offset int, v uint64) int { - offset -= sovWal(v) - base := offset - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ - } - dAtA[offset] = uint8(v) - return base -} -func (m *Series) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - l = len(m.UserId) - if l > 0 { - n += 1 + l + sovWal(uint64(l)) - } - if m.Fingerprint != 0 { - n += 1 + sovWal(uint64(m.Fingerprint)) - } - if len(m.Labels) > 0 { - for _, e := range m.Labels { - l = e.Size() - n += 1 + l + sovWal(uint64(l)) - } - } - if len(m.Chunks) > 0 { - for _, e := range m.Chunks { - l = e.Size() - n += 1 + l + sovWal(uint64(l)) - } - } - return n -} - -func sovWal(x uint64) (n int) { - return (math_bits.Len64(x|1) + 6) / 7 -} -func sozWal(x uint64) (n int) { - return sovWal(uint64((x << 1) ^ uint64((int64(x) >> 63)))) -} -func (this *Series) String() string { - if this == nil { - return "nil" - } - repeatedStringForChunks := "[]Chunk{" - for _, f := range this.Chunks { - repeatedStringForChunks += fmt.Sprintf("%v", f) + "," - } - repeatedStringForChunks += "}" - s := strings.Join([]string{`&Series{`, - `UserId:` + fmt.Sprintf("%v", this.UserId) + `,`, - `Fingerprint:` + fmt.Sprintf("%v", this.Fingerprint) + `,`, - `Labels:` + fmt.Sprintf("%v", this.Labels) + `,`, - `Chunks:` + repeatedStringForChunks + `,`, - `}`, - }, "") - return s -} -func valueToStringWal(v interface{}) string { - rv := reflect.ValueOf(v) - if rv.IsNil() { - return "nil" - } - pv := reflect.Indirect(rv).Interface() - return fmt.Sprintf("*%v", pv) -} -func (m *Series) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowWal - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: Series: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: Series: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field UserId", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowWal - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthWal - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthWal - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.UserId = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Fingerprint", wireType) - } - m.Fingerprint = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowWal - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Fingerprint |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowWal - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthWal - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthWal - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Labels = append(m.Labels, github_com_cortexproject_cortex_pkg_cortexpb.LabelAdapter{}) - if err := m.Labels[len(m.Labels)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 4: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Chunks", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowWal - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthWal - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthWal - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Chunks = append(m.Chunks, client.Chunk{}) - if err := m.Chunks[len(m.Chunks)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipWal(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthWal - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthWal - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func skipWal(dAtA []byte) (n int, err error) { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowWal - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - wireType := int(wire & 0x7) - switch wireType { - case 0: - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowWal - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - iNdEx++ - if dAtA[iNdEx-1] < 0x80 { - break - } - } - return iNdEx, nil - case 1: - iNdEx += 8 - return iNdEx, nil - case 2: - var length int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowWal - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - length |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if length < 0 { - return 0, ErrInvalidLengthWal - } - iNdEx += length - if iNdEx < 0 { - return 0, ErrInvalidLengthWal - } - return iNdEx, nil - case 3: - for { - var innerWire uint64 - var start int = iNdEx - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowWal - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - innerWire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - innerWireType := int(innerWire & 0x7) - if innerWireType == 4 { - break - } - next, err := skipWal(dAtA[start:]) - if err != nil { - return 0, err - } - iNdEx = start + next - if iNdEx < 0 { - return 0, ErrInvalidLengthWal - } - } - return iNdEx, nil - case 4: - return iNdEx, nil - case 5: - iNdEx += 4 - return iNdEx, nil - default: - return 0, fmt.Errorf("proto: illegal wireType %d", wireType) - } - } - panic("unreachable") -} - -var ( - ErrInvalidLengthWal = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowWal = fmt.Errorf("proto: integer overflow") -) diff --git a/pkg/ingester/wal.proto b/pkg/ingester/wal.proto deleted file mode 100644 index 1cd86f13c1..0000000000 --- a/pkg/ingester/wal.proto +++ /dev/null @@ -1,16 +0,0 @@ -syntax = "proto3"; - -package ingester; - -option go_package = "ingester"; - -import "github.com/gogo/protobuf/gogoproto/gogo.proto"; -import "github.com/cortexproject/cortex/pkg/cortexpb/cortex.proto"; -import "github.com/cortexproject/cortex/pkg/ingester/client/ingester.proto"; - -message Series { - string user_id = 1; - uint64 fingerprint = 2; - repeated cortexpb.LabelPair labels = 3 [(gogoproto.nullable) = false, (gogoproto.customtype) = "github.com/cortexproject/cortex/pkg/cortexpb.LabelAdapter"]; - repeated cortex.Chunk chunks = 4 [(gogoproto.nullable) = false]; -} diff --git a/pkg/ingester/wal_test.go b/pkg/ingester/wal_test.go deleted file mode 100644 index 97ee65605c..0000000000 --- a/pkg/ingester/wal_test.go +++ /dev/null @@ -1,313 +0,0 @@ -package ingester - -import ( - "context" - "fmt" - "io/ioutil" - "net/http" - "os" - "path/filepath" - "testing" - "time" - - prom_testutil "github.com/prometheus/client_golang/prometheus/testutil" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - "github.com/stretchr/testify/require" - "github.com/weaveworks/common/httpgrpc" - "github.com/weaveworks/common/user" - - "github.com/cortexproject/cortex/pkg/cortexpb" - "github.com/cortexproject/cortex/pkg/util/services" -) - -func TestWAL(t *testing.T) { - dirname := t.TempDir() - - cfg := defaultIngesterTestConfig(t) - cfg.WALConfig.WALEnabled = true - cfg.WALConfig.CheckpointEnabled = true - cfg.WALConfig.Recover = true - cfg.WALConfig.Dir = dirname - cfg.WALConfig.CheckpointDuration = 100 * time.Minute - cfg.WALConfig.checkpointDuringShutdown = true - - numSeries := 100 - numSamplesPerSeriesPerPush := 10 - numRestarts := 5 - - // Build an ingester, add some samples, then shut it down. - _, ing := newTestStore(t, cfg, defaultClientTestConfig(), defaultLimitsTestConfig(), nil) - userIDs, testData := pushTestSamples(t, ing, numSeries, numSamplesPerSeriesPerPush, 0) - // Checkpoint happens when stopping. - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing)) - - for r := 0; r < numRestarts; r++ { - if r == 2 { - // From 3rd restart onwards, we are disabling checkpointing during shutdown - // to test both checkpoint+WAL replay. - cfg.WALConfig.checkpointDuringShutdown = false - } - if r == numRestarts-1 { - cfg.WALConfig.WALEnabled = false - cfg.WALConfig.CheckpointEnabled = false - } - - // Start a new ingester and recover the WAL. - _, ing = newTestStore(t, cfg, defaultClientTestConfig(), defaultLimitsTestConfig(), nil) - - for i, userID := range userIDs { - testData[userID] = buildTestMatrix(numSeries, (r+1)*numSamplesPerSeriesPerPush, i) - } - // Check the samples are still there! - retrieveTestSamples(t, ing, userIDs, testData) - - if r != numRestarts-1 { - userIDs, testData = pushTestSamples(t, ing, numSeries, numSamplesPerSeriesPerPush, (r+1)*numSamplesPerSeriesPerPush) - } - - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing)) - } - - cfg.WALConfig.WALEnabled = true - cfg.WALConfig.CheckpointEnabled = true - - // Start a new ingester and recover the WAL. - _, ing = newTestStore(t, cfg, defaultClientTestConfig(), defaultLimitsTestConfig(), nil) - - userID := userIDs[0] - sampleStream := testData[userID][0] - lastSample := sampleStream.Values[len(sampleStream.Values)-1] - - // In-order and out of order sample in the same request. - metric := cortexpb.FromLabelAdaptersToLabels(cortexpb.FromMetricsToLabelAdapters(sampleStream.Metric)) - outOfOrderSample := cortexpb.Sample{TimestampMs: int64(lastSample.Timestamp - 10), Value: 99} - inOrderSample := cortexpb.Sample{TimestampMs: int64(lastSample.Timestamp + 10), Value: 999} - - ctx := user.InjectOrgID(context.Background(), userID) - _, err := ing.Push(ctx, cortexpb.ToWriteRequest( - []labels.Labels{metric, metric}, - []cortexpb.Sample{outOfOrderSample, inOrderSample}, nil, cortexpb.API)) - require.Equal(t, httpgrpc.Errorf(http.StatusBadRequest, wrapWithUser(makeMetricValidationError(sampleOutOfOrder, metric, - fmt.Errorf("sample timestamp out of order; last timestamp: %v, incoming timestamp: %v", lastSample.Timestamp, model.Time(outOfOrderSample.TimestampMs))), userID).Error()), err) - - // We should have logged the in-order sample. - testData[userID][0].Values = append(testData[userID][0].Values, model.SamplePair{ - Timestamp: model.Time(inOrderSample.TimestampMs), - Value: model.SampleValue(inOrderSample.Value), - }) - - // Check samples after restart from WAL. - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing)) - _, ing = newTestStore(t, cfg, defaultClientTestConfig(), defaultLimitsTestConfig(), nil) - retrieveTestSamples(t, ing, userIDs, testData) -} - -func TestCheckpointRepair(t *testing.T) { - cfg := defaultIngesterTestConfig(t) - cfg.WALConfig.WALEnabled = true - cfg.WALConfig.CheckpointEnabled = true - cfg.WALConfig.Recover = true - cfg.WALConfig.CheckpointDuration = 100 * time.Hour // Basically no automatic checkpoint. - - numSeries := 100 - numSamplesPerSeriesPerPush := 10 - for _, numCheckpoints := range []int{0, 1, 2, 3} { - cfg.WALConfig.Dir = t.TempDir() - - // Build an ingester, add some samples, then shut it down. - _, ing := newTestStore(t, cfg, defaultClientTestConfig(), defaultLimitsTestConfig(), nil) - - w, ok := ing.wal.(*walWrapper) - require.True(t, ok) - - var userIDs []string - // Push some samples for the 0 checkpoints case. - // We dont shutdown the ingester in that case, else it will create a checkpoint. - userIDs, _ = pushTestSamples(t, ing, numSeries, numSamplesPerSeriesPerPush, 0) - for i := 0; i < numCheckpoints; i++ { - // First checkpoint. - userIDs, _ = pushTestSamples(t, ing, numSeries, numSamplesPerSeriesPerPush, (i+1)*numSamplesPerSeriesPerPush) - if i == numCheckpoints-1 { - // Shutdown creates a checkpoint. This is only for the last checkpoint. - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing)) - } else { - require.NoError(t, w.performCheckpoint(true)) - } - } - - require.Equal(t, float64(numCheckpoints), prom_testutil.ToFloat64(w.checkpointCreationTotal)) - - // Verify checkpoint dirs. - files, err := ioutil.ReadDir(w.wal.Dir()) - require.NoError(t, err) - numDirs := 0 - for _, f := range files { - if f.IsDir() { - numDirs++ - } - } - if numCheckpoints <= 1 { - require.Equal(t, numCheckpoints, numDirs) - } else { - // At max there are last 2 checkpoints on the disk. - require.Equal(t, 2, numDirs) - } - - if numCheckpoints > 0 { - // Corrupt the last checkpoint. - lastChDir, _, err := lastCheckpoint(w.wal.Dir()) - require.NoError(t, err) - files, err = ioutil.ReadDir(lastChDir) - require.NoError(t, err) - - lastFile, err := os.OpenFile(filepath.Join(lastChDir, files[len(files)-1].Name()), os.O_WRONLY, os.ModeAppend) - require.NoError(t, err) - n, err := lastFile.WriteAt([]byte{1, 2, 3, 4}, 2) - require.NoError(t, err) - require.Equal(t, 4, n) - require.NoError(t, lastFile.Close()) - } - - // Open an ingester for the repair. - _, ing = newTestStore(t, cfg, defaultClientTestConfig(), defaultLimitsTestConfig(), nil) - w, ok = ing.wal.(*walWrapper) - require.True(t, ok) - // defer in case we hit an error though we explicitly close it later. - defer func() { - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing)) - }() - - if numCheckpoints > 0 { - require.Equal(t, 1.0, prom_testutil.ToFloat64(ing.metrics.walCorruptionsTotal)) - } else { - require.Equal(t, 0.0, prom_testutil.ToFloat64(ing.metrics.walCorruptionsTotal)) - } - - // Verify checkpoint dirs after the corrupt checkpoint is deleted. - files, err = ioutil.ReadDir(w.wal.Dir()) - require.NoError(t, err) - numDirs = 0 - for _, f := range files { - if f.IsDir() { - numDirs++ - } - } - if numCheckpoints <= 1 { - // The only checkpoint is removed (or) there was no checkpoint at all. - require.Equal(t, 0, numDirs) - } else { - // There is at max last 2 checkpoints. Hence only 1 should be remaining. - require.Equal(t, 1, numDirs) - } - - testData := map[string]model.Matrix{} - // Verify we did not lose any data. - for i, userID := range userIDs { - // 'numCheckpoints*' because we ingested the data 'numCheckpoints' number of time. - testData[userID] = buildTestMatrix(numSeries, (numCheckpoints+1)*numSamplesPerSeriesPerPush, i) - } - retrieveTestSamples(t, ing, userIDs, testData) - - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing)) - } - -} - -func TestCheckpointIndex(t *testing.T) { - tcs := []struct { - filename string - includeTmp bool - index int - shouldError bool - }{ - { - filename: "checkpoint.123456", - includeTmp: false, - index: 123456, - shouldError: false, - }, - { - filename: "checkpoint.123456", - includeTmp: true, - index: 123456, - shouldError: false, - }, - { - filename: "checkpoint.123456.tmp", - includeTmp: true, - index: 123456, - shouldError: false, - }, - { - filename: "checkpoint.123456.tmp", - includeTmp: false, - shouldError: true, - }, - { - filename: "not-checkpoint.123456.tmp", - includeTmp: true, - shouldError: true, - }, - { - filename: "checkpoint.123456.tmp2", - shouldError: true, - }, - { - filename: "checkpoints123456", - shouldError: true, - }, - { - filename: "012345", - shouldError: true, - }, - } - for _, tc := range tcs { - index, err := checkpointIndex(tc.filename, tc.includeTmp) - if tc.shouldError { - require.Error(t, err, "filename: %s, includeTmp: %t", tc.filename, tc.includeTmp) - continue - } - - require.NoError(t, err, "filename: %s, includeTmp: %t", tc.filename, tc.includeTmp) - require.Equal(t, tc.index, index) - } -} - -func BenchmarkWALReplay(b *testing.B) { - cfg := defaultIngesterTestConfig(b) - cfg.WALConfig.WALEnabled = true - cfg.WALConfig.CheckpointEnabled = true - cfg.WALConfig.Recover = true - cfg.WALConfig.Dir = b.TempDir() - cfg.WALConfig.CheckpointDuration = 100 * time.Minute - cfg.WALConfig.checkpointDuringShutdown = false - - numSeries := 10 - numSamplesPerSeriesPerPush := 2 - numPushes := 100000 - - _, ing := newTestStore(b, cfg, defaultClientTestConfig(), defaultLimitsTestConfig(), nil) - - // Add samples for the checkpoint. - for r := 0; r < numPushes; r++ { - _, _ = pushTestSamples(b, ing, numSeries, numSamplesPerSeriesPerPush, r*numSamplesPerSeriesPerPush) - } - w, ok := ing.wal.(*walWrapper) - require.True(b, ok) - require.NoError(b, w.performCheckpoint(true)) - - // Add samples for the additional WAL not in checkpoint. - for r := 0; r < numPushes; r++ { - _, _ = pushTestSamples(b, ing, numSeries, numSamplesPerSeriesPerPush, (numPushes+r)*numSamplesPerSeriesPerPush) - } - - require.NoError(b, services.StopAndAwaitTerminated(context.Background(), ing)) - - var ing2 *Ingester - b.Run("wal replay", func(b *testing.B) { - // Replay will happen here. - _, ing2 = newTestStore(b, cfg, defaultClientTestConfig(), defaultLimitsTestConfig(), nil) - }) - require.NoError(b, services.StopAndAwaitTerminated(context.Background(), ing2)) -} diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index 8993ea8069..6396d8fb8b 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -148,7 +148,7 @@ func NewChunkStoreQueryable(cfg Config, chunkStore chunkstore.ChunkStore) storag } // New builds a queryable and promql engine. -func New(cfg Config, limits *validation.Overrides, distributor Distributor, stores []QueryableWithFilter, tombstonesLoader *purger.TombstonesLoader, reg prometheus.Registerer, logger log.Logger) (storage.SampleAndChunkQueryable, storage.ExemplarQueryable, *promql.Engine) { +func New(cfg Config, limits *validation.Overrides, distributor Distributor, stores []QueryableWithFilter, tombstonesLoader purger.TombstonesLoader, reg prometheus.Registerer, logger log.Logger) (storage.SampleAndChunkQueryable, storage.ExemplarQueryable, *promql.Engine) { iteratorFunc := getChunksIteratorFunction(cfg) distributorQueryable := newDistributorQueryable(distributor, cfg.IngesterStreaming, cfg.IngesterMetadataStreaming, iteratorFunc, cfg.QueryIngestersWithin) @@ -221,7 +221,7 @@ type QueryableWithFilter interface { } // NewQueryable creates a new Queryable for cortex. -func NewQueryable(distributor QueryableWithFilter, stores []QueryableWithFilter, chunkIterFn chunkIteratorFunc, cfg Config, limits *validation.Overrides, tombstonesLoader *purger.TombstonesLoader) storage.Queryable { +func NewQueryable(distributor QueryableWithFilter, stores []QueryableWithFilter, chunkIterFn chunkIteratorFunc, cfg Config, limits *validation.Overrides, tombstonesLoader purger.TombstonesLoader) storage.Queryable { return storage.QueryableFunc(func(ctx context.Context, mint, maxt int64) (storage.Querier, error) { now := time.Now() @@ -289,7 +289,7 @@ type querier struct { ctx context.Context mint, maxt int64 - tombstonesLoader *purger.TombstonesLoader + tombstonesLoader purger.TombstonesLoader limits *validation.Overrides maxQueryIntoFuture time.Duration queryStoreForLabels bool diff --git a/pkg/querier/querier_test.go b/pkg/querier/querier_test.go index 6d1bbaf280..3dd303f37e 100644 --- a/pkg/querier/querier_test.go +++ b/pkg/querier/querier_test.go @@ -159,7 +159,7 @@ func TestQuerier(t *testing.T) { require.NoError(t, err) queryables := []QueryableWithFilter{UseAlwaysQueryable(NewChunkStoreQueryable(cfg, chunkStore)), UseAlwaysQueryable(db)} - queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewTombstonesLoader(nil, nil), nil, log.NewNopLogger()) + queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewNoopTombstonesLoader(), nil, log.NewNopLogger()) testRangeQuery(t, queryable, through, query) }) } @@ -271,7 +271,7 @@ func TestNoHistoricalQueryToIngester(t *testing.T) { overrides, err := validation.NewOverrides(DefaultLimitsConfig(), nil) require.NoError(t, err) - queryable, _, _ := New(cfg, overrides, distributor, []QueryableWithFilter{UseAlwaysQueryable(NewChunkStoreQueryable(cfg, chunkStore))}, purger.NewTombstonesLoader(nil, nil), nil, log.NewNopLogger()) + queryable, _, _ := New(cfg, overrides, distributor, []QueryableWithFilter{UseAlwaysQueryable(NewChunkStoreQueryable(cfg, chunkStore))}, purger.NewNoopTombstonesLoader(), nil, log.NewNopLogger()) query, err := engine.NewRangeQuery(queryable, nil, "dummy", c.mint, c.maxt, 1*time.Minute) require.NoError(t, err) @@ -360,7 +360,7 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryIntoFuture(t *testing.T) { require.NoError(t, err) queryables := []QueryableWithFilter{UseAlwaysQueryable(NewChunkStoreQueryable(cfg, chunkStore))} - queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewTombstonesLoader(nil, nil), nil, log.NewNopLogger()) + queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewNoopTombstonesLoader(), nil, log.NewNopLogger()) query, err := engine.NewRangeQuery(queryable, nil, "dummy", c.queryStartTime, c.queryEndTime, time.Minute) require.NoError(t, err) @@ -436,7 +436,7 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLength(t *testing.T) { distributor := &emptyDistributor{} queryables := []QueryableWithFilter{UseAlwaysQueryable(NewChunkStoreQueryable(cfg, chunkStore))} - queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewTombstonesLoader(nil, nil), nil, log.NewNopLogger()) + queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewNoopTombstonesLoader(), nil, log.NewNopLogger()) // Create the PromQL engine to execute the query. engine := promql.NewEngine(promql.EngineOpts{ @@ -572,7 +572,7 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLookback(t *testing.T) { distributor.On("Query", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(model.Matrix{}, nil) distributor.On("QueryStream", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&client.QueryStreamResponse{}, nil) - queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewTombstonesLoader(nil, nil), nil, log.NewNopLogger()) + queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewNoopTombstonesLoader(), nil, log.NewNopLogger()) require.NoError(t, err) query, err := engine.NewRangeQuery(queryable, nil, testData.query, testData.queryStartTime, testData.queryEndTime, time.Minute) @@ -601,7 +601,7 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLookback(t *testing.T) { distributor.On("MetricsForLabelMatchers", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]metric.Metric{}, nil) distributor.On("MetricsForLabelMatchersStream", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]metric.Metric{}, nil) - queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewTombstonesLoader(nil, nil), nil, log.NewNopLogger()) + queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewNoopTombstonesLoader(), nil, log.NewNopLogger()) q, err := queryable.Querier(ctx, util.TimeToMillis(testData.queryStartTime), util.TimeToMillis(testData.queryEndTime)) require.NoError(t, err) @@ -634,7 +634,7 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLookback(t *testing.T) { distributor.On("LabelNames", mock.Anything, mock.Anything, mock.Anything).Return([]string{}, nil) distributor.On("LabelNamesStream", mock.Anything, mock.Anything, mock.Anything).Return([]string{}, nil) - queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewTombstonesLoader(nil, nil), nil, log.NewNopLogger()) + queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewNoopTombstonesLoader(), nil, log.NewNopLogger()) q, err := queryable.Querier(ctx, util.TimeToMillis(testData.queryStartTime), util.TimeToMillis(testData.queryEndTime)) require.NoError(t, err) @@ -662,7 +662,7 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLookback(t *testing.T) { distributor.On("MetricsForLabelMatchers", mock.Anything, mock.Anything, mock.Anything, matchers).Return([]metric.Metric{}, nil) distributor.On("MetricsForLabelMatchersStream", mock.Anything, mock.Anything, mock.Anything, matchers).Return([]metric.Metric{}, nil) - queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewTombstonesLoader(nil, nil), nil, log.NewNopLogger()) + queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewNoopTombstonesLoader(), nil, log.NewNopLogger()) q, err := queryable.Querier(ctx, util.TimeToMillis(testData.queryStartTime), util.TimeToMillis(testData.queryEndTime)) require.NoError(t, err) @@ -689,7 +689,7 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLookback(t *testing.T) { distributor.On("LabelValuesForLabelName", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]string{}, nil) distributor.On("LabelValuesForLabelNameStream", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]string{}, nil) - queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewTombstonesLoader(nil, nil), nil, log.NewNopLogger()) + queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewNoopTombstonesLoader(), nil, log.NewNopLogger()) q, err := queryable.Querier(ctx, util.TimeToMillis(testData.queryStartTime), util.TimeToMillis(testData.queryEndTime)) require.NoError(t, err) @@ -922,7 +922,7 @@ func TestShortTermQueryToLTS(t *testing.T) { overrides, err := validation.NewOverrides(DefaultLimitsConfig(), nil) require.NoError(t, err) - queryable, _, _ := New(cfg, overrides, distributor, []QueryableWithFilter{UseAlwaysQueryable(NewChunkStoreQueryable(cfg, chunkStore))}, purger.NewTombstonesLoader(nil, nil), nil, log.NewNopLogger()) + queryable, _, _ := New(cfg, overrides, distributor, []QueryableWithFilter{UseAlwaysQueryable(NewChunkStoreQueryable(cfg, chunkStore))}, purger.NewNoopTombstonesLoader(), nil, log.NewNopLogger()) query, err := engine.NewRangeQuery(queryable, nil, "dummy", c.mint, c.maxt, 1*time.Minute) require.NoError(t, err) diff --git a/pkg/querier/series/series_set.go b/pkg/querier/series/series_set.go index 76896c6e8a..ee3a9a4ef0 100644 --- a/pkg/querier/series/series_set.go +++ b/pkg/querier/series/series_set.go @@ -197,11 +197,11 @@ func (b byLabels) Less(i, j int) bool { return labels.Compare(b[i].Labels(), b[j type DeletedSeriesSet struct { seriesSet storage.SeriesSet - tombstones *purger.TombstonesSet + tombstones purger.TombstonesSet queryInterval model.Interval } -func NewDeletedSeriesSet(seriesSet storage.SeriesSet, tombstones *purger.TombstonesSet, queryInterval model.Interval) storage.SeriesSet { +func NewDeletedSeriesSet(seriesSet storage.SeriesSet, tombstones purger.TombstonesSet, queryInterval model.Interval) storage.SeriesSet { return &DeletedSeriesSet{ seriesSet: seriesSet, tombstones: tombstones, diff --git a/pkg/ring/lifecycler.go b/pkg/ring/lifecycler.go index d356a742b2..1f0d4f0401 100644 --- a/pkg/ring/lifecycler.go +++ b/pkg/ring/lifecycler.go @@ -825,18 +825,6 @@ func (i *Lifecycler) SetUnregisterOnShutdown(enabled bool) { func (i *Lifecycler) processShutdown(ctx context.Context) { flushRequired := i.flushOnShutdown.Load() - transferStart := time.Now() - if err := i.flushTransferer.TransferOut(ctx); err != nil { - if err == ErrTransferDisabled { - level.Info(i.logger).Log("msg", "transfers are disabled") - } else { - level.Error(i.logger).Log("msg", "failed to transfer chunks to another instance", "ring", i.RingName, "err", err) - i.lifecyclerMetrics.shutdownDuration.WithLabelValues("transfer", "fail").Observe(time.Since(transferStart).Seconds()) - } - } else { - flushRequired = false - i.lifecyclerMetrics.shutdownDuration.WithLabelValues("transfer", "success").Observe(time.Since(transferStart).Seconds()) - } if flushRequired { flushStart := time.Now() diff --git a/pkg/ruler/ruler_test.go b/pkg/ruler/ruler_test.go index df9310b1dd..1528ec0e1f 100644 --- a/pkg/ruler/ruler_test.go +++ b/pkg/ruler/ruler_test.go @@ -125,7 +125,7 @@ func testQueryableFunc(querierTestConfig *querier.TestConfig, reg prometheus.Reg querierTestConfig.Cfg.ActiveQueryTrackerDir = "" overrides, _ := validation.NewOverrides(querier.DefaultLimitsConfig(), nil) - q, _, _ := querier.New(querierTestConfig.Cfg, overrides, querierTestConfig.Distributor, querierTestConfig.Stores, purger.NewTombstonesLoader(nil, nil), reg, logger) + q, _, _ := querier.New(querierTestConfig.Cfg, overrides, querierTestConfig.Distributor, querierTestConfig.Stores, purger.NewNoopTombstonesLoader(), reg, logger) return func(ctx context.Context, mint, maxt int64) (storage.Querier, error) { return q.Querier(ctx, mint, maxt) } diff --git a/pkg/util/chunkcompat/compat.go b/pkg/util/chunkcompat/compat.go index 8021497e48..ae6d22aaa2 100644 --- a/pkg/util/chunkcompat/compat.go +++ b/pkg/util/chunkcompat/compat.go @@ -13,20 +13,6 @@ import ( "github.com/cortexproject/cortex/pkg/util" ) -// StreamsToMatrix converts a slice of QueryStreamResponse to a model.Matrix. -func StreamsToMatrix(from, through model.Time, responses []*client.QueryStreamResponse) (model.Matrix, error) { - result := model.Matrix{} - for _, response := range responses { - series, err := SeriesChunksToMatrix(from, through, response.Chunkseries) - if err != nil { - return nil, err - } - - result = append(result, series...) - } - return result, nil -} - // SeriesChunksToMatrix converts slice of []client.TimeSeriesChunk to a model.Matrix. func SeriesChunksToMatrix(from, through model.Time, serieses []client.TimeSeriesChunk) (model.Matrix, error) { if serieses == nil { diff --git a/tools/doc-generator/main.go b/tools/doc-generator/main.go index 946760de96..5aaf9f1e73 100644 --- a/tools/doc-generator/main.go +++ b/tools/doc-generator/main.go @@ -15,7 +15,6 @@ import ( "github.com/cortexproject/cortex/pkg/alertmanager/alertstore" "github.com/cortexproject/cortex/pkg/chunk" "github.com/cortexproject/cortex/pkg/chunk/cache" - "github.com/cortexproject/cortex/pkg/chunk/purger" "github.com/cortexproject/cortex/pkg/chunk/storage" "github.com/cortexproject/cortex/pkg/compactor" "github.com/cortexproject/cortex/pkg/configs" @@ -194,11 +193,6 @@ var ( structType: reflect.TypeOf(storegateway.Config{}), desc: "The store_gateway_config configures the store-gateway service used by the blocks storage.", }, - { - name: "purger_config", - structType: reflect.TypeOf(purger.Config{}), - desc: "The purger_config configures the purger which takes care of delete requests.", - }, { name: "s3_sse_config", structType: reflect.TypeOf(s3.SSEConfig{}), From bc8895a656b58b4883e493c571508fcfc0e993dc Mon Sep 17 00:00:00 2001 From: Andrew Bloomgarden Date: Fri, 1 Apr 2022 13:43:40 -0400 Subject: [PATCH 03/28] Remove blocksconvert Signed-off-by: Andrew Bloomgarden --- Makefile | 1 - cmd/blocksconvert/Dockerfile | 9 - cmd/blocksconvert/main.go | 107 --- .../convert-stored-chunks-to-blocks.md | 115 --- tools/blocksconvert/allowed_users.go | 67 -- tools/blocksconvert/builder/builder.go | 482 ----------- tools/blocksconvert/builder/builder_test.go | 62 -- tools/blocksconvert/builder/fetcher.go | 52 -- tools/blocksconvert/builder/heap.go | 30 - tools/blocksconvert/builder/series.go | 224 ----- .../blocksconvert/builder/series_iterator.go | 163 ---- tools/blocksconvert/builder/series_test.go | 119 --- .../blocksconvert/builder/symbols_iterator.go | 172 ---- tools/blocksconvert/builder/tsdb.go | 427 ---------- tools/blocksconvert/cleaner/cleaner.go | 357 -------- tools/blocksconvert/plan_file.go | 112 --- tools/blocksconvert/plan_file_test.go | 27 - .../blocksconvert/planprocessor/heartbeat.go | 98 --- tools/blocksconvert/planprocessor/service.go | 412 ---------- .../scanner/bigtable_index_reader.go | 242 ------ .../scanner/bigtable_index_reader_test.go | 182 ----- .../scanner/cassandra_index_reader.go | 172 ---- tools/blocksconvert/scanner/files.go | 111 --- tools/blocksconvert/scanner/index_entry.go | 76 -- tools/blocksconvert/scanner/scanner.go | 649 --------------- .../scanner/scanner_processor.go | 149 ---- .../scanner/scanner_processor_test.go | 118 --- tools/blocksconvert/scanner/scanner_test.go | 115 --- tools/blocksconvert/scheduler.pb.go | 772 ------------------ tools/blocksconvert/scheduler.proto | 25 - tools/blocksconvert/scheduler/plan_status.go | 60 -- tools/blocksconvert/scheduler/scheduler.go | 508 ------------ .../blocksconvert/scheduler/scheduler_test.go | 122 --- tools/blocksconvert/shared_config.go | 44 - 34 files changed, 6381 deletions(-) delete mode 100644 cmd/blocksconvert/Dockerfile delete mode 100644 cmd/blocksconvert/main.go delete mode 100644 docs/blocks-storage/convert-stored-chunks-to-blocks.md delete mode 100644 tools/blocksconvert/allowed_users.go delete mode 100644 tools/blocksconvert/builder/builder.go delete mode 100644 tools/blocksconvert/builder/builder_test.go delete mode 100644 tools/blocksconvert/builder/fetcher.go delete mode 100644 tools/blocksconvert/builder/heap.go delete mode 100644 tools/blocksconvert/builder/series.go delete mode 100644 tools/blocksconvert/builder/series_iterator.go delete mode 100644 tools/blocksconvert/builder/series_test.go delete mode 100644 tools/blocksconvert/builder/symbols_iterator.go delete mode 100644 tools/blocksconvert/builder/tsdb.go delete mode 100644 tools/blocksconvert/cleaner/cleaner.go delete mode 100644 tools/blocksconvert/plan_file.go delete mode 100644 tools/blocksconvert/plan_file_test.go delete mode 100644 tools/blocksconvert/planprocessor/heartbeat.go delete mode 100644 tools/blocksconvert/planprocessor/service.go delete mode 100644 tools/blocksconvert/scanner/bigtable_index_reader.go delete mode 100644 tools/blocksconvert/scanner/bigtable_index_reader_test.go delete mode 100644 tools/blocksconvert/scanner/cassandra_index_reader.go delete mode 100644 tools/blocksconvert/scanner/files.go delete mode 100644 tools/blocksconvert/scanner/index_entry.go delete mode 100644 tools/blocksconvert/scanner/scanner.go delete mode 100644 tools/blocksconvert/scanner/scanner_processor.go delete mode 100644 tools/blocksconvert/scanner/scanner_processor_test.go delete mode 100644 tools/blocksconvert/scanner/scanner_test.go delete mode 100644 tools/blocksconvert/scheduler.pb.go delete mode 100644 tools/blocksconvert/scheduler.proto delete mode 100644 tools/blocksconvert/scheduler/plan_status.go delete mode 100644 tools/blocksconvert/scheduler/scheduler.go delete mode 100644 tools/blocksconvert/scheduler/scheduler_test.go delete mode 100644 tools/blocksconvert/shared_config.go diff --git a/Makefile b/Makefile index 82e560d353..6c61bdac76 100644 --- a/Makefile +++ b/Makefile @@ -104,7 +104,6 @@ pkg/ring/kv/memberlist/kv.pb.go: pkg/ring/kv/memberlist/kv.proto pkg/scheduler/schedulerpb/scheduler.pb.go: pkg/scheduler/schedulerpb/scheduler.proto pkg/storegateway/storegatewaypb/gateway.pb.go: pkg/storegateway/storegatewaypb/gateway.proto pkg/chunk/grpc/grpc.pb.go: pkg/chunk/grpc/grpc.proto -tools/blocksconvert/scheduler.pb.go: tools/blocksconvert/scheduler.proto pkg/alertmanager/alertmanagerpb/alertmanager.pb.go: pkg/alertmanager/alertmanagerpb/alertmanager.proto pkg/alertmanager/alertspb/alerts.pb.go: pkg/alertmanager/alertspb/alerts.proto diff --git a/cmd/blocksconvert/Dockerfile b/cmd/blocksconvert/Dockerfile deleted file mode 100644 index 100ee9aa3b..0000000000 --- a/cmd/blocksconvert/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM alpine:3.14 -RUN apk add --no-cache ca-certificates -COPY blocksconvert / -ENTRYPOINT ["/blocksconvert"] - -ARG revision -LABEL org.opencontainers.image.title="blocksconvert" \ - org.opencontainers.image.source="https://github.com/cortexproject/cortex/tree/master/tools/blocksconvert" \ - org.opencontainers.image.revision="${revision}" diff --git a/cmd/blocksconvert/main.go b/cmd/blocksconvert/main.go deleted file mode 100644 index f3f2b8a5e0..0000000000 --- a/cmd/blocksconvert/main.go +++ /dev/null @@ -1,107 +0,0 @@ -package main - -import ( - "context" - "flag" - "fmt" - "os" - "strings" - - "github.com/go-kit/log/level" - "github.com/prometheus/client_golang/prometheus" - "github.com/weaveworks/common/server" - "github.com/weaveworks/common/signals" - - "github.com/cortexproject/cortex/pkg/cortex" - util_log "github.com/cortexproject/cortex/pkg/util/log" - "github.com/cortexproject/cortex/pkg/util/services" - "github.com/cortexproject/cortex/tools/blocksconvert" - "github.com/cortexproject/cortex/tools/blocksconvert/builder" - "github.com/cortexproject/cortex/tools/blocksconvert/cleaner" - "github.com/cortexproject/cortex/tools/blocksconvert/scanner" - "github.com/cortexproject/cortex/tools/blocksconvert/scheduler" -) - -type Config struct { - Target string - ServerConfig server.Config - - SharedConfig blocksconvert.SharedConfig - ScannerConfig scanner.Config - BuilderConfig builder.Config - SchedulerConfig scheduler.Config - CleanerConfig cleaner.Config -} - -func main() { - cfg := Config{} - flag.StringVar(&cfg.Target, "target", "", "Module to run: Scanner, Scheduler, Builder") - cfg.SharedConfig.RegisterFlags(flag.CommandLine) - cfg.ScannerConfig.RegisterFlags(flag.CommandLine) - cfg.BuilderConfig.RegisterFlags(flag.CommandLine) - cfg.SchedulerConfig.RegisterFlags(flag.CommandLine) - cfg.CleanerConfig.RegisterFlags(flag.CommandLine) - cfg.ServerConfig.RegisterFlags(flag.CommandLine) - flag.Parse() - - util_log.InitLogger(&cfg.ServerConfig) - - cortex.DisableSignalHandling(&cfg.ServerConfig) - serv, err := server.New(cfg.ServerConfig) - if err != nil { - level.Error(util_log.Logger).Log("msg", "Unable to initialize server", "err", err.Error()) - os.Exit(1) - } - - cfg.Target = strings.ToLower(cfg.Target) - - registry := prometheus.DefaultRegisterer - - var targetService services.Service - switch cfg.Target { - case "scanner": - targetService, err = scanner.NewScanner(cfg.ScannerConfig, cfg.SharedConfig, util_log.Logger, registry) - case "builder": - targetService, err = builder.NewBuilder(cfg.BuilderConfig, cfg.SharedConfig, util_log.Logger, registry) - case "scheduler": - targetService, err = scheduler.NewScheduler(cfg.SchedulerConfig, cfg.SharedConfig, util_log.Logger, registry, serv.HTTP, serv.GRPC) - case "cleaner": - targetService, err = cleaner.NewCleaner(cfg.CleanerConfig, cfg.SharedConfig, util_log.Logger, registry) - default: - err = fmt.Errorf("unknown target") - } - - if err != nil { - level.Error(util_log.Logger).Log("msg", "failed to initialize", "err", err) - os.Exit(1) - } - - servService := cortex.NewServerService(serv, func() []services.Service { - return []services.Service{targetService} - }) - servManager, err := services.NewManager(servService, targetService) - if err == nil { - servManager.AddListener(services.NewManagerListener(nil, nil, func(service services.Service) { - servManager.StopAsync() - })) - - err = services.StartManagerAndAwaitHealthy(context.Background(), servManager) - } - if err != nil { - level.Error(util_log.Logger).Log("msg", "Unable to start", "err", err.Error()) - os.Exit(1) - } - - // Setup signal handler and ask service maanger to stop when signal arrives. - handler := signals.NewHandler(serv.Log) - go func() { - handler.Loop() - servManager.StopAsync() - }() - - // We only wait for target service. If any other service fails, listener will stop it (via manager) - if err := targetService.AwaitTerminated(context.Background()); err != nil { - level.Error(util_log.Logger).Log("msg", cfg.Target+" failed", "err", targetService.FailureCase()) - os.Exit(1) - } -} diff --git a/docs/blocks-storage/convert-stored-chunks-to-blocks.md b/docs/blocks-storage/convert-stored-chunks-to-blocks.md deleted file mode 100644 index fc6f98e16a..0000000000 --- a/docs/blocks-storage/convert-stored-chunks-to-blocks.md +++ /dev/null @@ -1,115 +0,0 @@ ---- -title: "Convert long-term storage from chunks to blocks" -linkTitle: "Convert long-term storage from chunks to blocks" -weight: 6 -slug: convert-long-term-storage-from-chunks-to-blocks ---- - -If you have [configured your cluster to write new data to blocks](./migrate-from-chunks-to-blocks.md), there is still a question about old data. -Cortex can query both chunks and the blocks at the same time, but converting old chunks to blocks still has some benefits, like being able to decommission the chunks storage backend and save costs. -This document presents set of tools for doing the conversion. - -_[Original design document](https://docs.google.com/document/d/1VI0cgaJmHD0pcrRb3UV04f8szXXGmFKQyqUJnFOcf6Q/edit?usp=sharing) for `blocksconvert` is also available._ - -## Tools - -Cortex provides a tool called `blocksconvert`, which is actually collection of three tools for converting chunks to blocks. - -Tools are: - -- [**Scanner**](#scanner)
- Scans the chunks index database and produces so-called "plan files", each file being a set of series and chunks for each series. Plan files are uploaded to the same object store bucket where blocks live. -- [**Scheduler**](#scheduler)
- Looks for plan files, and distributes them to builders. Scheduler has global view of overall conversion progress. -- [**Builder**](#builder)
- Asks scheduler for next plan file to work on, fetches chunks, puts them into TSDB block, and uploads the block to the object store. It repeats this process until there are no more plans. -- [**Cleaner**](#cleaner)
- Cleaner asks scheduler for next plan file to work on, but instead of building the block, it actually **REMOVES CHUNKS** and **INDEX ENTRIES** from the Index database. - -All tools start HTTP server (see `-server.http*` options) exposing the `/metrics` endpoint. -All tools also start gRPC server (`-server.grpc*` options), but only Scheduler exposes services on it. - -### Scanner - -Scanner is started by running `blocksconvert -target=scanner`. Scanner requires configuration for accessing Cortex Index: - -- `-schema-config-file` – this is standard Cortex schema file. -- `-bigtable.instance`, `-bigtable.project` – options for BigTable access. -- `-dynamodb.url` - for DynamoDB access. Example `dynamodb://us-east-1/` -- `-blocks-storage.backend` and corresponding `-blocks-storage.*` options for storing plan files. -- `-scanner.output-dir` – specifies local directory for writing plan files to. Finished plan files are deleted after upload to the bucket. List of scanned tables is also kept in this directory, to avoid scanning the same tables multiple times when Scanner is restarted. -- `-scanner.allowed-users` – comma-separated list of Cortex tenants that should have plans generated. If empty, plans for all found users are generated. -- `-scanner.ignore-users-regex` - If plans for all users are generated (`-scanner.allowed-users` is not set), then users matching this non-empty regular expression will be skipped. -- `-scanner.tables-limit` – How many tables should be scanned? By default all tables are scanned, but when testing scanner it may be useful to start with small number of tables first. -- `-scanner.tables` – Comma-separated list of tables to be scanned. Can be used to scan specific tables only. Note that schema is still used to find all tables first, and then this list is consulted to select only specified tables. -- `-scanner.scan-period-start` & `-scanner.scan-period-end` - limit the scan to a particular date range (format like `2020-12-31`) - -Scanner will read the Cortex schema file to discover Index tables, and then it will start scanning them from most-recent table first, going back. -For each table, it will fully read the table and generate a plan for each user and day stored in the table. -Plan files are then uploaded to the configured blocks-storage bucket (at the `-blocksconvert.bucket-prefix` location prefix), and local copies are deleted. -After that, scanner continues with the next table until it scans them all or `-scanner.tables-limit` is reached. - -Note that even though `blocksconvert` has options for configuring different Index store backends, **it only supports BigTable and DynamoDB at the moment.** - -It is expected that only single Scanner process is running. -Scanner does the scanning of multiple table subranges concurrently. - -Scanner exposes metrics with `cortex_blocksconvert_scanner_` prefix, eg. total number of scanned index entries of different type, number of open files (scanner doesn't close currently plan files until entire table has been scanned), scanned rows and parsed index entries. - -**Scanner only supports schema version v9 on DynamoDB; v9, v10 and v11 on BigTable. Earlier schema versions are currently not supported.** - -### Scheduler - -Scheduler is started by running `blocksconvert -target=scheduler`. It only needs to be configured with options to access the object store with blocks: - -- `-blocks-storage.*` - Blocks storage object store configuration. -- `-scheduler.scan-interval` – How often to scan for plan files and their status. -- `-scheduler.allowed-users` – Comma-separated list of Cortex tenants. If set, only plans for these tenants will be offered to Builders. - -It is expected that only single Scheduler process is running. Schedulers consume very little resources. - -Scheduler's metrics have `cortex_blocksconvert_scheduler` prefix (number of plans in different states, oldest/newest plan). -Scheduler HTTP server also exposes `/plans` page that shows currently queued plans, and all plans and their status for all users. - -### Builder - -Builder asks scheduler for next plan to work on, downloads the plan, builds the block and uploads the block to the blocks storage. It then repeats the process while there are still plans. - -Builder is started by `blocksconvert -target=builder`. It needs to be configured with Scheduler endpoint, Cortex schema file, chunk-store specific options and blocks storage to upload blocks to. - -- `-builder.scheduler-endpoint` - where to find scheduler, eg. "scheduler:9095" -- `-schema-config-file` - Cortex schema file, used to find out which chunks store to use for given plan -- `-gcs.bucketname` – when using GCS as chunks store (other chunks backend storages, like S3, are supported as well) -- `-blocks-storage.*` - blocks storage configuration -- `-builder.output-dir` - Local directory where Builder keeps the block while it is being built. Once block is uploaded to blocks storage, it is deleted from local directory. - -Multiple builders may run at the same time, each builder will receive different plan to work on from scheduler. -Builders are CPU intensive (decoding and merging chunks), and require fast disk IO for writing blocks. - -Builders's metrics have `cortex_blocksconvert_builder` prefix, and include total number of fetched chunks and their size, read position of the current plan and plan size, total number of written series and samples, number of chunks that couldn't be downloaded. - -### Cleaner - -Cleaner is similar to builder in that it asks scheduler for next plan to work on, but instead of building blocks, it actually **REMOVES CHUNKS and INDEX ENTRIES**. Use with caution. - -Cleaner is started by using `blocksconvert -target=cleaner`. Like Builder, it needs Scheduler endpoint, Cortex schema file, index and chunk-store specific options. Note that Cleaner works with any index store supported by Cortex, not just BigTable. - -- `-cleaner.scheduler-endpoint` – where to find scheduler -- `-blocks-storage.*` – blocks storage configuration, used for downloading plan files -- `-cleaner.plans-dir` – local directory to store plan file while it is being processed by Cleaner. -- `-schema-config-file` – Cortex schema file. - -Cleaner doesn't **scan** for index entries, but uses existing plan files to find chunks and index entries. For each series, Cleaner needs to download at least one chunk. This is because plan file doesn't contain label names and values, but chunks do. Cleaner will then delete all index entries associated with the series, and also all chunks. - -**WARNING:** If both Builder and Cleaner run at the same time and use use the same Scheduler, **some plans will be handled by builder, and some by cleaner!** This will result in a loss of data! - -Cleaner should only be deployed if no other Builder is running. Running multiple Cleaners at once is not supported, and will result in leftover chunks and index entries. Reason for this is that chunks can span multiple days, and chunk is fully deleted only when processing plan (day) when chunk started. Since cleaner also needs to download some chunks to be able to clean up all index entries, when using multiple cleaners, it can happen that cleaner processing older plans will delete chunks required to properly clean up data in newer plans. When using single cleaner only, this is not a problem, since scheduler sends plans to cleaner in time-reversed order. - -**Note:** Cleaner is designed for use in very special cases, eg. when deleting chunks and index entries for a specific customer. If `blocksconvert` was used to convert ALL chunks to blocks, it is simpler to just drop the index and chunks database afterwards. In such case, Cleaner is not needed. - -### Limitations - -The `blocksconvert` toolset currently has the following limitations: - -- Scanner supports only BigTable and DynamoDB for chunks index backend, and cannot currently scan other databases. -- Supports only chunks schema versions v9 for DynamoDB; v9, v10 and v11 for Bigtable. diff --git a/tools/blocksconvert/allowed_users.go b/tools/blocksconvert/allowed_users.go deleted file mode 100644 index d62143eb94..0000000000 --- a/tools/blocksconvert/allowed_users.go +++ /dev/null @@ -1,67 +0,0 @@ -package blocksconvert - -import ( - "bufio" - "os" - "strings" -) - -type AllowedUsers map[string]struct{} - -var AllowAllUsers = AllowedUsers(nil) - -func (a AllowedUsers) AllUsersAllowed() bool { - return a == nil -} - -func (a AllowedUsers) IsAllowed(user string) bool { - if a == nil { - return true - } - - _, ok := a[user] - return ok -} - -func (a AllowedUsers) GetAllowedUsers(users []string) []string { - if a == nil { - return users - } - - allowed := make([]string, 0, len(users)) - for _, u := range users { - if a.IsAllowed(u) { - allowed = append(allowed, u) - } - } - return allowed -} - -func ParseAllowedUsersFromFile(file string) (AllowedUsers, error) { - result := map[string]struct{}{} - - f, err := os.Open(file) - if err != nil { - return nil, err - } - - defer func() { _ = f.Close() }() - - s := bufio.NewScanner(f) - for s.Scan() { - result[s.Text()] = struct{}{} - } - return result, s.Err() -} - -func ParseAllowedUsers(commaSeparated string) AllowedUsers { - result := map[string]struct{}{} - - us := strings.Split(commaSeparated, ",") - for _, u := range us { - u = strings.TrimSpace(u) - result[u] = struct{}{} - } - - return result -} diff --git a/tools/blocksconvert/builder/builder.go b/tools/blocksconvert/builder/builder.go deleted file mode 100644 index 077f42036a..0000000000 --- a/tools/blocksconvert/builder/builder.go +++ /dev/null @@ -1,482 +0,0 @@ -package builder - -import ( - "context" - "flag" - "io/ioutil" - "os" - "path/filepath" - "sort" - "strings" - "time" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/prometheus/prometheus/model/labels" - "github.com/thanos-io/thanos/pkg/block" - "github.com/thanos-io/thanos/pkg/block/metadata" - "github.com/thanos-io/thanos/pkg/objstore" - "golang.org/x/sync/errgroup" - - "github.com/cortexproject/cortex/pkg/chunk" - "github.com/cortexproject/cortex/pkg/chunk/cache" - "github.com/cortexproject/cortex/pkg/chunk/storage" - "github.com/cortexproject/cortex/pkg/storage/bucket" - cortex_tsdb "github.com/cortexproject/cortex/pkg/storage/tsdb" - "github.com/cortexproject/cortex/pkg/util/backoff" - "github.com/cortexproject/cortex/pkg/util/services" - "github.com/cortexproject/cortex/tools/blocksconvert" - "github.com/cortexproject/cortex/tools/blocksconvert/planprocessor" -) - -// How many series are kept in the memory before sorting and writing them to the file. -const defaultSeriesBatchSize = 250000 - -type Config struct { - OutputDirectory string - Concurrency int - - ChunkCacheConfig cache.Config - UploadBlock bool - DeleteLocalBlock bool - SeriesBatchSize int - TimestampTolerance time.Duration - - PlanProcessorConfig planprocessor.Config -} - -func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - cfg.ChunkCacheConfig.RegisterFlagsWithPrefix("chunks.", "Chunks cache", f) - cfg.PlanProcessorConfig.RegisterFlags("builder", f) - - f.StringVar(&cfg.OutputDirectory, "builder.output-dir", "", "Local directory used for storing temporary plan files (will be created, if missing).") - f.IntVar(&cfg.Concurrency, "builder.concurrency", 128, "Number of concurrent series processors.") - f.BoolVar(&cfg.UploadBlock, "builder.upload", true, "Upload generated blocks to storage.") - f.BoolVar(&cfg.DeleteLocalBlock, "builder.delete-local-blocks", true, "Delete local files after uploading block.") - f.IntVar(&cfg.SeriesBatchSize, "builder.series-batch-size", defaultSeriesBatchSize, "Number of series to keep in memory before batch-write to temp file. Lower to decrease memory usage during the block building.") - f.DurationVar(&cfg.TimestampTolerance, "builder.timestamp-tolerance", 0, "Adjust sample timestamps by up to this to align them to an exact number of seconds apart.") -} - -func NewBuilder(cfg Config, scfg blocksconvert.SharedConfig, l log.Logger, reg prometheus.Registerer) (services.Service, error) { - err := scfg.SchemaConfig.Load() - if err != nil { - return nil, errors.Wrap(err, "failed to load schema") - } - - bucketClient, err := scfg.GetBucket(l, reg) - if err != nil { - return nil, err - } - - if cfg.OutputDirectory == "" { - return nil, errors.New("no output directory") - } - if err := os.MkdirAll(cfg.OutputDirectory, os.FileMode(0700)); err != nil { - return nil, errors.Wrap(err, "failed to create output directory") - } - - b := &Builder{ - cfg: cfg, - - bucketClient: bucketClient, - schemaConfig: scfg.SchemaConfig, - storageConfig: scfg.StorageConfig, - - fetchedChunks: promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "cortex_blocksconvert_builder_fetched_chunks_total", - Help: "Fetched chunks", - }), - fetchedChunksSize: promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "cortex_blocksconvert_builder_fetched_chunks_bytes_total", - Help: "Fetched chunks bytes", - }), - processedSeries: promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "cortex_blocksconvert_builder_series_total", - Help: "Processed series", - }), - writtenSamples: promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "cortex_blocksconvert_builder_written_samples_total", - Help: "Written samples", - }), - buildInProgress: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ - Name: "cortex_blocksconvert_builder_in_progress", - Help: "Build in progress", - }), - chunksNotFound: promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "cortex_blocksconvert_builder_chunks_not_found_total", - Help: "Number of chunks that were not found on the storage.", - }), - blocksSize: promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "cortex_blocksconvert_builder_block_size_bytes_total", - Help: "Total size of blocks generated by this builder.", - }), - seriesInMemory: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ - Name: "cortex_blocksconvert_builder_series_in_memory", - Help: "Number of series kept in memory at the moment. (Builder writes series to temp files in order to reduce memory usage.)", - }), - } - - return planprocessor.NewService(cfg.PlanProcessorConfig, filepath.Join(cfg.OutputDirectory, "plans"), bucketClient, b.cleanupFn, b.planProcessorFactory, l, reg) -} - -type Builder struct { - cfg Config - - bucketClient objstore.Bucket - schemaConfig chunk.SchemaConfig - storageConfig storage.Config - - fetchedChunks prometheus.Counter - fetchedChunksSize prometheus.Counter - processedSeries prometheus.Counter - writtenSamples prometheus.Counter - blocksSize prometheus.Counter - - buildInProgress prometheus.Gauge - chunksNotFound prometheus.Counter - seriesInMemory prometheus.Gauge -} - -func (b *Builder) cleanupFn(log log.Logger) error { - files, err := ioutil.ReadDir(b.cfg.OutputDirectory) - if err != nil { - return err - } - - // Delete directories with .tmp suffix (unfinished blocks). - for _, f := range files { - if strings.HasSuffix(f.Name(), ".tmp") && f.IsDir() { - toRemove := filepath.Join(b.cfg.OutputDirectory, f.Name()) - - level.Info(log).Log("msg", "deleting unfinished block", "dir", toRemove) - - err := os.RemoveAll(toRemove) - if err != nil { - return errors.Wrapf(err, "removing %s", toRemove) - } - } - } - - return nil -} - -func (b *Builder) planProcessorFactory(planLog log.Logger, userID string, start time.Time, end time.Time) planprocessor.PlanProcessor { - return &builderProcessor{ - builder: b, - log: planLog, - userID: userID, - dayStart: start, - dayEnd: end, - } -} - -type builderProcessor struct { - builder *Builder - - log log.Logger - userID string - dayStart time.Time - dayEnd time.Time -} - -func (p *builderProcessor) ProcessPlanEntries(ctx context.Context, planEntryCh chan blocksconvert.PlanEntry) (string, error) { - p.builder.buildInProgress.Set(1) - defer p.builder.buildInProgress.Set(0) - defer p.builder.seriesInMemory.Set(0) - - chunkClient, err := p.builder.createChunkClientForDay(p.dayStart) - if err != nil { - return "", errors.Wrap(err, "failed to create chunk client") - } - defer chunkClient.Stop() - - fetcher, err := newFetcher(p.userID, chunkClient, p.builder.fetchedChunks, p.builder.fetchedChunksSize) - if err != nil { - return "", errors.Wrap(err, "failed to create chunk fetcher") - } - - tsdbBuilder, err := newTsdbBuilder(p.builder.cfg.OutputDirectory, p.dayStart, p.dayEnd, p.builder.cfg.TimestampTolerance, p.builder.cfg.SeriesBatchSize, p.log, - p.builder.processedSeries, p.builder.writtenSamples, p.builder.seriesInMemory) - if err != nil { - return "", errors.Wrap(err, "failed to create TSDB builder") - } - - g, gctx := errgroup.WithContext(ctx) - for i := 0; i < p.builder.cfg.Concurrency; i++ { - g.Go(func() error { - return fetchAndBuild(gctx, fetcher, planEntryCh, tsdbBuilder, p.log, p.builder.chunksNotFound) - }) - } - - if err := g.Wait(); err != nil { - return "", errors.Wrap(err, "failed to build block") - } - - // Finish block. - ulid, err := tsdbBuilder.finishBlock("blocksconvert", map[string]string{ - cortex_tsdb.TenantIDExternalLabel: p.userID, - }) - if err != nil { - return "", errors.Wrap(err, "failed to finish block building") - } - - blockDir := filepath.Join(p.builder.cfg.OutputDirectory, ulid.String()) - blockSize, err := getBlockSize(blockDir) - if err != nil { - return "", errors.Wrap(err, "block size") - } - - level.Info(p.log).Log("msg", "successfully built block for a plan", "ulid", ulid.String(), "size", blockSize) - p.builder.blocksSize.Add(float64(blockSize)) - - if p.builder.cfg.UploadBlock { - // No per-tenant config provider because the blocksconvert tool doesn't support it. - userBucket := bucket.NewUserBucketClient(p.userID, p.builder.bucketClient, nil) - - err := uploadBlock(ctx, p.log, userBucket, blockDir) - if err != nil { - return "", errors.Wrap(err, "uploading block") - } - - level.Info(p.log).Log("msg", "block uploaded", "ulid", ulid.String()) - - if p.builder.cfg.DeleteLocalBlock { - if err := os.RemoveAll(blockDir); err != nil { - level.Warn(p.log).Log("msg", "failed to delete local block", "err", err) - } - } - } - - // All OK - return ulid.String(), nil -} - -func uploadBlock(ctx context.Context, planLog log.Logger, userBucket objstore.Bucket, blockDir string) error { - boff := backoff.New(ctx, backoff.Config{ - MinBackoff: 1 * time.Second, - MaxBackoff: 5 * time.Second, - MaxRetries: 5, - }) - - for boff.Ongoing() { - err := block.Upload(ctx, planLog, userBucket, blockDir, metadata.NoneFunc) - if err == nil { - return nil - } - - level.Warn(planLog).Log("msg", "failed to upload block", "err", err) - boff.Wait() - } - - return boff.Err() -} - -func getBlockSize(dir string) (int64, error) { - size := int64(0) - - err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - if !info.IsDir() { - size += info.Size() - } - - // Ignore directory with temporary series files. - if info.IsDir() && info.Name() == "series" { - return filepath.SkipDir - } - - return nil - }) - return size, err -} - -func fetchAndBuild(ctx context.Context, f *Fetcher, input chan blocksconvert.PlanEntry, tb *tsdbBuilder, log log.Logger, chunksNotFound prometheus.Counter) error { - b := backoff.New(ctx, backoff.Config{ - MinBackoff: 1 * time.Second, - MaxBackoff: 5 * time.Second, - MaxRetries: 5, - }) - - for { - select { - case <-ctx.Done(): - return nil - - case e, ok := <-input: - if !ok { - // End of input. - return nil - } - - var m labels.Labels - var cs []chunk.Chunk - var err error - - // Rather than aborting entire block build due to temporary errors ("connection reset by peer", "http2: client conn not usable"), - // try to fetch chunks multiple times. - for b.Reset(); b.Ongoing(); { - m, cs, err = fetchAndBuildSingleSeries(ctx, f, e.Chunks) - if err == nil { - break - } - - if b.Ongoing() { - level.Warn(log).Log("msg", "failed to fetch chunks for series", "series", e.SeriesID, "err", err, "retries", b.NumRetries()+1) - b.Wait() - } - } - - if err == nil { - err = b.Err() - } - if err != nil { - return errors.Wrapf(err, "failed to fetch chunks for series %s", e.SeriesID) - } - - if len(e.Chunks) > len(cs) { - chunksNotFound.Add(float64(len(e.Chunks) - len(cs))) - level.Warn(log).Log("msg", "chunks for series not found", "seriesID", e.SeriesID, "expected", len(e.Chunks), "got", len(cs)) - } - - if len(cs) == 0 { - continue - } - - err = tb.buildSingleSeries(m, cs) - if err != nil { - return errors.Wrapf(err, "failed to build series %s", e.SeriesID) - } - } - } -} - -func fetchAndBuildSingleSeries(ctx context.Context, fetcher *Fetcher, chunksIds []string) (labels.Labels, []chunk.Chunk, error) { - cs, err := fetcher.fetchChunks(ctx, chunksIds) - if err != nil && !errors.Is(err, chunk.ErrStorageObjectNotFound) { - return nil, nil, errors.Wrap(err, "fetching chunks") - } - - if len(cs) == 0 { - return nil, nil, nil - } - - m, err := normalizeLabels(cs[0].Metric) - if err != nil { - return nil, nil, errors.Wrapf(err, "chunk has invalid metrics: %v", cs[0].Metric.String()) - } - - // Verify that all chunks belong to the same series. - for _, c := range cs { - nm, err := normalizeLabels(c.Metric) - if err != nil { - return nil, nil, errors.Wrapf(err, "chunk has invalid metrics: %v", c.Metric.String()) - } - if !labels.Equal(m, nm) { - return nil, nil, errors.Errorf("chunks for multiple metrics: %v, %v", m.String(), c.Metric.String()) - } - } - - return m, cs, nil -} - -// Labels are already sorted, but there may be duplicate label names. -// This method verifies sortedness, and removes duplicate label names (if they have the same value). -func normalizeLabels(lbls labels.Labels) (labels.Labels, error) { - err := checkLabels(lbls) - if err == errLabelsNotSorted { - sort.Sort(lbls) - err = checkLabels(lbls) - } - - if err == errDuplicateLabelsSameValue { - lbls = removeDuplicateLabels(lbls) - err = checkLabels(lbls) - } - - return lbls, err -} - -var ( - errLabelsNotSorted = errors.New("labels not sorted") - errDuplicateLabelsSameValue = errors.New("duplicate labels, same value") - errDuplicateLabelsDifferentValue = errors.New("duplicate labels, different values") -) - -// Returns one of errLabelsNotSorted, errDuplicateLabelsSameValue, errDuplicateLabelsDifferentValue, -// or nil, if labels are fine. -func checkLabels(lbls labels.Labels) error { - prevName, prevValue := "", "" - - uniqueLabels := true - for _, l := range lbls { - switch { - case l.Name < prevName: - return errLabelsNotSorted - case l.Name == prevName: - if l.Value != prevValue { - return errDuplicateLabelsDifferentValue - } - - uniqueLabels = false - } - - prevName = l.Name - prevValue = l.Value - } - - if !uniqueLabels { - return errDuplicateLabelsSameValue - } - - return nil -} - -func removeDuplicateLabels(lbls labels.Labels) labels.Labels { - prevName, prevValue := "", "" - - for ix := 0; ix < len(lbls); { - l := lbls[ix] - if l.Name == prevName && l.Value == prevValue { - lbls = append(lbls[:ix], lbls[ix+1:]...) - continue - } - - prevName = l.Name - prevValue = l.Value - ix++ - } - - return lbls -} - -// Finds storage configuration for given day, and builds a client. -func (b *Builder) createChunkClientForDay(dayStart time.Time) (chunk.Client, error) { - for ix, s := range b.schemaConfig.Configs { - if dayStart.Unix() < s.From.Unix() { - continue - } - - if ix+1 < len(b.schemaConfig.Configs) && dayStart.Unix() > b.schemaConfig.Configs[ix+1].From.Unix() { - continue - } - - objectStoreType := s.ObjectType - if objectStoreType == "" { - objectStoreType = s.IndexType - } - // No registerer, to avoid problems with registering same metrics multiple times. - chunks, err := storage.NewChunkClient(objectStoreType, b.storageConfig, b.schemaConfig, nil) - if err != nil { - return nil, errors.Wrap(err, "error creating object client") - } - return chunks, nil - } - - return nil, errors.Errorf("no schema for day %v", dayStart.Format("2006-01-02")) -} diff --git a/tools/blocksconvert/builder/builder_test.go b/tools/blocksconvert/builder/builder_test.go deleted file mode 100644 index 2b5bc54a72..0000000000 --- a/tools/blocksconvert/builder/builder_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package builder - -import ( - "testing" - - "github.com/prometheus/prometheus/model/labels" - "github.com/stretchr/testify/assert" -) - -func TestNormalizeLabels(t *testing.T) { - for name, tc := range map[string]struct { - input labels.Labels - - expectedOutput labels.Labels - expectedError error - }{ - "good labels": { - input: fromStrings("__name__", "hello", "label1", "world"), - expectedOutput: fromStrings("__name__", "hello", "label1", "world"), - expectedError: nil, - }, - "not sorted": { - input: fromStrings("label1", "world", "__name__", "hello"), - expectedOutput: fromStrings("__name__", "hello", "label1", "world"), - expectedError: nil, - }, - "duplicate with same value": { - input: fromStrings("__name__", "hello", "label1", "world", "label1", "world"), - expectedOutput: fromStrings("__name__", "hello", "label1", "world"), - expectedError: nil, - }, - "not sorted, duplicate with same value": { - input: fromStrings("label1", "world", "__name__", "hello", "label1", "world"), - expectedOutput: fromStrings("__name__", "hello", "label1", "world"), - expectedError: nil, - }, - "duplicate with different value": { - input: fromStrings("label1", "world1", "__name__", "hello", "label1", "world2"), - expectedOutput: fromStrings("__name__", "hello", "label1", "world1", "label1", "world2"), // sorted - expectedError: errDuplicateLabelsDifferentValue, - }, - } { - t.Run(name, func(t *testing.T) { - out, err := normalizeLabels(tc.input) - - assert.Equal(t, tc.expectedOutput, out) - assert.Equal(t, tc.expectedError, err) - }) - } -} - -// Similar to labels.FromStrings, but doesn't do sorting. -func fromStrings(ss ...string) labels.Labels { - if len(ss)%2 != 0 { - panic("invalid number of strings") - } - var res labels.Labels - for i := 0; i < len(ss); i += 2 { - res = append(res, labels.Label{Name: ss[i], Value: ss[i+1]}) - } - return res -} diff --git a/tools/blocksconvert/builder/fetcher.go b/tools/blocksconvert/builder/fetcher.go deleted file mode 100644 index 61b3369ae7..0000000000 --- a/tools/blocksconvert/builder/fetcher.go +++ /dev/null @@ -1,52 +0,0 @@ -package builder - -import ( - "context" - - "github.com/prometheus/client_golang/prometheus" - - "github.com/cortexproject/cortex/pkg/chunk" -) - -type Fetcher struct { - userID string - - client chunk.Client - - fetchedChunks prometheus.Counter - fetchedChunksSize prometheus.Counter -} - -func newFetcher(userID string, client chunk.Client, fetchedChunks, fetchedChunksSize prometheus.Counter) (*Fetcher, error) { - return &Fetcher{ - client: client, - userID: userID, - fetchedChunks: fetchedChunks, - fetchedChunksSize: fetchedChunksSize, - }, nil -} - -func (f *Fetcher) fetchChunks(ctx context.Context, chunkIDs []string) ([]chunk.Chunk, error) { - chunks := make([]chunk.Chunk, 0, len(chunkIDs)) - - for _, cid := range chunkIDs { - c, err := chunk.ParseExternalKey(f.userID, cid) - if err != nil { - return nil, err - } - - chunks = append(chunks, c) - } - - cs, err := f.client.GetChunks(ctx, chunks) - for _, c := range cs { - f.fetchedChunks.Inc() - enc, nerr := c.Encoded() - if nerr != nil { - return nil, nerr - } - f.fetchedChunksSize.Add(float64(len(enc))) - } - - return cs, err -} diff --git a/tools/blocksconvert/builder/heap.go b/tools/blocksconvert/builder/heap.go deleted file mode 100644 index 7ee2c6c8e9..0000000000 --- a/tools/blocksconvert/builder/heap.go +++ /dev/null @@ -1,30 +0,0 @@ -package builder - -// Heap is a binary tree where parent node is "smaller" than its children nodes ("Heap Property"). -// Heap is stored in a slice, where children nodes for node at position ix are at positions 2*ix+1 and 2*ix+2. -// -// Heapify will maintain the heap property for node "ix". -// -// Building the heap for the first time must go from the latest element (last leaf) towards the element 0 (root of the tree). -// Once built, first element is the smallest. -// -// Element "ix" can be removed from the heap by moving the last element to position "ix", shrinking the heap -// and restoring the heap property from index "ix". (This is typically done from root, ix=0). -func heapify(length int, ix int, less func(i, j int) bool, swap func(i, j int)) { - smallest := ix - left := 2*ix + 1 - right := 2*ix + 2 - - if left < length && less(left, smallest) { - smallest = left - } - - if right < length && less(right, smallest) { - smallest = right - } - - if smallest != ix { - swap(ix, smallest) - heapify(length, smallest, less, swap) - } -} diff --git a/tools/blocksconvert/builder/series.go b/tools/blocksconvert/builder/series.go deleted file mode 100644 index 5fb3353dd2..0000000000 --- a/tools/blocksconvert/builder/series.go +++ /dev/null @@ -1,224 +0,0 @@ -package builder - -import ( - "encoding/gob" - "fmt" - "os" - "path/filepath" - "sort" - "sync" - - "github.com/golang/snappy" - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/tsdb/chunks" - tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" -) - -type series struct { - // All fields must be exported for serialization to work properly. - Metric labels.Labels - Chunks []chunks.Meta - MinTime int64 - MaxTime int64 - Samples uint64 -} - -// Keeps series in memory until limit is reached. Then series are sorted, and written to the file. -// Each batch goes to different file. -// When series are iterated, all files are merged (which is easy to do, as they are already sorted). -// Symbols are written to different set of files, they are also sorted, merged and deduplicated on iteration. -type seriesList struct { - limit int - dir string - - mu sync.Mutex - sers []series - seriesFiles []string - symbolsFiles []string -} - -func newSeriesList(limit int, dir string) *seriesList { - return &seriesList{ - limit: limit, - dir: dir, - } -} - -func (sl *seriesList) addSeries(m labels.Labels, cs []chunks.Meta, samples uint64, minTime, maxTime int64) error { - sl.mu.Lock() - defer sl.mu.Unlock() - - sl.sers = append(sl.sers, series{ - Metric: m, - Chunks: cs, - MinTime: minTime, - MaxTime: maxTime, - Samples: samples, - }) - - return sl.flushSeriesNoLock(false) -} - -func (sl *seriesList) unflushedSeries() int { - sl.mu.Lock() - defer sl.mu.Unlock() - - return len(sl.sers) -} - -func (sl *seriesList) flushSeries() error { - sl.mu.Lock() - defer sl.mu.Unlock() - - return sl.flushSeriesNoLock(true) -} - -func (sl *seriesList) flushSeriesNoLock(force bool) error { - if !force && len(sl.sers) < sl.limit { - return nil - } - - // Sort series by labels first. - sort.Slice(sl.sers, func(i, j int) bool { - return labels.Compare(sl.sers[i].Metric, sl.sers[j].Metric) < 0 - }) - - seriesFile := filepath.Join(sl.dir, fmt.Sprintf("series_%d", len(sl.seriesFiles))) - symbols, err := writeSeries(seriesFile, sl.sers) - if err != nil { - return err - } - - sl.sers = nil - sl.seriesFiles = append(sl.seriesFiles, seriesFile) - - // No error so far, let's write symbols too. - sortedSymbols := make([]string, 0, len(symbols)) - for k := range symbols { - sortedSymbols = append(sortedSymbols, k) - } - - sort.Strings(sortedSymbols) - - symbolsFile := filepath.Join(sl.dir, fmt.Sprintf("symbols_%d", len(sl.symbolsFiles))) - err = writeSymbols(symbolsFile, sortedSymbols) - if err == nil { - sl.symbolsFiles = append(sl.symbolsFiles, symbolsFile) - } - - return err -} - -func writeSymbols(filename string, symbols []string) error { - f, err := os.Create(filename) - if err != nil { - return err - } - - sn := snappy.NewBufferedWriter(f) - enc := gob.NewEncoder(sn) - - errs := tsdb_errors.NewMulti() - - for _, s := range symbols { - err := enc.Encode(s) - if err != nil { - errs.Add(err) - break - } - } - - errs.Add(sn.Close()) - errs.Add(f.Close()) - return errs.Err() -} - -func writeSeries(filename string, sers []series) (map[string]struct{}, error) { - f, err := os.Create(filename) - if err != nil { - return nil, err - } - - symbols := map[string]struct{}{} - - errs := tsdb_errors.NewMulti() - - sn := snappy.NewBufferedWriter(f) - enc := gob.NewEncoder(sn) - - // Write each series as a separate object, so that we can read them back individually. - for _, ser := range sers { - for _, sym := range ser.Metric { - symbols[sym.Name] = struct{}{} - symbols[sym.Value] = struct{}{} - } - - err := enc.Encode(ser) - if err != nil { - errs.Add(err) - break - } - } - - errs.Add(sn.Close()) - errs.Add(f.Close()) - - return symbols, errs.Err() -} - -// Returns iterator over sorted list of symbols. Each symbol is returned once. -func (sl *seriesList) symbolsIterator() (*symbolsIterator, error) { - sl.mu.Lock() - filenames := append([]string(nil), sl.symbolsFiles...) - sl.mu.Unlock() - - files, err := openFiles(filenames) - if err != nil { - return nil, err - } - - var result []*symbolsFile - for _, f := range files { - result = append(result, newSymbolsFile(f)) - } - - return newSymbolsIterator(result), nil -} - -// Returns iterator over sorted list of series. -func (sl *seriesList) seriesIterator() (*seriesIterator, error) { - sl.mu.Lock() - filenames := append([]string(nil), sl.seriesFiles...) - sl.mu.Unlock() - - files, err := openFiles(filenames) - if err != nil { - return nil, err - } - - var result []*seriesFile - for _, f := range files { - result = append(result, newSeriesFile(f)) - } - - return newSeriesIterator(result), nil -} - -func openFiles(filenames []string) ([]*os.File, error) { - var result []*os.File - - for _, fn := range filenames { - f, err := os.Open(fn) - - if err != nil { - // Close opened files so far. - for _, sf := range result { - _ = sf.Close() - } - return nil, err - } - - result = append(result, f) - } - return result, nil -} diff --git a/tools/blocksconvert/builder/series_iterator.go b/tools/blocksconvert/builder/series_iterator.go deleted file mode 100644 index 0b9cf4a871..0000000000 --- a/tools/blocksconvert/builder/series_iterator.go +++ /dev/null @@ -1,163 +0,0 @@ -package builder - -import ( - "encoding/gob" - "io" - "os" - - "github.com/golang/snappy" - "github.com/prometheus/prometheus/model/labels" - tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" -) - -type seriesIterator struct { - files []*seriesFile - errs []error -} - -func newSeriesIterator(files []*seriesFile) *seriesIterator { - si := &seriesIterator{ - files: files, - } - si.buildHeap() - return si -} - -func (sit *seriesIterator) buildHeap() { - // All files on the heap must have at least one element, so that "heapify" can order them. - // Here we verify that, and remove files with no more elements. - for ix := 0; ix < len(sit.files); { - f := sit.files[ix] - next, err := f.hasNext() - - if err != nil { - sit.errs = append(sit.errs, err) - return - } - - if !next { - if err := f.close(); err != nil { - sit.errs = append(sit.errs, err) - } - sit.files = append(sit.files[:ix], sit.files[ix+1:]...) - continue - } - - ix++ - } - - // Build heap, start with leaf nodes, and work towards to root. See comment at heapify for more details. - for ix := len(sit.files) - 1; ix >= 0; ix-- { - heapifySeries(sit.files, ix) - } -} - -// Next advances iterator forward, and returns next element. If there is no next element, returns false. -func (sit *seriesIterator) Next() (series, bool) { - if len(sit.errs) > 0 { - return series{}, false - } - - if len(sit.files) == 0 { - return series{}, false - } - - result := sit.files[0].pop() - - hasNext, err := sit.files[0].hasNext() - if err != nil { - sit.errs = append(sit.errs, err) - } - - if !hasNext { - if err := sit.files[0].close(); err != nil { - sit.errs = append(sit.errs, err) - } - - // Move last file to the front, and heapify from there. - sit.files[0] = sit.files[len(sit.files)-1] - sit.files = sit.files[:len(sit.files)-1] - } - - heapifySeries(sit.files, 0) - - return result, true -} - -func (sit *seriesIterator) Error() error { - return tsdb_errors.NewMulti(sit.errs...).Err() -} - -func (sit *seriesIterator) Close() error { - errs := tsdb_errors.NewMulti() - for _, f := range sit.files { - errs.Add(f.close()) - } - return errs.Err() -} - -func heapifySeries(files []*seriesFile, ix int) { - heapify(len(files), ix, func(i, j int) bool { - return labels.Compare(files[i].peek().Metric, files[j].peek().Metric) < 0 - }, func(i, j int) { - files[i], files[j] = files[j], files[i] - }) -} - -type seriesFile struct { - f *os.File - dec *gob.Decoder - - next bool - nextSeries series -} - -func newSeriesFile(f *os.File) *seriesFile { - sn := snappy.NewReader(f) - dec := gob.NewDecoder(sn) - - return &seriesFile{ - f: f, - dec: dec, - } -} - -func (sf *seriesFile) close() error { - return sf.f.Close() -} - -func (sf *seriesFile) hasNext() (bool, error) { - if sf.next { - return true, nil - } - - var s series - err := sf.dec.Decode(&s) - if err != nil { - if err == io.EOF { - return false, nil - } - return false, err - } - - sf.next = true - sf.nextSeries = s - return true, nil -} - -func (sf *seriesFile) peek() series { - if !sf.next { - panic("no next symbol") - } - - return sf.nextSeries -} - -func (sf *seriesFile) pop() series { - if !sf.next { - panic("no next symbol") - } - - sf.next = false - return sf.nextSeries -} diff --git a/tools/blocksconvert/builder/series_test.go b/tools/blocksconvert/builder/series_test.go deleted file mode 100644 index 278cc9a1a5..0000000000 --- a/tools/blocksconvert/builder/series_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package builder - -import ( - "bytes" - "math/rand" - "sort" - "testing" - "time" - - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/tsdb/chunks" - "github.com/stretchr/testify/require" -) - -type testSeries struct { - l labels.Labels - cs []chunks.Meta - samples uint64 - minTime, maxTime int64 -} - -func TestSeries(t *testing.T) { - series := map[string]testSeries{} - - r := rand.New(rand.NewSource(time.Now().UnixNano())) - - const seriesCount = 100 - for i := 0; i < seriesCount; i++ { - l := labels.Labels{labels.Label{Name: generateString(r), Value: generateString(r)}} - series[l.String()] = testSeries{ - l: l, - cs: []chunks.Meta{{Ref: chunks.ChunkRef(r.Uint64()), MinTime: r.Int63(), MaxTime: r.Int63()}}, - samples: r.Uint64(), - minTime: r.Int63(), - maxTime: r.Int63(), - } - } - - sl := newSeriesList(seriesCount/7, t.TempDir()) - - symbolsMap := map[string]bool{} - - for _, s := range series { - require.NoError(t, sl.addSeries(s.l, s.cs, s.samples, s.minTime, s.maxTime)) - - for _, l := range s.l { - symbolsMap[l.Name] = true - symbolsMap[l.Value] = true - } - } - require.NoError(t, sl.flushSeries()) - - symbols := make([]string, 0, len(symbolsMap)) - for s := range symbolsMap { - symbols = append(symbols, s) - } - sort.Strings(symbols) - - sit, err := sl.symbolsIterator() - require.NoError(t, err) - - for _, exp := range symbols { - s, ok := sit.Next() - require.True(t, ok) - require.Equal(t, exp, s) - } - _, ok := sit.Next() - require.False(t, ok) - require.NoError(t, sit.Error()) - require.NoError(t, sit.Close()) - - prevLabels := labels.Labels{} - - rit, err := sl.seriesIterator() - require.NoError(t, err) - - for len(series) > 0 { - s, ok := rit.Next() - require.True(t, ok) - - es, ok := series[s.Metric.String()] - require.True(t, ok) - require.True(t, labels.Compare(prevLabels, s.Metric) < 0) - - prevLabels = s.Metric - - require.Equal(t, 0, labels.Compare(es.l, s.Metric)) - - for ix, c := range es.cs { - require.True(t, ix < len(s.Chunks)) - require.Equal(t, c.Ref, s.Chunks[ix].Ref) - require.Equal(t, c.MinTime, s.Chunks[ix].MinTime) - require.Equal(t, c.MaxTime, s.Chunks[ix].MaxTime) - } - - require.Equal(t, es.minTime, s.MinTime) - require.Equal(t, es.maxTime, s.MaxTime) - require.Equal(t, es.samples, s.Samples) - - delete(series, s.Metric.String()) - } - - _, ok = rit.Next() - require.False(t, ok) - require.NoError(t, rit.Error()) - require.NoError(t, rit.Close()) -} - -func generateString(r *rand.Rand) string { - buf := bytes.Buffer{} - - chars := "abcdefghijklmnopqrstuvxyzABCDEFGHIJKLMNOPQRSTUVXYZ01234567890_" - - for l := 20 + r.Intn(100); l > 0; l-- { - buf.WriteByte(chars[r.Intn(len(chars))]) - } - - return buf.String() -} diff --git a/tools/blocksconvert/builder/symbols_iterator.go b/tools/blocksconvert/builder/symbols_iterator.go deleted file mode 100644 index d882ea12ef..0000000000 --- a/tools/blocksconvert/builder/symbols_iterator.go +++ /dev/null @@ -1,172 +0,0 @@ -package builder - -import ( - "encoding/gob" - "io" - "os" - - "github.com/golang/snappy" - tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" -) - -type symbolsIterator struct { - files []*symbolsFile - errs []error - - // To avoid returning duplicates, we remember last returned symbol. - lastReturned *string -} - -func newSymbolsIterator(files []*symbolsFile) *symbolsIterator { - si := &symbolsIterator{ - files: files, - } - si.buildHeap() - return si -} - -func (sit *symbolsIterator) buildHeap() { - // All files on the heap must have at least one element, so that "heapify" can order them. - // Here we verify that, and remove files with no more elements. - for ix := 0; ix < len(sit.files); { - f := sit.files[ix] - next, err := f.hasNext() - - if err != nil { - sit.errs = append(sit.errs, err) - return - } - - if !next { - if err := f.close(); err != nil { - sit.errs = append(sit.errs, err) - } - sit.files = append(sit.files[:ix], sit.files[ix+1:]...) - continue - } - - ix++ - } - - // Build heap, start with leaf nodes, and work towards to root. See comment at heapify for more details. - for ix := len(sit.files) - 1; ix >= 0; ix-- { - heapifySymbols(sit.files, ix) - } -} - -// Next advances iterator forward, and returns next element. If there is no next element, returns false. -func (sit *symbolsIterator) Next() (string, bool) { -again: - if len(sit.errs) > 0 { - return "", false - } - - if len(sit.files) == 0 { - return "", false - } - - result := sit.files[0].pop() - - hasNext, err := sit.files[0].hasNext() - if err != nil { - sit.errs = append(sit.errs, err) - } - - if !hasNext { - if err := sit.files[0].close(); err != nil { - sit.errs = append(sit.errs, err) - } - - // Move last file to the front, and heapify from there. - sit.files[0] = sit.files[len(sit.files)-1] - sit.files = sit.files[:len(sit.files)-1] - } - - heapifySymbols(sit.files, 0) - - if sit.lastReturned == nil || *sit.lastReturned != result { - sit.lastReturned = &result - return result, true - } - - // Duplicate symbol, try next one. - goto again -} - -func (sit *symbolsIterator) Error() error { - return tsdb_errors.NewMulti(sit.errs...).Err() -} - -func (sit *symbolsIterator) Close() error { - errs := tsdb_errors.NewMulti() - for _, f := range sit.files { - errs.Add(f.close()) - } - return errs.Err() -} - -func heapifySymbols(files []*symbolsFile, ix int) { - heapify(len(files), ix, func(i, j int) bool { - return files[i].peek() < files[j].peek() - }, func(i, j int) { - files[i], files[j] = files[j], files[i] - }) -} - -type symbolsFile struct { - f *os.File - dec *gob.Decoder - - next bool - nextSymbol string -} - -func newSymbolsFile(f *os.File) *symbolsFile { - sn := snappy.NewReader(f) - dec := gob.NewDecoder(sn) - - return &symbolsFile{ - f: f, - dec: dec, - } -} - -func (sf *symbolsFile) close() error { - return sf.f.Close() -} - -func (sf *symbolsFile) hasNext() (bool, error) { - if sf.next { - return true, nil - } - - var s string - err := sf.dec.Decode(&s) - if err != nil { - if err == io.EOF { - return false, nil - } - return false, err - } - - sf.next = true - sf.nextSymbol = s - return true, nil -} - -func (sf *symbolsFile) peek() string { - if !sf.next { - panic("no next symbol") - } - - return sf.nextSymbol -} - -func (sf *symbolsFile) pop() string { - if !sf.next { - panic("no next symbol") - } - - sf.next = false - return sf.nextSymbol -} diff --git a/tools/blocksconvert/builder/tsdb.go b/tools/blocksconvert/builder/tsdb.go deleted file mode 100644 index f2e4b0fb62..0000000000 --- a/tools/blocksconvert/builder/tsdb.go +++ /dev/null @@ -1,427 +0,0 @@ -package builder - -import ( - "context" - "crypto/rand" - "io" - "os" - "path/filepath" - "sync" - "time" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/oklog/ulid" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/storage" - "github.com/prometheus/prometheus/tsdb" - "github.com/prometheus/prometheus/tsdb/chunkenc" - "github.com/prometheus/prometheus/tsdb/chunks" - tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" - "github.com/prometheus/prometheus/tsdb/index" - "github.com/thanos-io/thanos/pkg/block" - "github.com/thanos-io/thanos/pkg/block/metadata" - - "github.com/cortexproject/cortex/pkg/chunk" - "github.com/cortexproject/cortex/pkg/querier/iterators" -) - -const ( - unsortedChunksDir = "unsorted_chunks" - readChunksConcurrency = 16 -) - -// This builder uses TSDB's chunk and index writer directly, without -// using TSDB Head. -type tsdbBuilder struct { - log log.Logger - - ulid ulid.ULID - outDir string - tmpBlockDir string - - unsortedChunksWriterMu sync.Mutex - unsortedChunksWriter tsdb.ChunkWriter - - startTime model.Time - endTime model.Time - timestampTolerance int // in milliseconds - - series *seriesList - seriesDir string - - writtenSamples prometheus.Counter - processedSeries prometheus.Counter - seriesInMemory prometheus.Gauge -} - -func newTsdbBuilder(outDir string, start, end time.Time, timestampTolerance time.Duration, seriesBatchLimit int, log log.Logger, processedSeries, writtenSamples prometheus.Counter, seriesInMemory prometheus.Gauge) (*tsdbBuilder, error) { - id, err := ulid.New(ulid.Now(), rand.Reader) - if err != nil { - return nil, errors.Wrap(err, "create ULID") - } - - blockDir := filepath.Join(outDir, id.String()+".tmp") - seriesDir := filepath.Join(blockDir, "series") - - err = os.RemoveAll(blockDir) - if err != nil { - return nil, err - } - - // Also makes blockDir, if missing - err = os.MkdirAll(seriesDir, 0777) - if err != nil { - return nil, err - } - - unsortedChunksWriter, err := chunks.NewWriter(filepath.Join(blockDir, unsortedChunksDir)) - if err != nil { - return nil, errors.Wrap(err, "chunks writer") - } - - return &tsdbBuilder{ - log: log, - ulid: id, - outDir: outDir, - tmpBlockDir: blockDir, - unsortedChunksWriter: unsortedChunksWriter, - startTime: model.TimeFromUnixNano(start.UnixNano()), - endTime: model.TimeFromUnixNano(end.UnixNano()), - timestampTolerance: int(timestampTolerance.Milliseconds()), - series: newSeriesList(seriesBatchLimit, seriesDir), - seriesDir: seriesDir, - - processedSeries: processedSeries, - writtenSamples: writtenSamples, - seriesInMemory: seriesInMemory, - }, err -} - -// Called concurrently with all chunks required to build a single series. -func (d *tsdbBuilder) buildSingleSeries(metric labels.Labels, cs []chunk.Chunk) error { - defer d.processedSeries.Inc() - - // Used by Prometheus, in head.go (with a reference to Gorilla paper). - const samplesPerChunk = 120 - - chs := make([]chunks.Meta, 0, 25) // On average, single series seem to have around 25 chunks. - seriesSamples := uint64(0) - - // current chunk and appender. If nil, new chunk will be created. - var ( - ch *chunks.Meta - app chunkenc.Appender - err error - ) - - // This will merge and deduplicate samples from chunks. - it := iterators.NewChunkMergeIterator(cs, d.startTime, d.endTime) - for it.Next() && it.Err() == nil { - t, v := it.At() - - mt := model.Time(t) - - if mt < d.startTime { - continue - } - if mt >= d.endTime { - break - } - - if ch == nil { - chs = append(chs, chunks.Meta{}) - ch = &chs[len(chs)-1] - ch.MinTime = t - - ch.Chunk = chunkenc.NewXORChunk() - app, err = ch.Chunk.Appender() - if err != nil { - return err - } - } - - // If gap since last scrape is very close to an exact number of seconds, tighten it up - if d.timestampTolerance != 0 && ch.MaxTime != 0 { - gap := t - ch.MaxTime - seconds := ((gap + 500) / 1000) - diff := int(gap - seconds*1000) - // Don't go past endTime. - if diff != 0 && diff >= -d.timestampTolerance && diff <= d.timestampTolerance && ch.MaxTime+seconds*1000 <= int64(d.endTime) { - t = ch.MaxTime + seconds*1000 - } - } - - ch.MaxTime = t - app.Append(t, v) - seriesSamples++ - - if ch.Chunk.NumSamples() == samplesPerChunk { - ch.Chunk.Compact() - ch = nil - } - } - - if ch != nil { - ch.Chunk.Compact() - ch = nil - } - - d.unsortedChunksWriterMu.Lock() - err = d.unsortedChunksWriter.WriteChunks(chs...) - d.unsortedChunksWriterMu.Unlock() - - if err != nil { - return err - } - - // Remove chunks data from memory, but keep reference. - for ix := range chs { - if chs[ix].Ref == 0 { - return errors.Errorf("chunk ref not set") - } - chs[ix].Chunk = nil - } - - // No samples, ignore. - if len(chs) == 0 { - return nil - } - - minTime := chs[0].MinTime - maxTime := chs[len(chs)-1].MaxTime - - err = d.series.addSeries(metric, chs, seriesSamples, minTime, maxTime) - - d.seriesInMemory.Set(float64(d.series.unflushedSeries())) - d.writtenSamples.Add(float64(seriesSamples)) - return err -} - -func (d *tsdbBuilder) finishBlock(source string, labels map[string]string) (ulid.ULID, error) { - if err := d.unsortedChunksWriter.Close(); err != nil { - return ulid.ULID{}, errors.Wrap(err, "closing chunks writer") - } - - if err := d.series.flushSeries(); err != nil { - return ulid.ULID{}, errors.Wrap(err, "flushing series") - } - d.seriesInMemory.Set(0) - - level.Info(d.log).Log("msg", "all chunks fetched, building block index") - - meta := &metadata.Meta{ - BlockMeta: tsdb.BlockMeta{ - ULID: d.ulid, - Version: 1, - MinTime: int64(d.startTime), - MaxTime: int64(d.endTime), - Compaction: tsdb.BlockMetaCompaction{ - Level: 1, - Sources: []ulid.ULID{d.ulid}, - }, - }, - - // We populate SegmentFiles (which is deprecated, but still used). The new Files property - // will be populated by Thanos block.Upload(). - Thanos: metadata.Thanos{ - Labels: labels, - Source: metadata.SourceType(source), - SegmentFiles: block.GetSegmentFiles(d.tmpBlockDir), - }, - } - - toClose := map[string]io.Closer{} - defer func() { - for k, c := range toClose { - err := c.Close() - if err != nil { - level.Error(d.log).Log("msg", "close failed", "name", k, "err", err) - } - } - }() - - const ( - indexWriterName = "index writer" - unsortedChunksReaderName = "unsorted chunks reader" - chunksWriterName = "chunks writer" - ) - - indexWriter, err := index.NewWriter(context.Background(), filepath.Join(d.tmpBlockDir, "index")) - if err != nil { - return ulid.ULID{}, errors.Wrap(err, indexWriterName) - } - toClose[indexWriterName] = indexWriter - - symbols, err := addSymbolsToIndex(indexWriter, d.series) - if err != nil { - return ulid.ULID{}, errors.Wrap(err, "adding symbols") - } - - level.Info(d.log).Log("msg", "added symbols to index", "count", symbols) - - unsortedChunksReader, err := chunks.NewDirReader(filepath.Join(d.tmpBlockDir, unsortedChunksDir), nil) - if err != nil { - return ulid.ULID{}, errors.Wrap(err, unsortedChunksReaderName) - } - toClose[unsortedChunksReaderName] = unsortedChunksReader - - chunksWriter, err := chunks.NewWriter(filepath.Join(d.tmpBlockDir, "chunks")) - if err != nil { - return ulid.ULID{}, errors.Wrap(err, chunksWriterName) - } - toClose[chunksWriterName] = chunksWriter - - stats, err := addSeriesToIndex(indexWriter, d.series, unsortedChunksReader, chunksWriter) - if err != nil { - return ulid.ULID{}, errors.Wrap(err, "adding series") - } - meta.Stats = stats - - level.Info(d.log).Log("msg", "added series to index", "series", stats.NumSeries, "chunks", stats.NumChunks, "samples", stats.NumSamples) - - // Close index writer, unsorted chunks reader and chunks writer. - for k, c := range toClose { - delete(toClose, k) - - err := c.Close() - if err != nil { - return ulid.ULID{}, errors.Wrapf(err, "closing %s", k) - } - } - - // Delete unsorted chunks, they are no longer needed. - if err := os.RemoveAll(filepath.Join(d.tmpBlockDir, unsortedChunksDir)); err != nil { - return ulid.ULID{}, errors.Wrap(err, "deleting unsorted chunks") - } - - if err := meta.WriteToDir(d.log, d.tmpBlockDir); err != nil { - return ulid.ULID{}, errors.Wrap(err, "writing meta.json") - } - - if err := os.Rename(d.tmpBlockDir, filepath.Join(d.outDir, d.ulid.String())); err != nil { - return ulid.ULID{}, errors.Wrap(err, "rename to final directory") - } - - return d.ulid, nil -} - -func addSeriesToIndex(indexWriter *index.Writer, sl *seriesList, unsortedChunksReader *chunks.Reader, chunksWriter *chunks.Writer) (tsdb.BlockStats, error) { - var stats tsdb.BlockStats - - it, err := sl.seriesIterator() - if err != nil { - return stats, errors.Wrap(err, "reading series") - } - - type chunkToRead struct { - ref chunks.ChunkRef - chunk *chunkenc.Chunk - err *error - } - - ch := make(chan chunkToRead) - defer close(ch) // To make sure that goroutines stop. - - // Number of chunks that should be loaded. - var pendingChunks sync.WaitGroup - - // These goroutines read chunks into memory. - for n := 0; n < readChunksConcurrency; n++ { - go func() { - for ctr := range ch { - c, e := unsortedChunksReader.Chunk(ctr.ref) - *ctr.chunk = c - *ctr.err = e - pendingChunks.Done() - } - }() - } - - seriesRef := storage.SeriesRef(0) - for s, ok := it.Next(); ok; s, ok = it.Next() { - l := s.Metric - cs := s.Chunks - - readErrors := make([]error, len(cs)) - - // Read chunks into memory by asking goroutines to load them. - for ix := range cs { - pendingChunks.Add(1) - ch <- chunkToRead{ - ref: cs[ix].Ref, - chunk: &cs[ix].Chunk, - err: &readErrors[ix], - } - cs[ix].Ref = 0 - } - - // Wait for all chunks to be fetched. - pendingChunks.Wait() - - multi := tsdb_errors.NewMulti() - for _, e := range readErrors { - if e != nil { - multi.Add(e) - } - } - if err := multi.Err(); err != nil { - return stats, errors.Wrap(err, "failed to read chunks") - } - - // Write chunks again. This time they will be written in the same order as series. - err = chunksWriter.WriteChunks(cs...) - if err != nil { - return stats, errors.Wrap(err, "failed to write sorted chunks") - } - - // Remove chunks data from memory, but keep reference for writing to index. - for ix := range cs { - if cs[ix].Ref == 0 { - return stats, errors.Errorf("chunk ref not set after writing sorted chunks") - } - cs[ix].Chunk = nil - } - - if err := indexWriter.AddSeries(seriesRef, l, cs...); err != nil { - return stats, errors.Wrapf(err, "adding series %v", l) - } - - seriesRef++ - - stats.NumSamples += s.Samples - stats.NumSeries++ - stats.NumChunks += uint64(len(cs)) - } - - return stats, nil -} - -func addSymbolsToIndex(indexWriter *index.Writer, sl *seriesList) (int, error) { - symbols := 0 - it, err := sl.symbolsIterator() - if err != nil { - return 0, errors.Wrap(err, "reading symbols") - } - - for s, ok := it.Next(); ok; s, ok = it.Next() { - symbols++ - if err := indexWriter.AddSymbol(s); err != nil { - _ = it.Close() // Make sure to close any open files. - return 0, errors.Wrapf(err, "adding symbol %v", s) - } - } - if err := it.Error(); err != nil { - _ = it.Close() // Make sure to close any open files. - return 0, err - } - - if err := it.Close(); err != nil { - return 0, err - } - - return symbols, nil -} diff --git a/tools/blocksconvert/cleaner/cleaner.go b/tools/blocksconvert/cleaner/cleaner.go deleted file mode 100644 index 0cfd24ab0e..0000000000 --- a/tools/blocksconvert/cleaner/cleaner.go +++ /dev/null @@ -1,357 +0,0 @@ -package cleaner - -import ( - "context" - "flag" - "time" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/prometheus/common/model" - "github.com/thanos-io/thanos/pkg/objstore" - "golang.org/x/sync/errgroup" - - "github.com/cortexproject/cortex/pkg/chunk" - "github.com/cortexproject/cortex/pkg/chunk/storage" - "github.com/cortexproject/cortex/pkg/util/backoff" - "github.com/cortexproject/cortex/pkg/util/services" - "github.com/cortexproject/cortex/tools/blocksconvert" - "github.com/cortexproject/cortex/tools/blocksconvert/planprocessor" -) - -type Config struct { - PlansDirectory string - Concurrency int - - PlanProcessorConfig planprocessor.Config -} - -func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - cfg.PlanProcessorConfig.RegisterFlags("cleaner", f) - - f.StringVar(&cfg.PlansDirectory, "cleaner.plans-dir", "", "Local directory used for storing temporary plan files.") - f.IntVar(&cfg.Concurrency, "cleaner.concurrency", 128, "Number of concurrent series cleaners.") -} - -func NewCleaner(cfg Config, scfg blocksconvert.SharedConfig, l log.Logger, reg prometheus.Registerer) (services.Service, error) { - err := scfg.SchemaConfig.Load() - if err != nil { - return nil, errors.Wrap(err, "failed to load schema") - } - - bucketClient, err := scfg.GetBucket(l, reg) - if err != nil { - return nil, err - } - - c := &Cleaner{ - cfg: cfg, - - bucketClient: bucketClient, - schemaConfig: scfg.SchemaConfig, - storageConfig: scfg.StorageConfig, - - deletedSeries: promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "cortex_blocksconvert_cleaner_deleted_series_total", - Help: "Deleted series", - }), - deletedSeriesErrors: promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "cortex_blocksconvert_cleaner_delete_series_errors_total", - Help: "Number of errors while deleting series.", - }), - deletedIndexEntries: promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "cortex_blocksconvert_cleaner_deleted_index_entries_total", - Help: "Deleted index entries", - }), - deletedChunks: promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "cortex_blocksconvert_cleaner_deleted_chunks_total", - Help: "Deleted chunks", - }), - deletedChunksMissing: promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "cortex_blocksconvert_cleaner_delete_chunks_missing_total", - Help: "Chunks that were missing when trying to delete them.", - }), - deletedChunksSkipped: promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "cortex_blocksconvert_cleaner_delete_chunks_skipped_total", - Help: "Number of skipped chunks during deletion.", - }), - deletedChunksErrors: promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "cortex_blocksconvert_cleaner_delete_chunks_errors_total", - Help: "Number of errors while deleting individual chunks.", - }), - } - - return planprocessor.NewService(cfg.PlanProcessorConfig, cfg.PlansDirectory, bucketClient, nil, c.planProcessorFactory, l, reg) -} - -type Cleaner struct { - cfg Config - - bucketClient objstore.Bucket - schemaConfig chunk.SchemaConfig - storageConfig storage.Config - - deletedChunks prometheus.Counter - deletedChunksSkipped prometheus.Counter - deletedChunksMissing prometheus.Counter - deletedChunksErrors prometheus.Counter - - deletedIndexEntries prometheus.Counter - deletedSeries prometheus.Counter - deletedSeriesErrors prometheus.Counter -} - -func (c *Cleaner) planProcessorFactory(planLog log.Logger, userID string, start time.Time, end time.Time) planprocessor.PlanProcessor { - return &cleanerProcessor{ - cleaner: c, - log: planLog, - userID: userID, - dayStart: start, - dayEnd: end, - } -} - -type cleanerProcessor struct { - cleaner *Cleaner - - log log.Logger - userID string - dayStart time.Time - dayEnd time.Time -} - -func (cp *cleanerProcessor) ProcessPlanEntries(ctx context.Context, planEntryCh chan blocksconvert.PlanEntry) (string, error) { - tableName, schema, chunkClient, indexClient, err := cp.cleaner.createClientsForDay(cp.dayStart) - if err != nil { - return "", errors.Wrap(err, "failed to create clients") - } - - defer chunkClient.Stop() - defer indexClient.Stop() - - seriesSchema, ok := schema.(chunk.SeriesStoreSchema) - if !ok || seriesSchema == nil { - return "", errors.Errorf("invalid schema, expected v9 or later") - } - - g, gctx := errgroup.WithContext(ctx) - for i := 0; i < cp.cleaner.cfg.Concurrency; i++ { - g.Go(func() error { - for { - select { - case <-ctx.Done(): - return nil - - case e, ok := <-planEntryCh: - if !ok { - // End of input. - return nil - } - - err := cp.deleteChunksForSeries(gctx, tableName, seriesSchema, chunkClient, indexClient, e) - if err != nil { - return err - } - } - } - }) - } - - if err := g.Wait(); err != nil { - return "", errors.Wrap(err, "failed to cleanup series") - } - - // "cleaned" will be appended as "block ID" to finished status file. - return "cleaned", nil -} - -func (cp *cleanerProcessor) deleteChunksForSeries(ctx context.Context, tableName string, schema chunk.SeriesStoreSchema, chunkClient chunk.Client, indexClient chunk.IndexClient, e blocksconvert.PlanEntry) error { - var c *chunk.Chunk - var err error - - b := backoff.New(ctx, backoff.Config{ - MinBackoff: 1 * time.Second, - MaxBackoff: 5 * time.Second, - MaxRetries: 5, - }) - for ; b.Ongoing(); b.Wait() { - c, err = fetchSingleChunk(ctx, cp.userID, chunkClient, e.Chunks) - if err == nil { - break - } - - level.Warn(cp.log).Log("msg", "failed to fetch chunk for series", "series", e.SeriesID, "err", err, "retries", b.NumRetries()+1) - } - - if err == nil { - err = b.Err() - } - if err != nil { - return errors.Wrapf(err, "error while fetching chunk for series %s", e.SeriesID) - } - - if c == nil { - cp.cleaner.deletedSeriesErrors.Inc() - // This can happen also when cleaner is restarted. Chunks deleted previously cannot be found anymore, - // but index entries should not exist. - // level.Warn(cp.log).Log("msg", "no chunk found for series, unable to delete series", "series", e.SeriesID) - return nil - } - - // All chunks belonging to the series use the same metric. - metric := c.Metric - - metricName := metric.Get(model.MetricNameLabel) - if metricName == "" { - return errors.Errorf("cannot find metric name for series %s", metric.String()) - } - - start := model.TimeFromUnixNano(cp.dayStart.UnixNano()) - end := model.TimeFromUnixNano(cp.dayEnd.UnixNano()) - - var chunksToDelete []string - - indexEntries := 0 - - // With metric, we find out which index entries to remove. - batch := indexClient.NewWriteBatch() - for _, cid := range e.Chunks { - c, err := chunk.ParseExternalKey(cp.userID, cid) - if err != nil { - return errors.Wrap(err, "failed to parse chunk key") - } - - // ChunkWriteEntries returns entries not only for this day-period, but all days that chunk covers. - // Since we process plans "backwards", more recent entries should already be cleaned up. - ents, err := schema.GetChunkWriteEntries(c.From, c.Through, cp.userID, metricName, metric, cid) - if err != nil { - return errors.Wrapf(err, "getting index entries to delete for chunkID=%s", cid) - } - for i := range ents { - // To avoid deleting entries from older tables, we check for table. This can still delete entries - // from different buckets in the same table, but we just accept that. - if tableName == ents[i].TableName { - batch.Delete(ents[i].TableName, ents[i].HashValue, ents[i].RangeValue) - } - } - indexEntries += len(ents) - - // Label entries in v9, v10 and v11 don't use from/through in encoded values, so instead of chunk From/Through values, - // we only pass start/end for current day, to avoid deleting entries in other buckets. - // As "end" is inclusive, we make it exclusive by -1. - _, perKeyEnts, err := schema.GetCacheKeysAndLabelWriteEntries(start, end-1, cp.userID, metricName, metric, cid) - if err != nil { - return errors.Wrapf(err, "getting index entries to delete for chunkID=%s", cid) - } - for _, ents := range perKeyEnts { - for i := range ents { - batch.Delete(ents[i].TableName, ents[i].HashValue, ents[i].RangeValue) - } - indexEntries += len(ents) - } - - // Only delete this chunk if it *starts* in plans' date-period. In general we process plans from most-recent - // to older, so if chunk starts in current plan's period, its index entries were already removed in later plans. - // This breaks when running multiple cleaners or cleaner crashes. - if c.From >= start { - chunksToDelete = append(chunksToDelete, cid) - } else { - cp.cleaner.deletedChunksSkipped.Inc() - continue - } - } - - // Delete index entries first. If we delete chunks first, and then cleaner is interrupted, - // chunks won't be find upon restart, and it won't be possible to clean up index entries. - if err := indexClient.BatchWrite(ctx, batch); err != nil { - level.Warn(cp.log).Log("msg", "failed to delete index entries for series", "series", e.SeriesID, "err", err) - cp.cleaner.deletedSeriesErrors.Inc() - } else { - cp.cleaner.deletedSeries.Inc() - cp.cleaner.deletedIndexEntries.Add(float64(indexEntries)) - } - - for _, cid := range chunksToDelete { - if err := chunkClient.DeleteChunk(ctx, cp.userID, cid); err != nil { - if errors.Is(err, chunk.ErrStorageObjectNotFound) { - cp.cleaner.deletedChunksMissing.Inc() - } else { - level.Warn(cp.log).Log("msg", "failed to delete chunk for series", "series", e.SeriesID, "chunk", cid, "err", err) - cp.cleaner.deletedChunksErrors.Inc() - } - } else { - cp.cleaner.deletedChunks.Inc() - } - } - - return nil -} - -func fetchSingleChunk(ctx context.Context, userID string, chunkClient chunk.Client, chunksIds []string) (*chunk.Chunk, error) { - // Fetch single chunk - for _, cid := range chunksIds { - c, err := chunk.ParseExternalKey(userID, cid) - if err != nil { - return nil, errors.Wrap(err, "fetching chunks") - } - - cs, err := chunkClient.GetChunks(ctx, []chunk.Chunk{c}) - - if errors.Is(err, chunk.ErrStorageObjectNotFound) { - continue - } - if err != nil { - return nil, errors.Wrap(err, "fetching chunks") - } - - if len(cs) > 0 { - return &cs[0], nil - } - } - - return nil, nil -} - -func (c *Cleaner) createClientsForDay(dayStart time.Time) (string, chunk.BaseSchema, chunk.Client, chunk.IndexClient, error) { - for ix, s := range c.schemaConfig.Configs { - if dayStart.Unix() < s.From.Unix() { - continue - } - - if ix+1 < len(c.schemaConfig.Configs) && dayStart.Unix() > c.schemaConfig.Configs[ix+1].From.Unix() { - continue - } - - tableName := s.IndexTables.TableFor(model.TimeFromUnixNano(dayStart.UnixNano())) - - schema, err := s.CreateSchema() - if err != nil { - return "", nil, nil, nil, errors.Wrap(err, "failed to create schema") - } - - // No registerer, to avoid problems with registering same metrics multiple times. - index, err := storage.NewIndexClient(s.IndexType, c.storageConfig, c.schemaConfig, nil) - if err != nil { - return "", nil, nil, nil, errors.Wrap(err, "error creating index client") - } - - objectStoreType := s.ObjectType - if objectStoreType == "" { - objectStoreType = s.IndexType - } - - // No registerer, to avoid problems with registering same metrics multiple times. - chunks, err := storage.NewChunkClient(objectStoreType, c.storageConfig, c.schemaConfig, nil) - if err != nil { - index.Stop() - return "", nil, nil, nil, errors.Wrap(err, "error creating object client") - } - - return tableName, schema, chunks, index, nil - } - - return "", nil, nil, nil, errors.Errorf("no schema for day %v", dayStart.Format("2006-01-02")) -} diff --git a/tools/blocksconvert/plan_file.go b/tools/blocksconvert/plan_file.go deleted file mode 100644 index 17c69f5c76..0000000000 --- a/tools/blocksconvert/plan_file.go +++ /dev/null @@ -1,112 +0,0 @@ -package blocksconvert - -import ( - "compress/gzip" - "fmt" - "io" - "regexp" - "strconv" - "strings" - "time" - - "github.com/golang/snappy" -) - -// Plan file describes which series must be included in a block for given user and day. -// It consists of JSON objects, each written on its own line. -// Plan file starts with single header, many plan entries and single footer. - -type PlanEntry struct { - // Header - User string `json:"user,omitempty"` - DayIndex int `json:"day_index,omitempty"` - - // Entries - SeriesID string `json:"sid,omitempty"` - Chunks []string `json:"cs,omitempty"` - - // Footer - Complete bool `json:"complete,omitempty"` -} - -func (pe *PlanEntry) Reset() { - *pe = PlanEntry{} -} - -// Returns true and "base name" or false and empty string. -func IsPlanFilename(name string) (bool, string) { - switch { - case strings.HasSuffix(name, ".plan.gz"): - return true, name[:len(name)-len(".plan.gz")] - - case strings.HasSuffix(name, ".plan.snappy"): - return true, name[:len(name)-len(".plan.snappy")] - - case strings.HasSuffix(name, ".plan"): - return true, name[:len(name)-len(".plan")] - } - - return false, "" -} - -func PreparePlanFileReader(planFile string, in io.Reader) (io.Reader, error) { - switch { - case strings.HasSuffix(planFile, ".snappy"): - return snappy.NewReader(in), nil - - case strings.HasSuffix(planFile, ".gz"): - return gzip.NewReader(in) - } - - return in, nil -} - -func StartingFilename(planBaseName string, t time.Time) string { - return fmt.Sprintf("%s.starting.%d", planBaseName, t.Unix()) -} - -func ProgressFilename(planBaseName string, t time.Time) string { - return fmt.Sprintf("%s.inprogress.%d", planBaseName, t.Unix()) -} - -var progress = regexp.MustCompile(`^(.+)\.(starting|progress|inprogress)\.(\d+)$`) - -func IsProgressFilename(name string) (bool, string, time.Time) { - m := progress.FindStringSubmatch(name) - if len(m) == 0 { - return false, "", time.Time{} - } - - ts, err := strconv.ParseInt(m[3], 10, 64) - if err != nil { - return false, "", time.Time{} - } - - return true, m[1], time.Unix(ts, 0) -} - -func FinishedFilename(planBaseName string, id string) string { - return fmt.Sprintf("%s.finished.%s", planBaseName, id) -} - -var finished = regexp.MustCompile(`^(.+)\.finished\.([a-zA-Z0-9]+)$`) - -func IsFinishedFilename(name string) (bool, string, string) { - m := finished.FindStringSubmatch(name) - if len(m) == 0 { - return false, "", "" - } - - return true, m[1], m[2] -} - -func ErrorFilename(planBaseName string) string { - return planBaseName + ".error" -} - -func IsErrorFilename(name string) (bool, string) { - if strings.HasSuffix(name, ".error") { - return true, name[:len(name)-len(".error")] - } - return false, "" -} diff --git a/tools/blocksconvert/plan_file_test.go b/tools/blocksconvert/plan_file_test.go deleted file mode 100644 index f1927b746c..0000000000 --- a/tools/blocksconvert/plan_file_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package blocksconvert - -import ( - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -func TestIsProgressFile(t *testing.T) { - for _, tc := range []struct { - input string - exp bool - base string - t time.Time - }{ - {input: "hello/world.progress.123456", exp: true, base: "hello/world", t: time.Unix(123456, 0)}, - {input: "hello/world.progress.123456123456123456123456123456123456", exp: false, base: "", t: time.Time{}}, - {input: "hello/world.notprogress.123456", exp: false, base: "", t: time.Time{}}, - {input: "hello/world.plan", exp: false, base: "", t: time.Time{}}, - } { - ok, base, tm := IsProgressFilename(tc.input) - require.Equal(t, tc.exp, ok, tc.input) - require.Equal(t, tc.base, base, tc.input) - require.Equal(t, tc.t, tm, tc.input) - } -} diff --git a/tools/blocksconvert/planprocessor/heartbeat.go b/tools/blocksconvert/planprocessor/heartbeat.go deleted file mode 100644 index cd4ed7888e..0000000000 --- a/tools/blocksconvert/planprocessor/heartbeat.go +++ /dev/null @@ -1,98 +0,0 @@ -package planprocessor - -import ( - "context" - "strconv" - "strings" - "time" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/pkg/errors" - "github.com/thanos-io/thanos/pkg/objstore" - - "github.com/cortexproject/cortex/pkg/util/services" - "github.com/cortexproject/cortex/tools/blocksconvert" -) - -type heartbeat struct { - services.Service - - log log.Logger - bucket objstore.Bucket - planFileBasename string - - lastProgressFile string -} - -func newHeartbeat(log log.Logger, bucket objstore.Bucket, interval time.Duration, planFileBasename, lastProgressFile string) *heartbeat { - hb := &heartbeat{ - log: log, - bucket: bucket, - planFileBasename: planFileBasename, - lastProgressFile: lastProgressFile, - } - - hb.Service = services.NewTimerService(interval, hb.heartbeat, hb.heartbeat, hb.stopping) - return hb -} - -func (hb *heartbeat) heartbeat(ctx context.Context) error { - if hb.lastProgressFile != "" { - ok, err := hb.bucket.Exists(ctx, hb.lastProgressFile) - if err != nil { - level.Warn(hb.log).Log("msg", "failed to check last progress file", "err", err) - return errors.Wrapf(err, "cannot check if progress file exists: %s", hb.lastProgressFile) - } - - if !ok { - level.Warn(hb.log).Log("msg", "previous progress file doesn't exist") - return errors.Errorf("previous progress file doesn't exist: %s", hb.lastProgressFile) - } - } - - now := time.Now() - newProgressFile := blocksconvert.ProgressFilename(hb.planFileBasename, now) - if newProgressFile == hb.lastProgressFile { - // when scheduler creates progress file, it can have the same timestamp. - return nil - } - - if err := hb.bucket.Upload(ctx, newProgressFile, strings.NewReader(strconv.FormatInt(now.Unix(), 10))); err != nil { - return errors.Wrap(err, "failed to upload new progress file") - } - - if hb.lastProgressFile != "" { - if err := hb.bucket.Delete(ctx, hb.lastProgressFile); err != nil { - return errors.Wrap(err, "failed to delete old progress file") - } - } - - level.Info(hb.log).Log("msg", "updated progress", "file", newProgressFile) - hb.lastProgressFile = newProgressFile - return nil -} - -func (hb *heartbeat) stopping(failure error) error { - // Only delete progress file if there was no failure until now. - if failure != nil { - return nil - } - - level.Info(hb.log).Log("msg", "deleting last progress file", "file", hb.lastProgressFile) - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - if hb.lastProgressFile != "" { - if err := hb.bucket.Delete(ctx, hb.lastProgressFile); err != nil { - return errors.Wrap(err, "failed to delete last progress file") - } - } - - return nil -} - -func (hb *heartbeat) String() string { - return "heartbeat" -} diff --git a/tools/blocksconvert/planprocessor/service.go b/tools/blocksconvert/planprocessor/service.go deleted file mode 100644 index 341fd22932..0000000000 --- a/tools/blocksconvert/planprocessor/service.go +++ /dev/null @@ -1,412 +0,0 @@ -package planprocessor - -import ( - "bufio" - "context" - "encoding/json" - "flag" - "io" - "io/ioutil" - "os" - "path/filepath" - "strings" - "time" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/thanos-io/thanos/pkg/objstore" - "golang.org/x/sync/errgroup" - "google.golang.org/grpc" - - "github.com/cortexproject/cortex/pkg/util/grpcclient" - "github.com/cortexproject/cortex/pkg/util/services" - "github.com/cortexproject/cortex/tools/blocksconvert" -) - -type PlanProcessor interface { - // Returns "id" that is appended to "finished" status file. - ProcessPlanEntries(ctx context.Context, entries chan blocksconvert.PlanEntry) (string, error) -} - -type Config struct { - // Exported config options. - Name string - HeartbeatPeriod time.Duration - SchedulerEndpoint string - NextPlanInterval time.Duration - GrpcConfig grpcclient.Config -} - -func (cfg *Config) RegisterFlags(prefix string, f *flag.FlagSet) { - cfg.GrpcConfig.RegisterFlagsWithPrefix(prefix+".client", f) - - host, _ := os.Hostname() - f.StringVar(&cfg.Name, prefix+".name", host, "Name passed to scheduler, defaults to hostname.") - f.DurationVar(&cfg.HeartbeatPeriod, prefix+".heartbeat", 5*time.Minute, "How often to update plan progress file.") - f.StringVar(&cfg.SchedulerEndpoint, prefix+".scheduler-endpoint", "", "Scheduler endpoint to ask for more plans to work on.") - f.DurationVar(&cfg.NextPlanInterval, prefix+".next-plan-interval", 1*time.Minute, "How often to ask for next plan (when idle)") -} - -// Creates new plan processor service. -// PlansDirectory is used for storing plan files. -// Bucket client used for downloading plan files. -// Cleanup function called on startup and after each build. Can be nil. -// Factory for creating PlanProcessor. Called for each new plan. -func NewService(cfg Config, plansDirectory string, bucket objstore.Bucket, cleanup func(logger log.Logger) error, factory func(planLog log.Logger, userID string, dayStart, dayEnd time.Time) PlanProcessor, l log.Logger, reg prometheus.Registerer) (*Service, error) { - if cfg.SchedulerEndpoint == "" { - return nil, errors.New("no scheduler endpoint") - } - - if bucket == nil || factory == nil { - return nil, errors.New("invalid config") - } - - if plansDirectory == "" { - return nil, errors.New("no directory for plans") - } - if err := os.MkdirAll(plansDirectory, os.FileMode(0700)); err != nil { - return nil, errors.Wrap(err, "failed to create plans directory") - } - - b := &Service{ - cfg: cfg, - plansDirectory: plansDirectory, - bucket: bucket, - cleanupFn: cleanup, - factory: factory, - log: l, - - currentPlanStartTime: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ - Name: "cortex_blocksconvert_plan_start_time_seconds", - Help: "Start time of current plan's time range (unix timestamp).", - }), - planFileReadPosition: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ - Name: "cortex_blocksconvert_plan_file_position", - Help: "Read bytes from the plan file.", - }), - planFileSize: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ - Name: "cortex_blocksconvert_plan_size", - Help: "Total size of plan file.", - }), - } - b.Service = services.NewBasicService(b.cleanup, b.running, nil) - return b, nil -} - -// This service implements common behaviour for plan-processing: 1) wait for next plan, 2) download plan, -// 3) process each plan entry, 4) delete local plan, 5) repeat. It gets plans from scheduler. During plan processing, -// this service maintains "progress" status file, and when plan processing finishes, it uploads "finished" plan. -type Service struct { - services.Service - - cfg Config - log log.Logger - - plansDirectory string - bucket objstore.Bucket - cleanupFn func(logger log.Logger) error - factory func(planLog log.Logger, userID string, dayStart time.Time, dayEnd time.Time) PlanProcessor - - planFileReadPosition prometheus.Gauge - planFileSize prometheus.Gauge - currentPlanStartTime prometheus.Gauge -} - -func (s *Service) cleanup(_ context.Context) error { - files, err := ioutil.ReadDir(s.plansDirectory) - if err != nil { - return err - } - - for _, f := range files { - toRemove := filepath.Join(s.plansDirectory, f.Name()) - - level.Info(s.log).Log("msg", "deleting unfinished local plan file", "file", toRemove) - err = os.Remove(toRemove) - if err != nil { - return errors.Wrapf(err, "removing %s", toRemove) - } - } - - if s.cleanupFn != nil { - return s.cleanupFn(s.log) - } - return nil -} - -func (s *Service) running(ctx context.Context) error { - ticker := time.NewTicker(s.cfg.NextPlanInterval) - defer ticker.Stop() - - var schedulerClient blocksconvert.SchedulerClient - var conn *grpc.ClientConn - - for { - select { - case <-ctx.Done(): - return nil - - case <-ticker.C: - // We may get "tick" even when we should stop. - if ctx.Err() != nil { - return nil - } - - if conn == nil { - opts, err := s.cfg.GrpcConfig.DialOption(nil, nil) - if err != nil { - return err - } - - conn, err = grpc.Dial(s.cfg.SchedulerEndpoint, opts...) - if err != nil { - level.Error(s.log).Log("msg", "failed to dial", "endpoint", s.cfg.SchedulerEndpoint, "err", err) - continue - } - - schedulerClient = blocksconvert.NewSchedulerClient(conn) - } - - resp, err := schedulerClient.NextPlan(ctx, &blocksconvert.NextPlanRequest{Name: s.cfg.Name}) - if err != nil { - level.Error(s.log).Log("msg", "failed to get next plan due to error, closing connection", "err", err) - _ = conn.Close() - conn = nil - schedulerClient = nil - continue - } - - // No plan to work on, ignore. - if resp.PlanFile == "" { - continue - } - - isPlanFile, planBaseName := blocksconvert.IsPlanFilename(resp.PlanFile) - if !isPlanFile { - level.Error(s.log).Log("msg", "got invalid plan file", "planFile", resp.PlanFile) - continue - } - - ok, base, _ := blocksconvert.IsProgressFilename(resp.ProgressFile) - if !ok || base != planBaseName { - level.Error(s.log).Log("msg", "got invalid progress file", "progressFile", resp.ProgressFile) - continue - } - - level.Info(s.log).Log("msg", "received plan file", "planFile", resp.PlanFile, "progressFile", resp.ProgressFile) - - err = s.downloadAndProcessPlanFile(ctx, resp.PlanFile, planBaseName, resp.ProgressFile) - if err != nil { - level.Error(s.log).Log("msg", "failed to process plan file", "planFile", resp.PlanFile, "err", err) - - // If context is canceled (blocksconvert is shutting down, or due to hearbeating failure), don't upload error. - if !errors.Is(err, context.Canceled) { - errorFile := blocksconvert.ErrorFilename(planBaseName) - err = s.bucket.Upload(ctx, errorFile, strings.NewReader(err.Error())) - if err != nil { - level.Error(s.log).Log("msg", "failed to upload error file", "errorFile", errorFile, "err", err) - } - } - } - - err = s.cleanup(ctx) - if err != nil { - level.Error(s.log).Log("msg", "failed to cleanup working directory", "err", err) - } - } - } -} - -func (s *Service) downloadAndProcessPlanFile(ctx context.Context, planFile, planBaseName, lastProgressFile string) error { - defer s.planFileSize.Set(0) - defer s.planFileReadPosition.Set(0) - defer s.currentPlanStartTime.Set(0) - - planLog := log.With(s.log, "plan", planFile) - - // Start heartbeating (updating of progress file). We setup new context used for the rest of the function. - // If hearbeating fails, we cancel this new context to abort quickly. - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - hb := newHeartbeat(planLog, s.bucket, s.cfg.HeartbeatPeriod, planBaseName, lastProgressFile) - hb.AddListener(services.NewListener(nil, nil, nil, nil, func(from services.State, failure error) { - level.Error(planLog).Log("msg", "heartbeating failed, aborting build", "failure", failure) - cancel() - })) - if err := services.StartAndAwaitRunning(ctx, hb); err != nil { - return errors.Wrap(err, "failed to start heartbeating") - } - - localPlanFile := filepath.Join(s.plansDirectory, filepath.Base(planFile)) - planSize, err := downloadPlanFile(ctx, s.bucket, planFile, localPlanFile) - if err != nil { - return errors.Wrapf(err, "failed to download plan file %s to %s", planFile, localPlanFile) - } - level.Info(planLog).Log("msg", "downloaded plan file", "localPlanFile", localPlanFile, "size", planSize) - - s.planFileSize.Set(float64(planSize)) - - f, err := os.Open(localPlanFile) - if err != nil { - return errors.Wrapf(err, "failed to read local plan file %s", localPlanFile) - } - defer func() { - _ = f.Close() - }() - - // Use a buffer for reading plan file. - r, err := blocksconvert.PreparePlanFileReader(planFile, bufio.NewReaderSize(&readPositionReporter{r: f, g: s.planFileReadPosition}, 1*1024*1024)) - if err != nil { - return err - } - - dec := json.NewDecoder(r) - - userID, dayStart, dayEnd, err := parsePlanHeader(dec) - if err != nil { - return err - } - - s.currentPlanStartTime.Set(float64(dayStart.Unix())) - - level.Info(planLog).Log("msg", "processing plan file", "user", userID, "dayStart", dayStart, "dayEnd", dayEnd) - - processor := s.factory(planLog, userID, dayStart, dayEnd) - - planEntryCh := make(chan blocksconvert.PlanEntry) - - idChan := make(chan string, 1) - - g, gctx := errgroup.WithContext(ctx) - g.Go(func() error { - id, err := processor.ProcessPlanEntries(gctx, planEntryCh) - idChan <- id - return err - }) - g.Go(func() error { - return parsePlanEntries(gctx, dec, planEntryCh) - }) - - if err := g.Wait(); err != nil { - return errors.Wrap(err, "failed to build block") - } - - err = os.Remove(localPlanFile) - if err != nil { - level.Warn(planLog).Log("msg", "failed to delete local plan file", "err", err) - } - - id := <-idChan - - // Upload finished status file - finishedFile := blocksconvert.FinishedFilename(planBaseName, id) - if err := s.bucket.Upload(ctx, finishedFile, strings.NewReader(id)); err != nil { - return errors.Wrap(err, "failed to upload finished status file") - } - level.Info(planLog).Log("msg", "uploaded finished file", "file", finishedFile) - - // Stop heartbeating. - if err := services.StopAndAwaitTerminated(ctx, hb); err != nil { - // No need to report this error to caller to avoid generating error file. - level.Warn(planLog).Log("msg", "hearbeating failed", "err", err) - } - - // All OK - return nil -} - -func downloadPlanFile(ctx context.Context, bucket objstore.Bucket, planFile string, localPlanFile string) (int64, error) { - f, err := os.Create(localPlanFile) - if err != nil { - return 0, err - } - - r, err := bucket.Get(ctx, planFile) - if err != nil { - _ = f.Close() - return 0, err - } - // Copy will read `r` until EOF, or error is returned. Any possible error from Close is irrelevant. - defer func() { _ = r.Close() }() - - n, err := io.Copy(f, r) - if err != nil { - _ = f.Close() - return 0, err - } - - return n, f.Close() -} - -func parsePlanHeader(dec *json.Decoder) (userID string, startTime, endTime time.Time, err error) { - header := blocksconvert.PlanEntry{} - if err = dec.Decode(&header); err != nil { - return - } - if header.User == "" || header.DayIndex == 0 { - err = errors.New("failed to read plan file header: no user or day index found") - return - } - - dayStart := time.Unix(int64(header.DayIndex)*int64(24*time.Hour/time.Second), 0).UTC() - dayEnd := dayStart.Add(24 * time.Hour) - return header.User, dayStart, dayEnd, nil -} - -func parsePlanEntries(ctx context.Context, dec *json.Decoder, planEntryCh chan blocksconvert.PlanEntry) error { - defer close(planEntryCh) - - var err error - complete := false - entry := blocksconvert.PlanEntry{} - for err = dec.Decode(&entry); err == nil; err = dec.Decode(&entry) { - if entry.Complete { - complete = true - entry.Reset() - continue - } - - if complete { - return errors.New("plan entries found after plan footer") - } - - if entry.SeriesID != "" && len(entry.Chunks) > 0 { - select { - case planEntryCh <- entry: - // ok - case <-ctx.Done(): - return nil - } - - } - - entry.Reset() - } - - if err == io.EOF { - if !complete { - return errors.New("plan is not complete") - } - err = nil - } - return errors.Wrap(err, "parsing plan entries") -} - -type readPositionReporter struct { - r io.Reader - g prometheus.Gauge - pos int64 -} - -func (r *readPositionReporter) Read(p []byte) (int, error) { - n, err := r.r.Read(p) - if n > 0 { - r.pos += int64(n) - r.g.Set(float64(r.pos)) - } - return n, err -} diff --git a/tools/blocksconvert/scanner/bigtable_index_reader.go b/tools/blocksconvert/scanner/bigtable_index_reader.go deleted file mode 100644 index 7c3e455e5f..0000000000 --- a/tools/blocksconvert/scanner/bigtable_index_reader.go +++ /dev/null @@ -1,242 +0,0 @@ -package scanner - -import ( - "bytes" - "context" - "io" - "sort" - "strings" - - "cloud.google.com/go/bigtable" - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "golang.org/x/sync/errgroup" - - "github.com/cortexproject/cortex/pkg/chunk" - "github.com/cortexproject/cortex/pkg/chunk/gcp" -) - -type bigtableIndexReader struct { - log log.Logger - project string - instance string - - rowsRead prometheus.Counter - parsedIndexEntries prometheus.Counter - currentTableRanges prometheus.Gauge - currentTableScannedRanges prometheus.Gauge -} - -func newBigtableIndexReader(project, instance string, l log.Logger, rowsRead prometheus.Counter, parsedIndexEntries prometheus.Counter, currentTableRanges, scannedRanges prometheus.Gauge) *bigtableIndexReader { - return &bigtableIndexReader{ - log: l, - project: project, - instance: instance, - - rowsRead: rowsRead, - parsedIndexEntries: parsedIndexEntries, - currentTableRanges: currentTableRanges, - currentTableScannedRanges: scannedRanges, - } -} - -func (r *bigtableIndexReader) IndexTableNames(ctx context.Context) ([]string, error) { - client, err := bigtable.NewAdminClient(ctx, r.project, r.instance) - if err != nil { - return nil, errors.Wrap(err, "create bigtable client failed") - } - defer closeCloser(r.log, "bigtable admin client", client) - - return client.Tables(ctx) -} - -// This reader supports both used versions of BigTable index client used by Cortex: -// -// 1) newStorageClientV1 ("gcp"), which sets -// - RowKey = entry.HashValue + \0 + entry.RangeValue -// - Column: "c" (in family "f") -// - Value: entry.Value -// -// 2) newStorageClientColumnKey ("gcp-columnkey", "bigtable", "bigtable-hashed"), which has two possibilities: -// - RowKey = entry.HashValue OR (if distribute key flag is enabled) hashPrefix(entry.HashValue) + "-" + entry.HashValue, where hashPrefix is 64-bit FNV64a hash, encoded as little-endian hex value -// - Column: entry.RangeValue (in family "f") -// - Value: entry.Value -// -// Index entries are returned in HashValue, RangeValue order. -// Entries for the same HashValue and RangeValue are passed to the same processor. -func (r *bigtableIndexReader) ReadIndexEntries(ctx context.Context, tableName string, processors []chunk.IndexEntryProcessor) error { - client, err := bigtable.NewClient(ctx, r.project, r.instance) - if err != nil { - return errors.Wrap(err, "create bigtable client failed") - } - defer closeCloser(r.log, "bigtable client", client) - - var rangesCh chan bigtable.RowRange - - tbl := client.Open(tableName) - if keys, err := tbl.SampleRowKeys(ctx); err == nil { - level.Info(r.log).Log("msg", "sampled row keys", "keys", strings.Join(keys, ", ")) - - rangesCh = make(chan bigtable.RowRange, len(keys)+1) - - start := "" - for _, k := range keys { - rangesCh <- bigtable.NewRange(start, k) - start = k - } - rangesCh <- bigtable.InfiniteRange(start) // Last segment from last key, to the end. - close(rangesCh) - } else { - level.Warn(r.log).Log("msg", "failed to sample row keys", "err", err) - - rangesCh = make(chan bigtable.RowRange, 1) - rangesCh <- bigtable.InfiniteRange("") - close(rangesCh) - } - - r.currentTableRanges.Set(float64(len(rangesCh))) - r.currentTableScannedRanges.Set(0) - - defer r.currentTableRanges.Set(0) - defer r.currentTableScannedRanges.Set(0) - - g, gctx := errgroup.WithContext(ctx) - - for ix := range processors { - p := processors[ix] - - g.Go(func() error { - for rng := range rangesCh { - var innerErr error - - level.Info(r.log).Log("msg", "reading rows", "range", rng) - - err := tbl.ReadRows(gctx, rng, func(row bigtable.Row) bool { - r.rowsRead.Inc() - - entries, err := parseRowKey(row, tableName) - if err != nil { - innerErr = errors.Wrapf(err, "failed to parse row: %s", row.Key()) - return false - } - - r.parsedIndexEntries.Add(float64(len(entries))) - - for _, e := range entries { - err := p.ProcessIndexEntry(e) - if err != nil { - innerErr = errors.Wrap(err, "processor error") - return false - } - } - - return true - }) - - if innerErr != nil { - return innerErr - } - - if err != nil { - return err - } - - r.currentTableScannedRanges.Inc() - } - - return p.Flush() - }) - } - - return g.Wait() -} - -func parseRowKey(row bigtable.Row, tableName string) ([]chunk.IndexEntry, error) { - var entries []chunk.IndexEntry - - rowKey := row.Key() - - rangeInRowKey := false - hashValue := row.Key() - rangeValue := "" - - // Remove hashPrefix, if used. Easy to check. - if len(hashValue) > 16 && hashValue[16] == '-' && hashValue[:16] == gcp.HashPrefix(hashValue[17:]) { - hashValue = hashValue[17:] - } else if ix := strings.IndexByte(hashValue, 0); ix > 0 { - // newStorageClientV1 uses - // - RowKey: entry.HashValue + \0 + entry.RangeValue - // - Column: "c" (in family "f") - // - Value: entry.Value - - rangeInRowKey = true - rangeValue = hashValue[ix+1:] - hashValue = hashValue[:ix] - } - - for family, columns := range row { - if family != "f" { - return nil, errors.Errorf("unknown family: %s", family) - } - - for _, colVal := range columns { - if colVal.Row != rowKey { - return nil, errors.Errorf("rowkey mismatch: %q, %q", colVal.Row, rowKey) - } - - if rangeInRowKey { - if colVal.Column != "f:c" { - return nil, errors.Errorf("found rangeValue in RowKey, but column is not 'f:c': %q", colVal.Column) - } - // we already have rangeValue - } else { - if !strings.HasPrefix(colVal.Column, "f:") { - return nil, errors.Errorf("invalid column prefix: %q", colVal.Column) - } - rangeValue = colVal.Column[2:] // With "f:" part removed - } - - entry := chunk.IndexEntry{ - TableName: tableName, - HashValue: hashValue, - RangeValue: []byte(rangeValue), - Value: colVal.Value, - } - - entries = append(entries, entry) - } - } - - if len(entries) > 1 { - // Sort entries by RangeValue. This is done to support `newStorageClientColumnKey` version properly: - // all index entries with same hashValue are in the same row, but map iteration over columns may - // have returned them in wrong order. - - sort.Sort(sortableIndexEntries(entries)) - } - - return entries, nil -} - -func closeCloser(log log.Logger, closerName string, closer io.Closer) { - err := closer.Close() - if err != nil { - level.Warn(log).Log("msg", "failed to close "+closerName, "err", err) - } -} - -type sortableIndexEntries []chunk.IndexEntry - -func (s sortableIndexEntries) Len() int { - return len(s) -} - -func (s sortableIndexEntries) Less(i, j int) bool { - return bytes.Compare(s[i].RangeValue, s[j].RangeValue) < 0 -} - -func (s sortableIndexEntries) Swap(i, j int) { - s[i], s[j] = s[j], s[i] -} diff --git a/tools/blocksconvert/scanner/bigtable_index_reader_test.go b/tools/blocksconvert/scanner/bigtable_index_reader_test.go deleted file mode 100644 index 389d9797c8..0000000000 --- a/tools/blocksconvert/scanner/bigtable_index_reader_test.go +++ /dev/null @@ -1,182 +0,0 @@ -package scanner - -import ( - "testing" - - "cloud.google.com/go/bigtable" - "github.com/stretchr/testify/require" - - "github.com/cortexproject/cortex/pkg/chunk" -) - -func TestParseRowKey(t *testing.T) { - tcs := map[string]struct { - row bigtable.Row - table string - - expectedEntries []chunk.IndexEntry - expectedError string - }{ - "newStorageClientV1 format": { - row: map[string][]bigtable.ReadItem{ - "f": { - {Row: "testUser:d18500:test_metric\u0000eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M\u0000\u0000\u00007\u0000", Column: "f:c", Value: []byte("-")}, - }, - }, - table: "test", - expectedEntries: []chunk.IndexEntry{ - { - TableName: "test", - HashValue: "testUser:d18500:test_metric", - RangeValue: []byte("eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M\x00\x00\x007\x00"), - Value: []byte("-"), - }, - }, - }, - // 2a) newStorageClientColumnKey, WITHOUT key distribution - // - RowKey = entry.HashValue - // - Column: entry.RangeValue (in family "f") - // - Value: entry.Value - "newStorageClientColumnKey without key distribution": { - row: map[string][]bigtable.ReadItem{ - "f": { - {Row: "testUser:d18500:test_metric", Column: "f:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M\u0000\u0000\u00007\u0000", Value: []byte("-")}, - }, - }, - table: "test", - expectedEntries: []chunk.IndexEntry{ - { - TableName: "test", - HashValue: "testUser:d18500:test_metric", - RangeValue: []byte("eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M\x00\x00\x007\x00"), - Value: []byte("-"), - }, - }, - }, - "newStorageClientColumnKey without key distribution, multiple columns": { - row: map[string][]bigtable.ReadItem{ - "f": { - {Row: "testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M", Column: "f:00a4cb80\u0000\u0000chunkID_1\u00003\u0000", Value: []byte("")}, - {Row: "testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M", Column: "f:05265c00\x00\x00chunkID_2\x003\x00", Value: []byte("")}, - {Row: "testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M", Column: "f:0036ee80\x00\x00chunkID_2\x003\x00", Value: []byte("")}, - }, - }, - table: "test", - expectedEntries: []chunk.IndexEntry{ - { - TableName: "test", - HashValue: "testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M", - RangeValue: []byte("0036ee80\x00\x00chunkID_2\x003\x00"), - Value: []byte(""), - }, - { - TableName: "test", - HashValue: "testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M", - RangeValue: []byte("00a4cb80\x00\x00chunkID_1\x003\x00"), - Value: []byte(""), - }, - { - TableName: "test", - HashValue: "testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M", - RangeValue: []byte("05265c00\x00\x00chunkID_2\x003\x00"), - Value: []byte(""), - }, - }, - }, - - // 2b) newStorageClientColumnKey, WITH key distribution - // - RowKey: hashPrefix(entry.HashValue) + "-" + entry.HashValue, where hashPrefix is 64-bit FNV64a hash, encoded as little-endian hex value - // - Column: entry.RangeValue (in family "f") - // - Value: entry.Value - "newStorageClientColumnKey with key distribution, multiple columns": { - row: map[string][]bigtable.ReadItem{ - "f": { - {Row: "15820f698f0f8d81-testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M", Column: "f:00a4cb80\u0000\u0000chunkID_1\u00003\u0000", Value: []byte("")}, - {Row: "15820f698f0f8d81-testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M", Column: "f:05265c00\x00\x00chunkID_2\x003\x00", Value: []byte("")}, - {Row: "15820f698f0f8d81-testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M", Column: "f:0036ee80\x00\x00chunkID_2\x003\x00", Value: []byte("")}, - }, - }, - table: "test", - expectedEntries: []chunk.IndexEntry{ - { - TableName: "test", - HashValue: "testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M", - RangeValue: []byte("0036ee80\x00\x00chunkID_2\x003\x00"), - Value: []byte(""), - }, - { - TableName: "test", - HashValue: "testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M", - RangeValue: []byte("00a4cb80\x00\x00chunkID_1\x003\x00"), - Value: []byte(""), - }, - { - TableName: "test", - HashValue: "testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M", - RangeValue: []byte("05265c00\x00\x00chunkID_2\x003\x00"), - Value: []byte(""), - }, - }, - }, - "different row keys": { - row: map[string][]bigtable.ReadItem{ - "f": { - {Row: "a", Column: "f:c", Value: []byte("-")}, - {Row: "b", Column: "f:c", Value: []byte("-")}, - }, - }, - table: "test", - expectedError: "rowkey mismatch: \"b\", \"a\"", - }, - - "newStorageClientV1, invalid column": { - row: map[string][]bigtable.ReadItem{ - "f": { - {Row: "testUser:d18500:test_metric\u0000eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M\u0000\u0000\u00007\u0000", Column: "wrong", Value: []byte("-")}, - }, - }, - table: "test", - expectedError: "found rangeValue in RowKey, but column is not 'f:c': \"wrong\"", - }, - - "newStorageClientColumnKey, invalid column family": { - row: map[string][]bigtable.ReadItem{ - "f": { - {Row: "testUser:d18500:test_metric", Column: "family:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M\u0000\u0000\u00007\u0000", Value: []byte("-")}, - }, - }, - table: "test", - expectedError: "invalid column prefix: \"family:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M\\x00\\x00\\x007\\x00\"", - }, - "newStorageClientColumnKey, invalid hash (hash ignored, not stripped)": { - row: map[string][]bigtable.ReadItem{ - "f": { - {Row: "1234567890123456-testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M", Column: "f:00a4cb80\u0000\u0000chunkID_1\u00003\u0000", Value: []byte("")}, - }, - }, - expectedEntries: []chunk.IndexEntry{ - { - TableName: "test", - HashValue: "1234567890123456-testUser:d18500:eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M", - RangeValue: []byte("00a4cb80\u0000\u0000chunkID_1\u00003\u0000"), - Value: []byte(""), - }, - }, - table: "test", - }, - } - - for name, tc := range tcs { - t.Run(name, func(t *testing.T) { - entries, err := parseRowKey(tc.row, tc.table) - - if tc.expectedError != "" { - require.EqualError(t, err, tc.expectedError) - require.Nil(t, entries) - } else { - require.NoError(t, err) - require.Equal(t, tc.expectedEntries, entries) - } - }) - } -} diff --git a/tools/blocksconvert/scanner/cassandra_index_reader.go b/tools/blocksconvert/scanner/cassandra_index_reader.go deleted file mode 100644 index 6a4fba5d41..0000000000 --- a/tools/blocksconvert/scanner/cassandra_index_reader.go +++ /dev/null @@ -1,172 +0,0 @@ -package scanner - -import ( - "context" - "fmt" - "math" - "strings" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "golang.org/x/sync/errgroup" - - "github.com/cortexproject/cortex/pkg/chunk" - "github.com/cortexproject/cortex/pkg/chunk/cassandra" -) - -/* Cassandra can easily run out of memory or timeout if we try to SELECT the - * entire table. Splitting into many smaller chunks help a lot. */ -const nbTokenRanges = 512 -const queryPageSize = 10000 - -type cassandraIndexReader struct { - log log.Logger - cassandraStorageConfig cassandra.Config - schemaCfg chunk.SchemaConfig - - rowsRead prometheus.Counter - parsedIndexEntries prometheus.Counter - currentTableRanges prometheus.Gauge - currentTableScannedRanges prometheus.Gauge -} - -func newCassandraIndexReader(cfg cassandra.Config, schemaCfg chunk.SchemaConfig, l log.Logger, rowsRead prometheus.Counter, parsedIndexEntries prometheus.Counter, currentTableRanges, scannedRanges prometheus.Gauge) *cassandraIndexReader { - return &cassandraIndexReader{ - log: l, - cassandraStorageConfig: cfg, - - rowsRead: rowsRead, - parsedIndexEntries: parsedIndexEntries, - currentTableRanges: currentTableRanges, - currentTableScannedRanges: scannedRanges, - } -} - -func (r *cassandraIndexReader) IndexTableNames(ctx context.Context) ([]string, error) { - client, err := cassandra.NewTableClient(ctx, r.cassandraStorageConfig, nil) - if err != nil { - return nil, errors.Wrap(err, "create cassandra client failed") - } - - defer client.Stop() - - return client.ListTables(ctx) -} - -type tokenRange struct { - start int64 - end int64 -} - -func (r *cassandraIndexReader) ReadIndexEntries(ctx context.Context, tableName string, processors []chunk.IndexEntryProcessor) error { - level.Debug(r.log).Log("msg", "scanning table", "table", tableName) - - client, err := cassandra.NewStorageClient(r.cassandraStorageConfig, r.schemaCfg, nil) - if err != nil { - return errors.Wrap(err, "create cassandra storage client failed") - } - - defer client.Stop() - - session := client.GetReadSession() - - rangesCh := make(chan tokenRange, nbTokenRanges) - - var step, n, start int64 - - step = int64(math.MaxUint64 / nbTokenRanges) - - for n = 0; n < nbTokenRanges; n++ { - start = math.MinInt64 + n*step - end := start + step - - if n == (nbTokenRanges - 1) { - end = math.MaxInt64 - } - - t := tokenRange{start: start, end: end} - rangesCh <- t - } - - close(rangesCh) - - r.currentTableRanges.Set(float64(len(rangesCh))) - r.currentTableScannedRanges.Set(0) - - defer r.currentTableRanges.Set(0) - defer r.currentTableScannedRanges.Set(0) - - g, gctx := errgroup.WithContext(ctx) - - for ix := range processors { - p := processors[ix] - g.Go(func() error { - for rng := range rangesCh { - level.Debug(r.log).Log("msg", "reading rows", "range_start", rng.start, "range_end", rng.end, "table_name", tableName) - - query := fmt.Sprintf("SELECT hash, range, value FROM %s WHERE token(hash) >= %v", tableName, rng.start) - - if rng.end < math.MaxInt64 { - query += fmt.Sprintf(" AND token(hash) < %v", rng.end) - } - - iter := session.Query(query).WithContext(gctx).PageSize(queryPageSize).Iter() - - if len(iter.Warnings()) > 0 { - level.Warn(r.log).Log("msg", "warnings from cassandra", "warnings", strings.Join(iter.Warnings(), " :: ")) - } - - scanner := iter.Scanner() - - oldHash := "" - oldRng := "" - - for scanner.Next() { - var hash, rng, value string - - err := scanner.Scan(&hash, &rng, &value) - if err != nil { - return errors.Wrap(err, "Cassandra scan error") - } - - r.rowsRead.Inc() - r.parsedIndexEntries.Inc() - - entry := chunk.IndexEntry{ - TableName: tableName, - HashValue: hash, - RangeValue: []byte(rng), - Value: []byte(value), - } - - if rng < oldRng && oldHash == hash { - level.Error(r.log).Log("msg", "new rng bad", "rng", rng, "old_rng", oldRng, "hash", hash, "old_hash", oldHash) - return fmt.Errorf("received range row in the wrong order for same hash: %v < %v", rng, oldRng) - } - - err = p.ProcessIndexEntry(entry) - if err != nil { - return errors.Wrap(err, "processor error") - } - - oldHash = hash - oldRng = rng - } - - // This will also close the iterator. - err := scanner.Err() - if err != nil { - return errors.Wrap(err, "Cassandra error during scan") - } - - r.currentTableScannedRanges.Inc() - } - - return p.Flush() - }) - } - - return g.Wait() -} diff --git a/tools/blocksconvert/scanner/files.go b/tools/blocksconvert/scanner/files.go deleted file mode 100644 index af4bba146e..0000000000 --- a/tools/blocksconvert/scanner/files.go +++ /dev/null @@ -1,111 +0,0 @@ -package scanner - -import ( - "encoding/json" - "io" - "os" - "path/filepath" - "sync" - - "github.com/golang/snappy" - "github.com/prometheus/client_golang/prometheus" - tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" -) - -type file struct { - mu sync.Mutex - file *os.File - comp io.WriteCloser - enc *json.Encoder -} - -// Provides serialized access to writing entries. -type openFiles struct { - mu sync.Mutex - files map[string]*file - - openFiles prometheus.Gauge -} - -func newOpenFiles(openFilesGauge prometheus.Gauge) *openFiles { - of := &openFiles{ - files: map[string]*file{}, - openFiles: openFilesGauge, - } - - return of -} - -func (of *openFiles) appendJSONEntryToFile(dir, filename string, data interface{}, headerFn func() interface{}) error { - f, err := of.getFile(dir, filename, headerFn) - if err != nil { - return err - } - - // To avoid mixed output from different writes, make sure to serialize access to the file. - f.mu.Lock() - defer f.mu.Unlock() - return f.enc.Encode(data) -} - -func (of *openFiles) getFile(dir, filename string, headerFn func() interface{}) (*file, error) { - of.mu.Lock() - defer of.mu.Unlock() - - name := filepath.Join(dir, filename+".snappy") - - f := of.files[name] - if f == nil { - err := os.MkdirAll(dir, os.FileMode(0700)) - if err != nil { - return nil, err - } - - fl, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) - if err != nil { - return nil, err - } - - comp := snappy.NewBufferedWriter(fl) - enc := json.NewEncoder(comp) - enc.SetEscapeHTML(false) - - if headerFn != nil { - err := enc.Encode(headerFn()) - if err != nil { - _ = fl.Close() - } - } - - f = &file{ - file: fl, - comp: comp, - enc: enc, - } - of.files[name] = f - of.openFiles.Inc() - } - - return f, nil -} - -func (of *openFiles) closeAllFiles(footerFn func() interface{}) error { - of.mu.Lock() - defer of.mu.Unlock() - - errs := tsdb_errors.NewMulti() - - for fn, f := range of.files { - delete(of.files, fn) - of.openFiles.Dec() - - if footerFn != nil { - errs.Add(f.enc.Encode(footerFn())) - } - - errs.Add(f.comp.Close()) - errs.Add(f.file.Close()) - } - - return errs.Err() -} diff --git a/tools/blocksconvert/scanner/index_entry.go b/tools/blocksconvert/scanner/index_entry.go deleted file mode 100644 index 19e4e6ce59..0000000000 --- a/tools/blocksconvert/scanner/index_entry.go +++ /dev/null @@ -1,76 +0,0 @@ -package scanner - -import ( - "bytes" - "fmt" - "strconv" - "strings" - - "github.com/pkg/errors" -) - -// v9 (hashValue, rangeRange -> value) -// :d:metricName, \0\0\0 "7" \0 -> "-" -// :d:metricName:labelName, sha256(labelValue)\0\0\0 "8" \0 -> labelValue -// :d:, throughBytes \0\0 chunkID \0 "3" \0 -> "" -// -// is base64 of SHA256(labels) - -// v10 (hashValue, rangeValue -> value) -// ::d:metricName, \0\0\0 "7" \0 -> "-" -// ::d:metricName:labelName, sha256(labelValue)\0\0\0 "8" \0 -> labelValue -// :d:, throughBytes \0\0 chunkID \0 "3" \0 -> "-" -// v11 adds: -// , \0\0\0 '9' \0 -> JSON array with label values. - -func IsMetricToSeriesMapping(RangeValue []byte) bool { - return bytes.HasSuffix(RangeValue, []byte("\0007\000")) -} - -func IsMetricLabelToLabelValueMapping(RangeValue []byte) bool { - return bytes.HasSuffix(RangeValue, []byte("\0008\000")) -} - -func IsSeriesToLabelValues(RangeValue []byte) bool { - return bytes.HasSuffix(RangeValue, []byte("\0009\000")) -} - -// Series to Chunk mapping uses \0 "3" \0 suffix of range value. -func IsSeriesToChunkMapping(RangeValue []byte) bool { - return bytes.HasSuffix(RangeValue, []byte("\0003\000")) -} - -func UnknownIndexEntryType(RangeValue []byte) string { - if len(RangeValue) < 3 { - return "too-short" - } - - // Take last three characters, and report it back. - return fmt.Sprintf("%x", RangeValue[len(RangeValue)-2:]) -} - -// e.RangeValue is: "userID:d:base64(sha256(labels))". Index is integer, base64 doesn't contain ':'. -func GetSeriesToChunkMapping(HashValue string, RangeValue []byte) (user string, index int, seriesID string, chunkID string, err error) { - s := bytes.Split(RangeValue, []byte("\000")) - chunkID = string(s[2]) - - parts := strings.Split(HashValue, ":") - if len(parts) < 3 { - err = errors.Errorf("not enough parts: %d", len(parts)) - return - } - - seriesID = parts[len(parts)-1] - indexStr := parts[len(parts)-2] - if !strings.HasPrefix(indexStr, "d") { // Schema v9 and later uses "day" buckets, prefixed with "d" - err = errors.Errorf("invalid index prefix") - return - } - index, err = strconv.Atoi(indexStr[1:]) - if err != nil { - err = errors.Wrapf(err, "failed to parse index") - return - } - user = strings.Join(parts[:len(parts)-2], ":") - return -} diff --git a/tools/blocksconvert/scanner/scanner.go b/tools/blocksconvert/scanner/scanner.go deleted file mode 100644 index aa1c258661..0000000000 --- a/tools/blocksconvert/scanner/scanner.go +++ /dev/null @@ -1,649 +0,0 @@ -package scanner - -import ( - "context" - "encoding/json" - "flag" - "io" - "io/ioutil" - "os" - "path" - "path/filepath" - "regexp" - "sort" - "strconv" - "strings" - "time" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/thanos-io/thanos/pkg/objstore" - "golang.org/x/sync/errgroup" - - "github.com/cortexproject/cortex/pkg/chunk" - "github.com/cortexproject/cortex/pkg/chunk/aws" - "github.com/cortexproject/cortex/pkg/chunk/storage" - "github.com/cortexproject/cortex/pkg/util/backoff" - "github.com/cortexproject/cortex/pkg/util/flagext" - "github.com/cortexproject/cortex/pkg/util/services" - "github.com/cortexproject/cortex/tools/blocksconvert" -) - -type Config struct { - TableNames string - TablesLimit int - - PeriodStart flagext.DayValue - PeriodEnd flagext.DayValue - - OutputDirectory string - Concurrency int - - VerifyPlans bool - UploadFiles bool - KeepFiles bool - - AllowedUsers string - IgnoredUserPattern string -} - -func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - f.StringVar(&cfg.TableNames, "scanner.tables", "", "Comma-separated tables to generate plan files from. If not used, all tables found via schema and scanning of Index store will be used.") - f.StringVar(&cfg.OutputDirectory, "scanner.output-dir", "", "Local directory used for storing temporary plan files (will be created if missing).") - f.IntVar(&cfg.Concurrency, "scanner.concurrency", 16, "Number of concurrent index processors, and plan uploads.") - f.BoolVar(&cfg.UploadFiles, "scanner.upload", true, "Upload plan files.") - f.BoolVar(&cfg.KeepFiles, "scanner.keep-files", false, "Keep plan files locally after uploading.") - f.IntVar(&cfg.TablesLimit, "scanner.tables-limit", 0, "Number of tables to convert. 0 = all.") - f.StringVar(&cfg.AllowedUsers, "scanner.allowed-users", "", "Allowed users that can be converted, comma-separated. If set, only these users have plan files generated.") - f.StringVar(&cfg.IgnoredUserPattern, "scanner.ignore-users-regex", "", "If set and user ID matches this regex pattern, it will be ignored. Checked after applying -scanner.allowed-users, if set.") - f.BoolVar(&cfg.VerifyPlans, "scanner.verify-plans", true, "Verify plans before uploading to bucket. Enabled by default for extra check. Requires extra memory for large plans.") - f.Var(&cfg.PeriodStart, "scanner.scan-period-start", "If specified, this is lower end of time period to scan. Specified date is included in the range. (format: \"2006-01-02\")") - f.Var(&cfg.PeriodEnd, "scanner.scan-period-end", "If specified, this is upper end of time period to scan. Specified date is not included in the range. (format: \"2006-01-02\")") -} - -type Scanner struct { - services.Service - - cfg Config - storageCfg storage.Config - - bucketPrefix string - bucket objstore.Bucket - - logger log.Logger - reg prometheus.Registerer - - series prometheus.Counter - openFiles prometheus.Gauge - indexEntries *prometheus.CounterVec - indexReaderRowsRead prometheus.Counter - indexReaderParsedIndexEntries prometheus.Counter - ignoredEntries prometheus.Counter - foundTables prometheus.Counter - processedTables prometheus.Counter - currentTableRanges prometheus.Gauge - currentTableScannedRanges prometheus.Gauge - - schema chunk.SchemaConfig - ignoredUsers *regexp.Regexp - allowedUsers blocksconvert.AllowedUsers -} - -func NewScanner(cfg Config, scfg blocksconvert.SharedConfig, l log.Logger, reg prometheus.Registerer) (*Scanner, error) { - err := scfg.SchemaConfig.Load() - if err != nil { - return nil, errors.Wrap(err, "no table name provided, and schema failed to load") - } - - if cfg.OutputDirectory == "" { - return nil, errors.Errorf("no output directory") - } - - var bucketClient objstore.Bucket - if cfg.UploadFiles { - var err error - bucketClient, err = scfg.GetBucket(l, reg) - if err != nil { - return nil, err - } - } - - var users = blocksconvert.AllowAllUsers - if cfg.AllowedUsers != "" { - users = blocksconvert.ParseAllowedUsers(cfg.AllowedUsers) - } - - var ignoredUserRegex *regexp.Regexp = nil - if cfg.IgnoredUserPattern != "" { - re, err := regexp.Compile(cfg.IgnoredUserPattern) - if err != nil { - return nil, errors.Wrap(err, "failed to compile ignored user regex") - } - ignoredUserRegex = re - } - - if err := os.MkdirAll(cfg.OutputDirectory, os.FileMode(0700)); err != nil { - return nil, errors.Wrapf(err, "failed to create new output directory %s", cfg.OutputDirectory) - } - - s := &Scanner{ - cfg: cfg, - schema: scfg.SchemaConfig, - storageCfg: scfg.StorageConfig, - logger: l, - bucket: bucketClient, - bucketPrefix: scfg.BucketPrefix, - reg: reg, - allowedUsers: users, - ignoredUsers: ignoredUserRegex, - - indexReaderRowsRead: promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "cortex_blocksconvert_bigtable_read_rows_total", - Help: "Number of rows read from BigTable", - }), - indexReaderParsedIndexEntries: promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "cortex_blocksconvert_bigtable_parsed_index_entries_total", - Help: "Number of parsed index entries", - }), - currentTableRanges: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ - Name: "cortex_blocksconvert_scanner_bigtable_ranges_in_current_table", - Help: "Number of ranges to scan from current table.", - }), - currentTableScannedRanges: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ - Name: "cortex_blocksconvert_scanner_bigtable_scanned_ranges_from_current_table", - Help: "Number of scanned ranges from current table. Resets to 0 every time a table is getting scanned or its scan has completed.", - }), - - series: promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "cortex_blocksconvert_scanner_series_written_total", - Help: "Number of series written to the plan files", - }), - - openFiles: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ - Name: "cortex_blocksconvert_scanner_open_files", - Help: "Number of series written to the plan files", - }), - - indexEntries: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ - Name: "cortex_blocksconvert_scanner_scanned_index_entries_total", - Help: "Number of various index entries scanned", - }, []string{"type"}), - ignoredEntries: promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "cortex_blocksconvert_scanner_ignored_index_entries_total", - Help: "Number of ignored index entries because of ignoring users.", - }), - - foundTables: promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "cortex_blocksconvert_scanner_found_tables_total", - Help: "Number of tables found for processing.", - }), - processedTables: promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Name: "cortex_blocksconvert_scanner_processed_tables_total", - Help: "Number of processed tables so far.", - }), - } - - s.Service = services.NewBasicService(nil, s.running, nil) - return s, nil -} - -func (s *Scanner) running(ctx context.Context) error { - allTables := []tableToProcess{} - - for ix, c := range s.schema.Configs { - if c.Schema != "v9" && c.Schema != "v10" && c.Schema != "v11" { - level.Warn(s.logger).Log("msg", "skipping unsupported schema version", "version", c.Schema, "schemaFrom", c.From.String()) - continue - } - - if c.IndexTables.Period%(24*time.Hour) != 0 { - level.Warn(s.logger).Log("msg", "skipping invalid index table period", "period", c.IndexTables.Period, "schemaFrom", c.From.String()) - continue - } - - var reader chunk.IndexReader - switch c.IndexType { - case "gcp", "gcp-columnkey", "bigtable", "bigtable-hashed": - bigTable := s.storageCfg.GCPStorageConfig - - if bigTable.Project == "" || bigTable.Instance == "" { - level.Error(s.logger).Log("msg", "cannot scan BigTable, missing configuration", "schemaFrom", c.From.String()) - continue - } - - reader = newBigtableIndexReader(bigTable.Project, bigTable.Instance, s.logger, s.indexReaderRowsRead, s.indexReaderParsedIndexEntries, s.currentTableRanges, s.currentTableScannedRanges) - case "aws-dynamo": - cfg := s.storageCfg.AWSStorageConfig - - if cfg.DynamoDB.URL == nil { - level.Error(s.logger).Log("msg", "cannot scan DynamoDB, missing configuration", "schemaFrom", c.From.String()) - continue - } - - var err error - reader, err = aws.NewDynamoDBIndexReader(cfg.DynamoDBConfig, s.schema, s.reg, s.logger, s.indexReaderRowsRead) - if err != nil { - level.Error(s.logger).Log("msg", "cannot scan DynamoDB", "err", err) - } - case "cassandra": - cass := s.storageCfg.CassandraStorageConfig - - reader = newCassandraIndexReader(cass, s.schema, s.logger, s.indexReaderRowsRead, s.indexReaderParsedIndexEntries, s.currentTableRanges, s.currentTableScannedRanges) - default: - level.Warn(s.logger).Log("msg", "unsupported index type", "type", c.IndexType, "schemaFrom", c.From.String()) - continue - } - - toTimestamp := time.Now().Add(24 * time.Hour).Truncate(24 * time.Hour).Unix() - if ix < len(s.schema.Configs)-1 { - toTimestamp = s.schema.Configs[ix+1].From.Unix() - } - - level.Info(s.logger).Log("msg", "scanning for schema tables", "schemaFrom", c.From.String(), "prefix", c.IndexTables.Prefix, "period", c.IndexTables.Period) - tables, err := s.findTablesToProcess(ctx, reader, c.From.Unix(), toTimestamp, c.IndexTables) - if err != nil { - return errors.Wrapf(err, "finding tables for schema %s", c.From.String()) - } - - level.Info(s.logger).Log("msg", "found tables", "count", len(tables)) - allTables = append(allTables, tables...) - } - - level.Info(s.logger).Log("msg", "total found tables", "count", len(allTables)) - - if s.cfg.TableNames != "" { - // Find tables from parameter. - tableNames := map[string]bool{} - for _, t := range strings.Split(s.cfg.TableNames, ",") { - tableNames[strings.TrimSpace(t)] = true - } - - for ix := 0; ix < len(allTables); { - t := allTables[ix] - if !tableNames[t.table] { - // remove table. - allTables = append(allTables[:ix], allTables[ix+1:]...) - continue - } - ix++ - } - - level.Error(s.logger).Log("msg", "applied tables filter", "selected", len(allTables)) - } - - // Recent tables go first. - sort.Slice(allTables, func(i, j int) bool { - return allTables[i].start.After(allTables[j].start) - }) - - for ix := 0; ix < len(allTables); { - t := allTables[ix] - if s.cfg.PeriodStart.IsSet() && !t.end.IsZero() && t.end.Unix() <= s.cfg.PeriodStart.Unix() { - level.Info(s.logger).Log("msg", "table ends before period-start, ignoring", "table", t.table, "table_start", t.start.String(), "table_end", t.end.String(), "period_start", s.cfg.PeriodStart.String()) - allTables = append(allTables[:ix], allTables[ix+1:]...) - continue - } - if s.cfg.PeriodEnd.IsSet() && t.start.Unix() >= s.cfg.PeriodEnd.Unix() { - level.Info(s.logger).Log("msg", "table starts after period-end, ignoring", "table", t.table, "table_start", t.start.String(), "table_end", t.end.String(), "period_end", s.cfg.PeriodEnd.String()) - allTables = append(allTables[:ix], allTables[ix+1:]...) - continue - } - ix++ - } - - if s.cfg.TablesLimit > 0 && len(allTables) > s.cfg.TablesLimit { - level.Info(s.logger).Log("msg", "applied tables limit", "limit", s.cfg.TablesLimit) - allTables = allTables[:s.cfg.TablesLimit] - } - - s.foundTables.Add(float64(len(allTables))) - - for _, t := range allTables { - if err := s.processTable(ctx, t.table, t.reader); err != nil { - return errors.Wrapf(err, "failed to process table %s", t.table) - } - s.processedTables.Inc() - } - - // All good, just wait until context is done, to avoid restarts. - level.Info(s.logger).Log("msg", "finished") - <-ctx.Done() - return nil -} - -type tableToProcess struct { - table string - reader chunk.IndexReader - start time.Time - end time.Time // Will not be set for non-periodic tables. Exclusive. -} - -func (s *Scanner) findTablesToProcess(ctx context.Context, indexReader chunk.IndexReader, fromUnixTimestamp, toUnixTimestamp int64, tablesConfig chunk.PeriodicTableConfig) ([]tableToProcess, error) { - tables, err := indexReader.IndexTableNames(ctx) - if err != nil { - return nil, err - } - - var result []tableToProcess - - for _, t := range tables { - if !strings.HasPrefix(t, tablesConfig.Prefix) { - continue - } - - var tp tableToProcess - if tablesConfig.Period == 0 { - tp = tableToProcess{ - table: t, - reader: indexReader, - start: time.Unix(fromUnixTimestamp, 0), - } - } else { - p, err := strconv.ParseInt(t[len(tablesConfig.Prefix):], 10, 64) - if err != nil { - level.Warn(s.logger).Log("msg", "failed to parse period index of table", "table", t) - continue - } - - start := time.Unix(p*int64(tablesConfig.Period/time.Second), 0) - tp = tableToProcess{ - table: t, - reader: indexReader, - start: start, - end: start.Add(tablesConfig.Period), - } - } - - if fromUnixTimestamp <= tp.start.Unix() && tp.start.Unix() < toUnixTimestamp { - result = append(result, tp) - } - } - - return result, nil -} - -func (s *Scanner) processTable(ctx context.Context, table string, indexReader chunk.IndexReader) error { - tableLog := log.With(s.logger, "table", table) - - tableProcessedFile := filepath.Join(s.cfg.OutputDirectory, table+".processed") - - if shouldSkipOperationBecauseFileExists(tableProcessedFile) { - level.Info(tableLog).Log("msg", "skipping table because it was already scanned") - return nil - } - - dir := filepath.Join(s.cfg.OutputDirectory, table) - level.Info(tableLog).Log("msg", "scanning table", "output", dir) - - ignoredUsers, err := scanSingleTable(ctx, indexReader, table, dir, s.cfg.Concurrency, s.allowedUsers, s.ignoredUsers, s.openFiles, s.series, s.indexEntries, s.ignoredEntries) - if err != nil { - return errors.Wrapf(err, "failed to scan table %s and generate plan files", table) - } - - tableLog.Log("msg", "ignored users", "count", len(ignoredUsers), "users", strings.Join(ignoredUsers, ",")) - - if s.cfg.VerifyPlans { - err = verifyPlanFiles(ctx, dir, tableLog) - if err != nil { - return errors.Wrap(err, "failed to verify plans") - } - } - - if s.bucket != nil { - level.Info(tableLog).Log("msg", "uploading generated plan files for table", "source", dir) - - err := uploadPlansConcurrently(ctx, tableLog, dir, s.bucket, s.bucketPrefix, s.cfg.Concurrency) - if err != nil { - return errors.Wrapf(err, "failed to upload plan files for table %s to bucket", table) - } - - level.Info(tableLog).Log("msg", "uploaded generated files for table") - if !s.cfg.KeepFiles { - if err := os.RemoveAll(dir); err != nil { - return errors.Wrapf(err, "failed to delete uploaded plan files for table %s", table) - } - } - } - - err = ioutil.WriteFile(tableProcessedFile, []byte("Finished on "+time.Now().String()+"\n"), 0600) - if err != nil { - return errors.Wrapf(err, "failed to create file %s", tableProcessedFile) - } - - level.Info(tableLog).Log("msg", "done processing table") - return nil -} - -func uploadPlansConcurrently(ctx context.Context, log log.Logger, dir string, bucket objstore.Bucket, bucketPrefix string, concurrency int) error { - df, err := os.Stat(dir) - if err != nil { - return errors.Wrap(err, "stat dir") - } - if !df.IsDir() { - return errors.Errorf("%s is not a directory", dir) - } - - // Path relative to dir, and only use Slash as separator. BucketPrefix is prepended to it when uploading. - paths := make(chan string) - - g, ctx := errgroup.WithContext(ctx) - for i := 0; i < concurrency; i++ { - g.Go(func() error { - for p := range paths { - src := filepath.Join(dir, filepath.FromSlash(p)) - dst := path.Join(bucketPrefix, p) - - boff := backoff.New(ctx, backoff.Config{ - MinBackoff: 1 * time.Second, - MaxBackoff: 5 * time.Second, - MaxRetries: 5, - }) - - for boff.Ongoing() { - err := objstore.UploadFile(ctx, log, bucket, src, dst) - - if err == nil { - break - } - - level.Warn(log).Log("msg", "failed to upload block", "err", err) - boff.Wait() - } - - if boff.Err() != nil { - return boff.Err() - } - } - return nil - }) - } - - g.Go(func() error { - defer close(paths) - - return filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error { - if err != nil { - return err - } - if ctx.Err() != nil { - return ctx.Err() - } - - if fi.IsDir() { - return nil - } - - relPath, err := filepath.Rel(dir, path) - if err != nil { - return err - } - - relPath = filepath.ToSlash(relPath) - - select { - case paths <- relPath: - return nil - case <-ctx.Done(): - return ctx.Err() - } - }) - }) - - return g.Wait() -} - -func shouldSkipOperationBecauseFileExists(file string) bool { - // If file exists, we should skip the operation. - _, err := os.Stat(file) - // Any error (including ErrNotExists) indicates operation should continue. - return err == nil -} - -func scanSingleTable( - ctx context.Context, - indexReader chunk.IndexReader, - tableName string, - outDir string, - concurrency int, - allowed blocksconvert.AllowedUsers, - ignored *regexp.Regexp, - openFiles prometheus.Gauge, - series prometheus.Counter, - indexEntries *prometheus.CounterVec, - ignoredEntries prometheus.Counter, -) ([]string, error) { - err := os.RemoveAll(outDir) - if err != nil { - return nil, errors.Wrapf(err, "failed to delete directory %s", outDir) - } - - err = os.MkdirAll(outDir, os.FileMode(0700)) - if err != nil { - return nil, errors.Wrapf(err, "failed to prepare directory %s", outDir) - } - - files := newOpenFiles(openFiles) - result := func(dir string, file string, entry blocksconvert.PlanEntry, header func() blocksconvert.PlanEntry) error { - return files.appendJSONEntryToFile(dir, file, entry, func() interface{} { - return header() - }) - } - - var ps []chunk.IndexEntryProcessor - - for i := 0; i < concurrency; i++ { - ps = append(ps, newProcessor(outDir, result, allowed, ignored, series, indexEntries, ignoredEntries)) - } - - err = indexReader.ReadIndexEntries(ctx, tableName, ps) - if err != nil { - return nil, err - } - - ignoredUsersMap := map[string]struct{}{} - for _, p := range ps { - for u := range p.(*processor).ignoredUsers { - ignoredUsersMap[u] = struct{}{} - } - } - - var ignoredUsers []string - for u := range ignoredUsersMap { - ignoredUsers = append(ignoredUsers, u) - } - - err = files.closeAllFiles(func() interface{} { - return blocksconvert.PlanEntry{Complete: true} - }) - return ignoredUsers, errors.Wrap(err, "closing files") -} - -func verifyPlanFiles(ctx context.Context, dir string, logger log.Logger) error { - return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if err := ctx.Err(); err != nil { - return err - } - - if info.IsDir() { - return nil - } - - ok, _ := blocksconvert.IsPlanFilename(info.Name()) - if !ok { - return nil - } - - r, err := os.Open(path) - if err != nil { - return errors.Wrapf(err, "failed to open %s", path) - } - defer func() { - _ = r.Close() - }() - - pr, err := blocksconvert.PreparePlanFileReader(info.Name(), r) - if err != nil { - return errors.Wrapf(err, "failed to prepare plan file for reading: %s", path) - } - - level.Info(logger).Log("msg", "verifying plan", "path", path) - return errors.Wrapf(verifyPlanFile(pr), "plan file: %s", path) - }) -} - -func verifyPlanFile(r io.Reader) error { - dec := json.NewDecoder(r) - - entry := blocksconvert.PlanEntry{} - if err := dec.Decode(&entry); err != nil { - return errors.Wrap(err, "failed to parse plan file header") - } - if entry.User == "" || entry.DayIndex == 0 { - return errors.New("failed to read plan file header: no user or day index found") - } - - series := map[string]struct{}{} - - var err error - footerFound := false - for err = dec.Decode(&entry); err == nil; err = dec.Decode(&entry) { - if entry.Complete { - footerFound = true - entry.Reset() - continue - } - - if footerFound { - return errors.New("plan entries found after plan footer") - } - - if entry.SeriesID == "" { - return errors.Errorf("plan contains entry without seriesID") - } - - if len(entry.Chunks) == 0 { - return errors.Errorf("entry for seriesID %s has no chunks", entry.SeriesID) - } - - if _, found := series[entry.SeriesID]; found { - return errors.Errorf("multiple entries for series %s found in plan", entry.SeriesID) - } - series[entry.SeriesID] = struct{}{} - - entry.Reset() - } - - if err == io.EOF { - if !footerFound { - return errors.New("no footer found in the plan") - } - err = nil - } - return err -} diff --git a/tools/blocksconvert/scanner/scanner_processor.go b/tools/blocksconvert/scanner/scanner_processor.go deleted file mode 100644 index fda99e1050..0000000000 --- a/tools/blocksconvert/scanner/scanner_processor.go +++ /dev/null @@ -1,149 +0,0 @@ -package scanner - -import ( - "path/filepath" - "regexp" - "strconv" - - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - - "github.com/cortexproject/cortex/pkg/chunk" - "github.com/cortexproject/cortex/tools/blocksconvert" -) - -// Results from processor are passed to this function. -type planEntryFn func(dir string, file string, entry blocksconvert.PlanEntry, header func() blocksconvert.PlanEntry) error - -// Processor implements IndexEntryProcessor. It caches chunks for single series until it finds -// that another series has arrived, at which point it writes it to the file. -// IndexReader guarantees correct order of entries. -type processor struct { - dir string - resultFn planEntryFn - - series prometheus.Counter - scanned *prometheus.CounterVec - - allowedUsers blocksconvert.AllowedUsers - ignoredUsersRegex *regexp.Regexp - ignoredUsers map[string]struct{} - ignoredEntries prometheus.Counter - - lastKey key - chunks []string -} - -// Key is full series ID, used by processor to find out whether subsequent index entries belong to the same series -// or not. -type key struct { - user string - dayIndex int - seriesID string -} - -func newProcessor(dir string, resultFn planEntryFn, allowed blocksconvert.AllowedUsers, ignoredUsers *regexp.Regexp, series prometheus.Counter, scannedEntries *prometheus.CounterVec, ignoredEntries prometheus.Counter) *processor { - w := &processor{ - dir: dir, - resultFn: resultFn, - series: series, - scanned: scannedEntries, - - allowedUsers: allowed, - ignoredUsersRegex: ignoredUsers, - ignoredUsers: map[string]struct{}{}, - ignoredEntries: ignoredEntries, - } - - return w -} - -func (w *processor) ProcessIndexEntry(indexEntry chunk.IndexEntry) error { - switch { - case IsMetricToSeriesMapping(indexEntry.RangeValue): - w.scanned.WithLabelValues("metric-to-series").Inc() - return nil - - case IsMetricLabelToLabelValueMapping(indexEntry.RangeValue): - w.scanned.WithLabelValues("metric-label-to-label-value").Inc() - return nil - - case IsSeriesToLabelValues(indexEntry.RangeValue): - w.scanned.WithLabelValues("series-to-label-values").Inc() - return nil - - case IsSeriesToChunkMapping(indexEntry.RangeValue): - w.scanned.WithLabelValues("series-to-chunk").Inc() - // We will process these, don't return yet. - - default: - // Should not happen. - w.scanned.WithLabelValues("unknown-" + UnknownIndexEntryType(indexEntry.RangeValue)).Inc() - return nil - } - - user, index, seriesID, chunkID, err := GetSeriesToChunkMapping(indexEntry.HashValue, indexEntry.RangeValue) - if err != nil { - return err - } - - if !w.AcceptUser(user) { - return nil - } - - k := key{ - user: user, - dayIndex: index, - seriesID: seriesID, - } - - if w.lastKey != k && len(w.chunks) > 0 { - err := w.Flush() - if err != nil { - return errors.Wrap(err, "failed to flush chunks") - } - } - - w.lastKey = k - w.chunks = append(w.chunks, chunkID) - return nil -} - -func (w *processor) AcceptUser(user string) bool { - if _, found := w.ignoredUsers[user]; found { - w.ignoredEntries.Inc() - return false - } - if !w.allowedUsers.IsAllowed(user) || (w.ignoredUsersRegex != nil && w.ignoredUsersRegex.MatchString(user)) { - w.ignoredEntries.Inc() - w.ignoredUsers[user] = struct{}{} - return false - } - return true -} - -func (w *processor) Flush() error { - if len(w.chunks) == 0 { - return nil - } - - k := w.lastKey - - err := w.resultFn(filepath.Join(w.dir, k.user), strconv.Itoa(k.dayIndex)+".plan", blocksconvert.PlanEntry{ - SeriesID: w.lastKey.seriesID, - Chunks: w.chunks, - }, func() blocksconvert.PlanEntry { - return blocksconvert.PlanEntry{ - User: k.user, - DayIndex: k.dayIndex, - } - }) - - if err != nil { - return err - } - - w.series.Inc() - w.chunks = nil - return nil -} diff --git a/tools/blocksconvert/scanner/scanner_processor_test.go b/tools/blocksconvert/scanner/scanner_processor_test.go deleted file mode 100644 index 8bade0c7e1..0000000000 --- a/tools/blocksconvert/scanner/scanner_processor_test.go +++ /dev/null @@ -1,118 +0,0 @@ -package scanner - -import ( - "fmt" - "path" - "testing" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - "github.com/stretchr/testify/require" - - "github.com/cortexproject/cortex/pkg/chunk" - "github.com/cortexproject/cortex/tools/blocksconvert" -) - -func TestProcessorError(t *testing.T) { - entries := map[string][]blocksconvert.PlanEntry{} - resultFn := func(dir string, file string, entry blocksconvert.PlanEntry, header func() blocksconvert.PlanEntry) error { - full := path.Join(dir, file) - - entries[full] = append(entries[full], entry) - return nil - } - - p := newProcessor("output", resultFn, nil, nil, prometheus.NewCounter(prometheus.CounterOpts{}), prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"type"}), prometheus.NewCounter(prometheus.CounterOpts{})) - - // Day 18500 - now := time.Unix(1598456178, 0) - startOfDay := model.TimeFromUnixNano(now.Truncate(24 * time.Hour).UnixNano()) - - pc := chunk.PeriodConfig{ - From: chunk.DayTime{Time: startOfDay}, - IndexType: "bigtable", - ObjectType: "gcs", - Schema: "v9", - IndexTables: chunk.PeriodicTableConfig{ - Prefix: "index_", - Period: 7 * 24 * time.Hour, - }, - ChunkTables: chunk.PeriodicTableConfig{ - Prefix: "chunks_", - Period: 7 * 24 * time.Hour, - }, - } - schema, err := pc.CreateSchema() - require.NoError(t, err) - - sschema := schema.(chunk.SeriesStoreSchema) - - // Label write entries are ignored by the processor - { - _, ies, err := sschema.GetCacheKeysAndLabelWriteEntries(startOfDay.Add(1*time.Hour), startOfDay.Add(2*time.Hour), "testUser", "test_metric", labels.Labels{ - {Name: "__name__", Value: "test_metric"}, - }, "chunkID") - require.NoError(t, err) - for _, es := range ies { - passEntriesToProcessor(t, p, es) - } - } - - // Processor expects all entries for same series to arrive in sequence, before receiving different series. BigTable reader guarantees that. - { - es, err := sschema.GetChunkWriteEntries(startOfDay.Add(2*time.Hour), startOfDay.Add(3*time.Hour), "testUser", "test_metric", labels.Labels{ - {Name: "__name__", Value: "test_metric"}, - }, "chunkID_1") - require.NoError(t, err) - passEntriesToProcessor(t, p, es) - } - - { - es, err := sschema.GetChunkWriteEntries(startOfDay.Add(23*time.Hour), startOfDay.Add(25*time.Hour), "testUser", "test_metric", labels.Labels{ - {Name: "__name__", Value: "test_metric"}, - }, "chunkID_2") - require.NoError(t, err) - passEntriesToProcessor(t, p, es) - } - - { - es, err := sschema.GetChunkWriteEntries(startOfDay.Add(5*time.Hour), startOfDay.Add(6*time.Hour), "testUser", "different_metric", labels.Labels{ - {Name: "__name__", Value: "different_metric"}, - }, "chunkID_5") - require.NoError(t, err) - passEntriesToProcessor(t, p, es) - } - - require.NoError(t, p.Flush()) - - // Now let's compare what we have received. - require.Equal(t, map[string][]blocksconvert.PlanEntry{ - "output/testUser/18500.plan": { - { - SeriesID: "eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M", - Chunks: []string{"chunkID_1", "chunkID_2"}, - }, - { - SeriesID: "+afMmul/w5PDAAWGPd7y8+xyBq5IN+Q2/ZnPnrMEI+k", - Chunks: []string{"chunkID_5"}, - }, - }, - - "output/testUser/18501.plan": { - { - SeriesID: "eg856WuFz2TNSApvcW7LrhiPKgkuU6KfI3nJPwLoA0M", - Chunks: []string{"chunkID_2"}, - }, - }, - }, entries) -} - -func passEntriesToProcessor(t *testing.T, p *processor, es []chunk.IndexEntry) { - for _, ie := range es { - fmt.Printf("%q %q %q\n", ie.HashValue, ie.RangeValue, ie.Value) - - require.NoError(t, p.ProcessIndexEntry(ie)) - } -} diff --git a/tools/blocksconvert/scanner/scanner_test.go b/tools/blocksconvert/scanner/scanner_test.go deleted file mode 100644 index 5d01ce7cb0..0000000000 --- a/tools/blocksconvert/scanner/scanner_test.go +++ /dev/null @@ -1,115 +0,0 @@ -package scanner - -import ( - "context" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/go-kit/log" - "github.com/prometheus/client_golang/prometheus" - "github.com/stretchr/testify/require" - "github.com/thanos-io/thanos/pkg/objstore" - - util_log "github.com/cortexproject/cortex/pkg/util/log" - "github.com/cortexproject/cortex/tools/blocksconvert" -) - -func TestVerifyPlanFile(t *testing.T) { - testCases := map[string]struct { - content string - errorMsg string - }{ - "Minimum valid plan file, with no series": { - content: `{"user": "test", "day_index": 12345}{"complete": true}`, - errorMsg: "", - }, - "no header": { - content: `{"complete": true}`, - errorMsg: "failed to read plan file header: no user or day index found", - }, - "no footer": { - content: `{"user": "test", "day_index": 12345}`, - errorMsg: "no footer found in the plan", - }, - "data after footer": { - content: `{"user": "test", "day_index": 12345}{"complete": true}{"sid": "some seriesID", "cs": ["chunk1", "chunk2"]}`, - errorMsg: "plan entries found after plan footer", - }, - "valid plan with single series": { - content: ` - {"user": "test", "day_index": 12345} - {"sid": "some seriesID", "cs": ["chunk1", "chunk2"]} - {"complete": true}`, - errorMsg: "", - }, - "series with no chunks": { - content: ` - {"user": "test", "day_index": 12345} - {"sid": "AAAAAA"} - {"complete": true}`, - errorMsg: fmt.Sprintf("entry for seriesID %s has no chunks", "AAAAAA"), - }, - "multiple series entries": { - content: ` - {"user": "test", "day_index": 12345} - {"sid": "AAA", "cs": ["chunk1", "chunk2"]} - {"sid": "AAA", "cs": ["chunk3", "chunk4"]} - {"complete": true}`, - errorMsg: "multiple entries for series AAA found in plan", - }, - } - - for name, tc := range testCases { - if tc.errorMsg == "" { - require.NoError(t, verifyPlanFile(strings.NewReader(tc.content)), name) - } else { - require.EqualError(t, verifyPlanFile(strings.NewReader(tc.content)), tc.errorMsg, name) - } - } -} - -func TestVerifyPlansDir(t *testing.T) { - dir := t.TempDir() - - of := newOpenFiles(prometheus.NewGauge(prometheus.GaugeOpts{})) - // This file is checked first, and no error is reported for it. - require.NoError(t, of.appendJSONEntryToFile(filepath.Join(dir, "user1"), "123.plan", blocksconvert.PlanEntry{User: "user1", DayIndex: 123}, nil)) - require.NoError(t, of.appendJSONEntryToFile(filepath.Join(dir, "user1"), "123.plan", blocksconvert.PlanEntry{SeriesID: "s1", Chunks: []string{"c1, c2"}}, nil)) - require.NoError(t, of.appendJSONEntryToFile(filepath.Join(dir, "user1"), "123.plan", blocksconvert.PlanEntry{Complete: true}, nil)) - - require.NoError(t, of.appendJSONEntryToFile(filepath.Join(dir, "user2"), "456.plan", blocksconvert.PlanEntry{User: "user2", DayIndex: 456}, nil)) - require.NoError(t, of.appendJSONEntryToFile(filepath.Join(dir, "user2"), "456.plan", blocksconvert.PlanEntry{SeriesID: "s1", Chunks: []string{"c1, c2"}}, nil)) - require.NoError(t, of.appendJSONEntryToFile(filepath.Join(dir, "user2"), "456.plan", blocksconvert.PlanEntry{SeriesID: "s1", Chunks: []string{"c3, c4"}}, nil)) - - require.NoError(t, of.closeAllFiles(nil)) - - err := verifyPlanFiles(context.Background(), dir, util_log.Logger) - require.Error(t, err) - require.True(t, strings.Contains(err.Error(), "456.plan")) - require.True(t, strings.Contains(err.Error(), "multiple entries for series s1 found in plan")) -} - -func TestUploadPlans(t *testing.T) { - dir := t.TempDir() - - require.NoError(t, os.MkdirAll(filepath.Join(dir, "user1"), 0700)) - require.NoError(t, os.MkdirAll(filepath.Join(dir, "user2"), 0700)) - require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "user1", "plan1"), []byte("plan1"), 0600)) - require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "user1", "plan2"), []byte("plan2"), 0600)) - require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "user2", "plan3"), []byte("plan3"), 0600)) - - inmem := objstore.NewInMemBucket() - - require.NoError(t, uploadPlansConcurrently(context.Background(), log.NewNopLogger(), dir, inmem, "bucket-prefix", 5)) - - objs := inmem.Objects() - require.Equal(t, objs, map[string][]byte{ - "bucket-prefix/user1/plan1": []byte("plan1"), - "bucket-prefix/user1/plan2": []byte("plan2"), - "bucket-prefix/user2/plan3": []byte("plan3"), - }) -} diff --git a/tools/blocksconvert/scheduler.pb.go b/tools/blocksconvert/scheduler.pb.go deleted file mode 100644 index b4b1d4e7aa..0000000000 --- a/tools/blocksconvert/scheduler.pb.go +++ /dev/null @@ -1,772 +0,0 @@ -// Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: scheduler.proto - -package blocksconvert - -import ( - context "context" - fmt "fmt" - _ "github.com/gogo/protobuf/gogoproto" - proto "github.com/gogo/protobuf/proto" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" - io "io" - math "math" - math_bits "math/bits" - reflect "reflect" - strings "strings" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package - -type NextPlanRequest struct { - // Name of service requesting the plan. - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` -} - -func (m *NextPlanRequest) Reset() { *m = NextPlanRequest{} } -func (*NextPlanRequest) ProtoMessage() {} -func (*NextPlanRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_2b3fc28395a6d9c5, []int{0} -} -func (m *NextPlanRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *NextPlanRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_NextPlanRequest.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *NextPlanRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_NextPlanRequest.Merge(m, src) -} -func (m *NextPlanRequest) XXX_Size() int { - return m.Size() -} -func (m *NextPlanRequest) XXX_DiscardUnknown() { - xxx_messageInfo_NextPlanRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_NextPlanRequest proto.InternalMessageInfo - -func (m *NextPlanRequest) GetName() string { - if m != nil { - return m.Name - } - return "" -} - -type NextPlanResponse struct { - PlanFile string `protobuf:"bytes,1,opt,name=planFile,proto3" json:"planFile,omitempty"` - ProgressFile string `protobuf:"bytes,2,opt,name=progressFile,proto3" json:"progressFile,omitempty"` -} - -func (m *NextPlanResponse) Reset() { *m = NextPlanResponse{} } -func (*NextPlanResponse) ProtoMessage() {} -func (*NextPlanResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_2b3fc28395a6d9c5, []int{1} -} -func (m *NextPlanResponse) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *NextPlanResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_NextPlanResponse.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *NextPlanResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_NextPlanResponse.Merge(m, src) -} -func (m *NextPlanResponse) XXX_Size() int { - return m.Size() -} -func (m *NextPlanResponse) XXX_DiscardUnknown() { - xxx_messageInfo_NextPlanResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_NextPlanResponse proto.InternalMessageInfo - -func (m *NextPlanResponse) GetPlanFile() string { - if m != nil { - return m.PlanFile - } - return "" -} - -func (m *NextPlanResponse) GetProgressFile() string { - if m != nil { - return m.ProgressFile - } - return "" -} - -func init() { - proto.RegisterType((*NextPlanRequest)(nil), "blocksconvert.NextPlanRequest") - proto.RegisterType((*NextPlanResponse)(nil), "blocksconvert.NextPlanResponse") -} - -func init() { proto.RegisterFile("scheduler.proto", fileDescriptor_2b3fc28395a6d9c5) } - -var fileDescriptor_2b3fc28395a6d9c5 = []byte{ - // 264 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x2f, 0x4e, 0xce, 0x48, - 0x4d, 0x29, 0xcd, 0x49, 0x2d, 0xd2, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x4d, 0xca, 0xc9, - 0x4f, 0xce, 0x2e, 0x4e, 0xce, 0xcf, 0x2b, 0x4b, 0x2d, 0x2a, 0x91, 0xd2, 0x4d, 0xcf, 0x2c, 0xc9, - 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x4f, 0xcf, 0x4f, 0xcf, 0xd7, 0x07, 0xab, 0x4a, 0x2a, - 0x4d, 0x03, 0xf3, 0xc0, 0x1c, 0x30, 0x0b, 0xa2, 0x5b, 0x49, 0x95, 0x8b, 0xdf, 0x2f, 0xb5, 0xa2, - 0x24, 0x20, 0x27, 0x31, 0x2f, 0x28, 0xb5, 0xb0, 0x34, 0xb5, 0xb8, 0x44, 0x48, 0x88, 0x8b, 0x25, - 0x2f, 0x31, 0x37, 0x55, 0x82, 0x51, 0x81, 0x51, 0x83, 0x33, 0x08, 0xcc, 0x56, 0x0a, 0xe2, 0x12, - 0x40, 0x28, 0x2b, 0x2e, 0xc8, 0xcf, 0x2b, 0x4e, 0x15, 0x92, 0xe2, 0xe2, 0x28, 0xc8, 0x49, 0xcc, - 0x73, 0xcb, 0xcc, 0x81, 0xa9, 0x85, 0xf3, 0x85, 0x94, 0xb8, 0x78, 0x0a, 0x8a, 0xf2, 0xd3, 0x8b, - 0x52, 0x8b, 0x8b, 0xc1, 0xf2, 0x4c, 0x60, 0x79, 0x14, 0x31, 0xa3, 0x28, 0x2e, 0xce, 0x60, 0x98, - 0x5f, 0x84, 0x7c, 0xb9, 0x38, 0x60, 0x16, 0x08, 0xc9, 0xe9, 0xa1, 0x78, 0x49, 0x0f, 0xcd, 0x81, - 0x52, 0xf2, 0x38, 0xe5, 0x21, 0x2e, 0x53, 0x62, 0x70, 0x72, 0xbe, 0xf0, 0x50, 0x8e, 0xe1, 0xc6, - 0x43, 0x39, 0x86, 0x0f, 0x0f, 0xe5, 0x18, 0x1b, 0x1e, 0xc9, 0x31, 0xae, 0x78, 0x24, 0xc7, 0x78, - 0xe2, 0x91, 0x1c, 0xe3, 0x85, 0x47, 0x72, 0x8c, 0x0f, 0x1e, 0xc9, 0x31, 0xbe, 0x78, 0x24, 0xc7, - 0xf0, 0xe1, 0x91, 0x1c, 0xe3, 0x84, 0xc7, 0x72, 0x0c, 0x17, 0x1e, 0xcb, 0x31, 0xdc, 0x78, 0x2c, - 0xc7, 0x10, 0x85, 0x1a, 0x94, 0x49, 0x6c, 0xe0, 0x20, 0x32, 0x06, 0x04, 0x00, 0x00, 0xff, 0xff, - 0xf7, 0x7b, 0xa4, 0x64, 0x73, 0x01, 0x00, 0x00, -} - -func (this *NextPlanRequest) Equal(that interface{}) bool { - if that == nil { - return this == nil - } - - that1, ok := that.(*NextPlanRequest) - if !ok { - that2, ok := that.(NextPlanRequest) - if ok { - that1 = &that2 - } else { - return false - } - } - if that1 == nil { - return this == nil - } else if this == nil { - return false - } - if this.Name != that1.Name { - return false - } - return true -} -func (this *NextPlanResponse) Equal(that interface{}) bool { - if that == nil { - return this == nil - } - - that1, ok := that.(*NextPlanResponse) - if !ok { - that2, ok := that.(NextPlanResponse) - if ok { - that1 = &that2 - } else { - return false - } - } - if that1 == nil { - return this == nil - } else if this == nil { - return false - } - if this.PlanFile != that1.PlanFile { - return false - } - if this.ProgressFile != that1.ProgressFile { - return false - } - return true -} -func (this *NextPlanRequest) GoString() string { - if this == nil { - return "nil" - } - s := make([]string, 0, 5) - s = append(s, "&blocksconvert.NextPlanRequest{") - s = append(s, "Name: "+fmt.Sprintf("%#v", this.Name)+",\n") - s = append(s, "}") - return strings.Join(s, "") -} -func (this *NextPlanResponse) GoString() string { - if this == nil { - return "nil" - } - s := make([]string, 0, 6) - s = append(s, "&blocksconvert.NextPlanResponse{") - s = append(s, "PlanFile: "+fmt.Sprintf("%#v", this.PlanFile)+",\n") - s = append(s, "ProgressFile: "+fmt.Sprintf("%#v", this.ProgressFile)+",\n") - s = append(s, "}") - return strings.Join(s, "") -} -func valueToGoStringScheduler(v interface{}, typ string) string { - rv := reflect.ValueOf(v) - if rv.IsNil() { - return "nil" - } - pv := reflect.Indirect(rv).Interface() - return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) -} - -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConn - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion4 - -// SchedulerClient is the client API for Scheduler service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type SchedulerClient interface { - // Returns next plan that builder should work on. - NextPlan(ctx context.Context, in *NextPlanRequest, opts ...grpc.CallOption) (*NextPlanResponse, error) -} - -type schedulerClient struct { - cc *grpc.ClientConn -} - -func NewSchedulerClient(cc *grpc.ClientConn) SchedulerClient { - return &schedulerClient{cc} -} - -func (c *schedulerClient) NextPlan(ctx context.Context, in *NextPlanRequest, opts ...grpc.CallOption) (*NextPlanResponse, error) { - out := new(NextPlanResponse) - err := c.cc.Invoke(ctx, "/blocksconvert.Scheduler/NextPlan", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -// SchedulerServer is the server API for Scheduler service. -type SchedulerServer interface { - // Returns next plan that builder should work on. - NextPlan(context.Context, *NextPlanRequest) (*NextPlanResponse, error) -} - -// UnimplementedSchedulerServer can be embedded to have forward compatible implementations. -type UnimplementedSchedulerServer struct { -} - -func (*UnimplementedSchedulerServer) NextPlan(ctx context.Context, req *NextPlanRequest) (*NextPlanResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method NextPlan not implemented") -} - -func RegisterSchedulerServer(s *grpc.Server, srv SchedulerServer) { - s.RegisterService(&_Scheduler_serviceDesc, srv) -} - -func _Scheduler_NextPlan_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(NextPlanRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SchedulerServer).NextPlan(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/blocksconvert.Scheduler/NextPlan", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SchedulerServer).NextPlan(ctx, req.(*NextPlanRequest)) - } - return interceptor(ctx, in, info, handler) -} - -var _Scheduler_serviceDesc = grpc.ServiceDesc{ - ServiceName: "blocksconvert.Scheduler", - HandlerType: (*SchedulerServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "NextPlan", - Handler: _Scheduler_NextPlan_Handler, - }, - }, - Streams: []grpc.StreamDesc{}, - Metadata: "scheduler.proto", -} - -func (m *NextPlanRequest) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *NextPlanRequest) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *NextPlanRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.Name) > 0 { - i -= len(m.Name) - copy(dAtA[i:], m.Name) - i = encodeVarintScheduler(dAtA, i, uint64(len(m.Name))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - -func (m *NextPlanResponse) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *NextPlanResponse) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *NextPlanResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.ProgressFile) > 0 { - i -= len(m.ProgressFile) - copy(dAtA[i:], m.ProgressFile) - i = encodeVarintScheduler(dAtA, i, uint64(len(m.ProgressFile))) - i-- - dAtA[i] = 0x12 - } - if len(m.PlanFile) > 0 { - i -= len(m.PlanFile) - copy(dAtA[i:], m.PlanFile) - i = encodeVarintScheduler(dAtA, i, uint64(len(m.PlanFile))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - -func encodeVarintScheduler(dAtA []byte, offset int, v uint64) int { - offset -= sovScheduler(v) - base := offset - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ - } - dAtA[offset] = uint8(v) - return base -} -func (m *NextPlanRequest) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - l = len(m.Name) - if l > 0 { - n += 1 + l + sovScheduler(uint64(l)) - } - return n -} - -func (m *NextPlanResponse) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - l = len(m.PlanFile) - if l > 0 { - n += 1 + l + sovScheduler(uint64(l)) - } - l = len(m.ProgressFile) - if l > 0 { - n += 1 + l + sovScheduler(uint64(l)) - } - return n -} - -func sovScheduler(x uint64) (n int) { - return (math_bits.Len64(x|1) + 6) / 7 -} -func sozScheduler(x uint64) (n int) { - return sovScheduler(uint64((x << 1) ^ uint64((int64(x) >> 63)))) -} -func (this *NextPlanRequest) String() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&NextPlanRequest{`, - `Name:` + fmt.Sprintf("%v", this.Name) + `,`, - `}`, - }, "") - return s -} -func (this *NextPlanResponse) String() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&NextPlanResponse{`, - `PlanFile:` + fmt.Sprintf("%v", this.PlanFile) + `,`, - `ProgressFile:` + fmt.Sprintf("%v", this.ProgressFile) + `,`, - `}`, - }, "") - return s -} -func valueToStringScheduler(v interface{}) string { - rv := reflect.ValueOf(v) - if rv.IsNil() { - return "nil" - } - pv := reflect.Indirect(rv).Interface() - return fmt.Sprintf("*%v", pv) -} -func (m *NextPlanRequest) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowScheduler - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: NextPlanRequest: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: NextPlanRequest: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowScheduler - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthScheduler - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthScheduler - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Name = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipScheduler(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthScheduler - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthScheduler - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *NextPlanResponse) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowScheduler - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: NextPlanResponse: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: NextPlanResponse: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field PlanFile", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowScheduler - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthScheduler - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthScheduler - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.PlanFile = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ProgressFile", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowScheduler - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthScheduler - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthScheduler - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.ProgressFile = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipScheduler(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthScheduler - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthScheduler - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func skipScheduler(dAtA []byte) (n int, err error) { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowScheduler - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - wireType := int(wire & 0x7) - switch wireType { - case 0: - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowScheduler - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - iNdEx++ - if dAtA[iNdEx-1] < 0x80 { - break - } - } - return iNdEx, nil - case 1: - iNdEx += 8 - return iNdEx, nil - case 2: - var length int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowScheduler - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - length |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if length < 0 { - return 0, ErrInvalidLengthScheduler - } - iNdEx += length - if iNdEx < 0 { - return 0, ErrInvalidLengthScheduler - } - return iNdEx, nil - case 3: - for { - var innerWire uint64 - var start int = iNdEx - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowScheduler - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - innerWire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - innerWireType := int(innerWire & 0x7) - if innerWireType == 4 { - break - } - next, err := skipScheduler(dAtA[start:]) - if err != nil { - return 0, err - } - iNdEx = start + next - if iNdEx < 0 { - return 0, ErrInvalidLengthScheduler - } - } - return iNdEx, nil - case 4: - return iNdEx, nil - case 5: - iNdEx += 4 - return iNdEx, nil - default: - return 0, fmt.Errorf("proto: illegal wireType %d", wireType) - } - } - panic("unreachable") -} - -var ( - ErrInvalidLengthScheduler = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowScheduler = fmt.Errorf("proto: integer overflow") -) diff --git a/tools/blocksconvert/scheduler.proto b/tools/blocksconvert/scheduler.proto deleted file mode 100644 index 4e02a6f74a..0000000000 --- a/tools/blocksconvert/scheduler.proto +++ /dev/null @@ -1,25 +0,0 @@ -syntax = "proto3"; - -package blocksconvert; - -option go_package = "blocksconvert"; - -import "github.com/gogo/protobuf/gogoproto/gogo.proto"; - -option (gogoproto.marshaler_all) = true; -option (gogoproto.unmarshaler_all) = true; - -service Scheduler { - // Returns next plan that builder should work on. - rpc NextPlan(NextPlanRequest) returns (NextPlanResponse) {}; -} - -message NextPlanRequest { - // Name of service requesting the plan. - string name = 1; -} - -message NextPlanResponse { - string planFile = 1; - string progressFile = 2; -} diff --git a/tools/blocksconvert/scheduler/plan_status.go b/tools/blocksconvert/scheduler/plan_status.go deleted file mode 100644 index 73310db335..0000000000 --- a/tools/blocksconvert/scheduler/plan_status.go +++ /dev/null @@ -1,60 +0,0 @@ -package scheduler - -import ( - "fmt" - "time" -) - -type planStatus int - -const ( - New planStatus = iota - InProgress - Finished - Error - Invalid -) - -func (s planStatus) String() string { - switch s { - case New: - return "New" - case InProgress: - return "InProgress" - case Finished: - return "Finished" - case Error: - return "Error" - case Invalid: - return "Invalid" - default: - panic(fmt.Sprintf("invalid status: %d", s)) - } -} - -type plan struct { - PlanFiles []string - ProgressFiles map[string]time.Time - Finished []string - ErrorFile string -} - -func (ps plan) Status() planStatus { - if len(ps.PlanFiles) != 1 || len(ps.Finished) > 1 || (len(ps.Finished) > 0 && ps.ErrorFile != "") { - return Invalid - } - - if len(ps.Finished) > 0 { - return Finished - } - - if ps.ErrorFile != "" { - return Error - } - - if len(ps.ProgressFiles) > 0 { - return InProgress - } - - return New -} diff --git a/tools/blocksconvert/scheduler/scheduler.go b/tools/blocksconvert/scheduler/scheduler.go deleted file mode 100644 index 392ec40092..0000000000 --- a/tools/blocksconvert/scheduler/scheduler.go +++ /dev/null @@ -1,508 +0,0 @@ -package scheduler - -import ( - "context" - "flag" - "html/template" - "net/http" - "path" - "regexp" - "sort" - "strconv" - "strings" - "sync" - "time" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/gorilla/mux" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/thanos-io/thanos/pkg/objstore" - "google.golang.org/grpc" - - "github.com/cortexproject/cortex/pkg/util" - "github.com/cortexproject/cortex/pkg/util/services" - "github.com/cortexproject/cortex/tools/blocksconvert" -) - -type Config struct { - ScanInterval time.Duration - PlanScanConcurrency int - MaxProgressFileAge time.Duration - AllowedUsers string - IgnoredUserPattern string -} - -func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - f.DurationVar(&cfg.ScanInterval, "scheduler.scan-interval", 5*time.Minute, "How often to scan for plans and their status.") - f.IntVar(&cfg.PlanScanConcurrency, "scheduler.plan-scan-concurrency", 5, "Limit of concurrent plan scans.") - f.DurationVar(&cfg.MaxProgressFileAge, "scheduler.max-progress-file-age", 30*time.Minute, "Progress files older than this duration are deleted.") - f.StringVar(&cfg.AllowedUsers, "scheduler.allowed-users", "", "Allowed users that can be converted, comma-separated") - f.StringVar(&cfg.IgnoredUserPattern, "scheduler.ignore-users-regex", "", "If set and user ID matches this regex pattern, it will be ignored. Checked after applying -scheduler.allowed-users, if set.") -} - -func NewScheduler(cfg Config, scfg blocksconvert.SharedConfig, l log.Logger, reg prometheus.Registerer, http *mux.Router, grpcServ *grpc.Server) (*Scheduler, error) { - b, err := scfg.GetBucket(l, reg) - if err != nil { - return nil, errors.Wrap(err, "create bucket") - } - - var users = blocksconvert.AllowAllUsers - if cfg.AllowedUsers != "" { - users = blocksconvert.ParseAllowedUsers(cfg.AllowedUsers) - } - - var ignoredUserRegex *regexp.Regexp = nil - if cfg.IgnoredUserPattern != "" { - re, err := regexp.Compile(cfg.IgnoredUserPattern) - if err != nil { - return nil, errors.Wrap(err, "failed to compile ignored user regex") - } - ignoredUserRegex = re - } - - s := newSchedulerWithBucket(l, b, scfg.BucketPrefix, users, ignoredUserRegex, cfg, reg) - blocksconvert.RegisterSchedulerServer(grpcServ, s) - http.HandleFunc("/plans", s.httpPlans) - return s, nil -} - -func newSchedulerWithBucket(l log.Logger, b objstore.Bucket, bucketPrefix string, users blocksconvert.AllowedUsers, ignoredUsers *regexp.Regexp, cfg Config, reg prometheus.Registerer) *Scheduler { - s := &Scheduler{ - log: l, - cfg: cfg, - bucket: b, - bucketPrefix: bucketPrefix, - allowedUsers: users, - ignoredUsers: ignoredUsers, - - planStatus: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ - Name: "cortex_blocksconvert_scheduler_scanned_plans", - Help: "Number of plans in different status", - }, []string{"status"}), - queuedPlansGauge: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ - Name: "cortex_blocksconvert_scheduler_queued_plans", - Help: "Number of queued plans", - }), - oldestPlanTimestamp: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ - Name: "cortex_blocksconvert_scheduler_oldest_queued_plan_seconds", - Help: "Unix timestamp of oldest plan.", - }), - newestPlanTimestamp: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ - Name: "cortex_blocksconvert_scheduler_newest_queued_plan_seconds", - Help: "Unix timestamp of newest plan", - }), - } - - s.Service = services.NewTimerService(cfg.ScanInterval, s.scanBucketForPlans, s.scanBucketForPlans, nil) - return s -} - -type Scheduler struct { - services.Service - cfg Config - log log.Logger - - allowedUsers blocksconvert.AllowedUsers - ignoredUsers *regexp.Regexp // Can be nil. - - bucket objstore.Bucket - bucketPrefix string - - planStatus *prometheus.GaugeVec - queuedPlansGauge prometheus.Gauge - oldestPlanTimestamp prometheus.Gauge - newestPlanTimestamp prometheus.Gauge - - // Used to avoid scanning while there is dequeuing happening. - dequeueWG sync.WaitGroup - - scanMu sync.Mutex - scanning bool - allUserPlans map[string]map[string]plan - plansQueue []queuedPlan // Queued plans are sorted by day index - more recent (higher day index) days go first. -} - -type queuedPlan struct { - DayIndex int - PlanFile string -} - -func (s *Scheduler) scanBucketForPlans(ctx context.Context) error { - s.scanMu.Lock() - s.scanning = true - s.scanMu.Unlock() - - defer func() { - s.scanMu.Lock() - s.scanning = false - s.scanMu.Unlock() - }() - - // Make sure that no dequeuing is happening when scanning. - // This is to avoid race when dequeing creates progress file, but scan will not find it. - s.dequeueWG.Wait() - - level.Info(s.log).Log("msg", "scanning for users") - - users, err := scanForUsers(ctx, s.bucket, s.bucketPrefix) - if err != nil { - level.Error(s.log).Log("msg", "failed to scan for users", "err", err) - return nil - } - - allUsers := len(users) - users = s.allowedUsers.GetAllowedUsers(users) - users = s.ignoreUsers(users) - - level.Info(s.log).Log("msg", "found users", "all", allUsers, "allowed", len(users)) - - var mu sync.Mutex - allPlans := map[string]map[string]plan{} - stats := map[planStatus]int{} - for _, k := range []planStatus{New, InProgress, Finished, Error, Invalid} { - stats[k] = 0 - } - var queue []queuedPlan - - runConcurrently(ctx, s.cfg.PlanScanConcurrency, users, func(user string) { - userPrefix := path.Join(s.bucketPrefix, user) + "/" - - userPlans, err := scanForPlans(ctx, s.bucket, userPrefix) - if err != nil { - level.Error(s.log).Log("msg", "failed to scan plans for user", "user", user, "err", err) - return - } - - mu.Lock() - allPlans[user] = map[string]plan{} - mu.Unlock() - - for base, plan := range userPlans { - st := plan.Status() - if st == InProgress { - s.deleteObsoleteProgressFiles(ctx, &plan, path.Join(userPrefix, base)) - - // After deleting old progress files, status might have changed from InProgress to Error. - st = plan.Status() - } - - mu.Lock() - allPlans[user][base] = plan - stats[st]++ - mu.Unlock() - - if st != New { - continue - } - - dayIndex, err := strconv.ParseInt(base, 10, 32) - if err != nil { - level.Warn(s.log).Log("msg", "unable to parse day-index", "planFile", plan.PlanFiles[0]) - continue - } - - mu.Lock() - queue = append(queue, queuedPlan{ - DayIndex: int(dayIndex), - PlanFile: plan.PlanFiles[0], - }) - mu.Unlock() - } - }) - - // Plans with higher day-index (more recent) are put at the beginning. - sort.Slice(queue, func(i, j int) bool { - return queue[i].DayIndex > queue[j].DayIndex - }) - - for st, c := range stats { - s.planStatus.WithLabelValues(st.String()).Set(float64(c)) - } - - s.scanMu.Lock() - s.allUserPlans = allPlans - s.plansQueue = queue - s.updateQueuedPlansMetrics() - s.scanMu.Unlock() - - totalPlans := 0 - for _, p := range allPlans { - totalPlans += len(p) - } - - level.Info(s.log).Log("msg", "plans scan finished", "queued", len(queue), "total_plans", totalPlans) - - return nil -} - -func (s *Scheduler) deleteObsoleteProgressFiles(ctx context.Context, plan *plan, planBaseName string) { - for pg, t := range plan.ProgressFiles { - if time.Since(t) < s.cfg.MaxProgressFileAge { - continue - } - - level.Warn(s.log).Log("msg", "found obsolete progress file, will be deleted and error uploaded", "path", pg) - - errFile := blocksconvert.ErrorFilename(planBaseName) - if err := s.bucket.Upload(ctx, blocksconvert.ErrorFilename(planBaseName), strings.NewReader("Obsolete progress file found: "+pg)); err != nil { - level.Error(s.log).Log("msg", "failed to create error for obsolete progress file", "err", err) - continue - } - - plan.ErrorFile = errFile - - if err := s.bucket.Delete(ctx, pg); err != nil { - level.Error(s.log).Log("msg", "failed to delete obsolete progress file", "path", pg, "err", err) - continue - } - - delete(plan.ProgressFiles, pg) - } -} - -// Returns next plan that builder should work on. -func (s *Scheduler) NextPlan(ctx context.Context, req *blocksconvert.NextPlanRequest) (*blocksconvert.NextPlanResponse, error) { - if s.State() != services.Running { - return &blocksconvert.NextPlanResponse{}, nil - } - - plan, progress := s.nextPlanNoRunningCheck(ctx) - if plan != "" { - level.Info(s.log).Log("msg", "sending plan file", "plan", plan, "service", req.Name) - } - return &blocksconvert.NextPlanResponse{ - PlanFile: plan, - ProgressFile: progress, - }, nil -} - -func (s *Scheduler) nextPlanNoRunningCheck(ctx context.Context) (string, string) { - p := s.getNextPlanAndIncreaseDequeuingWG() - if p == "" { - return "", "" - } - - // otherwise dequeueWG has been increased - defer s.dequeueWG.Done() - - // Before we return plan file, we create progress file. - ok, base := blocksconvert.IsPlanFilename(p) - if !ok { - // Should not happen - level.Error(s.log).Log("msg", "enqueued file is not a plan file", "path", p) - return "", "" - } - - pg := blocksconvert.StartingFilename(base, time.Now()) - err := s.bucket.Upload(ctx, pg, strings.NewReader("starting")) - if err != nil { - level.Error(s.log).Log("msg", "failed to create progress file", "path", pg, "err", err) - return "", "" - } - - level.Info(s.log).Log("msg", "uploaded new progress file", "progressFile", pg) - return p, pg -} - -func (s *Scheduler) getNextPlanAndIncreaseDequeuingWG() string { - s.scanMu.Lock() - defer s.scanMu.Unlock() - - if s.scanning { - return "" - } - - if len(s.plansQueue) == 0 { - return "" - } - - var p string - p, s.plansQueue = s.plansQueue[0].PlanFile, s.plansQueue[1:] - s.updateQueuedPlansMetrics() - - s.dequeueWG.Add(1) - return p -} - -func runConcurrently(ctx context.Context, concurrency int, users []string, userFunc func(user string)) { - wg := sync.WaitGroup{} - ch := make(chan string) - - for ix := 0; ix < concurrency; ix++ { - wg.Add(1) - go func() { - defer wg.Done() - - for userID := range ch { - userFunc(userID) - } - }() - } - -sendLoop: - for _, userID := range users { - select { - case ch <- userID: - // ok - case <-ctx.Done(): - // don't start new tasks. - break sendLoop - } - } - - close(ch) - - // wait for ongoing workers to finish. - wg.Wait() -} - -func scanForUsers(ctx context.Context, bucket objstore.Bucket, bucketPrefix string) ([]string, error) { - var users []string - err := bucket.Iter(ctx, bucketPrefix, func(entry string) error { - users = append(users, strings.TrimSuffix(entry[len(bucketPrefix)+1:], "/")) - return nil - }) - - return users, err -} - -// Returns map of "base name" -> plan. Base name is object name of the plan, with removed prefix -// and also stripped from suffixes. Scanner-produced base names are day indexes. -// Individual paths in plan struct are full paths. -func scanForPlans(ctx context.Context, bucket objstore.Bucket, prefix string) (map[string]plan, error) { - plans := map[string]plan{} - - err := bucket.Iter(ctx, prefix, func(fullPath string) error { - if !strings.HasPrefix(fullPath, prefix) { - return errors.Errorf("invalid prefix: %v", fullPath) - } - - filename := fullPath[len(prefix):] - if ok, base := blocksconvert.IsPlanFilename(filename); ok { - p := plans[base] - p.PlanFiles = append(p.PlanFiles, fullPath) - plans[base] = p - } else if ok, base, ts := blocksconvert.IsProgressFilename(filename); ok { - p := plans[base] - if p.ProgressFiles == nil { - p.ProgressFiles = map[string]time.Time{} - } - p.ProgressFiles[fullPath] = ts - plans[base] = p - } else if ok, base, id := blocksconvert.IsFinishedFilename(filename); ok { - p := plans[base] - p.Finished = append(p.Finished, id) - plans[base] = p - } else if ok, base := blocksconvert.IsErrorFilename(filename); ok { - p := plans[base] - p.ErrorFile = fullPath - plans[base] = p - } - - return nil - }) - - if err != nil { - return nil, err - } - - return plans, nil -} - -var plansTemplate = template.Must(template.New("plans").Parse(` - - - - - Queue, Plans - - -

Current time: {{ .Now }}

-

Queue

-
    - {{ range $i, $p := .Queue }} -
  • {{ .DayIndex }} - {{ .PlanFile }}
    • - {{ end }} -
- -

Users

- {{ range $u, $up := .Plans }} -

{{ $u }}

- - - - - - - - - - - {{ range $base, $planStatus := $up }} - {{ with $planStatus }} - - - - - - {{ end }} - {{ end }} - -
Plan FileStatusComment
{{ range .PlanFiles }}{{ . }}
{{ end }}		{{ .Status }} - {{ if .ErrorFile }} Error: {{ .ErrorFile }}
{{ end }} - {{ if .ProgressFiles }} Progress: {{ range $p, $t := .ProgressFiles }} {{ $p }} {{ end }}
{{ end }} - {{ if .Finished }} Finished: {{ .Finished }}
{{ end }} -
- {{ end }} - -`)) - -func (s *Scheduler) httpPlans(writer http.ResponseWriter, req *http.Request) { - s.scanMu.Lock() - plans := s.allUserPlans - queue := s.plansQueue - s.scanMu.Unlock() - - data := struct { - Now time.Time - Plans map[string]map[string]plan - Queue []queuedPlan - }{ - Now: time.Now(), - Plans: plans, - Queue: queue, - } - - util.RenderHTTPResponse(writer, data, plansTemplate, req) -} - -// This function runs with lock. -func (s *Scheduler) updateQueuedPlansMetrics() { - s.queuedPlansGauge.Set(float64(len(s.plansQueue))) - - if len(s.plansQueue) > 0 { - daySeconds := 24 * time.Hour.Seconds() - s.oldestPlanTimestamp.Set(float64(s.plansQueue[len(s.plansQueue)-1].DayIndex) * daySeconds) - s.newestPlanTimestamp.Set(float64(s.plansQueue[0].DayIndex) * daySeconds) - } else { - s.oldestPlanTimestamp.Set(0) - s.newestPlanTimestamp.Set(0) - } -} - -func (s *Scheduler) ignoreUsers(users []string) []string { - if s.ignoredUsers == nil { - return users - } - - result := make([]string, 0, len(users)) - for _, u := range users { - if !s.ignoredUsers.MatchString(u) { - result = append(result, u) - } - } - return result -} diff --git a/tools/blocksconvert/scheduler/scheduler_test.go b/tools/blocksconvert/scheduler/scheduler_test.go deleted file mode 100644 index b58b7e7817..0000000000 --- a/tools/blocksconvert/scheduler/scheduler_test.go +++ /dev/null @@ -1,122 +0,0 @@ -package scheduler - -import ( - "context" - "fmt" - "os" - "regexp" - "strings" - "testing" - "time" - - "github.com/go-kit/log" - "github.com/stretchr/testify/require" - "github.com/thanos-io/thanos/pkg/objstore" - - "github.com/cortexproject/cortex/tools/blocksconvert" -) - -func TestScanForPlans(t *testing.T) { - bucket := objstore.NewInMemBucket() - require.NoError(t, bucket.Upload(context.Background(), "migration/12345/1.plan", strings.NewReader(""))) - require.NoError(t, bucket.Upload(context.Background(), "migration/12345/1.starting.1234567", strings.NewReader(""))) - require.NoError(t, bucket.Upload(context.Background(), "migration/12345/1.inprogress.2345678", strings.NewReader(""))) - - require.NoError(t, bucket.Upload(context.Background(), "migration/12345/2.plan", strings.NewReader(""))) - require.NoError(t, bucket.Upload(context.Background(), "migration/12345/2.inprogress.93485345", strings.NewReader(""))) - require.NoError(t, bucket.Upload(context.Background(), "migration/12345/2.finished.01E8GCW9J0HV0992HSZ0N6RAMN", strings.NewReader(""))) - require.NoError(t, bucket.Upload(context.Background(), "migration/12345/2.finished.01EE9Y140JP4T58X8FGTG5T17F", strings.NewReader(""))) - - require.NoError(t, bucket.Upload(context.Background(), "migration/12345/3.plan", strings.NewReader(""))) - require.NoError(t, bucket.Upload(context.Background(), "migration/12345/3.error", strings.NewReader(""))) - - // Only error, progress or finished - require.NoError(t, bucket.Upload(context.Background(), "migration/12345/4.error", strings.NewReader(""))) - require.NoError(t, bucket.Upload(context.Background(), "migration/12345/5.progress.1234234", strings.NewReader(""))) - require.NoError(t, bucket.Upload(context.Background(), "migration/12345/6.finished.cleaned", strings.NewReader(""))) - - plans, err := scanForPlans(context.Background(), bucket, "migration/12345/") - require.NoError(t, err) - - require.Equal(t, map[string]plan{ - "1": { - PlanFiles: []string{"migration/12345/1.plan"}, - ProgressFiles: map[string]time.Time{ - "migration/12345/1.starting.1234567": time.Unix(1234567, 0), - "migration/12345/1.inprogress.2345678": time.Unix(2345678, 0), - }, - }, - "2": { - PlanFiles: []string{"migration/12345/2.plan"}, - ProgressFiles: map[string]time.Time{ - "migration/12345/2.inprogress.93485345": time.Unix(93485345, 0), - }, - Finished: []string{"01E8GCW9J0HV0992HSZ0N6RAMN", "01EE9Y140JP4T58X8FGTG5T17F"}, - }, - "3": { - PlanFiles: []string{"migration/12345/3.plan"}, - ErrorFile: "migration/12345/3.error", - }, - "4": { - ErrorFile: "migration/12345/4.error", - }, - "5": { - ProgressFiles: map[string]time.Time{ - "migration/12345/5.progress.1234234": time.Unix(1234234, 0), - }, - }, - "6": { - Finished: []string{"cleaned"}, - }, - }, plans) -} - -func TestSchedulerScan(t *testing.T) { - now := time.Now() - nowMinus1Hour := now.Add(-time.Hour) - - bucket := objstore.NewInMemBucket() - require.NoError(t, bucket.Upload(context.Background(), "migration/user1/1.plan", strings.NewReader(""))) - // This progress file is too old, will be removed. - require.NoError(t, bucket.Upload(context.Background(), fmt.Sprintf("migration/user1/1.inprogress.%d", nowMinus1Hour.Unix()), strings.NewReader(""))) - - require.NoError(t, bucket.Upload(context.Background(), "migration/user2/2.plan", strings.NewReader(""))) - require.NoError(t, bucket.Upload(context.Background(), fmt.Sprintf("migration/user2/2.inprogress.%d", now.Unix()), strings.NewReader(""))) - - require.NoError(t, bucket.Upload(context.Background(), "migration/user3/3.plan", strings.NewReader(""))) - require.NoError(t, bucket.Upload(context.Background(), "migration/user3/3.error", strings.NewReader(""))) - - require.NoError(t, bucket.Upload(context.Background(), "migration/user4/4.plan", strings.NewReader(""))) - require.NoError(t, bucket.Upload(context.Background(), "migration/user4/5.error", strings.NewReader(""))) - require.NoError(t, bucket.Upload(context.Background(), "migration/user4/6.finished.01E8GCW9J0HV0992HSZ0N6RAMN", strings.NewReader(""))) - - require.NoError(t, bucket.Upload(context.Background(), "migration/ignoredUser/7.plan", strings.NewReader(""))) - - ignoredUsers := regexp.MustCompile("ignored.*") - - s := newSchedulerWithBucket(log.NewLogfmtLogger(os.Stdout), bucket, "migration", blocksconvert.AllowAllUsers, ignoredUsers, Config{ - ScanInterval: 10 * time.Second, - PlanScanConcurrency: 5, - MaxProgressFileAge: 5 * time.Minute, - }, nil) - - require.NoError(t, s.scanBucketForPlans(context.Background())) - require.Equal(t, []queuedPlan{ - {DayIndex: 4, PlanFile: "migration/user4/4.plan"}, - }, s.plansQueue) - require.Equal(t, "migration/user1/1.error", s.allUserPlans["user1"]["1"].ErrorFile) - - { - p, pg := s.nextPlanNoRunningCheck(context.Background()) - require.Equal(t, "migration/user4/4.plan", p) - ok, err := bucket.Exists(context.Background(), pg) - require.NoError(t, err) - require.True(t, ok) - } - - { - p, pg := s.nextPlanNoRunningCheck(context.Background()) - require.Equal(t, "", p) - require.Equal(t, "", pg) - } -} diff --git a/tools/blocksconvert/shared_config.go b/tools/blocksconvert/shared_config.go deleted file mode 100644 index e6be735640..0000000000 --- a/tools/blocksconvert/shared_config.go +++ /dev/null @@ -1,44 +0,0 @@ -package blocksconvert - -import ( - "context" - "flag" - - "github.com/go-kit/log" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/thanos-io/thanos/pkg/objstore" - - "github.com/cortexproject/cortex/pkg/chunk" - "github.com/cortexproject/cortex/pkg/chunk/storage" - "github.com/cortexproject/cortex/pkg/storage/bucket" -) - -type SharedConfig struct { - SchemaConfig chunk.SchemaConfig // Flags registered by main.go - StorageConfig storage.Config - - Bucket bucket.Config - BucketPrefix string -} - -func (cfg *SharedConfig) RegisterFlags(f *flag.FlagSet) { - cfg.SchemaConfig.RegisterFlags(f) - cfg.Bucket.RegisterFlagsWithPrefix("blocks-storage.", f) - cfg.StorageConfig.RegisterFlags(f) - - f.StringVar(&cfg.BucketPrefix, "blocksconvert.bucket-prefix", "migration", "Prefix in the bucket for storing plan files.") -} - -func (cfg *SharedConfig) GetBucket(l log.Logger, reg prometheus.Registerer) (objstore.Bucket, error) { - if err := cfg.Bucket.Validate(); err != nil { - return nil, errors.Wrap(err, "invalid bucket config") - } - - bucket, err := bucket.NewClient(context.Background(), cfg.Bucket, "bucket", l, reg) - if err != nil { - return nil, errors.Wrap(err, "failed to create bucket") - } - - return bucket, nil -} From b42c43c9d9626c2d4cd478150abcd742476a3cfb Mon Sep 17 00:00:00 2001 From: Andrew Bloomgarden Date: Mon, 4 Apr 2022 11:39:02 -0400 Subject: [PATCH 04/28] Remove most all of chunks storage Signed-off-by: Andrew Bloomgarden --- Makefile | 6 +- .../migrate-from-chunks-to-blocks.md | 280 - docs/blocks-storage/querier.md | 9 - docs/chunks-storage/aws-tips.md | 91 - docs/chunks-storage/caching.md | 201 - .../chunks-storage-getting-started.md | 200 - docs/chunks-storage/ingesters-with-wal.md | 117 - .../running-chunks-storage-in-production.md | 85 - .../running-chunks-storage-with-cassandra.md | 182 - docs/chunks-storage/schema-config.md | 151 - .../chunks-storage/single-process-config.yaml | 98 - docs/chunks-storage/table-manager.md | 71 - docs/configuration/config-file-reference.md | 1086 +-- docs/guides/encryption-at-rest.md | 6 +- integration/backward_compatibility_test.go | 80 - integration/configs.go | 84 +- integration/querier_remote_read_test.go | 1 - integration/querier_test.go | 1 - pkg/chunk/aws/dynamodb_index_reader.go | 237 - pkg/chunk/aws/dynamodb_metrics.go | 60 - pkg/chunk/aws/dynamodb_storage_client.go | 819 --- pkg/chunk/aws/dynamodb_storage_client_test.go | 43 - pkg/chunk/aws/dynamodb_table_client.go | 387 - pkg/chunk/aws/fixtures.go | 96 - pkg/chunk/aws/metrics_autoscaling.go | 378 - pkg/chunk/aws/metrics_autoscaling_test.go | 554 -- pkg/chunk/aws/mock.go | 424 -- pkg/chunk/aws/retryer.go | 52 - pkg/chunk/bucket_client.go | 11 - pkg/chunk/cache/cache_test.go | 49 +- pkg/chunk/cassandra/authenticator.go | 43 - pkg/chunk/cassandra/fixtures.go | 76 - pkg/chunk/cassandra/instrumentation.go | 38 - pkg/chunk/cassandra/storage_client.go | 565 -- pkg/chunk/cassandra/storage_client_test.go | 155 - pkg/chunk/cassandra/table_client.go | 81 - pkg/chunk/cassandra/table_client_test.go | 48 - .../cassandra/testdata/example.com-key.pem | 27 - .../cassandra/testdata/example.com.ca.pem | 25 - pkg/chunk/cassandra/testdata/example.com.pem | 27 - .../password-with-trailing-newline.txt | 1 - .../password-without-trailing-newline.txt | 1 - pkg/chunk/chunk.go | 30 - pkg/chunk/chunk_store.go | 714 -- pkg/chunk/chunk_store_test.go | 1150 --- pkg/chunk/chunk_store_utils.go | 273 - pkg/chunk/chunk_test.go | 128 +- pkg/chunk/composite_store.go | 271 - pkg/chunk/composite_store_test.go | 291 - pkg/chunk/encoding/bigchunk.go | 352 - pkg/chunk/encoding/bigchunk_test.go | 96 - pkg/chunk/encoding/chunk.go | 191 +- pkg/chunk/encoding/chunk_test.go | 84 +- pkg/chunk/encoding/delta_helpers.go | 87 - pkg/chunk/encoding/doubledelta.go | 546 -- pkg/chunk/encoding/factory.go | 102 +- pkg/chunk/encoding/prometheus_chunk.go | 29 +- pkg/chunk/encoding/varbit.go | 1229 ---- pkg/chunk/encoding/varbit_helpers.go | 78 - pkg/chunk/encoding/varbit_test.go | 55 - pkg/chunk/fixtures.go | 24 - pkg/chunk/gcp/bigtable_index_client.go | 413 -- pkg/chunk/gcp/bigtable_object_client.go | 182 - pkg/chunk/gcp/fixtures.go | 111 - pkg/chunk/gcp/fnv.go | 36 - pkg/chunk/gcp/instrumentation.go | 34 - pkg/chunk/gcp/table_client.go | 126 - pkg/chunk/grpc/grpc.pb.go | 6481 ----------------- pkg/chunk/grpc/grpc.proto | 142 - pkg/chunk/grpc/grpc_client.go | 35 - pkg/chunk/grpc/grpc_client_test.go | 180 - pkg/chunk/grpc/grpc_server_mock_test.go | 185 - pkg/chunk/grpc/index_client.go | 107 - pkg/chunk/grpc/storage_client.go | 118 - pkg/chunk/grpc/table_client.go | 107 - pkg/chunk/index_reader.go | 33 - pkg/chunk/inmemory_storage_client.go | 391 +- pkg/chunk/local/boltdb_index_client.go | 366 - pkg/chunk/local/boltdb_index_client_test.go | 261 - pkg/chunk/local/boltdb_table_client.go | 61 - pkg/chunk/local/fixtures.go | 80 - pkg/chunk/local/fs_object_client.go | 211 - pkg/chunk/local/fs_object_client_test.go | 202 - pkg/chunk/objectclient/client.go | 119 - pkg/chunk/schema.go | 966 --- pkg/chunk/schema_caching.go | 73 - pkg/chunk/schema_caching_test.go | 70 - pkg/chunk/schema_config.go | 448 -- pkg/chunk/schema_config_test.go | 714 -- pkg/chunk/schema_test.go | 456 -- pkg/chunk/schema_util.go | 257 - pkg/chunk/schema_util_test.go | 149 - pkg/chunk/series_store.go | 588 -- pkg/chunk/storage/by_key_test.go | 12 - pkg/chunk/storage/bytes.go | 39 - pkg/chunk/storage/caching_fixtures.go | 48 - pkg/chunk/storage/caching_index_client.go | 308 - pkg/chunk/storage/caching_index_client.pb.go | 843 --- pkg/chunk/storage/caching_index_client.proto | 25 - .../storage/caching_index_client_test.go | 264 - pkg/chunk/storage/chunk_client_test.go | 61 - pkg/chunk/storage/factory.go | 341 +- pkg/chunk/storage/factory_test.go | 197 - pkg/chunk/storage/index_client_test.go | 230 - pkg/chunk/storage/metrics.go | 110 - pkg/chunk/storage/utils_test.go | 39 - pkg/chunk/storage_client.go | 46 - pkg/chunk/strings.go | 88 - pkg/chunk/table_client.go | 62 - pkg/chunk/table_manager.go | 574 -- pkg/chunk/table_manager_test.go | 769 -- pkg/chunk/table_provisioning.go | 111 - pkg/chunk/tags.go | 58 - pkg/chunk/testutils/testutils.go | 154 - pkg/chunk/util/parallel_chunk_fetch.go | 79 - pkg/chunk/util/parallel_chunk_fetch_test.go | 26 - pkg/chunk/util/util.go | 122 - pkg/cortex/cortex.go | 31 - pkg/cortex/modules.go | 72 +- pkg/distributor/distributor_test.go | 5 +- pkg/querier/batch/batch_test.go | 6 - pkg/querier/batch/chunk_test.go | 2 +- pkg/querier/chunk_store_queryable.go | 88 - pkg/querier/chunk_store_queryable_test.go | 2 +- pkg/querier/distributor_queryable_test.go | 6 +- pkg/querier/error_translate_queryable.go | 7 +- pkg/querier/error_translate_queryable_test.go | 7 - .../iterators/chunk_merge_iterator_test.go | 22 +- pkg/querier/querier.go | 37 +- pkg/querier/querier_benchmark_test.go | 38 - pkg/querier/querier_test.go | 70 +- pkg/querier/queryrange/queryable.go | 154 - pkg/querier/queryrange/queryable_test.go | 270 - pkg/querier/queryrange/querysharding.go | 262 - pkg/querier/queryrange/querysharding_test.go | 665 -- pkg/querier/queryrange/roundtrip.go | 37 - pkg/querier/queryrange/roundtrip_test.go | 29 - pkg/ruler/ruler_test.go | 33 +- tools/doc-generator/main.go | 11 - 139 files changed, 225 insertions(+), 32613 deletions(-) delete mode 100644 docs/blocks-storage/migrate-from-chunks-to-blocks.md delete mode 100644 docs/chunks-storage/aws-tips.md delete mode 100644 docs/chunks-storage/caching.md delete mode 100644 docs/chunks-storage/chunks-storage-getting-started.md delete mode 100644 docs/chunks-storage/ingesters-with-wal.md delete mode 100644 docs/chunks-storage/running-chunks-storage-in-production.md delete mode 100644 docs/chunks-storage/running-chunks-storage-with-cassandra.md delete mode 100644 docs/chunks-storage/schema-config.md delete mode 100644 docs/chunks-storage/single-process-config.yaml delete mode 100644 docs/chunks-storage/table-manager.md delete mode 100644 pkg/chunk/aws/dynamodb_index_reader.go delete mode 100644 pkg/chunk/aws/dynamodb_metrics.go delete mode 100644 pkg/chunk/aws/dynamodb_storage_client.go delete mode 100644 pkg/chunk/aws/dynamodb_storage_client_test.go delete mode 100644 pkg/chunk/aws/dynamodb_table_client.go delete mode 100644 pkg/chunk/aws/fixtures.go delete mode 100644 pkg/chunk/aws/metrics_autoscaling.go delete mode 100644 pkg/chunk/aws/metrics_autoscaling_test.go delete mode 100644 pkg/chunk/aws/mock.go delete mode 100644 pkg/chunk/aws/retryer.go delete mode 100644 pkg/chunk/bucket_client.go delete mode 100644 pkg/chunk/cassandra/authenticator.go delete mode 100644 pkg/chunk/cassandra/fixtures.go delete mode 100644 pkg/chunk/cassandra/instrumentation.go delete mode 100644 pkg/chunk/cassandra/storage_client.go delete mode 100644 pkg/chunk/cassandra/storage_client_test.go delete mode 100644 pkg/chunk/cassandra/table_client.go delete mode 100644 pkg/chunk/cassandra/table_client_test.go delete mode 100644 pkg/chunk/cassandra/testdata/example.com-key.pem delete mode 100644 pkg/chunk/cassandra/testdata/example.com.ca.pem delete mode 100644 pkg/chunk/cassandra/testdata/example.com.pem delete mode 100644 pkg/chunk/cassandra/testdata/password-with-trailing-newline.txt delete mode 100644 pkg/chunk/cassandra/testdata/password-without-trailing-newline.txt delete mode 100644 pkg/chunk/chunk_store.go delete mode 100644 pkg/chunk/chunk_store_test.go delete mode 100644 pkg/chunk/chunk_store_utils.go delete mode 100644 pkg/chunk/composite_store.go delete mode 100644 pkg/chunk/composite_store_test.go delete mode 100644 pkg/chunk/encoding/bigchunk.go delete mode 100644 pkg/chunk/encoding/bigchunk_test.go delete mode 100644 pkg/chunk/encoding/delta_helpers.go delete mode 100644 pkg/chunk/encoding/doubledelta.go delete mode 100644 pkg/chunk/encoding/varbit.go delete mode 100644 pkg/chunk/encoding/varbit_helpers.go delete mode 100644 pkg/chunk/encoding/varbit_test.go delete mode 100644 pkg/chunk/gcp/bigtable_index_client.go delete mode 100644 pkg/chunk/gcp/bigtable_object_client.go delete mode 100644 pkg/chunk/gcp/fixtures.go delete mode 100644 pkg/chunk/gcp/fnv.go delete mode 100644 pkg/chunk/gcp/table_client.go delete mode 100644 pkg/chunk/grpc/grpc.pb.go delete mode 100644 pkg/chunk/grpc/grpc.proto delete mode 100644 pkg/chunk/grpc/grpc_client.go delete mode 100644 pkg/chunk/grpc/grpc_client_test.go delete mode 100644 pkg/chunk/grpc/grpc_server_mock_test.go delete mode 100644 pkg/chunk/grpc/index_client.go delete mode 100644 pkg/chunk/grpc/storage_client.go delete mode 100644 pkg/chunk/grpc/table_client.go delete mode 100644 pkg/chunk/index_reader.go delete mode 100644 pkg/chunk/local/boltdb_index_client.go delete mode 100644 pkg/chunk/local/boltdb_index_client_test.go delete mode 100644 pkg/chunk/local/boltdb_table_client.go delete mode 100644 pkg/chunk/local/fixtures.go delete mode 100644 pkg/chunk/local/fs_object_client.go delete mode 100644 pkg/chunk/local/fs_object_client_test.go delete mode 100644 pkg/chunk/objectclient/client.go delete mode 100644 pkg/chunk/schema.go delete mode 100644 pkg/chunk/schema_caching.go delete mode 100644 pkg/chunk/schema_caching_test.go delete mode 100644 pkg/chunk/schema_config.go delete mode 100644 pkg/chunk/schema_config_test.go delete mode 100644 pkg/chunk/schema_test.go delete mode 100644 pkg/chunk/schema_util.go delete mode 100644 pkg/chunk/schema_util_test.go delete mode 100644 pkg/chunk/series_store.go delete mode 100644 pkg/chunk/storage/by_key_test.go delete mode 100644 pkg/chunk/storage/bytes.go delete mode 100644 pkg/chunk/storage/caching_fixtures.go delete mode 100644 pkg/chunk/storage/caching_index_client.go delete mode 100644 pkg/chunk/storage/caching_index_client.pb.go delete mode 100644 pkg/chunk/storage/caching_index_client.proto delete mode 100644 pkg/chunk/storage/caching_index_client_test.go delete mode 100644 pkg/chunk/storage/chunk_client_test.go delete mode 100644 pkg/chunk/storage/factory_test.go delete mode 100644 pkg/chunk/storage/index_client_test.go delete mode 100644 pkg/chunk/storage/metrics.go delete mode 100644 pkg/chunk/storage/utils_test.go delete mode 100644 pkg/chunk/strings.go delete mode 100644 pkg/chunk/table_client.go delete mode 100644 pkg/chunk/table_manager.go delete mode 100644 pkg/chunk/table_manager_test.go delete mode 100644 pkg/chunk/table_provisioning.go delete mode 100644 pkg/chunk/tags.go delete mode 100644 pkg/chunk/testutils/testutils.go delete mode 100644 pkg/chunk/util/parallel_chunk_fetch.go delete mode 100644 pkg/chunk/util/parallel_chunk_fetch_test.go delete mode 100644 pkg/querier/querier_benchmark_test.go delete mode 100644 pkg/querier/queryrange/queryable.go delete mode 100644 pkg/querier/queryrange/queryable_test.go delete mode 100644 pkg/querier/queryrange/querysharding.go delete mode 100644 pkg/querier/queryrange/querysharding_test.go diff --git a/Makefile b/Makefile index 6c61bdac76..2b874fea47 100644 --- a/Makefile +++ b/Makefile @@ -96,14 +96,12 @@ pkg/frontend/v1/frontendv1pb/frontend.pb.go: pkg/frontend/v1/frontendv1pb/fronte pkg/frontend/v2/frontendv2pb/frontend.pb.go: pkg/frontend/v2/frontendv2pb/frontend.proto pkg/querier/queryrange/queryrange.pb.go: pkg/querier/queryrange/queryrange.proto pkg/querier/stats/stats.pb.go: pkg/querier/stats/stats.proto -pkg/chunk/storage/caching_index_client.pb.go: pkg/chunk/storage/caching_index_client.proto pkg/distributor/ha_tracker.pb.go: pkg/distributor/ha_tracker.proto pkg/ruler/rulespb/rules.pb.go: pkg/ruler/rulespb/rules.proto pkg/ruler/ruler.pb.go: pkg/ruler/ruler.proto pkg/ring/kv/memberlist/kv.pb.go: pkg/ring/kv/memberlist/kv.proto pkg/scheduler/schedulerpb/scheduler.pb.go: pkg/scheduler/schedulerpb/scheduler.proto pkg/storegateway/storegatewaypb/gateway.pb.go: pkg/storegateway/storegatewaypb/gateway.proto -pkg/chunk/grpc/grpc.pb.go: pkg/chunk/grpc/grpc.proto pkg/alertmanager/alertmanagerpb/alertmanager.pb.go: pkg/alertmanager/alertmanagerpb/alertmanager.proto pkg/alertmanager/alertspb/alerts.pb.go: pkg/alertmanager/alertspb/alerts.proto @@ -358,7 +356,7 @@ dist/$(UPTODATE)-packages: dist $(wildcard packaging/deb/**) $(wildcard packagin --before-remove packaging/deb/control/prerm \ --package dist/cortex-$(VERSION)_$$arch.deb \ dist/cortex-linux-$$arch=/usr/local/bin/cortex \ - docs/chunks-storage/single-process-config.yaml=/etc/cortex/single-process-config.yaml \ + docs/configuration/single-process-config-blocks.yaml=/etc/cortex/single-process-config.yaml \ packaging/deb/default/cortex=/etc/default/cortex \ packaging/deb/systemd/cortex.service=/etc/systemd/system/cortex.service; \ $(FPM_OPTS) -t rpm \ @@ -367,7 +365,7 @@ dist/$(UPTODATE)-packages: dist $(wildcard packaging/deb/**) $(wildcard packagin --before-remove packaging/rpm/control/preun \ --package dist/cortex-$(VERSION)_$$arch.rpm \ dist/cortex-linux-$$arch=/usr/local/bin/cortex \ - docs/chunks-storage/single-process-config.yaml=/etc/cortex/single-process-config.yaml \ + docs/configuration/single-process-config-blocks.yaml=/etc/cortex/single-process-config.yaml \ packaging/rpm/sysconfig/cortex=/etc/sysconfig/cortex \ packaging/rpm/systemd/cortex.service=/etc/systemd/system/cortex.service; \ done diff --git a/docs/blocks-storage/migrate-from-chunks-to-blocks.md b/docs/blocks-storage/migrate-from-chunks-to-blocks.md deleted file mode 100644 index 3ebc122d06..0000000000 --- a/docs/blocks-storage/migrate-from-chunks-to-blocks.md +++ /dev/null @@ -1,280 +0,0 @@ ---- -title: "Migrate Cortex cluster from chunks to blocks" -linkTitle: "Migrate Cortex cluster from chunks to blocks" -weight: 5 -slug: migrate-cortex-cluster-from-chunks-to-blocks ---- - -This article describes how to migrate existing Cortex cluster from chunks storage to blocks storage, -and highlight possible issues you may encounter in the process. - -_This document replaces the [Cortex proposal](https://cortexmetrics.io/docs/proposals/ingesters-migration/), -which was written before support for migration was in place._ - -## Introduction - -This article **assumes** that: - -- Cortex cluster is managed by Kubernetes -- Cortex is using chunks storage -- Ingesters are using WAL -- Cortex version 1.4.0 or later. - -_If your ingesters are not using WAL, the documented procedure will still apply, but the presented migration script will not work properly without changes, as it assumes that ingesters are managed via StatefulSet._ - -The migration procedure is composed by 3 steps: - -1. [Preparation](#step-1-preparation) -1. [Ingesters migration](#step-2-ingesters-migration) -1. [Cleanup](#step-3-cleanup) - -_In case of any issue during or after the migration, this document also outlines a [Rollback](#rollback) strategy._ - -## Step 1: Preparation - -Before starting the migration of ingesters, we need to prepare other services. - -### Querier and Ruler - -_Everything discussed for querier applies to ruler as well, since it shares querier configuration – CLI flags prefix is `-querier` even when used by ruler._ - -Querier and ruler need to be reconfigured as follow: - -- `-querier.second-store-engine=blocks` -- `-querier.query-store-after=0` - -#### `-querier.second-store-engine=blocks` - -Querier (and ruler) needs to be reconfigured to query both chunks storage and blocks storage at the same time. -This is achieved by using `-querier.second-store-engine=blocks` option, and providing querier with full blocks configuration, but keeping "primary" store set to `-store.engine=chunks`. - -#### `-querier.query-store-after=0` - -Querier (and ruler) has an option `-querier.query-store-after` to query store only if query hits data older than some period of time. -For example, if ingesters keep 12h of data in memory, there is no need to hit the store for queries that only need last 1h of data. -During the migration, this flag needs to be set to 0, to make queriers always consult the store when handling queries. -As chunks ingesters shut down, they flush chunks to the storage. They are then replaced with new ingesters configured -to use blocks. Queriers cannot fetch recent chunks from ingesters directly (as blocks ingester don't reload chunks), -and need to use storage instead. - -### Query-frontend - -Query-frontend needs to be reconfigured as follow: - -- `-querier.parallelise-shardable-queries=false` - -#### `-querier.parallelise-shardable-queries=false` - -Query frontend has an option `-querier.parallelise-shardable-queries` to split some incoming queries into multiple queries based on sharding factor used in v11 schema of chunk storage. -As the description implies, it only works when using chunks storage. -During and after the migration to blocks (and also after possible rollback), this option needs to be disabled otherwise query-frontend will generate queries that cannot be satisfied by blocks storage. - -### Compactor and Store-gateway - -[Compactor](./compactor.md) and [store-gateway](./store-gateway.md) services should be deployed and successfully up and running before migrating ingesters. - -### Ingester – blocks - -Migration script presented in Step 2 assumes that there are two StatefulSets of ingesters: existing one configured with chunks, and the new one with blocks. -New StatefulSet with blocks ingesters should have 0 replicas at the beginning of migration. - -### Table-Manager - chunks - -If you use a store with provisioned IO, e.g. DynamoDB, scale up the provision before starting the migration. -Each ingester will need to flush all chunks before exiting, so will write to the store at many times the normal rate. - -Stop or reconfigure the table-manager to stop it adjusting the provision back to normal. -(Don't do the migration on Wednesday night when a new weekly table might be required.) - -## Step 2: Ingesters migration - -We have developed a script available in Cortex [`tools/migrate-ingester-statefulsets.sh`](https://github.com/cortexproject/cortex/blob/master/tools/migrate-ingester-statefulsets.sh) to migrate ingesters between two StatefulSets, shutting down ingesters one by one. - -It can be used like this: - -``` -$ tools/migrate-ingester-statefulsets.sh -``` - -Where parameters are: -- ``: Kubernetes namespace where the Cortex cluster is running -- ``: name of the ingesters StatefulSet to scale down (running chunks storage) -- ``: name of the ingesters StatefulSet to scale up (running blocks storage) -- ``: number of instances to scale down (in `ingester-old` statefulset) and scale up (in `ingester-new`), or "all" – which will scale down all remaining instances in `ingester-old` statefulset - -After starting new pod in `ingester-new` statefulset, script then triggers `/shutdown` endpoint on the old ingester. When the flushing on the old ingester is complete, scale down of statefulset continues, and process repeats. - -_The script supports both migration from chunks to blocks, and viceversa (eg. rollback)._ - -### Known issues - -There are few known issues with the script: - -- If expected messages don't appear in the log, but pod keeps on running, the script will never finish. -- Script doesn't verify that flush finished without any error. - -## Step 3: Cleanup - -When the ingesters migration finishes, there are still two StatefulSets, with original StatefulSet (running the chunks storage) having 0 instances now. - -At this point, we can delete the old StatefulSet and its persistent volumes and recreate it with final blocks storage configuration (eg. changing PVs), and use the script again to move pods from `ingester-blocks` to `ingester`. - -Querier (and ruler) can be reconfigured to use `blocks` as "primary" store to search, and `chunks` as secondary: - -- `-store.engine=blocks` -- `-querier.second-store-engine=chunks` -- `-querier.use-second-store-before-time=` -- `-querier.ingester-streaming=true` - -#### `-querier.use-second-store-before-time` - -The CLI flag `-querier.use-second-store-before-time` (or its respective YAML config option) is only available for secondary store. -This flag can be set to a timestamp when migration has finished, and it avoids querying secondary store (chunks) for data when running queries that don't need data before given time. - -Both primary and secondary stores are queried before this time, so the overlap where some data is in chunks and some in blocks is covered. - -## Rollback - -If rollback to chunks is needed for any reason, it is possible to use the same migration script with reversed arguments: - -- Scale down ingesters StatefulSet running blocks storage -- Scale up ingesters StatefulSet running chunks storage - -_Blocks ingesters support the same `/shutdown` endpoint for flushing data._ - -During the rollback, queriers and rulers need to use the same configuration changes as during migration. You should also make sure the following settings are applied: - -- `-store.engine=chunks` -- `-querier.second-store-engine=blocks` -- `-querier.use-second-store-before-time` should not be set -- `-querier.ingester-streaming=false` - -Once the rollback is complete, some configuration changes need to stay in place, because some data has already been stored to blocks: - -- The query sharding in the query-frontend must be kept disabled, otherwise querying blocks will not work correctly -- `store-gateway` needs to keep running, otherwise querying blocks will fail -- `compactor` may be shutdown, after it has no more compaction work to do - -Kubernetes resources related to the ingesters running the blocks storage may be deleted. - -### Known issues - -After rollback, chunks ingesters will replay their old Write-Ahead-Log, thus loading old chunks into memory. -WAL doesn't remember whether these old chunks were already flushed or not, so they will be flushed again to the storage. -Until that flush happens, Cortex reports those chunks as unflushed, which may trigger some alerts based on `cortex_oldest_unflushed_chunk_timestamp_seconds` metric. - -## Appendix - -### Jsonnet config - -This section shows how to use [cortex-jsonnet](https://github.com/grafana/cortex-jsonnet) to configure additional services. - -We will assume that `main.jsonnet` is main configuration for the cluster, that also imports `temp.jsonnet` – with our temporary configuration for migration. - -In `main.jsonnet` we have something like this: - -```jsonnet -local cortex = import 'cortex/cortex.libsonnet'; -local wal = import 'cortex/wal.libsonnet'; -local temp = import 'temp.jsonnet'; - -// Note that 'tsdb' is not imported here. -cortex + wal + temp { - _images+:: (import 'images.libsonnet'), - - _config+:: { - cluster: 'k8s-cluster', - namespace: 'k8s-namespace', - -... -``` - -To configure querier to use secondary store for querying, we need to add: - -``` - querier_second_storage_engine: 'blocks', - blocks_storage_bucket_name: 'bucket-for-storing-blocks', -``` - -to the `_config` object in main.jsonnet. - -Let's generate blocks configuration now in `temp.jsonnet`. -There are comments inside that should give you an idea about what's happening. -Most important thing is generating resources with blocks configuration, and exposing some of them. - - -```jsonnet -{ - local cortex = import 'cortex/cortex.libsonnet', - local tsdb = import 'cortex/tsdb.libsonnet', - local rootConfig = self._config, - local statefulSet = $.apps.v1beta1.statefulSet, - - // Prepare TSDB resources, but hide them. Cherry-picked resources will be exposed later. - tsdb_config:: cortex + tsdb + { - _config+:: { - cluster: rootConfig.cluster, - namespace: rootConfig.namespace, - external_url: rootConfig.external_url, - - // This Cortex cluster is using the blocks storage. - storage_tsdb_bucket_name: rootConfig.storage_tsdb_bucket_name, - cortex_store_gateway_data_disk_size: '100Gi', - cortex_compactor_data_disk_class: 'fast', - }, - - // We create another statefulset for ingesters here, with different name. - ingester_blocks_statefulset: self.newIngesterStatefulSet('ingester-blocks', self.ingester_container) + - statefulSet.mixin.spec.withReplicas(0), - - ingester_blocks_pdb: self.newIngesterPdb('ingester-blocks-pdb', 'ingester-blocks'), - ingester_blocks_service: $.util.serviceFor(self.ingester_blocks_statefulset, self.ingester_service_ignored_labels), - }, - - _config+: { - queryFrontend+: { - // Disabled because querying blocks-data breaks if query is rewritten for sharding. - sharded_queries_enabled: false, - }, - }, - - // Expose some services from TSDB configuration, needed for running Querier with Chunks as primary and TSDB as secondary store. - tsdb_store_gateway_pdb: self.tsdb_config.store_gateway_pdb, - tsdb_store_gateway_service: self.tsdb_config.store_gateway_service, - tsdb_store_gateway_statefulset: self.tsdb_config.store_gateway_statefulset, - - tsdb_memcached_metadata: self.tsdb_config.memcached_metadata, - - tsdb_ingester_statefulset: self.tsdb_config.ingester_blocks_statefulset, - tsdb_ingester_pdb: self.tsdb_config.ingester_blocks_pdb, - tsdb_ingester_service: self.tsdb_config.ingester_blocks_service, - - tsdb_compactor_statefulset: self.tsdb_config.compactor_statefulset, - - // Querier and ruler configuration used during migration, and after. - query_config_during_migration:: { - // Disable streaming, as it is broken when querying both chunks and blocks ingesters at the same time. - 'querier.ingester-streaming': 'false', - - // query-store-after is required during migration, since new ingesters running on blocks will not load any chunks from chunks-WAL. - // All such chunks are however flushed to the store. - 'querier.query-store-after': '0', - }, - - query_config_after_migration:: { - 'querier.ingester-streaming': 'true', - 'querier.query-ingesters-within': '13h', // TSDB ingesters have data for up to 4d. - 'querier.query-store-after': '12h', // Can be enabled once blocks ingesters are running for 12h. - - // Switch TSDB and chunks. TSDB is "primary" now so that we can skip querying chunks for old queries. - // We can do this, because querier/ruler have both configurations. - 'store.engine': 'blocks', - 'querier.second-store-engine': 'chunks', - - 'querier.use-second-store-before-time': '2020-07-28T17:00:00Z', // If migration from chunks finished around 18:10 CEST, no need to query chunk store for queries before this time. - }, - - querier_args+:: self.tsdb_config.blocks_metadata_caching_config + self.query_config_during_migration, // + self.query_config_after_migration, - ruler_args+:: self.tsdb_config.blocks_metadata_caching_config + self.query_config_during_migration, // + self.query_config_after_migration, -} -``` diff --git a/docs/blocks-storage/querier.md b/docs/blocks-storage/querier.md index e1592b9c89..9567713e3b 100644 --- a/docs/blocks-storage/querier.md +++ b/docs/blocks-storage/querier.md @@ -207,15 +207,6 @@ querier: # CLI flag: -querier.store-gateway-client.tls-insecure-skip-verify [tls_insecure_skip_verify: | default = false] - # Second store engine to use for querying. Empty = disabled. - # CLI flag: -querier.second-store-engine - [second_store_engine: | default = ""] - - # If specified, second store is only used for queries before this timestamp. - # Default value 0 means secondary store is always queried. - # CLI flag: -querier.use-second-store-before-time - [use_second_store_before_time: