From 60fcb12e9c9544dc497aff8f24611b0f9356f648 Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Thu, 19 Aug 2021 13:28:38 +0200 Subject: [PATCH 01/10] Add single_overrides_configmap support --- .../jsonnet/microservices/compactor.libsonnet | 7 ++++-- .../jsonnet/microservices/config.libsonnet | 4 ++++ .../jsonnet/microservices/configmap.libsonnet | 23 +++++++++++++------ .../microservices/distributor.libsonnet | 8 +++++-- .../jsonnet/microservices/ingester.libsonnet | 8 +++++-- 5 files changed, 37 insertions(+), 13 deletions(-) diff --git a/operations/jsonnet/microservices/compactor.libsonnet b/operations/jsonnet/microservices/compactor.libsonnet index 7b1e265e6fc..2dbdb79c303 100644 --- a/operations/jsonnet/microservices/compactor.libsonnet +++ b/operations/jsonnet/microservices/compactor.libsonnet @@ -11,6 +11,7 @@ local target_name = 'compactor', local tempo_config_volume = 'tempo-conf', local tempo_data_volume = 'tempo-data', + local tempo_overrides_config_volume = 'overrides', tempo_compactor_container:: container.new(target_name, $._images.tempo) + @@ -24,7 +25,7 @@ ]) + container.withVolumeMounts([ volumeMount.new(tempo_config_volume, '/conf'), - ]) + + ] + if $._config.use_overrides_configmap then [volumeMount.new(tempo_overrides_config_volume, '/overrides')] else []) + $.util.withResources($._config.compactor.resources) + $.util.readinessProbe, @@ -42,7 +43,9 @@ }) + deployment.mixin.spec.template.spec.withVolumes([ volume.fromConfigMap(tempo_config_volume, $.tempo_compactor_configmap.metadata.name), - ]), + ] + if $._config.use_overrides_configmap then + [volume.fromConfigMap(tempo_overrides_config_volume, $._config.overrides_configmap_name)] + else []), tempo_compactor_service: k.util.serviceFor($.tempo_compactor_deployment), diff --git a/operations/jsonnet/microservices/config.libsonnet b/operations/jsonnet/microservices/config.libsonnet index 7db8bf001ff..0dcd6d66a53 100644 --- a/operations/jsonnet/microservices/config.libsonnet +++ b/operations/jsonnet/microservices/config.libsonnet @@ -97,6 +97,10 @@ backend: error 'Must specify a backend', // gcs|s3 bucket: error 'Must specify a bucket', + // Eventually, once we enable it everywhere, we remove this and default will be true. + use_overrides_configmap: false, + overrides_configmap_name: 'overrides', + overrides+:: { super_user:: { max_traces_per_user: 100000, diff --git a/operations/jsonnet/microservices/configmap.libsonnet b/operations/jsonnet/microservices/configmap.libsonnet index 4f6f0c2b194..6311aa3ce03 100644 --- a/operations/jsonnet/microservices/configmap.libsonnet +++ b/operations/jsonnet/microservices/configmap.libsonnet @@ -42,7 +42,7 @@ }, }, overrides: { - per_tenant_override_config: '/conf/overrides.yaml', + per_tenant_override_config: if ! $._config.use_overrides_configmap then '/conf/overrides.yaml' else '/overrides/overrides.yaml', }, memberlist: { abort_if_cluster_join_fails: false, @@ -99,38 +99,47 @@ tempo_query_frontend_config:: $.tempo_config{}, + overrides_configmap: if $._config.use_overrides_configmap then + configMap.new('overrides') + + configMap.withDataMixin({ + 'overrides.yaml': k.util.manifestYaml({ + overrides: $._config.overrides, + }), + }) + else {}, + tempo_distributor_configmap: configMap.new('tempo-distributor') + configMap.withData({ 'tempo.yaml': k.util.manifestYaml($.tempo_distributor_config), }) + - configMap.withDataMixin({ + if ! $._config.use_overrides_configmap then configMap.withDataMixin({ 'overrides.yaml': k.util.manifestYaml({ overrides: $._config.overrides, }), - }), + }) else {}, tempo_ingester_configmap: configMap.new('tempo-ingester') + configMap.withData({ 'tempo.yaml': k.util.manifestYaml($.tempo_ingester_config), }) + - configMap.withDataMixin({ + if ! $._config.use_overrides_configmap then configMap.withDataMixin({ 'overrides.yaml': k.util.manifestYaml({ overrides: $._config.overrides, }), - }), + }) else {}, tempo_compactor_configmap: configMap.new('tempo-compactor') + configMap.withData({ 'tempo.yaml': k.util.manifestYaml($.tempo_compactor_config), }) + - configMap.withDataMixin({ + if ! $._config.use_overrides_configmap then configMap.withDataMixin({ 'overrides.yaml': k.util.manifestYaml({ overrides: $._config.overrides, }), - }), + }) else {}, tempo_querier_configmap: configMap.new('tempo-querier') + diff --git a/operations/jsonnet/microservices/distributor.libsonnet b/operations/jsonnet/microservices/distributor.libsonnet index c521952940d..0c918af73a8 100644 --- a/operations/jsonnet/microservices/distributor.libsonnet +++ b/operations/jsonnet/microservices/distributor.libsonnet @@ -11,6 +11,7 @@ local target_name = 'distributor', local tempo_config_volume = 'tempo-conf', + local tempo_overrides_config_volume = 'overrides', tempo_distributor_container:: container.new(target_name, $._images.tempo) + @@ -24,7 +25,7 @@ ]) + container.withVolumeMounts([ volumeMount.new(tempo_config_volume, '/conf'), - ]) + + ] + if $._config.use_overrides_configmap then [volumeMount.new(tempo_overrides_config_volume, '/overrides')] else []) + $.util.withResources($._config.distributor.resources) + $.util.readinessProbe, @@ -46,7 +47,10 @@ }) + deployment.mixin.spec.template.spec.withVolumes([ volume.fromConfigMap(tempo_config_volume, $.tempo_distributor_configmap.metadata.name), - ]), + ] + if $._config.use_overrides_configmap then + [volume.fromConfigMap(tempo_overrides_config_volume, $._config.overrides_configmap_name)] + else []), + tempo_distributor_service: k.util.serviceFor($.tempo_distributor_deployment), diff --git a/operations/jsonnet/microservices/ingester.libsonnet b/operations/jsonnet/microservices/ingester.libsonnet index 8cf4c4ce97f..560b708350e 100644 --- a/operations/jsonnet/microservices/ingester.libsonnet +++ b/operations/jsonnet/microservices/ingester.libsonnet @@ -12,6 +12,7 @@ local target_name = 'ingester', local tempo_config_volume = 'tempo-conf', local tempo_data_volume = 'ingester-data', + local tempo_overrides_config_volume = 'overrides', tempo_ingester_pvc:: pvc.new() @@ -35,7 +36,7 @@ container.withVolumeMounts([ volumeMount.new(tempo_config_volume, '/conf'), volumeMount.new(tempo_data_volume, '/var/tempo'), - ]) + + ] + if $._config.use_overrides_configmap then [volumeMount.new(tempo_overrides_config_volume, '/overrides')] else []) + $.util.withResources($._config.ingester.resources) + $.util.readinessProbe, @@ -57,7 +58,10 @@ }) + statefulset.mixin.spec.template.spec.withVolumes([ volume.fromConfigMap(tempo_config_volume, $.tempo_ingester_configmap.metadata.name), - ]), + ]+ if $._config.use_overrides_configmap then + [volume.fromConfigMap(tempo_overrides_config_volume, $._config.overrides_configmap_name)] + else []), + tempo_ingester_service: k.util.serviceFor($.tempo_ingester_statefulset), From 049263561cad106881e7fa3a423ff38506fd5848 Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Thu, 19 Aug 2021 17:09:05 +0200 Subject: [PATCH 02/10] Some helpful comments --- operations/jsonnet/microservices/config.libsonnet | 1 + operations/jsonnet/microservices/configmap.libsonnet | 1 + 2 files changed, 2 insertions(+) diff --git a/operations/jsonnet/microservices/config.libsonnet b/operations/jsonnet/microservices/config.libsonnet index 0dcd6d66a53..1ed8ff1d853 100644 --- a/operations/jsonnet/microservices/config.libsonnet +++ b/operations/jsonnet/microservices/config.libsonnet @@ -97,6 +97,7 @@ backend: error 'Must specify a backend', // gcs|s3 bucket: error 'Must specify a bucket', + // `use_overrides_configmap` enables the tempo workloads read `overrides.yaml` from single configmap // Eventually, once we enable it everywhere, we remove this and default will be true. use_overrides_configmap: false, overrides_configmap_name: 'overrides', diff --git a/operations/jsonnet/microservices/configmap.libsonnet b/operations/jsonnet/microservices/configmap.libsonnet index 6311aa3ce03..5f2a292259c 100644 --- a/operations/jsonnet/microservices/configmap.libsonnet +++ b/operations/jsonnet/microservices/configmap.libsonnet @@ -99,6 +99,7 @@ tempo_query_frontend_config:: $.tempo_config{}, + // This will be the single configmap that stores `overrides.yaml`. overrides_configmap: if $._config.use_overrides_configmap then configMap.new('overrides') + configMap.withDataMixin({ From d6c759a94068bcbd7e97d114796e7a282b011405 Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Thu, 19 Aug 2021 17:15:14 +0200 Subject: [PATCH 03/10] Add changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66f75cc62f2..8cf6888b9e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ * [BUGFIX] Cortex upgrade to fix an issue where unhealthy compactors can't be forgotten [#878](https://github.com/grafana/tempo/pull/878) (@joe-elliott) * [ENHANCEMENT] Added "query blocks" cli option. [#876](https://github.com/grafana/tempo/pull/876) (@joe-elliott) * [ENHANCEMENT] Added traceid to `trace too large message`. [#888](https://github.com/grafana/tempo/pull/888) (@mritunjaysharma394) +* [ENHANCEMENT] Add support to tempo workloads to `overrides` from single configmap in microservice mode. [#896](https://github.com/grafana/tempo/pull/896) (@kavirajk) ## v1.1.0-rc.0 / 2021-08-11 From 152c3f757771812d9ae7ec320f040f9832c19d90 Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Thu, 19 Aug 2021 17:57:12 +0200 Subject: [PATCH 04/10] Remove conditional flag --- .../jsonnet/microservices/compactor.libsonnet | 8 +++--- .../jsonnet/microservices/config.libsonnet | 4 --- .../jsonnet/microservices/configmap.libsonnet | 28 ++++--------------- .../microservices/distributor.libsonnet | 9 +++--- .../jsonnet/microservices/ingester.libsonnet | 9 +++--- 5 files changed, 18 insertions(+), 40 deletions(-) diff --git a/operations/jsonnet/microservices/compactor.libsonnet b/operations/jsonnet/microservices/compactor.libsonnet index 2dbdb79c303..7ddcd5afb53 100644 --- a/operations/jsonnet/microservices/compactor.libsonnet +++ b/operations/jsonnet/microservices/compactor.libsonnet @@ -25,7 +25,8 @@ ]) + container.withVolumeMounts([ volumeMount.new(tempo_config_volume, '/conf'), - ] + if $._config.use_overrides_configmap then [volumeMount.new(tempo_overrides_config_volume, '/overrides')] else []) + + volumeMount.new(tempo_overrides_config_volume, '/overrides'), + ]) + $.util.withResources($._config.compactor.resources) + $.util.readinessProbe, @@ -43,9 +44,8 @@ }) + deployment.mixin.spec.template.spec.withVolumes([ volume.fromConfigMap(tempo_config_volume, $.tempo_compactor_configmap.metadata.name), - ] + if $._config.use_overrides_configmap then - [volume.fromConfigMap(tempo_overrides_config_volume, $._config.overrides_configmap_name)] - else []), + volume.fromConfigMap(tempo_overrides_config_volume, $._config.overrides_configmap_name), + ]), tempo_compactor_service: k.util.serviceFor($.tempo_compactor_deployment), diff --git a/operations/jsonnet/microservices/config.libsonnet b/operations/jsonnet/microservices/config.libsonnet index 1ed8ff1d853..dcc38567457 100644 --- a/operations/jsonnet/microservices/config.libsonnet +++ b/operations/jsonnet/microservices/config.libsonnet @@ -97,11 +97,7 @@ backend: error 'Must specify a backend', // gcs|s3 bucket: error 'Must specify a bucket', - // `use_overrides_configmap` enables the tempo workloads read `overrides.yaml` from single configmap - // Eventually, once we enable it everywhere, we remove this and default will be true. - use_overrides_configmap: false, overrides_configmap_name: 'overrides', - overrides+:: { super_user:: { max_traces_per_user: 100000, diff --git a/operations/jsonnet/microservices/configmap.libsonnet b/operations/jsonnet/microservices/configmap.libsonnet index 5f2a292259c..825ac8a42e8 100644 --- a/operations/jsonnet/microservices/configmap.libsonnet +++ b/operations/jsonnet/microservices/configmap.libsonnet @@ -42,7 +42,7 @@ }, }, overrides: { - per_tenant_override_config: if ! $._config.use_overrides_configmap then '/conf/overrides.yaml' else '/overrides/overrides.yaml', + per_tenant_override_config: '/overrides/overrides.yaml', }, memberlist: { abort_if_cluster_join_fails: false, @@ -100,47 +100,31 @@ tempo_query_frontend_config:: $.tempo_config{}, // This will be the single configmap that stores `overrides.yaml`. - overrides_configmap: if $._config.use_overrides_configmap then + overrides_configmap: configMap.new('overrides') + configMap.withDataMixin({ 'overrides.yaml': k.util.manifestYaml({ overrides: $._config.overrides, }), - }) - else {}, + }), tempo_distributor_configmap: configMap.new('tempo-distributor') + configMap.withData({ 'tempo.yaml': k.util.manifestYaml($.tempo_distributor_config), - }) + - if ! $._config.use_overrides_configmap then configMap.withDataMixin({ - 'overrides.yaml': k.util.manifestYaml({ - overrides: $._config.overrides, - }), - }) else {}, + }), tempo_ingester_configmap: configMap.new('tempo-ingester') + configMap.withData({ 'tempo.yaml': k.util.manifestYaml($.tempo_ingester_config), - }) + - if ! $._config.use_overrides_configmap then configMap.withDataMixin({ - 'overrides.yaml': k.util.manifestYaml({ - overrides: $._config.overrides, - }), - }) else {}, + }), tempo_compactor_configmap: configMap.new('tempo-compactor') + configMap.withData({ 'tempo.yaml': k.util.manifestYaml($.tempo_compactor_config), - }) + - if ! $._config.use_overrides_configmap then configMap.withDataMixin({ - 'overrides.yaml': k.util.manifestYaml({ - overrides: $._config.overrides, - }), - }) else {}, + }), tempo_querier_configmap: configMap.new('tempo-querier') + diff --git a/operations/jsonnet/microservices/distributor.libsonnet b/operations/jsonnet/microservices/distributor.libsonnet index 0c918af73a8..443b441077a 100644 --- a/operations/jsonnet/microservices/distributor.libsonnet +++ b/operations/jsonnet/microservices/distributor.libsonnet @@ -25,7 +25,8 @@ ]) + container.withVolumeMounts([ volumeMount.new(tempo_config_volume, '/conf'), - ] + if $._config.use_overrides_configmap then [volumeMount.new(tempo_overrides_config_volume, '/overrides')] else []) + + volumeMount.new(tempo_overrides_config_volume, '/overrides'), + ]) + $.util.withResources($._config.distributor.resources) + $.util.readinessProbe, @@ -47,10 +48,8 @@ }) + deployment.mixin.spec.template.spec.withVolumes([ volume.fromConfigMap(tempo_config_volume, $.tempo_distributor_configmap.metadata.name), - ] + if $._config.use_overrides_configmap then - [volume.fromConfigMap(tempo_overrides_config_volume, $._config.overrides_configmap_name)] - else []), - + volume.fromConfigMap(tempo_overrides_config_volume, $._config.overrides_configmap_name), + ]), tempo_distributor_service: k.util.serviceFor($.tempo_distributor_deployment), diff --git a/operations/jsonnet/microservices/ingester.libsonnet b/operations/jsonnet/microservices/ingester.libsonnet index 560b708350e..da009381fcf 100644 --- a/operations/jsonnet/microservices/ingester.libsonnet +++ b/operations/jsonnet/microservices/ingester.libsonnet @@ -36,7 +36,8 @@ container.withVolumeMounts([ volumeMount.new(tempo_config_volume, '/conf'), volumeMount.new(tempo_data_volume, '/var/tempo'), - ] + if $._config.use_overrides_configmap then [volumeMount.new(tempo_overrides_config_volume, '/overrides')] else []) + + volumeMount.new(tempo_overrides_config_volume, '/overrides'), + ]) + $.util.withResources($._config.ingester.resources) + $.util.readinessProbe, @@ -58,10 +59,8 @@ }) + statefulset.mixin.spec.template.spec.withVolumes([ volume.fromConfigMap(tempo_config_volume, $.tempo_ingester_configmap.metadata.name), - ]+ if $._config.use_overrides_configmap then - [volume.fromConfigMap(tempo_overrides_config_volume, $._config.overrides_configmap_name)] - else []), - + volume.fromConfigMap(tempo_overrides_config_volume, $._config.overrides_configmap_name), + ]), tempo_ingester_service: k.util.serviceFor($.tempo_ingester_statefulset), From 7931a59a353905107cc85019f161c95b2e324285 Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Thu, 19 Aug 2021 19:26:52 +0200 Subject: [PATCH 05/10] Update operations/jsonnet/microservices/configmap.libsonnet Co-authored-by: Koenraad Verheyden --- operations/jsonnet/microservices/configmap.libsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operations/jsonnet/microservices/configmap.libsonnet b/operations/jsonnet/microservices/configmap.libsonnet index 825ac8a42e8..3d27dd6387a 100644 --- a/operations/jsonnet/microservices/configmap.libsonnet +++ b/operations/jsonnet/microservices/configmap.libsonnet @@ -102,7 +102,7 @@ // This will be the single configmap that stores `overrides.yaml`. overrides_configmap: configMap.new('overrides') + - configMap.withDataMixin({ + configMap.withData({ 'overrides.yaml': k.util.manifestYaml({ overrides: $._config.overrides, }), From d0e4500fb653a221b1da3d20cf7f01386765954a Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Thu, 19 Aug 2021 19:27:22 +0200 Subject: [PATCH 06/10] Update operations/jsonnet/microservices/ingester.libsonnet Co-authored-by: Koenraad Verheyden --- operations/jsonnet/microservices/ingester.libsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operations/jsonnet/microservices/ingester.libsonnet b/operations/jsonnet/microservices/ingester.libsonnet index da009381fcf..dfe0179780c 100644 --- a/operations/jsonnet/microservices/ingester.libsonnet +++ b/operations/jsonnet/microservices/ingester.libsonnet @@ -59,7 +59,7 @@ }) + statefulset.mixin.spec.template.spec.withVolumes([ volume.fromConfigMap(tempo_config_volume, $.tempo_ingester_configmap.metadata.name), - volume.fromConfigMap(tempo_overrides_config_volume, $._config.overrides_configmap_name), + volume.fromConfigMap(tempo_overrides_config_volume, $.overrides_configmap.metadata.name), ]), tempo_ingester_service: From c8414bf11d0ee67e0a73064b6af0fd522e8807ee Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Thu, 19 Aug 2021 19:27:33 +0200 Subject: [PATCH 07/10] Update operations/jsonnet/microservices/distributor.libsonnet Co-authored-by: Koenraad Verheyden --- operations/jsonnet/microservices/distributor.libsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operations/jsonnet/microservices/distributor.libsonnet b/operations/jsonnet/microservices/distributor.libsonnet index 443b441077a..19c7e460eae 100644 --- a/operations/jsonnet/microservices/distributor.libsonnet +++ b/operations/jsonnet/microservices/distributor.libsonnet @@ -48,7 +48,7 @@ }) + deployment.mixin.spec.template.spec.withVolumes([ volume.fromConfigMap(tempo_config_volume, $.tempo_distributor_configmap.metadata.name), - volume.fromConfigMap(tempo_overrides_config_volume, $._config.overrides_configmap_name), + volume.fromConfigMap(tempo_overrides_config_volume, $.overrides_configmap.metadata.name), ]), tempo_distributor_service: From 5aafa8d82acc987af0779ad87a5ddc4dc91743c1 Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Thu, 19 Aug 2021 19:27:44 +0200 Subject: [PATCH 08/10] Update operations/jsonnet/microservices/configmap.libsonnet Co-authored-by: Koenraad Verheyden --- operations/jsonnet/microservices/configmap.libsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operations/jsonnet/microservices/configmap.libsonnet b/operations/jsonnet/microservices/configmap.libsonnet index 3d27dd6387a..6b389c2b194 100644 --- a/operations/jsonnet/microservices/configmap.libsonnet +++ b/operations/jsonnet/microservices/configmap.libsonnet @@ -101,7 +101,7 @@ // This will be the single configmap that stores `overrides.yaml`. overrides_configmap: - configMap.new('overrides') + + configMap.new($._config.overrides_configmap_name) + configMap.withData({ 'overrides.yaml': k.util.manifestYaml({ overrides: $._config.overrides, From a8347748b88b95a60b57f7cb8b45fa7a97ce334d Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Thu, 19 Aug 2021 19:28:02 +0200 Subject: [PATCH 09/10] Update operations/jsonnet/microservices/config.libsonnet Co-authored-by: Koenraad Verheyden --- operations/jsonnet/microservices/config.libsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operations/jsonnet/microservices/config.libsonnet b/operations/jsonnet/microservices/config.libsonnet index dcc38567457..8b587be436a 100644 --- a/operations/jsonnet/microservices/config.libsonnet +++ b/operations/jsonnet/microservices/config.libsonnet @@ -97,7 +97,7 @@ backend: error 'Must specify a backend', // gcs|s3 bucket: error 'Must specify a bucket', - overrides_configmap_name: 'overrides', + overrides_configmap_name: 'tempo-overrides', overrides+:: { super_user:: { max_traces_per_user: 100000, From 5e207c7470656f7e47b6155208392afdbd8d7d4c Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Thu, 19 Aug 2021 19:28:18 +0200 Subject: [PATCH 10/10] Update operations/jsonnet/microservices/compactor.libsonnet Co-authored-by: Koenraad Verheyden --- operations/jsonnet/microservices/compactor.libsonnet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operations/jsonnet/microservices/compactor.libsonnet b/operations/jsonnet/microservices/compactor.libsonnet index 7ddcd5afb53..722e077b182 100644 --- a/operations/jsonnet/microservices/compactor.libsonnet +++ b/operations/jsonnet/microservices/compactor.libsonnet @@ -44,7 +44,7 @@ }) + deployment.mixin.spec.template.spec.withVolumes([ volume.fromConfigMap(tempo_config_volume, $.tempo_compactor_configmap.metadata.name), - volume.fromConfigMap(tempo_overrides_config_volume, $._config.overrides_configmap_name), + volume.fromConfigMap(tempo_overrides_config_volume, $.overrides_configmap.metadata.name), ]), tempo_compactor_service: