From 7f76bc5ff03fafd3b2f8bad15378bf065ec68ef0 Mon Sep 17 00:00:00 2001 From: Tom Donohue Date: Fri, 3 May 2024 14:13:51 +0100 Subject: [PATCH 1/8] fix(helm): Fix GEL image tag, bucket name and proxy URLs Set the correct enterprise-logs image tag to 3.0.1 (without the 'v') Use the correct proxy URLs on the gateway when deploying in SSD mode Use the user's configured admin bucket name Clarify README to note support for distributed mode --- production/helm/loki/Chart.yaml | 2 +- production/helm/loki/README.md | 2 +- .../gateway/deployment-gateway-enterprise.yaml | 14 +++++++++++++- production/helm/loki/values.yaml | 4 ++-- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/production/helm/loki/Chart.yaml b/production/helm/loki/Chart.yaml index 60ea27903dbbe..3d97f5516d927 100644 --- a/production/helm/loki/Chart.yaml +++ b/production/helm/loki/Chart.yaml @@ -3,7 +3,7 @@ name: loki description: Helm chart for Grafana Loki in simple, scalable mode type: application appVersion: 3.0.0 -version: 6.5.0 +version: 6.5.1 home: https://grafana.github.io/helm-charts sources: - https://github.com/grafana/loki diff --git a/production/helm/loki/README.md b/production/helm/loki/README.md index b5b5961421864..a1544544623c7 100644 --- a/production/helm/loki/README.md +++ b/production/helm/loki/README.md @@ -2,7 +2,7 @@ ![Version: 6.5.0](https://img.shields.io/badge/Version-6.5.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.0.0](https://img.shields.io/badge/AppVersion-3.0.0-informational?style=flat-square) -Helm chart for Grafana Loki in simple, scalable mode +Helm chart for Grafana Loki and Grafana Enterprise Logs supporting both simple, scalable and distributed modes. ## Source Code diff --git a/production/helm/loki/templates/gateway/deployment-gateway-enterprise.yaml b/production/helm/loki/templates/gateway/deployment-gateway-enterprise.yaml index 4f7dccac911ed..de8ba11058eb1 100644 --- a/production/helm/loki/templates/gateway/deployment-gateway-enterprise.yaml +++ b/production/helm/loki/templates/gateway/deployment-gateway-enterprise.yaml @@ -1,3 +1,5 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- $isSimpleScalable := eq (include "loki.deployment.isScalable" .) "true" -}} {{- if and .Values.gateway.enabled .Values.enterprise.enabled .Values.enterprise.gelGateway }} apiVersion: apps/v1 kind: Deployment @@ -69,7 +71,7 @@ spec: - -admin.client.s3.secret-access-key={{ .Values.minio.secretKey }} - -admin.client.s3.insecure=true {{- end }} - {{- if .Values.enterpriseGateway.useDefaultProxyURLs }} + {{- if and $isDistributed .Values.enterpriseGateway.useDefaultProxyURLs }} - -gateway.proxy.default.url=http://{{ template "loki.fullname" . }}-admin-api.{{ .Release.Namespace }}.svc:3100 - -gateway.proxy.admin-api.url=http://{{ template "loki.fullname" . }}-admin-api.{{ .Release.Namespace }}.svc:3100 - -gateway.proxy.distributor.url=dns:///{{ template "loki.fullname" . }}-distributor-headless.{{ .Release.Namespace }}.svc:9095 @@ -77,6 +79,16 @@ spec: - -gateway.proxy.query-frontend.url=http://{{ template "loki.fullname" . }}-query-frontend.{{ .Release.Namespace }}.svc:3100 - -gateway.proxy.ruler.url=http://{{ template "loki.fullname" . }}-ruler.{{ .Release.Namespace }}.svc:3100 {{- end }} + {{- if and $isSimpleScalable .Values.enterpriseGateway.useDefaultProxyURLs }} + - -gateway.proxy.default.url=http://{{ template "enterprise-logs.adminApiFullname" . }}.{{ .Release.Namespace }}.svc:3100 + - -gateway.proxy.admin-api.url=http://{{ template "enterprise-logs.adminApiFullname" . }}.{{ .Release.Namespace }}.svc:3100 + - -gateway.proxy.compactor.url=http://{{ template "loki.backendFullname" . }}-headless.{{ .Release.Namespace }}.svc:3100 + - -gateway.proxy.distributor.url=dns:///{{ template "loki.writeFullname" . }}-headless.{{ .Release.Namespace }}.svc:9095 + - -gateway.proxy.ingester.url=http://{{ template "loki.writeFullname" . }}.{{ .Release.Namespace }}.svc:3100 + - -gateway.proxy.query-frontend.url=http://{{ template "loki.readFullname" . }}.{{ .Release.Namespace }}.svc:3100 + - -gateway.proxy.ruler.url=http://{{ template "loki.backendFullname" . }}-headless.{{ .Release.Namespace }}.svc:3100 + - -gateway.proxy.query-scheduler.url=http://{{ template "loki.backendFullname" . }}-headless.{{ .Release.Namespace }}.svc:3100 + {{- end }} {{- range $key, $value := .Values.enterpriseGateway.extraArgs }} - "-{{ $key }}={{ $value }}" {{- end }} diff --git a/production/helm/loki/values.yaml b/production/helm/loki/values.yaml index 36d42cddc09c5..b204c9dd1bc83 100644 --- a/production/helm/loki/values.yaml +++ b/production/helm/loki/values.yaml @@ -449,7 +449,7 @@ enterprise: # Enable enterprise features, license must be provided enabled: false # Default verion of GEL to deploy - version: v3.0.1 + version: 3.0.1 # -- Optional name of the GEL cluster, otherwise will use .Release.Name # The cluster name must match what is in your GEL license cluster_name: null @@ -480,7 +480,7 @@ enterprise: admin_client: storage: s3: - bucket_name: admin + bucket_name: {{ .Values.loki.storage.bucketNames.admin }} {{- end }} {{- end }} auth: From 0f68697bfa403565a96fbea643309373e3ec0e95 Mon Sep 17 00:00:00 2001 From: Tom Donohue Date: Tue, 7 May 2024 14:17:28 +0100 Subject: [PATCH 2/8] fix(helm): Separate admin storage config and add minio-only defaults --- production/helm/loki/values.yaml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/production/helm/loki/values.yaml b/production/helm/loki/values.yaml index b204c9dd1bc83..31620d3cf7711 100644 --- a/production/helm/loki/values.yaml +++ b/production/helm/loki/values.yaml @@ -473,16 +473,24 @@ enterprise: # make sure auth is set to `type: trust`, or that `auth_enabled` is set to `false`. adminApi: enabled: true + # -- Storage configuration for GEL Admin objects + adminClientConfig: | + {{ if .Values.minio.enabled }} + storage: + s3: + endpoint: {{ include "loki.minio" $ }} + bucket_name: admin + secret_access_key: {{ $.Values.minio.rootPassword }} + access_key_id: {{ $.Values.minio.rootUser }} + s3forcepathstyle: true + insecure: true + {{ end }} # enterprise specific sections of the config.yaml file config: | {{- if .Values.enterprise.adminApi.enabled }} - {{- if or .Values.minio.enabled (eq .Values.loki.storage.type "s3") (eq .Values.loki.storage.type "gcs") (eq .Values.loki.storage.type "azure") }} admin_client: - storage: - s3: - bucket_name: {{ .Values.loki.storage.bucketNames.admin }} - {{- end }} - {{- end }} + {{ tpl .Values.enterprise.adminClientConfig . | nindent 2 }} + {{ end }} auth: type: {{ .Values.enterprise.adminApi.enabled | ternary "enterprise" "trust" }} auth_enabled: {{ .Values.loki.auth_enabled }} From f16aec68f3b0a76d3dfee9da966182ac9a533b1c Mon Sep 17 00:00:00 2001 From: Vladyslav Diachenko Date: Wed, 8 May 2024 17:47:33 +0300 Subject: [PATCH 3/8] extracted admin api storage config to helper templates Signed-off-by: Vladyslav Diachenko --- production/helm/loki/templates/_helpers.tpl | 76 +++++++++++++++++++++ production/helm/loki/values.yaml | 17 +---- 2 files changed, 79 insertions(+), 14 deletions(-) diff --git a/production/helm/loki/templates/_helpers.tpl b/production/helm/loki/templates/_helpers.tpl index d88b86e40b960..6c3ddb677702b 100644 --- a/production/helm/loki/templates/_helpers.tpl +++ b/production/helm/loki/templates/_helpers.tpl @@ -454,6 +454,82 @@ ruler: {{- end }} {{- end }} +{{/* Enterprise Logs Admin API storage config */}} +{{- define "enterprise-logs.adminAPIStorageConfig" }} +storage: + {{- if .Values.minio.enabled }} + backend: "s3" + s3: + bucketnames: admin + {{- else if eq .Values.loki.storage.type "s3" -}} + {{- with .Values.loki.storage.s3 }} + backend: "s3" + s3: + bucketnames: {{ $.Values.loki.storage.bucketNames.admin }} + {{- end -}} + {{- else if eq .Values.loki.storage.type "gcs" -}} + {{- with .Values.loki.storage.gcs }} + backend: "gcs" + gcs: + bucket_name: {{ $.Values.loki.storage.bucketNames.admin }} + {{- end -}} + {{- else if eq .Values.loki.storage.type "azure" -}} + {{- with .Values.loki.storage.azure }} + backend: "azure" + azure: + account_name: {{ .accountName }} + {{- with .accountKey }} + account_key: {{ . }} + {{- end }} + {{- with .connectionString }} + connection_string: {{ . }} + {{- end }} + container_name: {{ $.Values.loki.storage.bucketNames.admin }} + {{- with .endpointSuffix }} + endpoint_suffix: {{ . }} + {{- end }} + {{- end -}} + {{- else if eq .Values.loki.storage.type "swift" -}} + {{- with .Values.loki.storage.swift }} + backend: "swift" + swift: + {{- with .auth_version }} + auth_version: {{ . }} + {{- end }} + auth_url: {{ .auth_url }} + {{- with .internal }} + internal: {{ . }} + {{- end }} + username: {{ .username }} + user_domain_name: {{ .user_domain_name }} + {{- with .user_domain_id }} + user_domain_id: {{ . }} + {{- end }} + {{- with .user_id }} + user_id: {{ . }} + {{- end }} + password: {{ .password }} + {{- with .domain_id }} + domain_id: {{ . }} + {{- end }} + domain_name: {{ .domain_name }} + project_id: {{ .project_id }} + project_name: {{ .project_name }} + project_domain_id: {{ .project_domain_id }} + project_domain_name: {{ .project_domain_name }} + region_name: {{ .region_name }} + container_name: {{ .container_name }} + max_retries: {{ .max_retries | default 3 }} + connect_timeout: {{ .connect_timeout | default "10s" }} + request_timeout: {{ .request_timeout | default "5s" }} + {{- end -}} + {{- else }} + backend: "filesystem" + filesystem: + dir: {{ .Values.loki.storage.filesystem.admin_api_directory }} + {{- end -}} +{{- end }} + {{/* Calculate the config from structured and unstructured text input */}} diff --git a/production/helm/loki/values.yaml b/production/helm/loki/values.yaml index 31620d3cf7711..67544c99be53a 100644 --- a/production/helm/loki/values.yaml +++ b/production/helm/loki/values.yaml @@ -365,6 +365,7 @@ loki: filesystem: chunks_directory: /var/loki/chunks rules_directory: /var/loki/rules + admin_api_directory: /var/loki/admin # -- Configure memcached as an external cache for chunk and results cache. Disabled by default # must enable and specify a host for each cache you would like to use. memcached: @@ -473,23 +474,11 @@ enterprise: # make sure auth is set to `type: trust`, or that `auth_enabled` is set to `false`. adminApi: enabled: true - # -- Storage configuration for GEL Admin objects - adminClientConfig: | - {{ if .Values.minio.enabled }} - storage: - s3: - endpoint: {{ include "loki.minio" $ }} - bucket_name: admin - secret_access_key: {{ $.Values.minio.rootPassword }} - access_key_id: {{ $.Values.minio.rootUser }} - s3forcepathstyle: true - insecure: true - {{ end }} # enterprise specific sections of the config.yaml file config: | {{- if .Values.enterprise.adminApi.enabled }} - admin_client: - {{ tpl .Values.enterprise.adminClientConfig . | nindent 2 }} + admin_client: + {{ include "enterprise-logs.adminAPIStorageConfig" . | nindent 2 }} {{ end }} auth: type: {{ .Values.enterprise.adminApi.enabled | ternary "enterprise" "trust" }} From 6af2666545b6feba523d6c1cff4558f4f75d83e2 Mon Sep 17 00:00:00 2001 From: Vladyslav Diachenko Date: Wed, 8 May 2024 17:48:59 +0300 Subject: [PATCH 4/8] regenerated docs Signed-off-by: Vladyslav Diachenko --- docs/sources/setup/install/helm/reference.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/sources/setup/install/helm/reference.md b/docs/sources/setup/install/helm/reference.md index de875439bf2b8..e5f9befd65a7f 100644 --- a/docs/sources/setup/install/helm/reference.md +++ b/docs/sources/setup/install/helm/reference.md @@ -2572,7 +2572,7 @@ null }, "canarySecret": null, "cluster_name": null, - "config": "{{- if .Values.enterprise.adminApi.enabled }}\n{{- if or .Values.minio.enabled (eq .Values.loki.storage.type \"s3\") (eq .Values.loki.storage.type \"gcs\") (eq .Values.loki.storage.type \"azure\") }}\nadmin_client:\n storage:\n s3:\n bucket_name: admin\n{{- end }}\n{{- end }}\nauth:\n type: {{ .Values.enterprise.adminApi.enabled | ternary \"enterprise\" \"trust\" }}\nauth_enabled: {{ .Values.loki.auth_enabled }}\ncluster_name: {{ include \"loki.clusterName\" . }}\nlicense:\n path: /etc/loki/license/license.jwt\n", + "config": "{{- if .Values.enterprise.adminApi.enabled }}\nadmin_client: \n {{ include \"enterprise-logs.adminAPIStorageConfig\" . | nindent 2 }}\n{{ end }}\nauth:\n type: {{ .Values.enterprise.adminApi.enabled | ternary \"enterprise\" \"trust\" }}\nauth_enabled: {{ .Values.loki.auth_enabled }}\ncluster_name: {{ include \"loki.clusterName\" . }}\nlicense:\n path: /etc/loki/license/license.jwt\n", "enabled": false, "externalConfigName": "", "externalLicenseName": null, @@ -2630,7 +2630,7 @@ null "tolerations": [] }, "useExternalLicense": false, - "version": "v3.0.1" + "version": "3.0.1" } @@ -5594,6 +5594,7 @@ null "userAssignedId": null }, "filesystem": { + "admin_api_directory": "/var/loki/admin", "chunks_directory": "/var/loki/chunks", "rules_directory": "/var/loki/rules" }, From 80159d7a0c9320733a9a61f9ee27e2831bd5f2af Mon Sep 17 00:00:00 2001 From: Vladyslav Diachenko Date: Wed, 8 May 2024 17:52:55 +0300 Subject: [PATCH 5/8] regenerated docs Signed-off-by: Vladyslav Diachenko --- production/helm/loki/Chart.yaml | 2 +- production/helm/loki/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/production/helm/loki/Chart.yaml b/production/helm/loki/Chart.yaml index 3d97f5516d927..595c4b5710cd5 100644 --- a/production/helm/loki/Chart.yaml +++ b/production/helm/loki/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: loki -description: Helm chart for Grafana Loki in simple, scalable mode +description: Helm chart for Grafana Loki and Grafana Enterprise Logs supporting both simple, scalable and distributed modes. type: application appVersion: 3.0.0 version: 6.5.1 diff --git a/production/helm/loki/README.md b/production/helm/loki/README.md index a1544544623c7..9e0dce69d8385 100644 --- a/production/helm/loki/README.md +++ b/production/helm/loki/README.md @@ -1,6 +1,6 @@ # loki -![Version: 6.5.0](https://img.shields.io/badge/Version-6.5.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.0.0](https://img.shields.io/badge/AppVersion-3.0.0-informational?style=flat-square) +![Version: 6.5.1](https://img.shields.io/badge/Version-6.5.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.0.0](https://img.shields.io/badge/AppVersion-3.0.0-informational?style=flat-square) Helm chart for Grafana Loki and Grafana Enterprise Logs supporting both simple, scalable and distributed modes. From fc952cad10d280efb51cd3806aa537a9b5d79f5b Mon Sep 17 00:00:00 2001 From: Vladyslav Diachenko Date: Wed, 8 May 2024 17:57:54 +0300 Subject: [PATCH 6/8] removed trailing spaces Signed-off-by: Vladyslav Diachenko --- production/helm/loki/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production/helm/loki/values.yaml b/production/helm/loki/values.yaml index 67544c99be53a..11e579e6c885c 100644 --- a/production/helm/loki/values.yaml +++ b/production/helm/loki/values.yaml @@ -477,7 +477,7 @@ enterprise: # enterprise specific sections of the config.yaml file config: | {{- if .Values.enterprise.adminApi.enabled }} - admin_client: + admin_client: {{ include "enterprise-logs.adminAPIStorageConfig" . | nindent 2 }} {{ end }} auth: From fc92f9ab0bafe46cb56098c9d2a737d53011b4d0 Mon Sep 17 00:00:00 2001 From: Vladyslav Diachenko Date: Wed, 8 May 2024 18:00:22 +0300 Subject: [PATCH 7/8] regenerated docs Signed-off-by: Vladyslav Diachenko --- docs/sources/setup/install/helm/reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/setup/install/helm/reference.md b/docs/sources/setup/install/helm/reference.md index e5f9befd65a7f..2a538a7516175 100644 --- a/docs/sources/setup/install/helm/reference.md +++ b/docs/sources/setup/install/helm/reference.md @@ -2572,7 +2572,7 @@ null }, "canarySecret": null, "cluster_name": null, - "config": "{{- if .Values.enterprise.adminApi.enabled }}\nadmin_client: \n {{ include \"enterprise-logs.adminAPIStorageConfig\" . | nindent 2 }}\n{{ end }}\nauth:\n type: {{ .Values.enterprise.adminApi.enabled | ternary \"enterprise\" \"trust\" }}\nauth_enabled: {{ .Values.loki.auth_enabled }}\ncluster_name: {{ include \"loki.clusterName\" . }}\nlicense:\n path: /etc/loki/license/license.jwt\n", + "config": "{{- if .Values.enterprise.adminApi.enabled }}\nadmin_client:\n {{ include \"enterprise-logs.adminAPIStorageConfig\" . | nindent 2 }}\n{{ end }}\nauth:\n type: {{ .Values.enterprise.adminApi.enabled | ternary \"enterprise\" \"trust\" }}\nauth_enabled: {{ .Values.loki.auth_enabled }}\ncluster_name: {{ include \"loki.clusterName\" . }}\nlicense:\n path: /etc/loki/license/license.jwt\n", "enabled": false, "externalConfigName": "", "externalLicenseName": null, From 16f62a6339c199dcd01f83f9131bad4ad85acb20 Mon Sep 17 00:00:00 2001 From: Vladyslav Diachenko Date: Wed, 8 May 2024 18:04:52 +0300 Subject: [PATCH 8/8] fixed bucket_name property name Signed-off-by: Vladyslav Diachenko --- production/helm/loki/templates/_helpers.tpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/production/helm/loki/templates/_helpers.tpl b/production/helm/loki/templates/_helpers.tpl index 6c3ddb677702b..c71c6c23aff88 100644 --- a/production/helm/loki/templates/_helpers.tpl +++ b/production/helm/loki/templates/_helpers.tpl @@ -460,12 +460,12 @@ storage: {{- if .Values.minio.enabled }} backend: "s3" s3: - bucketnames: admin + bucket_name: admin {{- else if eq .Values.loki.storage.type "s3" -}} {{- with .Values.loki.storage.s3 }} backend: "s3" s3: - bucketnames: {{ $.Values.loki.storage.bucketNames.admin }} + bucket_name: {{ $.Values.loki.storage.bucketNames.admin }} {{- end -}} {{- else if eq .Values.loki.storage.type "gcs" -}} {{- with .Values.loki.storage.gcs }}