From ecfb4cfd979e1404cf474a16bba1e860fc88097b Mon Sep 17 00:00:00 2001 From: metron2 Date: Thu, 30 May 2024 14:03:26 -0400 Subject: [PATCH 1/4] clean-up kubectl since graylog detects the masternode now, supply data_dir for graylog 6 Signed-off-by: metron2 --- charts/graylog/Chart.yaml | 2 +- charts/graylog/templates/configmap.yaml | 31 +---------------------- charts/graylog/templates/statefulset.yaml | 27 ++++++-------------- charts/graylog/values.yaml | 7 ----- 4 files changed, 10 insertions(+), 57 deletions(-) diff --git a/charts/graylog/Chart.yaml b/charts/graylog/Chart.yaml index a51b84d..0548309 100755 --- a/charts/graylog/Chart.yaml +++ b/charts/graylog/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: graylog home: https://www.graylog.org -version: 2.3.7 +version: 2.3.8 appVersion: 5.2.6 description: Graylog is the centralized log management solution built to open standards for capturing, storing, and enabling real-time analysis of terabytes diff --git a/charts/graylog/templates/configmap.yaml b/charts/graylog/templates/configmap.yaml index acbb6b7..5a6c434 100644 --- a/charts/graylog/templates/configmap.yaml +++ b/charts/graylog/templates/configmap.yaml @@ -149,6 +149,7 @@ data: {{- end }} {{- if .Values.graylog.trustedProxies }} trusted_proxies = {{.Values.graylog.trustedProxies}} + data_dir = /usr/share/graylog/data {{- end }} {{- if .Values.graylog.config }} {{ .Values.graylog.config | indent 4 }} @@ -162,36 +163,6 @@ data: find ${GRAYLOG_HOME}/plugins-default/ -type f -exec cp {} ${GRAYLOG_PLUGIN_DIR} \; # Looking for Master IP retry=1 - for i in {0..2} - do - MASTER_IP=`/k8s/kubectl --namespace {{ .Release.Namespace }} get pod -o jsonpath='{range .items[*]}{.metadata.name} {.status.podIP}{"\n"}{end}' -l graylog-role=master --field-selector=status.phase=Running|awk '{print $2}'` - SELF_IP=`/k8s/kubectl --namespace {{ .Release.Namespace }} get pod $HOSTNAME -o jsonpath='{.status.podIP}'` - echo "Current master is $MASTER_IP" - echo "Self IP is $SELF_IP" - retry=$((retry+1)) - [[ ! -z "$MASTER_IP" ]] && break - echo "[Try ${retry}/3] Waiting for master node..." - sleep 2 - done - if [[ -z "$MASTER_IP" ]]; then - echo "Launching $HOSTNAME as master" - export GRAYLOG_IS_MASTER="true" - export GRAYLOG_IS_LEADER="true" - /k8s/kubectl --namespace {{ .Release.Namespace }} label --overwrite pod $HOSTNAME graylog-role="master" - else - # When container was recreated or restart, MASTER_IP == SELF_IP, running as master and no need to change label graylog-role="master" - if [ "$SELF_IP" == "$MASTER_IP" ];then - echo "Launching $HOSTNAME as master" - export GRAYLOG_IS_MASTER="true" - export GRAYLOG_IS_LEADER="true" - else - # MASTER_IP != SELF_IP, running as coordinating - echo "Launching $HOSTNAME as coordinating" - export GRAYLOG_IS_MASTER="false" - export GRAYLOG_IS_LEADER="false" - /k8s/kubectl --namespace {{ .Release.Namespace }} label --overwrite pod $HOSTNAME graylog-role="coordinating" - fi - fi # Download plugins {{- if .Values.graylog.plugins.proxy.enabled }} export https_proxy={{ .Values.graylog.plugins.proxy.host }} diff --git a/charts/graylog/templates/statefulset.yaml b/charts/graylog/templates/statefulset.yaml index 2ef3885..549ec38 100644 --- a/charts/graylog/templates/statefulset.yaml +++ b/charts/graylog/templates/statefulset.yaml @@ -74,14 +74,6 @@ spec: rm -rf /usr/share/graylog/data/journal/messagejournal-0 rm -rf /usr/share/graylog/data/journal/recovery-point-offset-checkpoint {{- end }} - {{- if .Values.graylog.init.kubectlLocation }} - wget {{ .Values.graylog.init.kubectlLocation }} -O /k8s/kubectl - {{- else }} - {{.Capabilities.KubeVersion}} - wget https://dl.k8s.io/release/{{ .Values.graylog.init.kubectlVersion | default (regexReplaceAll "-.+" .Capabilities.KubeVersion.Version "") }}/bin/linux/amd64/kubectl -O /k8s/kubectl - {{- end }} - chmod +x /k8s/kubectl - GRAYLOG_HOME=/usr/share/graylog chown -R 1100:1100 ${GRAYLOG_HOME}/data/ securityContext: @@ -94,8 +86,6 @@ spec: volumeMounts: - name: journal mountPath: /usr/share/graylog/data/journal - - name: kubectl - mountPath: /k8s {{- if .Values.graylog.init.resources }} resources: {{ toYaml .Values.graylog.init.resources | indent 12 }} @@ -110,6 +100,12 @@ spec: command: - /entrypoint.sh env: + # Kubernetes Auto Master Selection + # https://go2docs.graylog.org/5-0/downloading_and_installing_graylog/docker_installation.htm#KubernetesAutomaticMasterSelection + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name - name: GRAYLOG_SERVER_JAVA_OPTS {{- $javaOpts := .Values.graylog.javaOpts }} {{- if .Values.graylog.heapSize }} @@ -231,13 +227,10 @@ spec: - name: files mountPath: /etc/graylog/server {{- end }} - - name: kubectl - mountPath: /k8s + {{- if .Values.graylog.extraVolumeMounts }} {{ toYaml .Values.graylog.extraVolumeMounts | nindent 12 }} {{- end }} - {{ $graylogVersion := .Values.graylog.image.tag | default .Chart.AppVersion }} - {{- if semverCompare "< 4.2.0-0" ( $graylogVersion ) }} lifecycle: preStop: exec: @@ -245,12 +238,10 @@ spec: - bash - -ec - | - ROOT_PASSWORD=`/k8s/kubectl get secret {{ template "graylog.fullname" . }} -o "jsonpath={.data['graylog-password-secret']}" | base64 -d` curl {{ if .Values.graylog.tls.enabled }}-k{{ end }} -XPOST -sS \ - -u "{{ .Values.graylog.rootUsername }}:${ROOT_PASSWORD}" \ + -u "{{ .Values.graylog.rootUsername }}:${GRAYLOG_PASSWORD_SECRET}" \ -H "X-Requested-By: {{ template "graylog.fullname" . }}" \ {{ template "graylog.formatUrl" (list . "localhost:9000/api/system/shutdown/shutdown") }} - {{- end }} {{- if .Values.graylog.sidecarContainers }} {{ toYaml .Values.graylog.sidecarContainers | nindent 8 }} {{- end }} @@ -278,8 +269,6 @@ spec: configMap: name: {{ template "graylog.fullname" . }}-files {{- end }} - - name: kubectl - emptyDir: {} {{- if .Values.graylog.extraVolumes }} {{ toYaml .Values.graylog.extraVolumes | nindent 8 }} {{- end }} diff --git a/charts/graylog/values.yaml b/charts/graylog/values.yaml index 2dc132a..7520815 100644 --- a/charts/graylog/values.yaml +++ b/charts/graylog/values.yaml @@ -502,13 +502,6 @@ graylog: repository: "alpine" pullPolicy: "IfNotPresent" - ## Set kubectl location to download and use on init-container. If the value is not set, the https://dl.k8s.io/release/ will be used. - ## - kubectlLocation: "" - ## Set kubectl command version to download from https://dl.k8s.io/release/. If the value is not set, default value is .Capabilities.KubeVersion.Version - ## - # kubectlVersion: "v1.20" - # Additional environment variables to be added to Graylog initContainer env: {} From c2a7864fda13e24523bb4c83018a5e3586f4922a Mon Sep 17 00:00:00 2001 From: metron2 Date: Thu, 30 May 2024 16:27:39 -0400 Subject: [PATCH 2/4] fix shutdown call, no longer needed on graylog 5+ Signed-off-by: metron2 --- charts/graylog/templates/statefulset.yaml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/charts/graylog/templates/statefulset.yaml b/charts/graylog/templates/statefulset.yaml index 549ec38..814ebb6 100644 --- a/charts/graylog/templates/statefulset.yaml +++ b/charts/graylog/templates/statefulset.yaml @@ -231,17 +231,6 @@ spec: {{- if .Values.graylog.extraVolumeMounts }} {{ toYaml .Values.graylog.extraVolumeMounts | nindent 12 }} {{- end }} - lifecycle: - preStop: - exec: - command: - - bash - - -ec - - | - curl {{ if .Values.graylog.tls.enabled }}-k{{ end }} -XPOST -sS \ - -u "{{ .Values.graylog.rootUsername }}:${GRAYLOG_PASSWORD_SECRET}" \ - -H "X-Requested-By: {{ template "graylog.fullname" . }}" \ - {{ template "graylog.formatUrl" (list . "localhost:9000/api/system/shutdown/shutdown") }} {{- if .Values.graylog.sidecarContainers }} {{ toYaml .Values.graylog.sidecarContainers | nindent 8 }} {{- end }} From fb79ffea955c8459f4c2df8a07e5c8f244c56023 Mon Sep 17 00:00:00 2001 From: metron2 Date: Wed, 5 Jun 2024 10:25:59 -0400 Subject: [PATCH 3/4] chart needs to export the same variable as the image entrypoint in order for leader detection to work Signed-off-by: metron2 --- charts/graylog/templates/configmap.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/charts/graylog/templates/configmap.yaml b/charts/graylog/templates/configmap.yaml index 5a6c434..e54db84 100644 --- a/charts/graylog/templates/configmap.yaml +++ b/charts/graylog/templates/configmap.yaml @@ -141,7 +141,9 @@ data: transport_email_web_interface_url = {{ $externalUri }} {{- end }} content_packs_dir = /usr/share/graylog/data/contentpacks + {{- if semverCompare "~5" ( $graylogVersion ) }} content_packs_auto_load = grok-patterns.json + {{- end}} proxied_requests_thread_pool_size = 32 {{- if .Values.graylog.metrics.enabled }} prometheus_exporter_enabled = true @@ -199,6 +201,15 @@ data: {{- end }} # Start Graylog echo "Starting graylog" + if [[ ! -z "${POD_NAME}" ]] + then + if echo "${POD_NAME}" | grep "\\-0$" >/dev/null + then + export GRAYLOG_IS_LEADER="true" + else + export GRAYLOG_IS_LEADER="false" + fi + fi # Original docker-entrypoint.sh in Graylog Docker will error while executing since you can't chown readonly files in `config` # exec /docker-entrypoint.sh graylog {{- if or (.Values.graylog.opensearch.uriSecretKey) (.Values.graylog.mongodb.uriSecretKey) }} @@ -209,6 +220,7 @@ data: export GRAYLOG_ELASTICSEARCH_VERSION={{ .Values.graylog.opensearch.version }} {{- end }} echo "Graylog Home ${GRAYLOG_HOME}" + echo "Graylog Leader ${GRAYLOG_IS_LEADER}" echo "Graylog Plugin Dir ${GRAYLOG_PLUGIN_DIR}" echo "Graylog Elasticsearch Version ${GRAYLOG_ELASTICSEARCH_VERSION}" "${JAVA_HOME}/bin/java" \ From 29979fa998842e0a9bea56d75c95e3177075c456 Mon Sep 17 00:00:00 2001 From: metron2 Date: Wed, 5 Jun 2024 13:13:17 -0400 Subject: [PATCH 4/4] codereview feedback Signed-off-by: metron2 --- charts/graylog/templates/configmap.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/charts/graylog/templates/configmap.yaml b/charts/graylog/templates/configmap.yaml index e54db84..ace61e6 100644 --- a/charts/graylog/templates/configmap.yaml +++ b/charts/graylog/templates/configmap.yaml @@ -163,8 +163,6 @@ data: export GRAYLOG_PLUGIN_DIR=${GRAYLOG_HOME}/plugin # Graylog 4.0.2 images move plugin dir to `plugins-default` find ${GRAYLOG_HOME}/plugins-default/ -type f -exec cp {} ${GRAYLOG_PLUGIN_DIR} \; - # Looking for Master IP - retry=1 # Download plugins {{- if .Values.graylog.plugins.proxy.enabled }} export https_proxy={{ .Values.graylog.plugins.proxy.host }}