[bitnami/airflow] Add support for extending default configuration (#3…

…0525)
bitnami · Nov 19, 2024 · e84395d · e84395d
1 parent d7b125b
commit e84395d
Show file tree

Hide file tree

Showing 8 changed files with 114 additions and 65 deletions.
diff --git a/.vib/airflow/goss/goss.yaml b/.vib/airflow/goss/goss.yaml
@@ -6,11 +6,22 @@ command:
     exec: airflow dags trigger example_bash_operator
     exit-status: 0
     timeout: 40000
+  {{- $uid := .Vars.web.containerSecurityContext.runAsUser }}
+  {{- $gid := .Vars.web.podSecurityContext.fsGroup }}
+  check-user-info:
+    # The UID and GID should always be either the one specified as vars (always a bigger number that the default)
+    # or the one randomly defined by openshift (larger values). Otherwise, the chart is still using the default value.
+    exec: if [ $(id -u) -lt {{ $uid }} ] || [ $(id -G | awk '{print $2}') -lt {{ $gid }} ]; then exit 1; fi
+    exit-status: 0
 file:
   /opt/bitnami/airflow/airflow.cfg:
     mode: "0644"
     filetype: file
     exists: true
+    contents:
+      - /statsd_port.*{{ .Vars.metrics.service.ports.ingest }}/
+      # Ensure overwritten configuration is applied
+      - /colored_console_log.*True/
   /opt/bitnami/airflow/webserver_config.py:
     mode: "0644"
     filetype: file
@@ -26,11 +37,3 @@ http:
     status: 200
     timeout: 30000
   {{- end }}
-command:
-  {{- $uid := .Vars.web.containerSecurityContext.runAsUser }}
-  {{- $gid := .Vars.web.podSecurityContext.fsGroup }}
-  check-user-info:
-    # The UID and GID should always be either the one specified as vars (always a bigger number that the default)
-    # or the one randomly defined by openshift (larger values). Otherwise, the chart is still using the default value.
-    exec: if [ $(id -u) -lt {{ $uid }} ] || [ $(id -G | awk '{print $2}') -lt {{ $gid }} ]; then exit 1; fi
-    exit-status: 0
diff --git a/.vib/airflow/runtime-parameters.yaml b/.vib/airflow/runtime-parameters.yaml
@@ -1,5 +1,8 @@
 auth:
   password: ComplicatedPassword123!4
+overrideConfiguration:
+  logging:
+    colored_console_log: 'True'
 loadExamples: true
 web:
   containerPorts:
@@ -27,4 +30,5 @@ metrics:
   enabled: true
   service:
     ports:
+      ingest: 9125
       metrics: 9102
diff --git a/bitnami/airflow/CHANGELOG.md b/bitnami/airflow/CHANGELOG.md
@@ -1,8 +1,12 @@
 # Changelog
 
+## 22.2.0 (2024-11-19)
+
+* [bitnami/airflow] Add support for extending default configuration ([#30525](https://github.com/bitnami/charts/pull/30525))
+
 ## 22.1.0 (2024-11-18)
 
-* [bitnami/airflow] feat: add support for custom local settings ([#30496](https://github.com/bitnami/charts/pull/30496))
+* [bitnami/airflow] feat: add support for custom local settings (#30496) ([6c3060e](https://github.com/bitnami/charts/commit/6c3060e024a7fdcfe5b4eb07ec699cf425858e9f)), closes [#30496](https://github.com/bitnami/charts/issues/30496)
 
 ## <small>22.0.1 (2024-11-18)</small>
 

diff --git a/bitnami/airflow/Chart.yaml b/bitnami/airflow/Chart.yaml
@@ -39,4 +39,4 @@ maintainers:
 name: airflow
 sources:
 - https://github.com/bitnami/charts/tree/main/bitnami/airflow
-version: 22.1.0
+version: 22.2.0
diff --git a/bitnami/airflow/README.md b/bitnami/airflow/README.md
@@ -91,6 +91,27 @@ executor=SequentialExecutor
 redis.enabled=false
 ```
 
+### Airflow configuration file
+
+By default, the Airflow configuration file is auto-generated based on the chart parameters you set. For instance, the `executor` parameter will be used to set the `executor` class under the `[core]` section.
+
+You can also provider your own configuration by setting the `configuration` parameter. This parameter expects the configuration as a sections/keys/values dictionary on YAML format, then it's converted to .cfg format by the chart. For instance, using a configuration like the one below...
+
+```yaml
+configuration:
+  core:
+    dags_folder: "/opt/bitnami/airflow/dags"
+```
+
+... the chart will translate it to the following configuration file:
+
+```ini
+[core]
+dags_folder = "/opt/bitnami/airflow/dags"
+```
+
+As an alternative to providing the whole configuration, you can also extend the default configuration using the `overrideConfiguration` parameter. The values set in this parameter, which also expects YAML format, will be merged with the default configuration or those set in the `configuration` parameter taking precedence.
+
 ### Scaling worker pods
 
 Sometime when using large workloads a fixed number of worker pods may make task to take a long time to be executed. This chart provide two ways for scaling worker pods.
@@ -374,7 +395,8 @@ The Bitnami Airflow chart relies on the PostgreSQL chart persistence. This means
 | `auth.existingSecret`                                                                         | Name of an existing secret to use for Airflow credentials                                                                                                                                                                                                                                                                            | `""`                      |
 | `executor`                                                                                    | Airflow executor. Allowed values: `SequentialExecutor`, `LocalExecutor`, `CeleryExecutor`, `KubernetesExecutor`, `CeleryKubernetesExecutor` and `LocalKubernetesExecutor`                                                                                                                                                            | `CeleryExecutor`          |
 | `loadExamples`                                                                                | Switch to load some Airflow examples                                                                                                                                                                                                                                                                                                 | `false`                   |
-| `configuration`                                                                               | Specify content for Airflow config file (auto-generated based on other env. vars otherwise)                                                                                                                                                                                                                                          | `""`                      |
+| `configuration`                                                                               | Specify content for Airflow config file (auto-generated based on other parameters otherwise)                                                                                                                                                                                                                                         | `{}`                      |
+| `overrideConfiguration`                                                                       | Airflow common configuration override. Values defined here takes precedence over the ones defined at `configuration`                                                                                                                                                                                                                 | `{}`                      |
 | `localSettings`                                                                               | Specify content for Airflow local settings (airflow_local_settings.py)                                                                                                                                                                                                                                                               | `""`                      |
 | `existingConfigmap`                                                                           | Name of an existing ConfigMap with the Airflow config file and, optionally, the local settings file                                                                                                                                                                                                                                  | `""`                      |
 | `dags.enabled`                                                                                | Enable loading DAGs from a ConfigMap or Git repositories                                                                                                                                                                                                                                                                             | `false`                   |
@@ -1243,6 +1265,10 @@ Find more information about how to deal with common errors related to Bitnami's
 
 ## Upgrading
 
+### To 22.2.0
+
+This minor version no longer expects custom Airflow configuration (set via the `configuration` parameter) to be provided as a string. Instead, it expects a dictionary with the configuration sections/keys/values. Find more info in the [section](#airflow-configuration-file) above.
+
 ### To 22.0.0
 
 This major version replaces exposing Prometheus metrics using the [Airflow prometheus exporter](https://github.com/PBWebMedia/airflow-prometheus-exporter), that exposes metrics based on the data retrieved from the database, by configuring Airflow components to send StatsD metrics to the [StatsD exporter](https://github.com/prometheus/statsd_exporter) that transforms them into Prometheus metrics. Find more information about this approach in the [Apache Airflow official documentation](https://airflow.apache.org/docs/apache-airflow/stable/administration-and-deployment/logging-monitoring/metrics.html#setup-statsd).

diff --git a/bitnami/airflow/templates/config/configmap-pod-template.yaml b/bitnami/airflow/templates/config/configmap-pod-template.yaml
@@ -219,12 +219,6 @@ data:
             {{- if .Values.worker.extraVolumeMounts }}
             {{- include "common.tplvalues.render" (dict "value" .Values.worker.extraVolumeMounts "context" $) | nindent 12 }}
             {{- end }}
-        {{- if .Values.dags.enabled }}
-        {{- include "airflow.defaultSidecars.syncDAGs" . | nindent 8 }}
-        {{- end }}
-        {{- if .Values.plugins.enabled }}
-        {{- include "airflow.defaultSidecars.syncPlugins" . | nindent 8 }}
-        {{- end }}
         {{- if .Values.sidecars }}
         {{- include "common.tplvalues.render" (dict "value" .Values.sidecars "context" $) | trim | nindent 8 }}
         {{- end }}

diff --git a/bitnami/airflow/templates/config/configmap.yaml b/bitnami/airflow/templates/config/configmap.yaml
@@ -3,6 +3,48 @@ Copyright Broadcom, Inc. All Rights Reserved.
 SPDX-License-Identifier: APACHE-2.0
 */}}
 
+{{/*
+Return the Airflow common configuration.
+ref: https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html
+*/}}
+{{- define "airflow.configuration" -}}
+{{- if .Values.configuration }}
+{{- include "common.tplvalues.render" (dict "value" .Values.configuration "context" .) }}
+{{- else }}
+core:
+  load_examples: {{ ternary "True" "False" .Values.loadExamples | squote }}
+  executor: {{ .Values.executor | quote }}
+logging:
+  colored_console_log: 'False'
+metrics:
+  statsd_on: {{ ternary "True" "False" (.Values.metrics.enabled) | squote }}
+  statsd_port: {{ .Values.metrics.service.ports.ingest | quote }}
+  statsd_prefix: airflow
+  statsd_host: {{ include "airflow.metrics.fullname" . | quote }}
+scheduler:
+  standalone_dag_processor: {{ ternary "True" "False" .Values.dagProcessor.enabled | squote }}
+triggerer:
+  default_capacity: {{ .Values.triggerer.defaultCapacity | quote }}
+webserver:
+  base_url: {{ include "airflow.baseUrl" . | quote }}
+  enable_proxy_fix: {{ ternary "True" "False" (and .Values.ingress.enabled .Values.ingress.tls) | squote }}
+  web_server_port: {{ .Values.web.containerPorts.http | quote }}
+  {{- if .Values.web.tls.enabled }}
+  web_server_ssl_cert: "/opt/bitnami/airflow/certs/tls.crt"
+  web_server_ssl_key: "/opt/bitnami/airflow/certs/tls.key"
+  {{- end }}
+{{- if contains "KubernetesExecutor" .Values.executor }}
+kubernetes_executor:
+  namespace: {{ include "common.names.namespace" . | quote }}
+  worker_container_repository: {{ printf "%s/%s" .Values.image.registry .Values.image.repository | quote }}
+  worker_container_tag: {{ .Values.image.tag | quote }}
+  delete_worker_pods: 'True'
+  delete_worker_pods_on_failure: 'True'
+  pod_template_file: "/opt/bitnami/airflow/config/pod_template.yaml"
+{{- end }}
+{{- end }}
+{{- end }}
+
 {{- if not .Values.existingConfigmap }}
 apiVersion: v1
 kind: ConfigMap
@@ -14,48 +56,18 @@ metadata:
   annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }}
   {{- end }}
 data:
-  airflow.cfg: |-
-  {{- if .Values.configuration }}
-    {{- include "common.tplvalues.render" (dict "value" .Values.configuration "context" .) | nindent 4 }}
-  {{- else }}
-    [core]
-    load_examples={{ ternary "True" "False" .Values.loadExamples }}
-    executor={{ .Values.executor }}
-
-    [logging]
-    colored_console_log=False
-
-    [metrics]
-    statsd_on={{ ternary "True" "False" (.Values.metrics.enabled) }}
-    statsd_port={{ .Values.metrics.service.ports.ingest }}
-    statsd_prefix=airflow
-    statsd_host={{ include "airflow.metrics.fullname" . }}
-
-    [scheduler]
-    standalone_dag_processor={{ ternary "True" "False" .Values.dagProcessor.enabled }}
-
-    [triggerer]
-    default_capacity={{ .Values.triggerer.defaultCapacity }}
-
-    [webserver]
-    base_url={{ include "airflow.baseUrl" . }}
-    enable_proxy_fix={{ ternary "True" "False" (and .Values.ingress.enabled .Values.ingress.tls) }}
-    web_server_port={{ .Values.web.containerPorts.http }}
-  {{- if .Values.web.tls.enabled }}
-    web_server_ssl_cert="/opt/bitnami/airflow/certs/tls.crt"
-    web_server_ssl_key="/opt/bitnami/airflow/certs/tls.key"
-  {{- end }}
-
-  {{- if contains "KubernetesExecutor" .Values.executor }}
-    [kubernetes_executor]
-    namespace={{ include "common.names.namespace" . }}
-    worker_container_repository={{ printf "%s/%s" .Values.image.registry .Values.image.repository }}
-    worker_container_tag={{ .Values.image.tag }}
-    delete_worker_pods=True
-    delete_worker_pods_on_failure=True
-    pod_template_file="/opt/bitnami/airflow/config/pod_template.yaml"
-  {{- end }}
+  {{- $configuration := include "airflow.configuration" . | fromYaml -}}
+  {{- if .Values.overrideConfiguration }}
+  {{- $overrideConfiguration := include "common.tplvalues.render" (dict "value" .Values.overrideConfiguration "context" .) | fromYaml }}
+  {{- $configuration = mustMergeOverwrite $configuration $overrideConfiguration }}
   {{- end }}
+  airflow.cfg: |-
+    {{- range $section, $settings := $configuration }}
+    [{{ $section }}]
+    {{- range $key, $val := $settings }}
+    {{ $key }} = {{ $val }}
+    {{- end }}
+    {{- end }}
   {{- if .Values.localSettings }}
   airflow_local_settings.py: |-
     {{- include "common.tplvalues.render" (dict "value" .Values.localSettings "context" .) | nindent 4 }}

diff --git a/bitnami/airflow/values.yaml b/bitnami/airflow/values.yaml
@@ -134,14 +134,20 @@ executor: CeleryExecutor
 ## @param loadExamples Switch to load some Airflow examples
 ##
 loadExamples: false
-## @param configuration Specify content for Airflow config file (auto-generated based on other env. vars otherwise)
-## e.g:
-## configuration: |-
-##   [core]
-##   dags_folder=/opt/bitnami/airflow/dags
-##   ...
+## @param configuration Specify content for Airflow config file (auto-generated based on other parameters otherwise)
+## ref: https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html
+## Use YAML format, then it's transformed to .cfg format by the chart. e.g:
+##    configuration:
+##      core:
+##        dags_folder: "/opt/bitnami/airflow/dags"
+## ... will be transformed to:
+##    [core]
+##    dags_folder = "/opt/bitnami/airflow/dags"
+##
+configuration: {}
+## @param overrideConfiguration Airflow common configuration override. Values defined here takes precedence over the ones defined at `configuration`
 ##
-configuration: ""
+overrideConfiguration: {}
 ## @param localSettings Specify content for Airflow local settings (airflow_local_settings.py)
 ## ref: https://airflow.apache.org/docs/apache-airflow/stable/howto/set-config.html#configuring-local-settings
 ## e.g: