Cluster Operator dynamic logging configuration

CHANGELOG.md

@@ -7,6 +7,7 @@
 * Cruise Control metrics integration:
   * enable metrics JMX exporter configuration in the `cruiseControl` property of the Kafka custom resource
   * new Grafana dashboard for the Cruise Control metrics
+* Configure Cluster Operator logging using ConfigMap instead of environment variable and support dynamic changes  
 
 ## 0.19.0
 
@@ -30,6 +31,7 @@
   * enable/disable metrics in the KafkaBridge custom resource
   * new Grafana dashboard for the bridge metrics
 * Support dynamically changeable logging in the Entity Operator and Kafka Bridge 
+* Configure Cluster Operator logging using ConfigMap instead of environment variable and support dynamic changes
 
 ### Deprecations and removals
 

Jenkinsfile-pr

@@ -166,13 +166,13 @@ pipeline {
                         if ("${env.TEST_PROFILE}".contains("upgrade")) {
                             println("[INFO] Update files for upgrade procedure")
                             sh(script: """
-                                sed -i 's#:latest#:${env.DOCKER_TAG}#g' ${env.WORKSPACE}/systemtest/src/main/resources/StrimziUpgradeST.json ${env.WORKSPACE}/install/cluster-operator/050-Deployment-strimzi-cluster-operator.yaml
+                                sed -i 's#:latest#:${env.DOCKER_TAG}#g' ${env.WORKSPACE}/systemtest/src/main/resources/StrimziUpgradeST.json ${env.WORKSPACE}/install/cluster-operator/060-Deployment-strimzi-cluster-operator.yaml
                                 sed -i 's#strimzi/test-client:${env.DOCKER_TAG}#${env.DOCKER_REGISTRY}/strimzi/test-client:${env.DOCKER_TAG}#g' ${env.WORKSPACE}/systemtest/src/main/resources/StrimziUpgradeST.json
-                                sed -i 's#strimzi/#${env.DOCKER_REGISTRY}/strimzi/#g' ${env.WORKSPACE}/install/cluster-operator/050-Deployment-strimzi-cluster-operator.yaml
-                                sed -i 's#/opt/${env.DOCKER_REGISTRY}#/opt#g' ${env.WORKSPACE}/install/cluster-operator/050-Deployment-strimzi-cluster-operator.yaml
+                                sed -i 's#strimzi/#${env.DOCKER_REGISTRY}/strimzi/#g' ${env.WORKSPACE}/install/cluster-operator/060-Deployment-strimzi-cluster-operator.yaml
+                                sed -i 's#/opt/${env.DOCKER_REGISTRY}#/opt#g' ${env.WORKSPACE}/install/cluster-operator/060-Deployment-strimzi-cluster-operator.yaml
                             """)
                             sh(script: "cat ${env.WORKSPACE}/systemtest/src/main/resources/StrimziUpgradeST.json")
-                            sh(script: "cat ${env.WORKSPACE}/install/cluster-operator/050-Deployment-strimzi-cluster-operator.yaml")
+                            sh(script: "cat ${env.WORKSPACE}/install/cluster-operator/060-Deployment-strimzi-cluster-operator.yaml")
                         }
                         lib.runSystemTests(env.WORKSPACE, env.TEST_CASE, env.TEST_PROFILE, env.EXCLUDE_GROUPS)
                     }

cluster-operator/scripts/cluster_operator_run.sh

@@ -1,4 +1,9 @@
 #!/usr/bin/env bash
 export JAVA_CLASSPATH=lib/io.strimzi.@project.build.finalName@.@project.packaging@:@project.dist.classpath@
 export JAVA_MAIN=io.strimzi.operator.cluster.Main
+
+if [ -f /opt/strimzi/custom-config/log4j2.properties ]; then
+    # if ConfigMap was not mounted and thus this file was not created, use properties file from the classpath
+    export JAVA_OPTS="${JAVA_OPTS} -Dlog4j2.configurationFile=file:/opt/strimzi/custom-config/log4j2.properties"
+fi
 exec "${STRIMZI_HOME}/bin/launch_java.sh"

cluster-operator/src/main/resources/log4j2.properties

@@ -1,5 +1,5 @@
 name = COConfig
- 
+
 appender.console.type = Console
 appender.console.name = STDOUT
 appender.console.layout.type = PatternLayout
@@ -15,7 +15,7 @@ logger.kafka.name = org.apache.kafka
 logger.kafka.level = ${env:STRIMZI_AC_LOG_LEVEL:-WARN}
 logger.kafka.additivity = false
 
-# Zookeeper is very verbose even on INFO level -> We set it to WARN by default
+# Zookeeper is very verbose on INFO level , set it to WARN by default
 logger.zookeepertrustmanager.name = org.apache.zookeeper
 logger.zookeepertrustmanager.level = ${env:STRIMZI_ZOOKEEPER_LOG_LEVEL:-WARN}
 logger.zookeepertrustmanager.additivity = false

development-docs/DEV_GUIDE.md

@@ -47,15 +47,15 @@ To build Strimzi from source the operator and Kafka code needs to be compiled in
         make clean
         make MVN_ARGS='-DskipTests -DskipIT' all
 
-4. In order to use the newly built images, you need to update the `install/cluster-operator/050-Deployment-strimzi-cluster-operator.yml` to obtain the images from your repositories on Docker Hub rather than the official Strimzi images. That can be done using the following command:
+4. To use the newly built images, update the `install/cluster-operator/060-Deployment-strimzi-cluster-operator.yml` to obtain the images from your repositories on Docker Hub rather than the official Strimzi images:
 
     ```
     sed -Ei -e "s#(image|value): strimzi/([a-z0-9-]+):latest#\1: $DOCKER_ORG/\2:latest#" \
             -e "s#([0-9.]+)=strimzi/([a-zA-Z0-9-]+:[a-zA-Z0-9.-]+-kafka-[0-9.]+)#\1=$DOCKER_ORG/\2#" \
-            install/cluster-operator/050-Deployment-strimzi-cluster-operator.yaml
+            install/cluster-operator/060-Deployment-strimzi-cluster-operator.yaml
     ```
 
-   This will update `050-Deployment-strimzi-cluster-operator.yaml` replacing all the image references (in `image` and `value` properties) with ones with the same name but with the repository changed.
+   This updates `060-Deployment-strimzi-cluster-operator.yaml`, replacing all the image references (in `image` and `value` properties) with ones with the same name but with the repository changed.
 
 5. Then you can deploy the Cluster Operator by running: 
 
@@ -232,15 +232,15 @@ By default the `docker_push` target will build the images under the strimzi orga
 
 When the Docker images are build, they will be labeled in the form: `docker_hub_username/operator:latest` in your local repository and pushed to your Docker Hub account under the same label.
 
-In order to use these newly built images, you need to update the `install/cluster-operator/050-Deployment-strimzi-cluster-operator.yml` to obtain the images from your repositories on Docker Hub rather than the official Strimzi images. That can be done using the following command and replacing `docker_hub_username` with the relevant value:
+To use these newly built images, update the `install/cluster-operator/060-Deployment-strimzi-cluster-operator.yml` to obtain the images from your repositories on Docker Hub rather than the official Strimzi images, replacing `docker_hub_username` with the relevant value:
 
 ```
 sed -Ei -e 's#(image|value): strimzi/([a-z0-9-]+):latest#\1: docker_hub_username/\2:latest#' \
         -e 's#([0-9.]+)=strimzi/([a-zA-Z0-9-]+:[a-zA-Z0-9.-]+-kafka-[0-9.]+)#\1=docker_hub_username/\2#' \
-        install/cluster-operator/050-Deployment-strimzi-cluster-operator.yaml
+        install/cluster-operator/060-Deployment-strimzi-cluster-operator.yaml
 ```
 
-This will update `050-Deployment-strimzi-cluster-operator.yaml` replacing all the image references (in `image` and `value` properties) with ones with the same name but with the repository changed.
+This updates `060-Deployment-strimzi-cluster-operator.yaml`, replacing all the image references (in `image` and `value` properties) with ones with the same name but with the repository changed.
 
 Then you can deploy the Cluster Operator by running (for an OpenShift cluster):
 
@@ -279,15 +279,15 @@ Assuming your OpenShift login is `developer` (a user with the `cluster-admin` ro
 
         DOCKER_REGISTRY=172.30.1.1:5000 DOCKER_ORG=$(oc project -q) make docker_push
 
-5. In order to use the built images in a deployment, you need to update the `install/cluster-operator/050-Deployment-strimzi-cluster-operator.yml` to obtain the images from the registry at `172.30.1.1:5000`, rather than from DockerHub. That can be done using the following command:
+5. To use the built images in a deployment, update the `install/cluster-operator/060-Deployment-strimzi-cluster-operator.yml` to obtain the images from the registry at `172.30.1.1:5000`, rather than from DockerHub:
 
     ```
     sed -Ei -e 's#(image|value): strimzi/([a-z0-9-]+):latest#\1: 172.30.1.1:5000/myproject/\2:latest#' \
             -e 's#([0-9.]+)=strimzi/([a-zA-Z0-9-]+:[a-zA-Z0-9.-]+-kafka-[0-9.]+)#\1=172.30.1.1:5000/myproject/\2#' \
-            install/cluster-operator/050-Deployment-strimzi-cluster-operator.yaml
+            install/cluster-operator/060-Deployment-strimzi-cluster-operator.yaml
     ```
 
-    This will update `050-Deployment-strimzi-cluster-operator.yaml` replacing all the image references (in `image` and `value` properties) with ones with the same name from `172.30.1.1:5000/myproject`.
+    This updates `060-Deployment-strimzi-cluster-operator.yaml`, replacing all the image references (in `image` and `value` properties) with ones with the same name from `172.30.1.1:5000/myproject`.
 
 6. Then you can deploy the Cluster Operator by running:
 

documentation/modules/deploying/proc-deploy-cluster-operator-watch-multiple-namespaces.adoc

@@ -20,7 +20,7 @@ For example, in this procedure the Cluster Operator is installed into the namesp
 +
 include::snip-cluster-operator-namespace-sed.adoc[]
 
-. Edit the `install/cluster-operator/050-Deployment-strimzi-cluster-operator.yaml` file
+. Edit the `install/cluster-operator/060-Deployment-strimzi-cluster-operator.yaml` file
 to add a list of all the namespaces the Cluster Operator will watch to the `STRIMZI_NAMESPACE` environment variable.
 +
 For example, in this  procedure the Cluster Operator will watch the namespaces `watched-namespace-1`, `watched-namespace-2`, `watched-namespace-3`.

documentation/modules/deploying/proc-deploy-cluster-operator-watch-whole-cluster.adoc

@@ -22,7 +22,7 @@ For example, in this procedure the Cluster Operator is installed into the namesp
 +
 include::snip-cluster-operator-namespace-sed.adoc[]
 
-. Edit the `install/cluster-operator/050-Deployment-strimzi-cluster-operator.yaml` file to set the value of the `STRIMZI_NAMESPACE` environment variable to `*`.
+. Edit the `install/cluster-operator/060-Deployment-strimzi-cluster-operator.yaml` file to set the value of the `STRIMZI_NAMESPACE` environment variable to `*`.
 +
 [source,yaml,subs="attributes"]
 ----

documentation/modules/deploying/proc-deploy-kafka-connect-new-image-from-base.adoc

@@ -96,7 +96,7 @@ spec: <1>
 +
 or
 +
-* In the `install/cluster-operator/050-Deployment-strimzi-cluster-operator.yaml` file, edit the `STRIMZI_KAFKA_CONNECT_IMAGES` variable to point to the new container image, and then reinstall the Cluster Operator.
+* In the `install/cluster-operator/060-Deployment-strimzi-cluster-operator.yaml` file, edit the `STRIMZI_KAFKA_CONNECT_IMAGES` variable to point to the new container image, and then reinstall the Cluster Operator.
 
 .Additional resources
 

documentation/modules/quickstart/proc-installing-product.adoc

@@ -51,7 +51,7 @@ sed -i '' 's/namespace: .\*/namespace: kafka/' install/cluster-operator/*RoleBin
 kubectl create ns my-kafka-project
 ----
 
-. Edit the `install/cluster-operator/050-Deployment-strimzi-cluster-operator.yaml` file and set the `STRIMZI_NAMESPACE` environment variable to the namespace `my-kafka-project`.
+. Edit the `install/cluster-operator/060-Deployment-strimzi-cluster-operator.yaml` file and set the `STRIMZI_NAMESPACE` environment variable to the namespace `my-kafka-project`.
 +
 [source, yaml, subs=+quotes ]
 ----

documentation/modules/ref-operator-cluster.adoc

@@ -5,7 +5,7 @@
 [id='ref-operator-cluster-{context}']
 = Cluster Operator configuration
 
-The Cluster Operator can be configured through the following supported environment variables.
+The Cluster Operator can be configured through the following supported environment variables and through the logging configuration.
 
 `STRIMZI_NAMESPACE`:: A comma-separated list of namespaces that the operator should operate in.
 When not set, set to empty string, or to `*` the Cluster Operator will operate in all namespaces.
@@ -23,9 +23,6 @@ env:
 
 [[STRIMZI_FULL_RECONCILIATION_INTERVAL_MS]] `STRIMZI_FULL_RECONCILIATION_INTERVAL_MS`:: Optional, default is 120000 ms. The interval between periodic reconciliations, in milliseconds.
 
-`STRIMZI_LOG_LEVEL`:: Optional, default `INFO`.
-The level for printing logging messages. The value can be set to: `ERROR`, `WARNING`, `INFO`, `DEBUG`, and `TRACE`.
-
 `STRIMZI_OPERATION_TIMEOUT_MS`:: Optional, default 300000 ms.
 The timeout for internal operations, in milliseconds. This value should be
 increased when using Strimzi on clusters where regular Kubernetes operations take longer than usual (because of slow downloading of Docker images, for example).
@@ -121,6 +118,32 @@ The DNS domain name is added to the Kafka broker certificates used for hostname
 +
 If you are using a different DNS domain name suffix in your cluster, change the `KUBERNETES_SERVICE_DNS_DOMAIN` environment variable from the default to the one you are using in order to establish a connection with the Kafka brokers.
 
+[discrete]
+== Configuration by ConfigMap
+
+Currently only Cluster Operator logging can be configured by ConfigMap.
+
+A `ConfigMap` containing logging configuration is created when installing the Cluster Operator.
+This `ConfigMap` is described in the file `install/cluster-operator/050-ConfigMap-strimzi-cluster-operator.yaml`.
+You configure Cluster Operator logging by changing the data field `log4j2.properties` in this `ConfigMap`.
+
+To update the logging configuration, you can edit the `050-ConfigMap-strimzi-cluster-operator.yaml` file and then run the following command:
+[source,shell,subs=+quotes]
+kubectl apply -f _install/cluster-operator/050-ConfigMap-strimzi-cluster-operator.yaml_
+
+Alternatively, edit the `ConfigMap` directly:
+[source,shell,subs=+quotes]
+kubectl edit cm strimzi-cluster-operator
+
+To change the frequency of the reload interval, set a time in seconds in the monitorInterval option in the created ConfigMap.
+
+If the `ConfigMap` is missing when the Cluster Operator is deployed, the default logging values are used.
+
+If the `ConfigMap` is accidentally deleted after the Cluster Operator is deployed, the most recently loaded logging configuration is used.
+Create a new `ConfigMap` to load a new logging configuration.
+
+NOTE: Do not remove the monitorInterval option from the ConfigMap.
+
 == Periodic reconciliation
 
 Although the Cluster Operator reacts to all notifications about the desired cluster resources received from the Kubernetes cluster,
@@ -174,7 +197,7 @@ The `Deployment` of the operator then needs to specify this in its `spec.templat
 [source,yaml,numbered,options="nowrap",highlight='12']
 .Partial example of `Deployment` for the Cluster Operator
 ----
-include::install/cluster-operator/050-Deployment-strimzi-cluster-operator.yaml[lines=1..13]
+include::install/cluster-operator/060-Deployment-strimzi-cluster-operator.yaml[lines=1..13]
       # ...
 ----
 

documentation/modules/upgrading/proc-upgrade-cluster-operator.adoc

@@ -24,7 +24,7 @@ Any changes will be *overwritten* by the new version of the Cluster Operator.
 include::snip-cluster-operator-namespace-sed.adoc[]
 +
 .. If you modified one or more environment variables in your existing Cluster Operator `Deployment`, edit the
-`install/cluster-operator/050-Deployment-cluster-operator.yaml` file to use those environment variables.
+`install/cluster-operator/060-Deployment-cluster-operator.yaml` file to use those environment variables.
 
 . When you have an updated configuration, deploy it along with the rest of the installation resources:
 +

helm-charts/helm2/Makefile

@@ -7,8 +7,6 @@ CHART_SEMANTIC_RELEASE_VERSION ?= $(shell cat ../../release.version | tr A-Z a-z
 CHART_NAME = strimzi-kafka-operator
 CHART_PATH ?= ../helm2/strimzi-kafka-operator/
 CHART_RENDERED_TEMPLATES_TMP = ../../target/charts
-CHART_RENDERED_TEMPLATES_CLUSTERROLES = ../../cluster-operator/src/main/resources/cluster-roles/
-CHART_RENDERED_TEMPLATES_INSTALL = ../../install/cluster-operator/
 HELM_CLI = helm2
 
 helm_clean:

helm-charts/helm2/strimzi-kafka-operator/README.md

@@ -89,7 +89,6 @@ the documentation for more details.
 | `image.tag`                          | Cluster Operator image tag                | `latest`                                             |
 | `image.imagePullPolicy`              | Image pull policy for all pods deployed by Cluster Operator       | `IfNotPresent`                                       |
 | `image.imagePullSecrets`              | Docker registry pull secret              | `nil`                                                 |
-| `logLevel`                           | Cluster Operator log level                | `INFO`                                               |
 | `fullReconciliationIntervalMs`       | Full reconciliation interval in milliseconds | 120000                                            |
 | `operationTimeoutMs`                 | Operation timeout in milliseconds         | 300000                                               |
 | `zookeeper.image.repository`         | ZooKeeper image repository                | `strimzi`                                            |

helm-charts/helm2/strimzi-kafka-operator/templates/050-ConfigMap-strimzi-cluster-operator.yaml

@@ -0,0 +1,28 @@
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: {{ .Values.logConfigMap }}
+data:
+  log4j2.properties: |
+    name = COConfig
+
+    appender.console.type = Console
+    appender.console.name = STDOUT
+    appender.console.layout.type = PatternLayout
+    appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+
+    rootLogger.level = INFO
+    rootLogger.appenderRefs = stdout
+    rootLogger.appenderRef.console.ref = STDOUT
+    rootLogger.additivity = false
+
+    # Kafka AdminClient logging is a bit noisy at INFO level
+    logger.kafka.name = org.apache.kafka
+    logger.kafka.level = WARN
+    logger.kafka.additivity = false
+
+    # Zookeeper is very verbose even on INFO level -> We set it to WARN by default
+    logger.zookeepertrustmanager.name = org.apache.zookeeper
+    logger.zookeepertrustmanager.level = WARN
+    logger.zookeepertrustmanager.additivity = false
+    monitorInterval = 30

helm-charts/helm2/strimzi-kafka-operator/templates/050-Deployment-strimzi-cluster-operator.yaml → helm-charts/helm2/strimzi-kafka-operator/templates/060-Deployment-strimzi-cluster-operator.yaml

@@ -25,6 +25,10 @@ spec:
       imagePullSecrets:
         - name: {{ .Values.image.imagePullSecrets }}
       {{- end }}
+      volumes:
+        - name: {{ .Values.logVolume }}
+          configMap:
+            name: {{ .Values.logConfigMap }}
       containers:
         - name: strimzi-cluster-operator
           image: {{ default .Values.image.repository .Values.imageRepositoryOverride }}/{{ .Values.image.name }}:{{ default .Values.image.tag .Values.imageTagOverride }}
@@ -36,6 +40,9 @@ spec:
           {{- end }}
           args:
             - /opt/strimzi/bin/cluster_operator_run.sh
+          volumeMounts:
+            - name: {{ .Values.logVolume }}
+              mountPath: /opt/strimzi/custom-config/
           env:
             - name: STRIMZI_NAMESPACE
               {{- if .Values.watchAnyNamespace }}
@@ -66,8 +73,6 @@ spec:
               value: {{ default .Values.kafkaBridge.image.repository .Values.imageRepositoryOverride }}/{{ .Values.kafkaBridge.image.name }}:{{ default .Values.kafkaBridge.image.tag .Values.imageTagOverride }}
             - name: STRIMZI_DEFAULT_JMXTRANS_IMAGE
               value: {{ default .Values.jmxTrans.image.repository .Values.imageRepositoryOverride }}/{{ .Values.jmxTrans.image.name }}:{{ default .Values.jmxTrans.image.tag .Values.imageTagOverride }}
-            - name: STRIMZI_LOG_LEVEL
-              value: {{ .Values.logLevel | quote }}
             {{- if .Values.image.imagePullSecrets }}
             - name: STRIMZI_IMAGE_PULL_SECRETS
               value: {{ .Values.image.imagePullSecrets }}

helm-charts/helm2/strimzi-kafka-operator/values.yaml

@@ -9,7 +9,8 @@ image:
   repository: strimzi
   name: operator
   tag: latest
-logLevel: INFO
+logVolume: cluster-operator-logging-volume
+logConfigMap: strimzi-cluster-operator
 fullReconciliationIntervalMs: 120000
 operationTimeoutMs: 300000
 kubernetesServiceDnsDomain: cluster.local