Add kafka metrics + updated monitoring image versions

.github/workflows/super-linter.yml

@@ -43,6 +43,7 @@ jobs:
           VALIDATE_SQLFLUFF: false
           VALIDATE_MARKDOWN_PRETTIER: false
           VALIDATE_YAML_PRETTIER: false
+          VALIDATE_JSON_PRETTIER: false
           FILTER_REGEX_EXCLUDE: "gradlew"
           DEFAULT_BRANCH: main
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

build.gradle

@@ -45,6 +45,10 @@ spotless {
         trimTrailingWhitespace()
         endWithNewline()
     }
+    json {
+        target 'monitoring/**/*.json', '**/*.json'
+        jackson()
+    }
 }
 
 

docker-compose.yaml

@@ -23,8 +23,11 @@ services:
       - KAFKA_KRAFT_CLUSTER_ID=L0ZEQh1yTbGhNNUE7-6wSQ
       - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true
       - KAFKA_ADVERTISED_HOST_NAME=kafka
+     # - JMX_PORT=12345
+      - "KAFKA_JMX_OPTS=-javaagent:/prometheus/jmx_prometheus_javaagent-1.0.1.jar=12345:/prometheus/kafka-kraft-config-3_0_0.yml -Dcom.sun.management.jmxremote.rmi.port=12345 -Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false  -Dcom.sun.management.jmxremote.ssl=false"
     volumes:
       - kafka_volume:/bitnami/kafka
+      - ./monitoring/kafka:/prometheus
     networks:
       - jforwarder-network
     healthcheck:

monitoring/README.md

@@ -13,3 +13,38 @@
    0 * * * * sudo docker exec --user root zipkin-dependencies /bin/sh /etc/periodic/hourly/zipkin-dependencies-today
    0 2 * * * sudo docker exec --user root zipkin-dependencies /bin/sh /etc/periodic/daily/zipkin-dependencies-yesterday
    ```
+6) JMX exporter for kafka. Seems like docker container jmx exporter working as http server doesn't work with custom configs and tries to connect to localhost for some damn reason. So doesn't need to be used. Used javaagent for kafka container 
+   ```docker
+      - "KAFKA_JMX_OPTS=-javaagent:/prometheus/jmx_prometheus_javaagent-1.0.1.jar=12345:/prometheus/kafka-kraft-config-3_0_0.yml -Dcom.sun.management.jmxremote.rmi.port=12345 -Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false  -Dcom.sun.management.jmxremote.ssl=false"
+   ```
+
+   ```yml
+  # for some unknown reason exporter tries to connect to localhost even if you define hostPort in config.yml.
+  # broken stuff.
+  jmx-exporter:
+    image: 'docker.io/bitnami/jmx-exporter:1.0.1'
+    container_name: jmx-exporter
+    ports:
+      - 5556:5556
+    # host and port is necessary to pass in command because otherwise agent can't connect to localhost
+    command:
+      [
+        "0.0.0.0:5556",
+        "/jmx_exporter/config.yml"
+      ]
+    volumes:
+      - ./kafka/kafka-kraft-config-3_0_0.yml:/jmx_exporter/config.yml
+    networks:
+      - jforwarder-network
+   ```
+
+
+7) Update loki storage schema
+
+   evel=error ts=2024-10-13T08:20:00.176323295Z caller=main.go:70 msg="validating config" err="MULTIPLE CONFIG ERRORS FOUND, PLEASE READ CAREFULLY\n
+
+   CONFIG ERROR: schema v13 is required to store Structured Metadata and use native OTLP ingestion, your schema version is v11. Set `allow_structured_metadata: false` in the `limits_config` section or set the command line argument `-validation.allow-structured-metadata=false` and restart Loki. 
+
+   Then proceed to update to schema v13 or newer before re-enabling this config, search for 'Storage Schema' in the docs for the schema update procedure\n
+
+   CONFIG ERROR: `tsdb` index type is required to store Structured Metadata and use native OTLP ingestion, your index type is `boltdb-shipper` (defined in the `store` parameter of the schema_config). Set `allow_structured_metadata: false` in the `limits_config` section or set the command line argument `-validation.allow-structured-metadata=false` and restart Loki. Then proceed to update the schema to use index type `tsdb` before re-enabling this config, search for 'Storage Schema' in the docs for the schema update procedure"

monitoring/docker-compose.yaml

@@ -28,7 +28,7 @@ services:
   #      - jforwarder-network
   #
   loki:
-    image: grafana/loki:2.9.8
+    image: grafana/loki:3.2.0
     container_name: grafana-loki
     command: [ "-config.file=/etc/loki/local-config.yaml" ]
     ports:
@@ -43,7 +43,7 @@ services:
       - jforwarder-network
 
   prometheus-node-exporter:
-    image: prom/node-exporter:v1.8.0
+    image: prom/node-exporter:v1.8.2
     container_name: prometheus-node-exporter
     user: root
     volumes:
@@ -60,7 +60,7 @@ services:
   grafana:
     env_file:
       - .env
-    image: 'grafana/grafana:10.4.2'
+    image: 'grafana/grafana:11.2.2'
     container_name: grafana
     ports:
       - '3000:3000'
@@ -75,7 +75,7 @@ services:
       - jforwarder-network
 
   prometheus:
-    image: 'prom/prometheus:v2.51.2'
+    image: 'prom/prometheus:v2.54.1'
     container_name: prometheus
     ports:
       - '9090:9090'