chore: Added metrics to liteprotocoltester (#3002)

* Added metrics to tests Fix liteprotocoltester docker files with libnegentropy COPY docker compose with waku-sim simulation now having test performance dashboard and localhost:3033 Mention dashboard in Readme * Update apps/liteprotocoltester/statistics.nim Co-authored-by: Ivan FB <128452529+Ivansete-status@users.noreply.github.com> * indent fix, more stable finding of service/bootstrap nodes, pre-set for TWN --------- Co-authored-by: Ivan FB <128452529+Ivansete-status@users.noreply.github.com>
waku-org · Sep 4, 2024 · 8baf627 · 8baf627
1 parent 19feb6b
commit 8baf627
Show file tree

Hide file tree

Showing 20 changed files with 1,240 additions and 6,169 deletions.
diff --git a/apps/liteprotocoltester/.env b/apps/liteprotocoltester/.env
@@ -1,4 +1,4 @@
-START_PUBLISHING_AFTER=10
+START_PUBLISHING_AFTER=45
 # can add some seconds delay before SENDER starts publishing
 
 NUM_MESSAGES=0
@@ -12,16 +12,16 @@ MIN_MESSAGE_SIZE=15Kb
 MAX_MESSAGE_SIZE=145Kb
 
 ## for wakusim
-PUBSUB=/waku/2/rs/66/0
-CONTENT_TOPIC=/tester/2/light-pubsub-test/wakusim
-CLUSTER_ID=66
+#PUBSUB=/waku/2/rs/66/0
+#CONTENT_TOPIC=/tester/2/light-pubsub-test/wakusim
+#CLUSTER_ID=66
 
 ## for status.prod
 #PUBSUB=/waku/2/rs/16/32
 #CONTENT_TOPIC=/tester/2/light-pubsub-test/fleet
 #CLUSTER_ID=16
 
 ## for TWN
-#PUBSUB=/waku/2/rs/1/4
-#CONTENT_TOPIC=/tester/2/light-pubsub-test/twn
-#CLUSTER_ID=1
+PUBSUB=/waku/2/rs/1/4
+CONTENT_TOPIC=/tester/2/light-pubsub-test/twn
+CLUSTER_ID=1
diff --git a/apps/liteprotocoltester/Dockerfile.liteprotocoltester b/apps/liteprotocoltester/Dockerfile.liteprotocoltester
@@ -29,6 +29,8 @@
 
   COPY build/liteprotocoltester /usr/bin/
   COPY apps/liteprotocoltester/run_tester_node.sh /usr/bin/
+  COPY ./libnegentropy.so /usr/lib/
+
 
   ENTRYPOINT ["/usr/bin/run_tester_node.sh", "/usr/bin/liteprotocoltester"]
 

diff --git a/apps/liteprotocoltester/Dockerfile.liteprotocoltester.compile b/apps/liteprotocoltester/Dockerfile.liteprotocoltester.compile
@@ -51,6 +51,7 @@
 
   # Copy migration scripts for DB upgrades
   COPY --from=nim-build /app/migrations/ /app/migrations/
+  COPY --from=nim-build /app/libnegentropy.so /usr/lib/
 
   ENTRYPOINT ["/usr/bin/liteprotocoltester"]
 

diff --git a/apps/liteprotocoltester/Dockerfile.liteprotocoltester.copy b/apps/liteprotocoltester/Dockerfile.liteprotocoltester.copy
diff --git a/apps/liteprotocoltester/README.md b/apps/liteprotocoltester/README.md
@@ -96,6 +96,10 @@ docker compose -f docker-compose-on-simularor.yml logs -f receivernode
 - Notice there is a configurable wait before start publishing messages as it is noticed time is needed for the service nodes to get connected to full nodes from simulator
 - light clients will print report on their and the connected service node's connectivity to the network in every 20 secs.
 
+#### Test monitoring
+
+Navigate to http://localhost:3033 to see the lite-protocol-tester dashboard.
+
 ### Phase 3
 
 > Run independently on a chosen waku fleet

diff --git a/apps/liteprotocoltester/docker-compose-on-simularor.yml b/apps/liteprotocoltester/docker-compose-on-simularor.yml
@@ -62,7 +62,7 @@ services:
     image: waku.liteprotocoltester:latest
     build:
       context: ../..
-      dockerfile: ./apps/liteprotocoltester/Dockerfile.liteprotocoltester.copy
+      dockerfile: ./apps/liteprotocoltester/Dockerfile.liteprotocoltester
     deploy:
       replicas: ${NUM_PUBLISHER_NODES:-3}
     # ports:
@@ -84,13 +84,12 @@ services:
         - *rln_env
         - *test_running_conditions
     volumes:
-      - ./run_tester_node.sh:/opt/run_tester_node.sh:Z
       - ${CERTS_DIR:-./certs}:/etc/letsencrypt/:Z
       - ./rln_tree:/etc/rln_tree/:Z
       - ./keystore:/keystore:Z
     entrypoint: sh
     command:
-      - /opt/run_tester_node.sh
+      - /usr/bin/run_tester_node.sh
       - /usr/bin/liteprotocoltester
       - SENDER
       - waku-sim
@@ -139,7 +138,7 @@ services:
     image: waku.liteprotocoltester:latest
     build:
       context: ../..
-      dockerfile: ./apps/liteprotocoltester/Dockerfile.liteprotocoltester.copy
+      dockerfile: ./apps/liteprotocoltester/Dockerfile.liteprotocoltester
     deploy:
       replicas: ${NUM_RECEIVER_NODES:-1}
     # ports:
@@ -161,13 +160,12 @@ services:
         - *rln_env
         - *test_running_conditions
     volumes:
-      - ./run_tester_node.sh:/opt/run_tester_node.sh:Z
       - ${CERTS_DIR:-./certs}:/etc/letsencrypt/:Z
       - ./rln_tree:/etc/rln_tree/:Z
       - ./keystore:/keystore:Z
     entrypoint: sh
     command:
-      - /opt/run_tester_node.sh
+      - /usr/bin/run_tester_node.sh
       - /usr/bin/liteprotocoltester
       - RECEIVER
       - waku-sim
@@ -180,34 +178,44 @@ services:
     networks:
       - waku-simulator_simulation
 
-  ## We have prometheus and grafana defined in waku-simulator already
-  # prometheus:
-  #   image: docker.io/prom/prometheus:latest
-  #   volumes:
-  #     - ./monitoring/prometheus-config.yml:/etc/prometheus/prometheus.yml:Z
-  #   command:
-  #     - --config.file=/etc/prometheus/prometheus.yml
-  #   ports:
-  #     - 127.0.0.1:9090:9090
-  #   depends_on:
-  #     - servicenode
+  # We have prometheus and grafana defined in waku-simulator already
+  prometheus:
+    image: docker.io/prom/prometheus:latest
+    volumes:
+      - ./monitoring/prometheus-config.yml:/etc/prometheus/prometheus.yml:Z
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --web.listen-address=:9099
+    # ports:
+    #   - 127.0.0.1:9090:9090
+    restart: on-failure:5
+    depends_on:
+      - filter-service
+      - lightpush-service
+      - publishernode
+      - receivernode
+    networks:
+      - waku-simulator_simulation
 
-  # grafana:
-  #   image: docker.io/grafana/grafana:latest
-  #   env_file:
-  #     - ./monitoring/configuration/grafana-plugins.env
-  #   volumes:
-  #     - ./monitoring/configuration/grafana.ini:/etc/grafana/grafana.ini:Z
-  #     - ./monitoring/configuration/dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml:Z
-  #     - ./monitoring/configuration/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml:Z
-  #     - ./monitoring/configuration/dashboards:/var/lib/grafana/dashboards/:Z
-  #     - ./monitoring/configuration/customizations/custom-logo.svg:/usr/share/grafana/public/img/grafana_icon.svg:Z
-  #     - ./monitoring/configuration/customizations/custom-logo.svg:/usr/share/grafana/public/img/grafana_typelogo.svg:Z
-  #     - ./monitoring/configuration/customizations/custom-logo.png:/usr/share/grafana/public/img/fav32.png:Z
-  #   ports:
-  #     - 0.0.0.0:3000:3000
-  #   depends_on:
-  #     - prometheus
+  grafana:
+    image: docker.io/grafana/grafana:latest
+    env_file:
+      - ./monitoring/configuration/grafana-plugins.env
+    volumes:
+      - ./monitoring/configuration/grafana.ini:/etc/grafana/grafana.ini:Z
+      - ./monitoring/configuration/dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml:Z
+      - ./monitoring/configuration/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml:Z
+      - ./monitoring/configuration/dashboards:/var/lib/grafana/dashboards/:Z
+      - ./monitoring/configuration/customizations/custom-logo.svg:/usr/share/grafana/public/img/grafana_icon.svg:Z
+      - ./monitoring/configuration/customizations/custom-logo.svg:/usr/share/grafana/public/img/grafana_typelogo.svg:Z
+      - ./monitoring/configuration/customizations/custom-logo.png:/usr/share/grafana/public/img/fav32.png:Z
+    ports:
+      - 0.0.0.0:3033:3033
+    restart: on-failure:5
+    depends_on:
+      - prometheus
+    networks:
+      - waku-simulator_simulation
 
 configs:
   cfg_tester_node.toml:

diff --git a/apps/liteprotocoltester/docker-compose.yml b/apps/liteprotocoltester/docker-compose.yml
@@ -59,7 +59,7 @@ services:
     image: waku.liteprotocoltester:latest
     build:
       context: ../..
-      dockerfile: ./apps/liteprotocoltester/Dockerfile.liteprotocoltester.copy
+      dockerfile: ./apps/liteprotocoltester/Dockerfile.liteprotocoltester
     ports:
       # - 30304:30304/tcp
       # - 30304:30304/udp
@@ -79,13 +79,12 @@ services:
         - *rln_env
         - *test_running_conditions
     volumes:
-      - ./run_tester_node.sh:/opt/run_tester_node.sh:Z
       - ${CERTS_DIR:-./certs}:/etc/letsencrypt/:Z
       - ./rln_tree:/etc/rln_tree/:Z
       - ./keystore:/keystore:Z
     entrypoint: sh
     command:
-      - /opt/run_tester_node.sh
+      - /usr/bin/run_tester_node.sh
       - /usr/bin/liteprotocoltester
       - SENDER
       - servicenode
@@ -99,7 +98,7 @@ services:
     image: waku.liteprotocoltester:latest
     build:
       context: ../..
-      dockerfile: ./apps/liteprotocoltester/Dockerfile.liteprotocoltester.copy
+      dockerfile: ./apps/liteprotocoltester/Dockerfile.liteprotocoltester
     ports:
       # - 30304:30304/tcp
       # - 30304:30304/udp
@@ -125,7 +124,7 @@ services:
       - ./keystore:/keystore:Z
     entrypoint: sh
     command:
-      - /opt/run_tester_node.sh
+      - /usr/bin/run_tester_node.sh
       - /usr/bin/liteprotocoltester
       - RECEIVER
       - servicenode

diff --git a/apps/liteprotocoltester/lightpush_publisher.nim b/apps/liteprotocoltester/lightpush_publisher.nim
@@ -17,7 +17,8 @@ import
     common/utils/parse_size_units,
   ],
   ./tester_config,
-  ./tester_message
+  ./tester_message,
+  ./lpt_metrics
 
 randomize()
 
@@ -141,12 +142,15 @@ proc publishMessages(
         pubsubTopic = lightpushPubsubTopic,
         hash = msgHash
       inc(messagesSent)
+      lpt_publisher_sent_messages_count.inc()
+      lpt_publisher_sent_bytes.inc(amount = msgSize.int64)
     else:
       sentMessages[messagesSent] = (hash: msgHash, relayed: false)
       failedToSendCause.mgetOrPut(wlpRes.error, 1).inc()
       error "failed to publish message using lightpush",
         err = wlpRes.error, hash = msgHash
       inc(failedToSendCount)
+      lpt_publisher_failed_messages_count.inc(labelValues = [wlpRes.error])
 
     await sleepAsync(delayMessages)
 

diff --git a/apps/liteprotocoltester/liteprotocoltester.nim b/apps/liteprotocoltester/liteprotocoltester.nim
@@ -96,6 +96,10 @@ when isMainModule:
   wakuConf.clusterId = conf.clusterId
   ## TODO: Depending on the tester needs we might extend here with shards, clusterId, etc...
 
+  wakuConf.metricsServer = true
+  wakuConf.metricsServerAddress = parseIpAddress("0.0.0.0")
+  wakuConf.metricsServerPort = 8003
+
   if conf.testFunc == TesterFunctionality.SENDER:
     wakuConf.lightpushnode = conf.serviceNode
   else:
@@ -108,9 +112,6 @@ when isMainModule:
 
   wakuConf.rest = false
 
-  wakuConf.metricsServer = true
-  wakuConf.metricsServerAddress = parseIpAddress("0.0.0.0")
-
   # NOTE: {.threadvar.} is used to make the global variable GC safe for the closure uses it
   # It will always be called from main thread anyway.
   # Ref: https://nim-lang.org/docs/manual.html#threads-gc-safety

diff --git a/apps/liteprotocoltester/lpt_metrics.nim b/apps/liteprotocoltester/lpt_metrics.nim
@@ -0,0 +1,30 @@
+## Example showing how a resource restricted client may
+## subscribe to messages without relay
+
+import metrics
+
+export metrics
+
+declarePublicGauge lpt_receiver_sender_peer_count, "count of sender peers"
+
+declarePublicCounter lpt_receiver_received_messages_count,
+  "number of messages received per peer", ["peer"]
+
+declarePublicCounter lpt_receiver_received_bytes,
+  "number of received bytes per peer", ["peer"]
+
+declarePublicGauge lpt_receiver_missing_messages_count,
+  "number of missing messages per peer", ["peer"]
+
+declarePublicCounter lpt_receiver_duplicate_messages_count,
+  "number of duplicate messages per peer", ["peer"]
+
+declarePublicGauge lpt_receiver_distinct_duplicate_messages_count,
+  "number of distinct duplicate messages per peer", ["peer"]
+
+declarePublicCounter lpt_publisher_sent_messages_count, "number of messages published"
+
+declarePublicCounter lpt_publisher_failed_messages_count,
+  "number of messages failed to publish per failure cause", ["cause"]
+
+declarePublicCounter lpt_publisher_sent_bytes, "number of total bytes sent"