fix: alerts

docker/prometheus/rules/common.yml

@@ -3,44 +3,28 @@ groups:
     rules:
 
       - alert: HeadBlockIsNotChanging
-        expr: absent(ethereum_slashing_watcher_slot_number) OR changes(ethereum_slashing_watcher_slot_number[15m]) == 0
-        for: 1m
+        expr: absent(ethereum_head_watcher_slot_number) OR changes(ethereum_head_watcher_slot_number[1h]) == 0
+        for: 5m
         labels:
           severity: critical
         annotations:
-          emoji: ⏳🐦
-          summary: "Head block doesn't change for more than 15 minutes"
-          resolved_summary: "Head block was changed"
+          summary: "⏳🐦 Head block doesn't change for more than 1 hour"
           description: "It's not OK. Please, check app health"
-          resolved_description: "It's OK"
-          footer_text: 'Slot • {{ with query "ethereum_slashing_watcher_slot_number" }}{{ . | first | value | printf "%.0f" }}{{ end }}'
-          footer_icon_url: "https://cryptologos.cc/logos/steth-steth-logo.png"
 
       - alert: ValidatorsIndexSlotIsNotChanging
-        expr: absent(ethereum_slashing_watcher_validators_index_slot_number) OR changes(ethereum_slashing_watcher_validators_index_slot_number[1h]) == 0
-        for: 1m
+        expr: absent(ethereum_head_watcher_validators_index_slot_number) OR changes(ethereum_head_watcher_validators_index_slot_number[1h]) == 0
+        for: 5m
         labels:
           severity: critical
         annotations:
-          emoji: ⏳👥
-          summary: "Validators index slot doesn't change for more than 1 hour"
-          resolved_summary: "Validators index slot was changed"
+          summary: "⏳👥 Validators index slot doesn't change for more than 1 hour"
           description: "It's not OK. Please, check app health"
-          resolved_description: "It's OK"
-          footer_text: 'Slot • {{ with query "ethereum_slashing_watcher_slot_number" }}{{ . | first | value | printf "%.0f" }}{{ end }}'
-          footer_icon_url: "https://cryptologos.cc/logos/steth-steth-logo.png"
-
 
       - alert: KeysAPISlotIsNotChanging
-        expr: absent(ethereum_slashing_watcher_keys_api_slot_number) OR changes(ethereum_slashing_watcher_keys_api_slot_number[1h]) == 0
-        for: 1m
+        expr: absent(ethereum_head_watcher_keys_api_slot_number) OR changes(ethereum_head_watcher_keys_api_slot_number[1h]) == 0
+        for: 5m
         labels:
           severity: critical
         annotations:
-          emoji: ⏳🔑
-          summary: "KeysAPI index slot doesn't change for more than 1 hour"
-          resolved_summary: "KeysAPI index slot was changed"
-          description: "It's not OK. Please, check app health"
-          resolved_description: "It's OK"
-          footer_text: 'Slot • {{ with query "ethereum_slashing_watcher_slot_number" }}{{ . | first | value | printf "%.0f" }}{{ end }}'
-          footer_icon_url: "https://cryptologos.cc/logos/steth-steth-logo.png"
+          summary: "⏳🔑 KeysAPI index slot doesn't change for more than 1 hour"
+          description: "It's not OK. Please, check app health"

src/variables.py

@@ -36,7 +36,7 @@
 
 # - Metrics -
 PROMETHEUS_PORT = int(os.getenv('PROMETHEUS_PORT', 9000))
-PROMETHEUS_PREFIX = os.getenv("PROMETHEUS_PREFIX", "ethereum_slashing_watcher")
+PROMETHEUS_PREFIX = os.getenv("PROMETHEUS_PREFIX", "ethereum_head_watcher")
 
 HEALTHCHECK_SERVER_PORT = int(os.getenv('HEALTHCHECK_SERVER_PORT', 9010))
 

src/watcher.py

@@ -124,7 +124,7 @@ def _update_validators(self):
         now = time.time()
         diff = now - self.genesis_time
         slot = int(diff / SECONDS_PER_SLOT)
-        if self.indexed_validators_keys and SLOTS_PER_EPOCH != 0:
+        if self.indexed_validators_keys and slot % SLOTS_PER_EPOCH != 0:
             return
         logger.info({'msg': 'Updating indexed validators keys'})
         try:
@@ -155,7 +155,8 @@ def _update_lido_keys(self, header: BlockHeaderResponseData) -> None:
             modules = self.keys_api.get_modules()[0]
 
             current_nonce = sum(module['nonce'] for module in modules)
-            if self.keys_api_nonce >= current_nonce:
+            if self.keys_api_nonce == current_nonce:
+                KEYS_API_SLOT_NUMBER.set(int(header.header.message.slot))
                 return
             self.keys_api_nonce = current_nonce