diff --git a/docker/prometheus/rules/common.yml b/docker/prometheus/rules/common.yml index fa2cfc9..bd5157a 100644 --- a/docker/prometheus/rules/common.yml +++ b/docker/prometheus/rules/common.yml @@ -3,44 +3,28 @@ groups: rules: - alert: HeadBlockIsNotChanging - expr: absent(ethereum_slashing_watcher_slot_number) OR changes(ethereum_slashing_watcher_slot_number[15m]) == 0 - for: 1m + expr: absent(ethereum_head_watcher_slot_number) OR changes(ethereum_head_watcher_slot_number[1h]) == 0 + for: 5m labels: severity: critical annotations: - emoji: ⏳🐦 - summary: "Head block doesn't change for more than 15 minutes" - resolved_summary: "Head block was changed" + summary: "⏳🐦 Head block doesn't change for more than 1 hour" description: "It's not OK. Please, check app health" - resolved_description: "It's OK" - footer_text: 'Slot • {{ with query "ethereum_slashing_watcher_slot_number" }}{{ . | first | value | printf "%.0f" }}{{ end }}' - footer_icon_url: "https://cryptologos.cc/logos/steth-steth-logo.png" - alert: ValidatorsIndexSlotIsNotChanging - expr: absent(ethereum_slashing_watcher_validators_index_slot_number) OR changes(ethereum_slashing_watcher_validators_index_slot_number[1h]) == 0 - for: 1m + expr: absent(ethereum_head_watcher_validators_index_slot_number) OR changes(ethereum_head_watcher_validators_index_slot_number[1h]) == 0 + for: 5m labels: severity: critical annotations: - emoji: ⏳👥 - summary: "Validators index slot doesn't change for more than 1 hour" - resolved_summary: "Validators index slot was changed" + summary: "⏳👥 Validators index slot doesn't change for more than 1 hour" description: "It's not OK. Please, check app health" - resolved_description: "It's OK" - footer_text: 'Slot • {{ with query "ethereum_slashing_watcher_slot_number" }}{{ . | first | value | printf "%.0f" }}{{ end }}' - footer_icon_url: "https://cryptologos.cc/logos/steth-steth-logo.png" - - alert: KeysAPISlotIsNotChanging - expr: absent(ethereum_slashing_watcher_keys_api_slot_number) OR changes(ethereum_slashing_watcher_keys_api_slot_number[1h]) == 0 - for: 1m + expr: absent(ethereum_head_watcher_keys_api_slot_number) OR changes(ethereum_head_watcher_keys_api_slot_number[1h]) == 0 + for: 5m labels: severity: critical annotations: - emoji: ⏳🔑 - summary: "KeysAPI index slot doesn't change for more than 1 hour" - resolved_summary: "KeysAPI index slot was changed" - description: "It's not OK. Please, check app health" - resolved_description: "It's OK" - footer_text: 'Slot • {{ with query "ethereum_slashing_watcher_slot_number" }}{{ . | first | value | printf "%.0f" }}{{ end }}' - footer_icon_url: "https://cryptologos.cc/logos/steth-steth-logo.png" \ No newline at end of file + summary: "⏳🔑 KeysAPI index slot doesn't change for more than 1 hour" + description: "It's not OK. Please, check app health" \ No newline at end of file diff --git a/src/variables.py b/src/variables.py index 31a6302..bbbaf11 100644 --- a/src/variables.py +++ b/src/variables.py @@ -36,7 +36,7 @@ # - Metrics - PROMETHEUS_PORT = int(os.getenv('PROMETHEUS_PORT', 9000)) -PROMETHEUS_PREFIX = os.getenv("PROMETHEUS_PREFIX", "ethereum_slashing_watcher") +PROMETHEUS_PREFIX = os.getenv("PROMETHEUS_PREFIX", "ethereum_head_watcher") HEALTHCHECK_SERVER_PORT = int(os.getenv('HEALTHCHECK_SERVER_PORT', 9010)) diff --git a/src/watcher.py b/src/watcher.py index 75099a1..6042551 100644 --- a/src/watcher.py +++ b/src/watcher.py @@ -124,7 +124,7 @@ def _update_validators(self): now = time.time() diff = now - self.genesis_time slot = int(diff / SECONDS_PER_SLOT) - if self.indexed_validators_keys and SLOTS_PER_EPOCH != 0: + if self.indexed_validators_keys and slot % SLOTS_PER_EPOCH != 0: return logger.info({'msg': 'Updating indexed validators keys'}) try: @@ -155,7 +155,8 @@ def _update_lido_keys(self, header: BlockHeaderResponseData) -> None: modules = self.keys_api.get_modules()[0] current_nonce = sum(module['nonce'] for module in modules) - if self.keys_api_nonce >= current_nonce: + if self.keys_api_nonce == current_nonce: + KEYS_API_SLOT_NUMBER.set(int(header.header.message.slot)) return self.keys_api_nonce = current_nonce