fix: configs. multi-run

docker-compose.yml

@@ -1,10 +1,16 @@
 version: "3.7"
+
+networks:
+  head-watcher: {}
+
 services:
 
-  ethereum-head-watcher:
-    container_name: ethereum-head-watcher
+  app-mainnet:
+    container_name: ethereum-head-watcher-mainnet
     build: .
     restart: unless-stopped
+    networks:
+      - head-watcher
     deploy:
       resources:
         limits:
@@ -13,20 +19,48 @@ services:
       - prometheus
       - alertmanager
       - alertmanager-discord
-    ports:
-      - "9000:9000"
+    expose:
+      - 9000
     environment:
       - CONSENSUS_CLIENT_URI=${CONSENSUS_CLIENT_URI}
       - KEYS_API_URI=${KEYS_API_URI}
       - ALERTMANAGER_URI=${ALERTMANAGER_URI}
       - LOG_LEVEL=${LOG_LEVEL}
+      - SLOTS_RANGE=${SLOTS_RANGE}
+
+  app-goerli:
+    container_name: ethereum-head-watcher-goerli
+    build: .
+    restart: unless-stopped
+    networks:
+      - head-watcher
+    deploy:
+      resources:
+        limits:
+          memory: 2g
+    depends_on:
+      - prometheus
+      - alertmanager
+      - alertmanager-discord
+    expose:
+      - 19000
+    environment:
+      - HTTP_PORT=19000
+      - CONSENSUS_CLIENT_URI=${CONSENSUS_CLIENT_URI_GOERLI}
+      - KEYS_API_URI=${KEYS_API_URI_GOERLI}
+      - ALERTMANAGER_URI=${ALERTMANAGER_URI}
+      - LOG_LEVEL=${LOG_LEVEL}
+      - SLOTS_RANGE=${SLOTS_RANGE_GOERLI}
+      - NETWORK_NAME=goerli
 
   prometheus:
     image: prom/prometheus:latest
     sysctls:
       - net.ipv6.conf.lo.disable_ipv6=0
       - net.ipv6.conf.all.disable_ipv6=0
     container_name: prometheus
+    networks:
+      - head-watcher
     restart: unless-stopped
     deploy:
       resources:
@@ -37,13 +71,13 @@ services:
       - ./docker/prometheus/:/etc/prometheus/
     expose:
       - 9090
-    ports:
-      - "0.0.0.0:9090:9090/tcp"
 
   alertmanager:
     image: prom/alertmanager:latest
     container_name: alertmanager
     restart: unless-stopped
+    networks:
+      - head-watcher
     deploy:
       resources:
         limits:
@@ -55,17 +89,15 @@ services:
       - '--config.file=/etc/alertmanager/alertmanager.yml'
     expose:
       - 9093
-    ports:
-      - "0.0.0.0:9093:9093/tcp"
 
   alertmanager-discord:
-    image: vgorkavenko/alertmanager-discord-balval:1.4.0
+    image: lidofinance/alertmanager-discord:2.3.0
+    platform: linux/x86_64
     container_name: alertmanager-discord
     restart: unless-stopped
+    networks:
+      - head-watcher
     expose:
-      - 9094
-    ports:
-      - "0.0.0.0:9094:9094/tcp"
-    environment:
-      - "PORT=9094"
-      - "DISCORD_WEBHOOK=${DISCORD_WEBHOOK_URL}"
+      - 5001
+    volumes:
+      - ./docker/alertmanager-discord/alertmanager-discord.yml:/etc/alertmanager-discord.yml

docker/alertmanager-discord/alertmanager-discord.yml

@@ -0,0 +1,5 @@
+hooks:
+  - slug: slashing-watcher-mainnet
+    hook: https://discord.com/api/webhooks/1
+  - slug: slashing-watcher-goerli
+    hook: https://discord.com/api/webhooks/2

docker/alertmanager/alertmanager.yml

@@ -4,56 +4,28 @@ route:
   group_by: ['alertname']
   receiver: empty
   routes:
-    - receiver: discord_webhook_send_resolved
-      group_wait: 25s
-      group_interval: 35s
+    - receiver: discord_webhook_mainnet
+      group_wait: 100ms
+      group_interval: 100ms
       repeat_interval: 1h
       matchers:
-        - alertname =~ "DataActuality"
-#      continue: true
-
-#    - receiver: telegram_resolved
-#      matchers:
-#        - alertname =~ "DataActuality"
+        - network="mainnet"
 
-#    - receiver: telegram
-#      continue: true
-
-    - receiver: discord_webhook_default
+    - receiver: discord_webhook_goerli
       group_wait: 100ms
       group_interval: 100ms
       repeat_interval: 1h
+      matchers:
+        - network="goerli"
+
 
 receivers:
   - name: 'empty'
-  - name: 'discord_webhook_default'
+  - name: 'discord_webhook_mainnet'
     webhook_configs:
-      - url: 'http://alertmanager-discord:9094/'
+      - url: 'http://alertmanager-discord:5001/hook/slashing-watcher-mainnet'
         send_resolved: false
-
-  - name: 'discord_webhook_send_resolved'
+  - name: 'discord_webhook_goerli'
     webhook_configs:
-      - url: 'http://alertmanager-discord:9094/'
-        send_resolved: true
-
-#  - name: 'telegram'
-#    telegram_configs:
-#      - send_resolved: false
-#        disable_notifications: false
-#        parse_mode: "HTML"
-#        api_url: 'https://api.telegram.org'
-#        message: '{{ template "telegram.custom.message" . }}'
-#        bot_token: 'YOUR_TOKEN'
-#        chat_id: 0
-#
-#  - name: 'telegram_resolved'
-#    telegram_configs:
-#      - send_resolved: true
-#        disable_notifications: false
-#        parse_mode: "HTML"
-#        api_url: 'https://api.telegram.org'
-#        message: '{{ template "telegram.custom.message" . }}'
-#        bot_token: 'YOUR_TOKEN'
-#        chat_id: 0
-#
-#templates: ['/etc/alertmanager/templates/telegram.tmpl']
+      - url: 'http://alertmanager-discord:5001/hook/slashing-watcher-goerli'
+        send_resolved: false

docker/prometheus/prometheus.yml

@@ -15,6 +15,10 @@ scrape_configs:
     static_configs:
       - targets: ['localhost:9090']
 
-  - job_name: 'app'
+  - job_name: 'app-mainnet'
     static_configs:
-      - targets: ['ethereum-head-watcher:9000']
+      - targets: ['ethereum-head-watcher-mainnet:9000']
+
+  - job_name: 'app-goerli'
+    static_configs:
+      - targets: ['ethereum-head-watcher-goerli:19000']

src/providers/alertmanager/client.py

@@ -4,6 +4,7 @@
 from src.metrics.prometheus.basic import ALERTMANAGER_REQUESTS_DURATION
 from src.providers.alertmanager.typings import AlertBody
 from src.providers.http_provider import HTTPProvider
+from src.variables import NETWORK_NAME
 
 logger = logging.getLogger()
 
@@ -14,5 +15,8 @@ class AlertmanagerClient(HTTPProvider):
     ALERTS = "api/v1/alerts"
 
     def send_alerts(self, alerts: list[AlertBody]):
-        logger.debug({'msg': f'Sending {len(alerts)} alerts', 'alerts': [asdict(alert) for alert in alerts]})
-        self._post(self.ALERTS, query_body=[asdict(alert) for alert in alerts])
+        alerts = [asdict(alert) for alert in alerts]
+        for alert in alerts:
+            alert['labels']['network'] = NETWORK_NAME
+        logger.debug({'msg': f'Sending {len(alerts)} alerts', 'alerts': alerts})
+        self._post(self.ALERTS, query_body=alerts)

src/variables.py

@@ -7,6 +7,9 @@
 KEYS_API_URI = os.getenv('KEYS_API_URI', '').split(',')
 ALERTMANAGER_URI = os.getenv('ALERTMANAGER_URI', '').split(',')
 
+NETWORK_NAME = os.getenv('NETWORK_NAME', 'mainnet')
+
+SLOTS_RANGE = os.getenv('SLOTS_RANGE')
 
 CYCLE_SLEEP_IN_SECONDS = int(os.getenv('CYCLE_SLEEP_IN_SECONDS', 1))
 

src/watcher.py

@@ -21,7 +21,7 @@
 from src.providers.keys.client import KeysAPIClient
 from src.providers.keys.typings import KeysApiStatus, LidoNamedKey
 from src.typings import BlockNumber, SlotNumber
-from src.variables import CYCLE_SLEEP_IN_SECONDS
+from src.variables import CYCLE_SLEEP_IN_SECONDS, SLOTS_RANGE
 
 logger = logging.getLogger()
 
@@ -58,14 +58,13 @@ def __init__(self, handlers: list[WatcherHandler]):
 
     def run(self):
         logger.info({'msg': f'Watcher started. Handlers: {[handler.__class__.__name__ for handler in self.handlers]}'})
-        # goeli: 5486325
-        # mainnet: 6213852 - 6213857
-        while True:
-            current_head = self._get_head_block()
+
+        def wrapped(slot_to_handle='head'):
+            current_head = self._get_head_block(slot_to_handle)
             if self.head is not None and current_head.message.slot == self.head.message.slot:
                 logger.info({'msg': f'No new head, waiting {CYCLE_SLEEP_IN_SECONDS} seconds'})
                 time.sleep(CYCLE_SLEEP_IN_SECONDS)
-                continue
+                return
 
             if self.keys_updater is not None and self.keys_updater.done():
                 self.keys_updater.result()
@@ -90,6 +89,14 @@ def run(self):
 
             time.sleep(CYCLE_SLEEP_IN_SECONDS)
 
+        if SLOTS_RANGE:
+            start, end = SLOTS_RANGE.split('-')
+            for slot in range(int(start), int(end) + 1):
+                wrapped(str(slot))
+        else:
+            while True:
+                wrapped()
+
     @duration_meter()
     def _handle_head(self, head: BlockDetailsResponse):
         tasks = [h.handle(self, head) for h in self.handlers]
@@ -130,6 +137,8 @@ def _update_validators(self):
     def _update_lido_keys(self, block: BlockDetailsResponse) -> None:
         """Return dict with `publickey` as key and `LidoNamedKey` as value"""
 
+        # todo: reforge to streams
+
         current_keys_status = self.keys_api.get_status()
         if self.keys_api_status is not None and (
             current_keys_status.elBlockSnapshot['timestamp'] >= self.keys_api_status.elBlockSnapshot['timestamp']