Merge pull request #57 from lidofinance/develop

Develop

README.md

@@ -1,6 +1,6 @@
 ## Lido Node Operators Keys Service
 
-Simple Lido keys and validators HTTP API. It will be possible to launch it on someone machine and fetch actual Lido keys from all sources. API also provides http methods for Automating Validator Exits.
+Simple Lido keys and validators HTTP API.
 
 ## How it works
 
@@ -487,6 +487,35 @@ interface Response {
 }
 ```
 
+## Installation
+
+`$ yarn install`
+
+## Running
+
+`$ yarn start:dev`
+
+For running locally in container run
+
+1. `docker-compose -f docker-compose.yml build`
+2. `docker-compose -f docker-compose.yml up`
+3. `docker-compose -f docker-compose.metrics.yml build`
+4. `docker-compose -f docker-compose.metrics.yml up`
+
+To configure grafana go to `http://localhost:8000/dashboards` and dashboards from `./grafana` folder.
+
+## Tests
+
+Unit tests
+
+`$ yarn test`
+
+## Environment variable
+
+Check `sample.env` file.
+
+Pay attention that API by default running job for fetching and updating Validators. If you are not planning to use `validators` endpoints, you could disable this job by setting `VALIDATOR_REGISTRY_ENABLE=false`.
+
 ## Release flow
 
 To create new release:

alerts/keys-api-outdated-keys.rule.yml

@@ -0,0 +1,11 @@
+groups:
+  - name: Keys API. Keys
+    rules:
+      - alert: KeysApiOutdatedKeys
+        expr: time() - lido_keys_api_last_update_timestamp{} >= 5 * 60
+        labels:
+          severity: warning
+          app_team: tooling
+        annotations:
+          summary: Keys are outdated
+          description: Keys were not updated for {{ $value | humanizeDuration }}

alerts/keys-api-outdated-keys.test.yml

@@ -0,0 +1,59 @@
+rule_files:
+  - keys-api-outdated-keys.rule.yml
+
+evaluation_interval: 1m
+
+tests:
+  # Outdated keys list. Outdated
+  - interval: 5m
+    input_series:
+      - series: lido_keys_api_last_update_timestamp{}
+        values: 0 0 0 400
+    alert_rule_test:
+      - eval_time: 10m
+        alertname: KeysApiOutdatedKeys
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              app_team: tooling
+            exp_annotations:
+              summary: Keys are outdated
+              description: Keys were not updated for 10m 0s
+      - eval_time: 15m
+        alertname: KeysApiOutdatedKeys
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              app_team: tooling
+            exp_annotations:
+              summary: Keys are outdated
+              description: Keys were not updated for 8m 20s
+
+  #Outdated keys list. Up to date
+  - interval: 5m
+    input_series:
+      - series: lido_keys_api_last_update_timestamp{}
+        values: 0 300 600 601
+    alert_rule_test:
+      - eval_time: 10m
+        alertname: KeysApiOutdatedKeys
+        exp_alerts: []
+      - eval_time: 15m
+        alertname: KeysApiOutdatedKeys
+        exp_alerts: []
+
+  # Outdated keys list. Initial start
+  - interval: 10m
+    input_series:
+      - series: lido_keys_api_last_update_timestamp{}
+        values: _ _ _ 1501
+    alert_rule_test:
+      - eval_time: 10m
+        alertname: KeysApiOutdatedKeys
+        exp_alerts: []
+      - eval_time: 20m
+        alertname: KeysApiOutdatedKeys
+        exp_alerts: []
+      - eval_time: 30m
+        alertname: KeysApiOutdatedKeys
+        exp_alerts: []

alerts/keys-api-outdated-validators.rule.yml

@@ -0,0 +1,11 @@
+groups:
+  - name: Keys API. Validators
+    rules:
+      - alert: KeysApiOutdatedValidators
+        expr: time() - lido_keys_api_validators_registry_last_update_block_timestamp{} >=  60 * 30
+        labels:
+          severity: warning
+          app_team: tooling
+        annotations:
+          summary: Validators are outdated
+          description: Validators were not updated for {{ $value | humanizeDuration }}

alerts/keys-api-outdated-validators.test.yml

@@ -0,0 +1,56 @@
+rule_files:
+  - keys-api-outdated-validators.rule.yml
+
+evaluation_interval: 1m
+
+tests:
+  # Outdated validators list. Outdated
+  - interval: 15m
+    input_series:
+      - series: lido_keys_api_validators_registry_last_update_block_timestamp{}
+        values: 0 0 0 900
+    alert_rule_test:
+      - eval_time: 30m
+        alertname: KeysApiOutdatedValidators
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              app_team: tooling
+            exp_annotations:
+              summary: Validators are outdated
+              description: Validators were not updated for 30m 0s
+      - eval_time: 45m
+        alertname: KeysApiOutdatedValidators
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              app_team: tooling
+            exp_annotations:
+              summary: Validators are outdated
+              description: Validators were not updated for 30m 0s
+
+  # Outdated validators list. Up to date
+  - interval: 15m
+    input_series:
+      - series: lido_keys_api_validators_registry_last_update_block_timestamp{}
+        values: 0 0 1 901
+    alert_rule_test:
+      - eval_time: 30m
+        alertname: KeysApiOutdatedValidators
+        exp_alerts: []
+      - eval_time: 45m
+        alertname: KeysApiOutdatedValidators
+        exp_alerts: []
+
+  # Outdated validators list. Initial start
+  - interval: 15m
+    input_series:
+      - series: lido_keys_api_validators_registry_last_update_block_timestamp{}
+        values: _ _ _ 901
+    alert_rule_test:
+      - eval_time: 30m
+        alertname: KeysApiOutdatedValidators
+        exp_alerts: []
+      - eval_time: 45m
+        alertname: KeysApiOutdatedValidators
+        exp_alerts: []

alerts/keys-api.rule.yml

@@ -0,0 +1,20 @@
+groups:
+  - name: Keys API
+    rules:
+      - alert: KeysApiExecutionLayerRPCErrors
+        expr: sum by (instance, project) (increase(el_rpc_requests_duration_seconds_count{result="error", service="lido-keys-api"}[5m])) > 40
+        labels:
+          severity: warning
+          app_team: tooling
+        annotations:
+          summary: Execution Layer RPC errors
+          description: '{{ $value }} errors in the last 5 minutes appeared in EL RPC queries'
+
+      - alert: KeysApiConsensusLayerAPIErrors
+        expr: sum by (instance, project) (increase(cl_api_requests_duration_seconds_count{result="error", service="lido-keys-api"}[5m])) > 60
+        labels:
+          severity: warning
+          app_team: tooling
+        annotations:
+          summary: Consensus Layer API errors
+          description: '{{ $value }} errors in the last 5 minutes appeared in CL API queries'

alerts/keys-api.test.yml

@@ -0,0 +1,61 @@
+rule_files:
+  - keys-api.rule.yml
+
+evaluation_interval: 1m
+
+tests:
+  # Execution Layer RPC errors
+  - interval: 5m
+    input_series:
+      - series: el_rpc_requests_duration_seconds_count{result="error", service="lido-keys-api"}
+        values: 10 70 111 151
+    alert_rule_test:
+      - eval_time: 5m
+        alertname: KeysApiExecutionLayerRPCErrors
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              app_team: tooling
+            exp_annotations:
+              summary: Execution Layer RPC errors
+              description: 60 errors in the last 5 minutes appeared in EL RPC queries
+      - eval_time: 10m
+        alertname: KeysApiExecutionLayerRPCErrors
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              app_team: tooling
+            exp_annotations:
+              summary: Execution Layer RPC errors
+              description: 41 errors in the last 5 minutes appeared in EL RPC queries
+      - eval_time: 15m
+        alertname: KeysApiExecutionLayerRPCErrors
+        exp_alerts: []
+
+  # Consensus Layer API errors
+  - interval: 5m
+    input_series:
+      - series: cl_api_requests_duration_seconds_count{result="error", service="lido-keys-api"}
+        values: 0 61 152 212
+    alert_rule_test:
+      - eval_time: 5m
+        alertname: KeysApiConsensusLayerAPIErrors
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              app_team: tooling
+            exp_annotations:
+              summary: Consensus Layer API errors
+              description: 61 errors in the last 5 minutes appeared in CL API queries
+      - eval_time: 10m
+        alertname: KeysApiConsensusLayerAPIErrors
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              app_team: tooling
+            exp_annotations:
+              summary: Consensus Layer API errors
+              description: 91 errors in the last 5 minutes appeared in CL API queries
+      - eval_time: 15m
+        alertname: KeysApiConsensusLayerAPIErrors
+        exp_alerts: []

docker-compose.metrics.yml

@@ -0,0 +1,23 @@
+version: '3.7'
+
+services:
+  keys_api_prometheus:
+    image: prom/prometheus:v2.17.2
+    container_name: keys_api_prometheus
+    ports:
+      - 9090:9090
+    volumes:
+      - ./prometheus/:/etc/prometheus/
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+
+  keys_api_grafana:
+    image: grafana/grafana-oss:9.1.5
+    container_name: keys_api_grafana
+    restart: unless-stopped
+    ports:
+      - 8000:3000
+    volumes:
+      - ./grafana/datasources.yml:/etc/grafana/provisioning/datasources/datasources.yml
+    depends_on:
+      - keys_api_prometheus

docker-compose.yml

@@ -13,7 +13,7 @@ services:
     volumes:
       - ./.volumes/pgdata-${CHAIN_ID}/:/var/lib/postgresql/data
 
-  main:
+  keys_api:
     build: ./
     restart: always
     environment:
@@ -34,5 +34,9 @@ services:
       - DB_USER=${DB_USER}
       - DB_PASSWORD=${DB_PASSWORD}
       - JOB_INTERVAL_REGISTRY=${JOB_INTERVAL_REGISTRY}
+      - JOB_INTERVAL_VALIDATORS_REGISTRY=${JOB_INTERVAL_VALIDATORS_REGISTRY}
+      - VALIDATOR_REGISTRY_ENABLE=${VALIDATOR_REGISTRY_ENABLE}
     ports:
       - '${PORT}:${PORT}'
+    depends_on:
+      - db

grafana/.gitignore

@@ -0,0 +1,8 @@
+# Ignore everything
+*
+
+# But not these files...
+!.gitignore
+!keys_api.json
+!datasources.yml
+!nodejs.json

grafana/datasources.yml

@@ -0,0 +1,15 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    orgId: 1
+    url: http://docker.for.mac.host.internal:9090
+    basicAuth: false
+    isDefault: true
+    editable: true
+    jsonData:
+      graphiteVersion: '1.1'
+      tlsAuth: false
+      tlsAuthWithCACert: false