Add monitoring to dev engine

Makefile

@@ -1,4 +1,4 @@
-.PHONY: all e2e check-version docker-publish docker-publish-dev docker-tools dev dev-stop dev-start lint dep build test mock protobuf changelog clean genesis clean-build clean-docker build-cmd-cosmos
+.PHONY: all build build-cmd-cosmos changelog check-version clean clean-build clean-docker dep dev dev-mon dev-start dev-stop docker-build docker-dev docker-publish docker-publish-dev docker-tools genesis lint mock protobuf test
 
 MAJOR_VERSION := $(shell echo $(version) | cut -d . -f 1)	
 MINOR_VERSION := $(shell echo $(version) | cut -d . -f 1-2)
@@ -39,11 +39,14 @@ docker-tools:
 dev: docker-dev
 	- ./scripts/dev.sh
 
+dev-mon: docker-dev
+	- ./scripts/dev.sh -m
+
 dev-start: docker-dev
 	./scripts/dev.sh -q
 
 dev-stop: docker-dev
-	./scripts/dev.sh stop
+	./scripts/dev.sh -m stop
 
 dep:
 	go mod download

scripts/datasource.yml

@@ -0,0 +1,52 @@
+# grafana datasource
+
+# config file version
+apiVersion: 1
+
+# list of datasources that should be deleted from the database
+deleteDatasources:
+  - name: Prometheus
+    orgId: 1
+
+# list of datasources to insert/update depending
+# whats available in the database
+datasources:
+  # <string, required> name of the datasource. Required
+- name: Prometheus
+  # <string, required> datasource type. Required
+  type: prometheus
+  # <string, required> access mode. direct or proxy. Required
+  access: proxy
+  # <int> org id. will default to orgId 1 if not specified
+  orgId: 1
+  # <string> url
+  url: http://engine-prometheus:9090
+  # <string> database password, if used
+  password:
+  # <string> database user, if used
+  user:
+  # <string> database name, if used
+  database:
+  # <bool> enable/disable basic auth
+  basicAuth: true
+  # <string> basic auth username
+  basicAuthUser: admin
+  # <string> basic auth password
+  basicAuthPassword: foobar
+  # <bool> enable/disable with credentials headers
+  withCredentials:
+  # <bool> mark as default datasource. Max one per org
+  isDefault:
+  # <map> fields that will be converted to json and stored in json_data
+  jsonData:
+     graphiteVersion: "1.1"
+     tlsAuth: false
+     tlsAuthWithCACert: false
+  # <string> json object of data that will be encrypted.
+  secureJsonData:
+    tlsCACert: "..."
+    tlsClientCert: "..."
+    tlsClientKey: "..."
+  version: 1
+  # <bool> allow users to edit datasources from the UI.
+  editable: true

scripts/dev.sh

@@ -12,10 +12,17 @@ MESG_TENDERMINT_PORT=${MESG_TENDERMINT_PORT:-"26656"}
 MESG_TENDERMINT_RPC_PORT=${MESG_TENDERMINT_RPC_PORT:-"26657"}
 MESG_PROMETHEUS_RPC_PORT=${MESG_PROMETHEUS_RPC_PORT:-"26660"}
 
+# cmd args
+quiet=false
+monitor=false
+
 function onexit {
   set +e
   echo -e "\nshutting down, please wait..."
   docker_service_remove "$MESG_NAME"
+  if $monitor; then
+    docker service rm engine-grafana engine-prometheus
+  fi
   docker_network_remove "$MESG_NAME"
   docker_network_remove "$MESG_TENDERMINT_NETWORK"
 }
@@ -48,6 +55,11 @@ function start_engine {
     docker_network_create "$MESG_TENDERMINT_NETWORK"
   fi
 
+  if $monitor; then
+    docker service create -p 3001:3000 --network=engine --name=engine-grafana --mount type=bind,source=$(pwd)/scripts/datasource.yml,destination=/etc/grafana/provisioning/datasources/datasource.yml grafana/grafana
+    docker service create -p 9090:9090 --network=engine --name=engine-prometheus --mount type=bind,source=$(pwd)/scripts/prometheus.yml,destination=/etc/prometheus/prometheus.yml prom/prometheus
+  fi
+
   mkdir -p $MESG_PATH
 
   echo "create docker service: "
@@ -72,13 +84,14 @@ function stop_engine {
   onexit
 }
 
-quiet=false
-
-while getopts "q" o; do
+while getopts "qm" o; do
   case $o in
     q)
       quiet=true
       ;;
+    m)
+      monitor=true
+      ;;
     *)
       echo "unknown flag $0"
       exit 1

scripts/prometheus.yml

@@ -0,0 +1,13 @@
+global:
+  scrape_interval:     15s
+  evaluation_interval: 15s
+scrape_configs:
+  - job_name: 'prometheus'
+    static_configs:
+      - targets: ['localhost:9090']
+  - job_name: 'testnet'
+    scrape_interval: 15s
+    static_configs:
+      - targets: ['engine:26660']
+        labels:
+          instance: 'engine'