Automatically restart Hermes relayer on cored version change (#286)

Since in master `cored` we migrate to IBC v7 and Hermes has this bug
informalsystems/hermes#3579 when it is not
able to detect IBC version change we have to restart the relayer on
cored version change.
This PR adds functionality to automatically restart Hermes if cored
version returned by API changes.

.github/workflows/ci.yml

@@ -44,9 +44,7 @@ jobs:
           - ci_step: "integration tests coreum-upgrade & coreum-ibc"
             command: |
               crust build/integration-tests/coreum/upgrade build/integration-tests/coreum/ibc images
-              crust znet test --cored-version=v2.0.2 --test-groups=coreum-upgrade --timeout-commit 1s
-              docker restart znet-ibc-hermes-gaiad
-              crust znet test --test-groups=coreum-ibc --timeout-commit 1s
+              crust znet test --cored-version=v2.0.2 --test-groups=coreum-upgrade,coreum-ibc --timeout-commit 1s
             go-cache: true
             wasm-cache: true
             linter-cache: false

build/docker/basic/Dockerfile.tmpl

@@ -1,5 +1,9 @@
 FROM {{ .From }}
 
+{{ if .Run }}
+RUN {{ .Run }}
+{{ end }}
+
 COPY {{ .Binary }} /{{ .Binary }}
 
 ENTRYPOINT ["/{{ .Binary }}"]

build/docker/basic/template.go

@@ -19,6 +19,9 @@ type Data struct {
 
 	// Binary is the name of binary file to copy from build context
 	Binary string
+
+	// Run is an extra command to be run during image build. RUN directive will be used to execute it.
+	Run string
 }
 
 // Execute executes dockerfile template and returns complete dockerfile.

build/hermes/image.go

@@ -30,6 +30,7 @@ func BuildDockerImage(ctx context.Context, deps build.DepsFunc) error {
 	dockerfile, err := dockerbasic.Execute(dockerbasic.Data{
 		From:   docker.UbuntuImage,
 		Binary: binaryPath,
+		Run:    "apt update && apt install curl jq -y",
 	})
 	if err != nil {
 		return err

infra/apps/hermes/hermes.go

@@ -172,6 +172,12 @@ func (h Hermes) Deployment() infra.Deployment {
 		},
 		PrepareFunc: h.prepare,
 		Entrypoint:  filepath.Join(targets.AppHomeDir, dockerEntrypoint),
+		DockerArgs: []string{
+			// Restart is needed to handle chain upgrade by Hermes.
+			// Since v2.0.2 -> v3 upgrade changes IBC version we have to restart Hermes because it stops working.
+			// https://github.com/informalsystems/hermes/issues/3579
+			"--restart", "on-failure:1000",
+		},
 	}
 }
 
@@ -208,7 +214,7 @@ func (h Hermes) saveConfigFile() error {
 		CoreumGRPCURL:       infra.JoinNetAddr("http", h.config.Cored.Info().HostFromContainer, h.config.Cored.Config().Ports.GRPC),
 		CoreumWebsocketURL:  infra.JoinNetAddr("ws", h.config.Cored.Info().HostFromContainer, h.config.Cored.Config().Ports.RPC) + "/websocket",
 		CoreumAccountPrefix: h.config.Cored.Config().NetworkConfig.Provider.GetAddressPrefix(),
-		// TODO(dzmitryhil) move gas price for host and peed chains to prams
+		// TODO(dzmitryhil) move gas price for host and peer chains to prams
 		CoreumGasPrice: sdk.NewDecCoinFromDec("udevcore", sdk.MustNewDecFromStr("0.0625")),
 
 		PeerChanID:        h.config.PeeredChain.AppConfig().ChainID,
@@ -244,6 +250,7 @@ func (h Hermes) saveRunScriptFile() error {
 
 		CoreumChanID          string
 		CoreumRelayerMnemonic string
+		CoreumRPCURL          string
 		CoreumRelayerCoinType uint32
 
 		PeerChanID          string
@@ -253,6 +260,7 @@ func (h Hermes) saveRunScriptFile() error {
 
 		CoreumChanID:          string(h.config.Cored.Config().NetworkConfig.ChainID()),
 		CoreumRelayerMnemonic: h.config.CoreumRelayerMnemonic,
+		CoreumRPCURL:          infra.JoinNetAddr("http", h.config.Cored.Info().HostFromContainer, h.config.Cored.Config().Ports.RPC),
 		CoreumRelayerCoinType: coreumconstant.CoinType,
 
 		PeerChanID:          h.config.PeeredChain.AppConfig().ChainID,

infra/apps/hermes/run.tmpl

@@ -19,6 +19,35 @@ if [ ! -d "$RELAYER_KEYS_PATH" ]; then
 
 fi
 
-
 echo "Starting the relayer."
-hermes start
+hermes start &
+
+# Capture the process ID to kill it later if needed
+PID=$!
+
+INITIAL_VERSION=$(curl -s {{ .CoreumRPCURL }}/abci_info\? | jq '.result.response.version')
+
+# If initial version is empty kill the process.
+if [ -z "$INITIAL_VERSION" ]; then
+  echo "Failed to fetch the initial API version. Exiting."
+  kill $PID
+  exit 1
+fi
+
+while true; do
+  sleep 3
+
+  CURRENT_VERSION=$(curl -s {{ .CoreumRPCURL }}/abci_info\? | jq '.result.response.version')
+
+  # If fetching of version fails, skip this iteration
+  if [ -z "$CURRENT_VERSION" ]; then
+    echo "API version is not available. Skipping this check."
+    continue
+  fi
+
+  if [ "$INITIAL_VERSION" != "$CURRENT_VERSION" ]; then
+    kill $PID
+    echo "API version changed from $INITIAL_VERSION to $CURRENT_VERSION. Process killed."
+    exit 1
+  fi
+done