Automatically restart Hermes relayer on cored version change

.github/workflows/ci.yml

@@ -44,9 +44,7 @@ jobs:
           - ci_step: "integration tests coreum-upgrade & coreum-ibc"
             command: |
               crust build/integration-tests/coreum/upgrade build/integration-tests/coreum/ibc images
-              crust znet test --cored-version=v2.0.2 --test-groups=coreum-upgrade --timeout-commit 1s
-              docker restart znet-ibc-hermes-gaiad
-              crust znet test --test-groups=coreum-ibc --timeout-commit 1s
+              crust znet test --cored-version=v2.0.2 --test-groups=coreum-upgrade,coreum-ibc --timeout-commit 1s
             go-cache: true
             wasm-cache: true
             linter-cache: false

build/docker/basic/Dockerfile.tmpl

@@ -1,5 +1,9 @@
 FROM {{ .From }}
 
+{{ if .Run }}
+RUN {{ .Run }}
+{{ end }}
+
 COPY {{ .Binary }} /{{ .Binary }}
 
 ENTRYPOINT ["/{{ .Binary }}"]

build/docker/basic/template.go

@@ -19,6 +19,9 @@ type Data struct {
 
 	// Binary is the name of binary file to copy from build context
 	Binary string
+
+	// Run is an extra command to be run during image build. RUN directive will be used to execute it.
+	Run string
 }
 
 // Execute executes dockerfile template and returns complete dockerfile.

build/hermes/image.go

@@ -30,6 +30,7 @@ func BuildDockerImage(ctx context.Context, deps build.DepsFunc) error {
 	dockerfile, err := dockerbasic.Execute(dockerbasic.Data{
 		From:   docker.UbuntuImage,
 		Binary: binaryPath,
+		Run:    "apt update && apt install curl jq -y",
 	})
 	if err != nil {
 		return err

infra/apps/hermes/hermes.go

@@ -172,6 +172,12 @@ func (h Hermes) Deployment() infra.Deployment {
 		},
 		PrepareFunc: h.prepare,
 		Entrypoint:  filepath.Join(targets.AppHomeDir, dockerEntrypoint),
+		DockerArgs: []string{
+			// Restart is needed to handle chain upgrade by Hermes.
+			// Since v2.0.2 -> v3 upgrade changes IBC version we have to restart Hermes because it stops working.
+			// https://github.com/informalsystems/hermes/issues/3579
+			"--restart", "on-failure:1000",
+		},
 	}
 }
 
@@ -208,7 +214,7 @@ func (h Hermes) saveConfigFile() error {
 		CoreumGRPCURL:       infra.JoinNetAddr("http", h.config.Cored.Info().HostFromContainer, h.config.Cored.Config().Ports.GRPC),
 		CoreumWebsocketURL:  infra.JoinNetAddr("ws", h.config.Cored.Info().HostFromContainer, h.config.Cored.Config().Ports.RPC) + "/websocket",
 		CoreumAccountPrefix: h.config.Cored.Config().NetworkConfig.Provider.GetAddressPrefix(),
-		// TODO(dzmitryhil) move gas price for host and peed chains to prams
+		// TODO(dzmitryhil) move gas price for host and peer chains to prams
 		CoreumGasPrice: sdk.NewDecCoinFromDec("udevcore", sdk.MustNewDecFromStr("0.0625")),
 
 		PeerChanID:        h.config.PeeredChain.AppConfig().ChainID,
@@ -244,6 +250,7 @@ func (h Hermes) saveRunScriptFile() error {
 
 		CoreumChanID          string
 		CoreumRelayerMnemonic string
+		CoreumRPCURL          string
 		CoreumRelayerCoinType uint32
 
 		PeerChanID          string
@@ -253,6 +260,7 @@ func (h Hermes) saveRunScriptFile() error {
 
 		CoreumChanID:          string(h.config.Cored.Config().NetworkConfig.ChainID()),
 		CoreumRelayerMnemonic: h.config.CoreumRelayerMnemonic,
+		CoreumRPCURL:          infra.JoinNetAddr("http", h.config.Cored.Info().HostFromContainer, h.config.Cored.Config().Ports.RPC),
 		CoreumRelayerCoinType: coreumconstant.CoinType,
 
 		PeerChanID:          h.config.PeeredChain.AppConfig().ChainID,

infra/apps/hermes/run.tmpl

@@ -19,6 +19,35 @@ if [ ! -d "$RELAYER_KEYS_PATH" ]; then
 
 fi
 
-
 echo "Starting the relayer."
-hermes start
+hermes start &
+
+# Capture the process ID to kill it later if needed
+PID=$!
+
+INITIAL_VERSION=$(curl -s {{ .CoreumRPCURL }}/abci_info\? | jq '.result.response.version')
+
+# If initial version is empty kill the process.
+if [ -z "$INITIAL_VERSION" ]; then
+  echo "Failed to fetch the initial API version. Exiting."
+  kill $PID
+  exit 1
+fi
+
+while true; do
+  sleep 3
+
+  CURRENT_VERSION=$(curl -s {{ .CoreumRPCURL }}/abci_info\? | jq '.result.response.version')
+
+  # If fetching of version fails, skip this iteration
+  if [ -z "$CURRENT_VERSION" ]; then
+    echo "API version is not available. Skipping this check."
+    continue
+  fi
+
+  if [ "$INITIAL_VERSION" != "$CURRENT_VERSION" ]; then
+    kill $PID
+    echo "API version changed from $INITIAL_VERSION to $CURRENT_VERSION. Process killed."
+    exit 1
+  fi
+done