diff --git a/tests/integration/actions.go b/tests/integration/actions.go index 32578bc72d..4250187c11 100644 --- a/tests/integration/actions.go +++ b/tests/integration/actions.go @@ -377,7 +377,7 @@ func (tr TestRun) startConsumerChain( "query", "provider", "consumer-genesis", string(tr.chainConfigs[action.consumerChain].chainId), - `--node`, tr.getValidatorNode(action.providerChain, tr.getDefaultValidator(action.providerChain)), + `--node`, tr.getQueryNode(action.providerChain), `-o`, `json`, ) @@ -438,12 +438,12 @@ func (tr TestRun) addChainToRelayer( action addChainToRelayerAction, verbose bool, ) { - valIp := tr.getValidatorIP(action.chain, action.validator) + queryNodeIP := tr.getQueryNodeIP(action.chain) chainId := tr.chainConfigs[action.chain].chainId - keyName := "validator" + fmt.Sprint(action.validator) - rpcAddr := "http://" + valIp + ":26658" - grpcAddr := "tcp://" + valIp + ":9091" - wsAddr := "ws://" + valIp + ":26657/websocket" + keyName := "query" + rpcAddr := "http://" + queryNodeIP + ":26658" + grpcAddr := "tcp://" + queryNodeIP + ":9091" + wsAddr := "ws://" + queryNodeIP + ":26657/websocket" chainConfig := fmt.Sprintf(hermesChainConfigTemplate, grpcAddr, diff --git a/tests/integration/config.go b/tests/integration/config.go index ffebd002dc..95f748f90b 100644 --- a/tests/integration/config.go +++ b/tests/integration/config.go @@ -158,7 +158,11 @@ func (s *TestRun) ValidateStringLiterals() { panic(fmt.Sprintf("ip suffix must be an int: %v\n", err)) } - if ipSuffix < 1 || ipSuffix > 253 { + if ipSuffix == 253 { + panic("ip suffix 253 is reserved for query node") + } + + if ipSuffix < 1 || ipSuffix > 252 { panic("ip suffix out of range, need to change config") } } diff --git a/tests/integration/state.go b/tests/integration/state.go index 23498ce1e5..3e0785cd78 100644 --- a/tests/integration/state.go +++ b/tests/integration/state.go @@ -6,7 +6,6 @@ import ( "os/exec" "regexp" "strconv" - "strings" "time" clienttypes "github.com/cosmos/ibc-go/v3/modules/core/02-client/types" @@ -127,7 +126,7 @@ func (tr TestRun) getBlockHeight(chain chainID) uint { "query", "tendermint-validator-set", - `--node`, tr.getValidatorNode(chain, tr.getDefaultValidator(chain)), + `--node`, tr.getQueryNode(chain), ).CombinedOutput() if err != nil { @@ -229,7 +228,7 @@ func (tr TestRun) getReward(chain chainID, validator validatorID, blockHeight ui tr.validatorConfigs[validator].delAddress, `--height`, fmt.Sprint(blockHeight), - `--node`, tr.getValidatorNode(chain, tr.getDefaultValidator(chain)), + `--node`, tr.getQueryNode(chain), `-o`, `json`, ).CombinedOutput() @@ -252,7 +251,7 @@ func (tr TestRun) getBalance(chain chainID, validator validatorID) uint { "query", "bank", "balances", tr.validatorConfigs[validator].delAddress, - `--node`, tr.getValidatorNode(chain, tr.getDefaultValidator(chain)), + `--node`, tr.getQueryNode(chain), `-o`, `json`, ).CombinedOutput() @@ -275,7 +274,7 @@ func (tr TestRun) getProposal(chain chainID, proposal uint) Proposal { "query", "gov", "proposal", fmt.Sprint(proposal), - `--node`, tr.getValidatorNode(chain, tr.getDefaultValidator(chain)), + `--node`, tr.getQueryNode(chain), `-o`, `json`, ).CombinedOutput() @@ -360,7 +359,7 @@ func (tr TestRun) getValPower(chain chainID, validator validatorID) uint { "query", "tendermint-validator-set", - `--node`, tr.getValidatorNode(chain, tr.getDefaultValidator(chain)), + `--node`, tr.getQueryNode(chain), ).CombinedOutput() if err != nil { @@ -406,7 +405,7 @@ func (tr TestRun) getRepresentativePower(chain chainID, validator validatorID) u "query", "staking", "validator", tr.validatorConfigs[validator].valoperAddress, - `--node`, tr.getValidatorNode(chain, tr.getDefaultValidator(chain)), + `--node`, tr.getQueryNode(chain), `-o`, `json`, ).CombinedOutput() @@ -427,7 +426,7 @@ func (tr TestRun) getParam(chain chainID, param Param) string { param.Subspace, param.Key, - `--node`, tr.getValidatorNode(chain, tr.getDefaultValidator(chain)), + `--node`, tr.getQueryNode(chain), `-o`, `json`, ).CombinedOutput() @@ -440,31 +439,6 @@ func (tr TestRun) getParam(chain chainID, param Param) string { return value.String() } -// Gets a default validator for txs and queries using the first subdirectory -// of the directory of the input chain, which will be the home directory -// of one of the validators. -// TODO: Best solution for default validator fulfilling queries etc. is a dedicated, non validating, full node. -// See https://github.com/cosmos/interchain-security/issues/263 -func (s TestRun) getDefaultValidator(chain chainID) validatorID { - //#nosec G204 -- Bypass linter warning for spawning subprocess with cmd arguments. - bz, err := exec.Command("docker", "exec", s.containerConfig.instanceName, "bash", "-c", - `cd /`+string(s.chainConfigs[chain].chainId)+ - `; ls -d */ | awk '{print $1}' | head -n 1`).CombinedOutput() - - if err != nil { - log.Fatal(err, "\n", string(bz)) - } - - // Returned string will be of format: "validator/" - bzPrefixTrimmed := strings.TrimPrefix(string(bz), "validator") - bzFullyTrimmed := bzPrefixTrimmed[:len(bzPrefixTrimmed)-2] - if bzPrefixTrimmed == string(bz) || bzFullyTrimmed == string(bz) { - log.Fatal("unexpected validator subdirectory name: ", bz) - } - - return validatorID(bzFullyTrimmed) -} - func (tr TestRun) getValidatorNode(chain chainID, validator validatorID) string { return "tcp://" + tr.getValidatorIP(chain, validator) + ":26658" } @@ -476,3 +450,14 @@ func (tr TestRun) getValidatorIP(chain chainID, validator validatorID) string { func (tr TestRun) getValidatorHome(chain chainID, validator validatorID) string { return `/` + string(tr.chainConfigs[chain].chainId) + `/validator` + fmt.Sprint(validator) } + +// getQueryNode returns query node tcp address on chain. +func (tr TestRun) getQueryNode(chain chainID) string { + return fmt.Sprintf("tcp://%s:26658", tr.getQueryNodeIP(chain)) +} + +// getQueryNodeIP returns query node IP for chain, +// ipSuffix is hardcoded to be 253 on all query nodes. +func (tr TestRun) getQueryNodeIP(chain chainID) string { + return fmt.Sprintf("%s.253", tr.chainConfigs[chain].ipPrefix) +} diff --git a/tests/integration/testnet-scripts/start-chain.sh b/tests/integration/testnet-scripts/start-chain.sh index e61aa3cec8..aa2a78187f 100644 --- a/tests/integration/testnet-scripts/start-chain.sh +++ b/tests/integration/testnet-scripts/start-chain.sh @@ -230,12 +230,65 @@ do ip netns exec $NET_NAMESPACE_NAME $BIN $ARGS start &> /$CHAIN_ID/validator$VAL_ID/logs & done +## SETUP QUERY NODE NETWORK NAMESPACE +QUERY_NODE_ID="query" +QUERY_IP_SUFFIX="253" +QUERY_NET_NAMESPACE_NAME="$CHAIN_ID-$QUERY_NODE_ID" +QUERY_IP_ADDR="$CHAIN_IP_PREFIX.$QUERY_IP_SUFFIX/24" + +ip netns add $QUERY_NET_NAMESPACE_NAME +ip link add $QUERY_NET_NAMESPACE_NAME-in type veth peer name $QUERY_NET_NAMESPACE_NAME-out +ip link set $QUERY_NET_NAMESPACE_NAME-in netns $QUERY_NET_NAMESPACE_NAME +ip netns exec $QUERY_NET_NAMESPACE_NAME ip addr add $QUERY_IP_ADDR dev $QUERY_NET_NAMESPACE_NAME-in +ip link set $QUERY_NET_NAMESPACE_NAME-out master virtual-bridge +## DONE ADD SETUP QUERY NODE NETWORK NAMESPACE + +## QUERY NODE ENABLE DEVICE +ip link set $QUERY_NET_NAMESPACE_NAME-out up +ip netns exec $QUERY_NET_NAMESPACE_NAME ip link set dev $QUERY_NET_NAMESPACE_NAME-in up +ip netns exec $QUERY_NET_NAMESPACE_NAME ip link set dev lo up +## DONE QUERY NODE ENABLE DEVICE + +## INIT QUERY NODE +$BIN init --home /$CHAIN_ID/$QUERY_NODE_ID --chain-id=$CHAIN_ID $QUERY_NODE_ID > /dev/null +cp /$CHAIN_ID/genesis.json /$CHAIN_ID/$QUERY_NODE_ID/config/genesis.json +## DONE INIT QUERY NODE + + +## START QUERY NODE +QUERY_GAIA_HOME="--home /$CHAIN_ID/$QUERY_NODE_ID" +QUERY_RPC_ADDRESS="--rpc.laddr tcp://$CHAIN_IP_PREFIX.$QUERY_IP_SUFFIX:26658" +QUERY_GRPC_ADDRESS="--grpc.address $CHAIN_IP_PREFIX.$QUERY_IP_SUFFIX:9091" +QUERY_LISTEN_ADDRESS="--address tcp://$CHAIN_IP_PREFIX.$QUERY_IP_SUFFIX:26655" +QUERY_P2P_ADDRESS="--p2p.laddr tcp://$CHAIN_IP_PREFIX.$QUERY_IP_SUFFIX:26656" +QUERY_LOG_LEVEL="--log_level trace" # Keep as "trace" to see panic messages +QUERY_ENABLE_WEBGRPC="--grpc-web.enable=false" + +QUERY_PERSISTENT_PEERS="" + +## add validators to persistend peers of QUERY node +for j in $(seq 0 $(($NODES - 1))); +do + PEER_VAL_ID=$(echo "$VALIDATORS" | jq -r ".[$j].val_id") + PEER_VAL_IP_SUFFIX=$(echo "$VALIDATORS" | jq -r ".[$j].ip_suffix") + NODE_ID=$($BIN tendermint show-node-id --home /$CHAIN_ID/validator$PEER_VAL_ID) + ADDRESS="$NODE_ID@$CHAIN_IP_PREFIX.$PEER_VAL_IP_SUFFIX:26656" + QUERY_PERSISTENT_PEERS="$QUERY_PERSISTENT_PEERS,$ADDRESS" +done + +# Remove leading comma and concat to flag +QUERY_PERSISTENT_PEERS="--p2p.persistent_peers ${QUERY_PERSISTENT_PEERS:1}" + +## START NODE +ARGS="$QUERY_GAIA_HOME $QUERY_LISTEN_ADDRESS $QUERY_RPC_ADDRESS $QUERY_GRPC_ADDRESS $QUERY_LOG_LEVEL $QUERY_P2P_ADDRESS $QUERY_ENABLE_WEBGRPC $QUERY_PERSISTENT_PEERS" +ip netns exec $QUERY_NET_NAMESPACE_NAME $BIN $ARGS start &> /$CHAIN_ID/$QUERY_NODE_ID/logs & +## DONE START NODE # poll for chain start set +e -until $BIN query block --node "tcp://$CHAIN_IP_PREFIX.$FIRST_VAL_IP_SUFFIX:26658" | grep -q -v '{"block_id":{"hash":"","parts":{"total":0,"hash":""}},"block":null}'; do sleep 0.3 ; done +until $BIN query block --node "tcp://$CHAIN_IP_PREFIX.$QUERY_IP_SUFFIX:26658" | grep -q -v '{"block_id":{"hash":"","parts":{"total":0,"hash":""}},"block":null}'; do sleep 0.3 ; done set -e echo "done!!!!!!!!" diff --git a/tests/integration/testnet-scripts/start-docker.sh b/tests/integration/testnet-scripts/start-docker.sh index 179d38ac52..c4a016309d 100755 --- a/tests/integration/testnet-scripts/start-docker.sh +++ b/tests/integration/testnet-scripts/start-docker.sh @@ -36,5 +36,4 @@ rm -rf ./cosmos-sdk/ # Run new test container instance with extended privileges. # Extended privileges are granted to the container here to allow for network namespace manipulation (bringing a node up/down) # See: https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities -docker run --name "$INSTANCE_NAME" --cap-add=NET_ADMIN --privileged "$CONTAINER_NAME" /bin/bash /testnet-scripts/beacon.sh -` +docker run --name "$INSTANCE_NAME" --cap-add=NET_ADMIN --privileged "$CONTAINER_NAME" /bin/bash /testnet-scripts/beacon.sh &