From a04c1f1f910e3eb248a55ec583bb50509a486fb9 Mon Sep 17 00:00:00 2001 From: sampocs Date: Sun, 21 Jul 2024 00:04:12 -0500 Subject: [PATCH 01/24] updated add-consumer-section to be more flexible --- cmd/consumer.go | 135 +++++++++++++++++++++++++++--------- dockernet/src/init_chain.sh | 7 +- scripts/start_local_node.sh | 6 +- 3 files changed, 108 insertions(+), 40 deletions(-) diff --git a/cmd/consumer.go b/cmd/consumer.go index a7d79fa43..68235e819 100644 --- a/cmd/consumer.go +++ b/cmd/consumer.go @@ -1,13 +1,17 @@ package cmd import ( + "encoding/base64" "encoding/json" + "errors" "fmt" - "strconv" + "strings" errorsmod "cosmossdk.io/errors" types1 "github.com/cometbft/cometbft/abci/types" + "github.com/cometbft/cometbft/config" pvm "github.com/cometbft/cometbft/privval" + tmprotocrypto "github.com/cometbft/cometbft/proto/tendermint/crypto" tmtypes "github.com/cometbft/cometbft/types" "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/client/flags" @@ -22,48 +26,110 @@ import ( "github.com/Stride-Labs/stride/v23/testutil" ) +const ( + FlagValidatorPublicKeys = "validator-public-keys" + FlagValidatorHomeDirectories = "validator-home-directories" +) + +// Builds the list of validator Ed25519 pubkeys from a comma separate list of base64 encoded pubkeys +func buildPublicKeysFromString(publicKeysRaw string) (publicKeys []tmprotocrypto.PublicKey, err error) { + for _, publicKeyEncoded := range strings.Split(publicKeysRaw, ",") { + if publicKeyEncoded == "" { + continue + } + publicKeyBytes, err := base64.StdEncoding.DecodeString(publicKeyEncoded) + if err != nil { + return nil, errorsmod.Wrapf(err, "unable to decode public key") + } + publicKeys = append(publicKeys, tmprotocrypto.PublicKey{ + Sum: &tmprotocrypto.PublicKey_Ed25519{ + Ed25519: publicKeyBytes, + }, + }) + } + + return publicKeys, nil +} + +// Builds the list validator Ed25519 pubkeys from a comma separated list of validator home directories +func buildPublicKeysFromHomeDirectories(config *config.Config, homeDirectories string) (publicKeys []tmprotocrypto.PublicKey, err error) { + for _, homeDir := range strings.Split(homeDirectories, ",") { + if homeDir == "" { + continue + } + config.SetRoot(homeDir) + + privValidator := pvm.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()) + pk, err := privValidator.GetPubKey() + if err != nil { + return nil, err + } + sdkPublicKey, err := cryptocodec.FromTmPubKeyInterface(pk) + if err != nil { + return nil, err + } + tmProtoPublicKey, err := cryptocodec.ToTmProtoPublicKey(sdkPublicKey) + if err != nil { + return nil, err + } + publicKeys = append(publicKeys, tmProtoPublicKey) + } + + return publicKeys, nil +} + func AddConsumerSectionCmd(defaultNodeHome string) *cobra.Command { genesisMutator := NewDefaultGenesisIO() - txCmd := &cobra.Command{ - Use: "add-consumer-section [num_nodes]", - Args: cobra.ExactArgs(1), + cmd := &cobra.Command{ + Use: "add-consumer-section", + Args: cobra.ExactArgs(0), Short: "ONLY FOR TESTING PURPOSES! Modifies genesis so that chain can be started locally with one node.", SuggestionsMinimumDistance: 2, RunE: func(cmd *cobra.Command, args []string) error { - numNodes, err := strconv.Atoi(args[0]) + // We need to public keys for each validator - they can either be provided explicitly + // or indirectly by providing the validator home directories + publicKeysRaw, err := cmd.Flags().GetString(FlagValidatorPublicKeys) if err != nil { - return errorsmod.Wrap(err, "invalid number of nodes") - } else if numNodes == 0 { - return errorsmod.Wrap(nil, "num_nodes can not be zero") + return errorsmod.Wrapf(err, "unable to parse validator public key flag") + } + homeDirectoriesRaw, err := cmd.Flags().GetString(FlagValidatorHomeDirectories) + if err != nil { + return errorsmod.Wrapf(err, "unable to parse validator home directories flag") + } + if (publicKeysRaw == "" && homeDirectoriesRaw == "") || (publicKeysRaw != "" && homeDirectoriesRaw != "") { + return fmt.Errorf("must specified either --%s or --%s", FlagValidatorPublicKeys, FlagValidatorHomeDirectories) + } + + // Build up a list of the validator public keys + // If the public keys were passed directly, decode them and create the tm proto pub keys + // Otherwise, derrive them from the private keys in each validator's home directory + var tmPublicKeys []tmprotocrypto.PublicKey + if publicKeysRaw != "" { + tmPublicKeys, err = buildPublicKeysFromString(publicKeysRaw) + if err != nil { + return err + } + } else { + serverCtx := server.GetServerContextFromCmd(cmd) + config := serverCtx.Config + + tmPublicKeys, err = buildPublicKeysFromHomeDirectories(config, homeDirectoriesRaw) + if err != nil { + return err + } + } + + if len(tmPublicKeys) == 0 { + return errors.New("no valid public keys or validator home directories provided") } return genesisMutator.AlterConsumerModuleState(cmd, func(state *GenesisData, _ map[string]json.RawMessage) error { initialValset := []types1.ValidatorUpdate{} genesisState := testutil.CreateMinimalConsumerTestGenesis() - clientCtx := client.GetClientContextFromCmd(cmd) - serverCtx := server.GetServerContextFromCmd(cmd) - config := serverCtx.Config - homeDir := clientCtx.HomeDir - for i := 1; i <= numNodes; i++ { - homeDir = fmt.Sprintf("%s%d", homeDir[:len(homeDir)-1], i) - config.SetRoot(homeDir) - - privValidator := pvm.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()) - pk, err := privValidator.GetPubKey() - if err != nil { - return err - } - sdkPublicKey, err := cryptocodec.FromTmPubKeyInterface(pk) - if err != nil { - return err - } - tmProtoPublicKey, err := cryptocodec.ToTmProtoPublicKey(sdkPublicKey) - if err != nil { - return err - } - - initialValset = append(initialValset, types1.ValidatorUpdate{PubKey: tmProtoPublicKey, Power: 100}) + + for _, publicKey := range tmPublicKeys { + initialValset = append(initialValset, types1.ValidatorUpdate{PubKey: publicKey, Power: 100}) } vals, err := tmtypes.PB2TM.ValidatorUpdates(initialValset) @@ -80,10 +146,11 @@ func AddConsumerSectionCmd(defaultNodeHome string) *cobra.Command { }, } - txCmd.Flags().String(flags.FlagHome, defaultNodeHome, "The application home directory") - flags.AddQueryFlagsToCmd(txCmd) + cmd.Flags().String(flags.FlagHome, defaultNodeHome, "The application home directory") + cmd.Flags().String(FlagValidatorPublicKeys, "", "Comma separated, base64-encoded public keys for each validator") + cmd.Flags().String(FlagValidatorHomeDirectories, "", "Comma separated list of home directories for each validator") - return txCmd + return cmd } type GenesisMutator interface { diff --git a/dockernet/src/init_chain.sh b/dockernet/src/init_chain.sh index ae8f4890c..cc46ef94f 100644 --- a/dockernet/src/init_chain.sh +++ b/dockernet/src/init_chain.sh @@ -139,7 +139,12 @@ set_consumer_genesis() { genesis_config=$1 # add consumer genesis - $MAIN_CMD add-consumer-section $NUM_NODES + home_directories="" + for (( i=1; i <= $NUM_NODES; i++ )); do + home_directories+="${STATE}/stride${i}," + done + + $MAIN_CMD add-consumer-section --validator-home-directories $home_directories jq '.app_state.ccvconsumer.params.unbonding_period = $newVal' --arg newVal "$UNBONDING_TIME" $genesis_config > json.tmp && mv json.tmp $genesis_config } diff --git a/scripts/start_local_node.sh b/scripts/start_local_node.sh index fa9ae2a12..827bb9a0d 100644 --- a/scripts/start_local_node.sh +++ b/scripts/start_local_node.sh @@ -49,11 +49,7 @@ jq "del(.app_state.interchain_accounts)" $genesis_json > json.tmp && mv json.tmp interchain_accts=$(cat dockernet/config/ica_controller.json) jq ".app_state += $interchain_accts" $genesis_json > json.tmp && mv json.tmp $genesis_json -# hack since add-comsumer-section is built for dockernet -rm -rf ~/.stride-loca1 -cp -r ${STRIDE_HOME} ~/.stride-loca1 - -$STRIDED add-consumer-section 1 +$STRIDED add-consumer-section --validator-home-directories $STRIDE_HOME jq '.app_state.ccvconsumer.params.unbonding_period = $newVal' --arg newVal "$UNBONDING_TIME" $genesis_json > json.tmp && mv json.tmp $genesis_json rm -rf ~/.stride-loca1 From abef2372945d19f111d8e785ef5c54e5b456b519 Mon Sep 17 00:00:00 2001 From: sampocs Date: Sun, 21 Jul 2024 18:27:42 -0500 Subject: [PATCH 02/24] startup working with 1 stride validator --- .gitignore | 3 + integration-tests/Makefile | 31 ++++++++ integration-tests/config/keys.json | 30 ++++++++ integration-tests/scripts/setup.sh | 106 ++++++++++++++++++++++++++ integration-tests/validator.yaml | 116 +++++++++++++++++++++++++++++ 5 files changed, 286 insertions(+) create mode 100644 integration-tests/Makefile create mode 100644 integration-tests/config/keys.json create mode 100644 integration-tests/scripts/setup.sh create mode 100644 integration-tests/validator.yaml diff --git a/.gitignore b/.gitignore index 8153ce13e..433a4c1c5 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,6 @@ vue/* !.vscode/settings.json .ipynb_checkpoints/* + +integration-tests/state +integration-tests/shared \ No newline at end of file diff --git a/integration-tests/Makefile b/integration-tests/Makefile new file mode 100644 index 000000000..b570ebd26 --- /dev/null +++ b/integration-tests/Makefile @@ -0,0 +1,31 @@ +K8S_NAMESPACE=integration + +KUBECTL := $(shell which kubectl) +DOCKER := $(shell which docker) + +upload: docker-build docker-push + +docker-build: + @echo "Building docker image: stride-test" + @$(DOCKER) buildx build --platform linux/amd64 --tag stride-test .. + @$(DOCKER) tag stride-test gcr.io/stride-nodes/stride-test + +docker-push: + @echo "Pushing image to GCR" + @$(DOCKER) push gcr.io/stride-nodes/stride-test + +update-scripts: + @echo "Updating scripts configmap" + @$(KUBECTL) delete configmap scripts -n $(K8S_NAMESPACE) --ignore-not-found=true + @$(KUBECTL) create configmap scripts --from-file=scripts -n $(K8S_NAMESPACE) + +update-config: + @echo "Updating config configmap" + @$(KUBECTL) delete configmap config -n $(K8S_NAMESPACE) --ignore-not-found=true + @$(KUBECTL) create configmap config --from-file=config -n $(K8S_NAMESPACE) + +start: + @$(KUBECTL) apply -f validator.yaml + +stop: + @$(KUBECTL) delete -f validator.yaml \ No newline at end of file diff --git a/integration-tests/config/keys.json b/integration-tests/config/keys.json new file mode 100644 index 000000000..33aace07f --- /dev/null +++ b/integration-tests/config/keys.json @@ -0,0 +1,30 @@ +{ + "validators": [ + { + "name": "val1", + "mnemonic": "close soup mirror crew erode defy knock trigger gather eyebrow tent farm gym gloom base lemon sleep weekend rich forget diagram hurt prize fly" + }, + { + "name": "val2", + "mnemonic": "turkey miss hurry unable embark hospital kangaroo nuclear outside term toy fall buffalo book opinion such moral meadow wing olive camp sad metal banner" + }, + { + "name": "val3", + "mnemonic": "tenant neck ask season exist hill churn rice convince shock modify evidence armor track army street stay light program harvest now settle feed wheat" + }, + { + "name": "val4", + "mnemonic": "tail forward era width glory magnet knock shiver cup broken turkey upgrade cigar story agent lake transfer misery sustain fragile parrot also air document" + }, + { + "name": "val5", + "mnemonic": "crime lumber parrot enforce chimney turtle wing iron scissors jealous indicate peace empty game host protect juice submit motor cause second picture nuclear area" + } + ], + "faucet": [ + { + "name": "faucet", + "mnemonic": "chimney become stuff spoil resource supply picture divorce casual curve check web valid survey zebra various pet sphere timber friend faint blame mansion film" + } + ] + } diff --git a/integration-tests/scripts/setup.sh b/integration-tests/scripts/setup.sh new file mode 100644 index 000000000..08a736174 --- /dev/null +++ b/integration-tests/scripts/setup.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +set -e + +LOCAL_MODE=${1:-false} + +CHAIN_NAME=stride +BINARY=strided +DENOM=ustrd +VALIDATOR_BALANCE=10000000 +MICRO_DENOM_UNITS=000000 +CHAIN_HOME=.stride +SHARED_DIR=shared +NUM_VALIDATORS=1 + +STRIDE_DAY_EPOCH_DURATION="140s" +STRIDE_EPOCH_EPOCH_DURATION="35s" +MAX_DEPOSIT_PERIOD="30s" +VOTING_PERIOD="30s" +UNBONDING_TIME="240s" +BLOCK_TIME=1s + +# check if the binary has genesis subcommand or not, if not, set CHAIN_GENESIS_CMD to empty +CHAIN_GENESIS_CMD=$($BINARY 2>&1 | grep -q "genesis-related subcommands" && echo "genesis" || echo "") + +jq_inplace() { + jq_filter="$1" + file="$2" + + jq "$jq_filter" "$file" > /tmp/$(basename $file) && mv /tmp/$(basename $file) ${file} +} + +# If this is being run locally, don't overwrite the main chain folder +if [[ "$LOCAL_MODE" == "true" ]]; then + CHAIN_HOME=state + rm -rf state + BINARY="$BINARY --home $CHAIN_HOME" +fi + +moniker=${CHAIN_NAME}1 +chain_id=${CHAIN_NAME}-test-1 + +$BINARY init $moniker --chain-id $chain_id --overwrite + +$BINARY config keyring-backend test + +validator_public_keys="" +for (( i=1; i <= $NUM_VALIDATORS; i++ )); do + validator_config=$(jq -r '.validators[$index]' --argjson index "$((i-1))" config/keys.json) + name=$(echo $validator_config | jq -r '.name') + mnemonic=$(echo $validator_config | jq -r '.mnemonic') + + echo $mnemonic | $BINARY keys add $name --recover + address=$($BINARY keys show $name -a) + + genesis_balance=${VALIDATOR_BALANCE}${MICRO_DENOM_UNITS}${DENOM} + $BINARY $CHAIN_GENESIS_CMD add-genesis-account $address $genesis_balance + + validator_home=/tmp/${CHAIN_NAME}-${name} + $BINARY init $CHAIN_NAME-$name --chain-id $chain_id --overwrite --home ${validator_home} &> /dev/null + node_id=$($BINARY tendermint show-node-id --home ${validator_home}) + + mkdir -p ${SHARED_DIR}/validator_keys + mkdir -p ${SHARED_DIR}/node_ids + + mv ${validator_home}/config/priv_validator_key.json ${SHARED_DIR}/validator_keys/${name}.json + echo $node_id > ${SHARED_DIR}/node_ids/${name}.json + + validator_public_keys+="$(jq -r '.pub_key.value' ${SHARED_DIR}/validator_keys/${name}.json)," +done + +config_toml="${CHAIN_HOME}/config/config.toml" +client_toml="${CHAIN_HOME}/config/client.toml" +app_toml="${CHAIN_HOME}/config/app.toml" +genesis_json=${CHAIN_HOME}/config/genesis.json + +sed -i -E "s|\"stake\"|\"${DENOM}\"|g" $genesis_json + +jq_inplace '(.app_state.epochs.epochs[] | select(.identifier=="day") ).duration |= "'$STRIDE_DAY_EPOCH_DURATION'"' $genesis_json +jq_inplace '(.app_state.epochs.epochs[] | select(.identifier=="stride_epoch") ).duration |= "'$STRIDE_EPOCH_EPOCH_DURATION'"' $genesis_json +jq_inplace '.app_state.gov.params.max_deposit_period |= "'$MAX_DEPOSIT_PERIOD'"' $genesis_json +jq_inplace '.app_state.gov.params.voting_period |= "'$VOTING_PERIOD'"' $genesis_json + +$BINARY add-consumer-section --validator-public-keys $validator_public_keys + +# wipe out the persistent peers for the main node (these are incorrectly autogenerated for each validator during collect-gentxs) +sed -i -E "s|^persistent_peers = .*|persistent_peers = \"\"|g" $config_toml + +sed -i -E "s|cors_allowed_origins = \[\]|cors_allowed_origins = [\"\*\"]|g" $config_toml +sed -i -E "s|127.0.0.1|0.0.0.0|g" $config_toml +sed -i -E "s|timeout_commit = \"5s\"|timeout_commit = \"${BLOCK_TIME}\"|g" $config_toml +sed -i -E "s|prometheus = false|prometheus = true|g" $config_toml + +sed -i -E "s|minimum-gas-prices = \".*\"|minimum-gas-prices = \"0${DENOM}\"|g" $app_toml +sed -i -E '/\[api\]/,/^enable = .*$/ s/^enable = .*$/enable = true/' $app_toml +sed -i -E 's|unsafe-cors = .*|unsafe-cors = true|g' $app_toml +sed -i -E 's|localhost|0.0.0.0|g' $app_toml + +sed -i -E "s|chain-id = \"\"|chain-id = \"${CHAIN_ID}\"|g" $client_toml +sed -i -E "s|keyring-backend = \"os\"|keyring-backend = \"test\"|g" $client_toml +sed -i -E "s|node = \".*\"|node = \"tcp://localhost:$RPC_PORT\"|g" $client_toml + +sed -i -E "s|\"stake\"|\"${DENOM}\"|g" $genesis_json +sed -i -E "s|\"aphoton\"|\"${DENOM}\"|g" $genesis_json # ethermint default + +mv ${SHARED_DIR}/validator_keys/val1.json ${CHAIN_HOME}/config/priv_validator_key.json diff --git a/integration-tests/validator.yaml b/integration-tests/validator.yaml new file mode 100644 index 000000000..570226460 --- /dev/null +++ b/integration-tests/validator.yaml @@ -0,0 +1,116 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: stride-validator + namespace: integration +spec: + replicas: 1 + selector: + matchLabels: + app: stride-validator + serviceName: stride-validator + template: + metadata: + labels: + app: stride-validator + spec: + securityContext: + fsGroup: 1000 + containers: + - name: stride-validator + image: gcr.io/stride-nodes/stride-test + imagePullPolicy: Always + command: + - bash + - "-c" + - | + bash scripts/setup.sh + strided start --reject-config-defaults + ports: + - name: rpc + containerPort: 26657 + protocol: TCP + - name: api + containerPort: 1317 + protocol: TCP + - name: grpc + containerPort: 9090 + protocol: TCP + volumeMounts: + - name: state + mountPath: /home/stride/.stride + - name: shared + mountPath: /home/stride/shared + - name: scripts + mountPath: /home/stride/scripts + - name: config + mountPath: /home/stride/config + resources: + limits: + cpu: "1000m" + memory: "2Gi" + requests: + cpu: "1000m" + memory: "2Gi" + # readinessProbe: + # exec: + # command: + # - bash + # - "-c" + # - | + # set -e + + # BINARY=strided + # STRIDE_HOME=/home/stride/.stride + + # # If chain hasn't been initialized yet, exit immediately + # if [ ! -d $STRIDE_HOME/config ]; then + # echo "READINESS CHECK FAILED - Chain has not been initialized yet." + # exit 1 + # fi + + # # Check that the node is running + # if ! $($BINARY status &> /dev/null); then + # echo "READINESS CHECK FAILED - Chain is down" + # exit 1 + # fi + + # # Then check if the node is synced according to it's status query + # CATCHING_UP=$($BINARY status 2>&1 | jq ".SyncInfo.catching_up") + # if [[ "$CATCHING_UP" != "false" ]]; then + # echo "READINESS CHECK FAILED - Node is still syncing" + # exit 1 + # fi + # periodSeconds: 5 + volumes: + - name: state + emptyDir: + medium: "Memory" + - name: shared + emptyDir: + medium: "Memory" + - name: scripts + configMap: + name: scripts + - name: config + configMap: + name: config +--- +apiVersion: v1 +kind: Service +metadata: + name: stride-validator + namespace: integration +spec: + selector: + app: stride-validator + ports: + - port: 26657 + name: rpc + protocol: TCP + - port: 1317 + name: api + protocol: TCP + - port: 9090 + name: grpc + protocol: TCP From 48e13416eb275bc6c0175311981b743b8297de23 Mon Sep 17 00:00:00 2001 From: sampocs Date: Mon, 22 Jul 2024 18:32:03 -0500 Subject: [PATCH 03/24] expanded setup to support multiple nodes --- integration-tests/scripts/init-chain.sh | 110 ++++++++++++++++++++++++ integration-tests/scripts/init-node.sh | 101 ++++++++++++++++++++++ integration-tests/scripts/liveness.sh | 23 +++++ integration-tests/scripts/setup.sh | 106 ----------------------- integration-tests/validator.yaml | 103 ++++++++++++---------- 5 files changed, 292 insertions(+), 151 deletions(-) create mode 100644 integration-tests/scripts/init-chain.sh create mode 100644 integration-tests/scripts/init-node.sh create mode 100644 integration-tests/scripts/liveness.sh delete mode 100644 integration-tests/scripts/setup.sh diff --git a/integration-tests/scripts/init-chain.sh b/integration-tests/scripts/init-chain.sh new file mode 100644 index 000000000..ce59ca20b --- /dev/null +++ b/integration-tests/scripts/init-chain.sh @@ -0,0 +1,110 @@ +#!/bin/bash + +set -e + +LOCAL_MODE=${1:-false} + +CHAIN_NAME=stride +BINARY=strided +DENOM=ustrd +CHAIN_ID=${CHAIN_NAME}-test-1 +VALIDATOR_BALANCE=10000000 +MICRO_DENOM_UNITS=000000 +CHAIN_HOME=.stride +SHARED_DIR=shared +NUM_VALIDATORS=3 + +STRIDE_DAY_EPOCH_DURATION="140s" +STRIDE_EPOCH_EPOCH_DURATION="35s" +MAX_DEPOSIT_PERIOD="30s" +VOTING_PERIOD="30s" +UNBONDING_TIME="240s" + +# If this is being run locally, don't overwrite the main chain folder +if [[ "$LOCAL_MODE" == "true" ]]; then + CHAIN_HOME=state + rm -rf state + BINARY="$BINARY --home $CHAIN_HOME" +fi + +# Helper to update a json attribute in-place +jq_inplace() { + jq_filter="$1" + file="$2" + + jq "$jq_filter" "$file" > /tmp/$(basename $file) && mv /tmp/$(basename $file) ${file} +} + +# Initializes the chain config +init_config() { + moniker=${CHAIN_NAME}1 + $BINARY init $moniker --chain-id $CHAIN_ID --overwrite + $BINARY config keyring-backend test +} + +# Adds each validator to the genesis file, and also saves down the public keys +# which are needed for ICS +# Each validators public private key and node ID are saved in the shared directory +add_validators() { + echo "Adding validators..." + + # check if the binary has genesis subcommand or not, if not, set CHAIN_GENESIS_CMD to empty + chain_genesis_command=$($BINARY 2>&1 | grep -q "genesis-related subcommands" && echo "genesis" || echo "") + + validator_public_keys="" + for (( i=1; i <= $NUM_VALIDATORS; i++ )); do + # Extract the validator name and mnemonic from keys.json + validator_config=$(jq -r '.validators[$index]' --argjson index "$((i-1))" config/keys.json) + name=$(echo $validator_config | jq -r '.name') + mnemonic=$(echo $validator_config | jq -r '.mnemonic') + + # Add the key to the keyring + echo "$mnemonic" | $BINARY keys add $name --recover + address=$($BINARY keys show $name -a) + + # Add the genesis account + genesis_balance=${VALIDATOR_BALANCE}${MICRO_DENOM_UNITS}${DENOM} + $BINARY $chain_genesis_command add-genesis-account $address $genesis_balance + + # Save the node-id and validator private keys to the shared directory + validator_home=/tmp/${CHAIN_NAME}-${name} + $BINARY init $CHAIN_NAME-$name --chain-id $CHAIN_ID --overwrite --home ${validator_home} &> /dev/null + node_id=$($BINARY tendermint show-node-id --home ${validator_home}) + + mkdir -p ${SHARED_DIR}/validator-keys + mkdir -p ${SHARED_DIR}/node-keys + mkdir -p ${SHARED_DIR}/node-ids + + mv ${validator_home}/config/priv_validator_key.json ${SHARED_DIR}/validator-keys/${name}.json + mv ${validator_home}/config/node_key.json ${SHARED_DIR}/node-keys/${name}.json + echo $node_id > ${SHARED_DIR}/node-ids/${name}.txt + + # Save the comma separted public keys for the ICS genesis update + validator_public_keys+="$(jq -r '.pub_key.value' ${SHARED_DIR}/validator-keys/${name}.json)," + done +} + +# Updates the genesis config with defaults +update_genesis() { + echo "Updating genesis.json" + + genesis_json=${CHAIN_HOME}/config/genesis.json + + sed -i -E "s|\"stake\"|\"${DENOM}\"|g" $genesis_json + sed -i -E "s|\"aphoton\"|\"${DENOM}\"|g" $genesis_json # ethermint default + + jq_inplace '(.app_state.epochs.epochs[] | select(.identifier=="day") ).duration |= "'$STRIDE_DAY_EPOCH_DURATION'"' $genesis_json + jq_inplace '(.app_state.epochs.epochs[] | select(.identifier=="stride_epoch") ).duration |= "'$STRIDE_EPOCH_EPOCH_DURATION'"' $genesis_json + jq_inplace '.app_state.gov.params.max_deposit_period |= "'$MAX_DEPOSIT_PERIOD'"' $genesis_json + jq_inplace '.app_state.gov.params.voting_period |= "'$VOTING_PERIOD'"' $genesis_json + + $BINARY add-consumer-section --validator-public-keys $validator_public_keys + + cp $genesis_json ${SHARED_DIR}/genesis.json +} + +echo "Initializing chain..." +init_config +add_validators +update_genesis +echo "Done" diff --git a/integration-tests/scripts/init-node.sh b/integration-tests/scripts/init-node.sh new file mode 100644 index 000000000..4ef8d1098 --- /dev/null +++ b/integration-tests/scripts/init-node.sh @@ -0,0 +1,101 @@ +#!/bin/bash + +set -eu + +BINARY=strided +CHAIN_NAME=stride +CHAIN_HOME=.stride +DENOM=ustrd +BLOCK_TIME=1s +CHAIN_ID=${CHAIN_NAME}-test-1 +STAKE_TOKENS=1000000 +MICRO_DENOM_UNITS=000000 + +PEER_PORT=26656 +RPC_PORT=26657 +POD_INDEX=${POD_NAME##*-} +VALIDATOR_INDEX=$((POD_INDEX+1)) +VALIDATOR_NAME=val${VALIDATOR_INDEX} +SHARED_DIR=shared + +config_toml="${CHAIN_HOME}/config/config.toml" +client_toml="${CHAIN_HOME}/config/client.toml" +app_toml="${CHAIN_HOME}/config/app.toml" + +# Initialize the config directory and validator key if it's not the main node +init_config() { + if [[ "$VALIDATOR_INDEX" != "1" ]]; then + moniker=${CHAIN_NAME}${VALIDATOR_INDEX} + $BINARY init $moniker --chain-id $CHAIN_ID --overwrite + $BINARY config keyring-backend test + + validator_config=$(jq -r '.validators[$index]' --argjson index "$POD_INDEX" config/keys.json) + mnemonic=$(echo $validator_config | jq -r '.mnemonic') + name=$(echo $validator_config | jq -r '.name') + + # echo "$mnemonic" | $BINARY keys add $name --recover + fi +} + +# Update config.toml, app.toml, and client.toml +update_config() { + echo "Updating config.toml..." + sed -i -E "s|cors_allowed_origins = \[\]|cors_allowed_origins = [\"\*\"]|g" $config_toml + sed -i -E "s|127.0.0.1|0.0.0.0|g" $config_toml + sed -i -E "s|timeout_commit = \"5s\"|timeout_commit = \"${BLOCK_TIME}\"|g" $config_toml + sed -i -E "s|prometheus = false|prometheus = true|g" $config_toml + + echo "Updating app.toml..." + sed -i -E "s|minimum-gas-prices = \".*\"|minimum-gas-prices = \"0${DENOM}\"|g" $app_toml + sed -i -E '/\[api\]/,/^enable = .*$/ s/^enable = .*$/enable = true/' $app_toml + sed -i -E 's|unsafe-cors = .*|unsafe-cors = true|g' $app_toml + sed -i -E 's|localhost|0.0.0.0|g' $app_toml + + echo "Updating client.toml" + sed -i -E "s|chain-id = \"\"|chain-id = \"${CHAIN_ID}\"|g" $client_toml + sed -i -E "s|keyring-backend = \"os\"|keyring-backend = \"test\"|g" $client_toml + sed -i -E "s|node = \".*\"|node = \"tcp://localhost:${RPC_PORT}\"|g" $client_toml +} + +# Extract private keys and genesis +download_shared() { + echo "Retrieving private keys and genesis.json..." + ls -la ${SHARED_DIR} + ls -la ${SHARED_DIR}/validator-keys + cp ${SHARED_DIR}/validator-keys/val${VALIDATOR_INDEX}.json ${CHAIN_HOME}/config/priv_validator_key.json + cp ${SHARED_DIR}/node-keys/val${VALIDATOR_INDEX}.json ${CHAIN_HOME}/config/node_key.json + cp ${SHARED_DIR}/genesis.json ${CHAIN_HOME}/config/genesis.json +} + +# Update the persistent peers conditionally based on which node it is +add_peers() { + echo "Setting peers..." + if [[ "$VALIDATOR_INDEX" == "1" ]]; then + # For the main node, wipe out the persistent peers that are incorrectly generated + sed -i -E "s|^persistent_peers = .*|persistent_peers = \"\"|g" $config_toml + else + # For the other nodes, add the main node as the persistent peer + main_node_id=$(cat ${SHARED_DIR}/node-ids/val1.txt) + main_pod_id=${CHAIN_NAME}-validator-0 + service=${CHAIN_NAME}-validator + persistent_peer=${main_node_id}@${main_pod_id}.${service}.${NAMESPACE}.svc.cluster.local:${PEER_PORT} + sed -i -E "s|^persistent_peers = .*|persistent_peers = \"${persistent_peer}\"|g" $config_toml + fi +} + +# Create the governor +create_governor() { + echo "Creating governor..." + pub_key=$($BINARY tendermint show-validator) + $BINARY tx staking create-validator --amount ${STAKE_TOKENS}${MICRO_DENOM_UNITS}${DENOM} --from ${VALIDATOR_NAME} \ + --pubkey=$pub_key --commission-rate="0.10" --commission-max-rate="0.20" \ + --commission-max-change-rate="0.01" --min-self-delegation="1" -y +} + +echo "Initializing node..." +init_config +update_config +download_shared +add_peers +# create_governor +echo "Done" \ No newline at end of file diff --git a/integration-tests/scripts/liveness.sh b/integration-tests/scripts/liveness.sh new file mode 100644 index 000000000..7878ea741 --- /dev/null +++ b/integration-tests/scripts/liveness.sh @@ -0,0 +1,23 @@ +set -e + +BINARY=strided +STRIDE_HOME=/home/stride/.stride + +# If chain hasn't been initialized yet, exit immediately +if [ ! -d $STRIDE_HOME/config ]; then + echo "READINESS CHECK FAILED - Chain has not been initialized yet." + exit 1 +fi + +# Check that the node is running +if ! $($BINARY status &> /dev/null); then + echo "READINESS CHECK FAILED - Chain is down" + exit 1 +fi + +# Then check if the node is synced according to it's status query +CATCHING_UP=$($BINARY status 2>&1 | jq ".SyncInfo.catching_up") +if [[ "$CATCHING_UP" != "false" ]]; then + echo "READINESS CHECK FAILED - Node is still syncing" + exit 1 +fi \ No newline at end of file diff --git a/integration-tests/scripts/setup.sh b/integration-tests/scripts/setup.sh deleted file mode 100644 index 08a736174..000000000 --- a/integration-tests/scripts/setup.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/bin/bash - -set -e - -LOCAL_MODE=${1:-false} - -CHAIN_NAME=stride -BINARY=strided -DENOM=ustrd -VALIDATOR_BALANCE=10000000 -MICRO_DENOM_UNITS=000000 -CHAIN_HOME=.stride -SHARED_DIR=shared -NUM_VALIDATORS=1 - -STRIDE_DAY_EPOCH_DURATION="140s" -STRIDE_EPOCH_EPOCH_DURATION="35s" -MAX_DEPOSIT_PERIOD="30s" -VOTING_PERIOD="30s" -UNBONDING_TIME="240s" -BLOCK_TIME=1s - -# check if the binary has genesis subcommand or not, if not, set CHAIN_GENESIS_CMD to empty -CHAIN_GENESIS_CMD=$($BINARY 2>&1 | grep -q "genesis-related subcommands" && echo "genesis" || echo "") - -jq_inplace() { - jq_filter="$1" - file="$2" - - jq "$jq_filter" "$file" > /tmp/$(basename $file) && mv /tmp/$(basename $file) ${file} -} - -# If this is being run locally, don't overwrite the main chain folder -if [[ "$LOCAL_MODE" == "true" ]]; then - CHAIN_HOME=state - rm -rf state - BINARY="$BINARY --home $CHAIN_HOME" -fi - -moniker=${CHAIN_NAME}1 -chain_id=${CHAIN_NAME}-test-1 - -$BINARY init $moniker --chain-id $chain_id --overwrite - -$BINARY config keyring-backend test - -validator_public_keys="" -for (( i=1; i <= $NUM_VALIDATORS; i++ )); do - validator_config=$(jq -r '.validators[$index]' --argjson index "$((i-1))" config/keys.json) - name=$(echo $validator_config | jq -r '.name') - mnemonic=$(echo $validator_config | jq -r '.mnemonic') - - echo $mnemonic | $BINARY keys add $name --recover - address=$($BINARY keys show $name -a) - - genesis_balance=${VALIDATOR_BALANCE}${MICRO_DENOM_UNITS}${DENOM} - $BINARY $CHAIN_GENESIS_CMD add-genesis-account $address $genesis_balance - - validator_home=/tmp/${CHAIN_NAME}-${name} - $BINARY init $CHAIN_NAME-$name --chain-id $chain_id --overwrite --home ${validator_home} &> /dev/null - node_id=$($BINARY tendermint show-node-id --home ${validator_home}) - - mkdir -p ${SHARED_DIR}/validator_keys - mkdir -p ${SHARED_DIR}/node_ids - - mv ${validator_home}/config/priv_validator_key.json ${SHARED_DIR}/validator_keys/${name}.json - echo $node_id > ${SHARED_DIR}/node_ids/${name}.json - - validator_public_keys+="$(jq -r '.pub_key.value' ${SHARED_DIR}/validator_keys/${name}.json)," -done - -config_toml="${CHAIN_HOME}/config/config.toml" -client_toml="${CHAIN_HOME}/config/client.toml" -app_toml="${CHAIN_HOME}/config/app.toml" -genesis_json=${CHAIN_HOME}/config/genesis.json - -sed -i -E "s|\"stake\"|\"${DENOM}\"|g" $genesis_json - -jq_inplace '(.app_state.epochs.epochs[] | select(.identifier=="day") ).duration |= "'$STRIDE_DAY_EPOCH_DURATION'"' $genesis_json -jq_inplace '(.app_state.epochs.epochs[] | select(.identifier=="stride_epoch") ).duration |= "'$STRIDE_EPOCH_EPOCH_DURATION'"' $genesis_json -jq_inplace '.app_state.gov.params.max_deposit_period |= "'$MAX_DEPOSIT_PERIOD'"' $genesis_json -jq_inplace '.app_state.gov.params.voting_period |= "'$VOTING_PERIOD'"' $genesis_json - -$BINARY add-consumer-section --validator-public-keys $validator_public_keys - -# wipe out the persistent peers for the main node (these are incorrectly autogenerated for each validator during collect-gentxs) -sed -i -E "s|^persistent_peers = .*|persistent_peers = \"\"|g" $config_toml - -sed -i -E "s|cors_allowed_origins = \[\]|cors_allowed_origins = [\"\*\"]|g" $config_toml -sed -i -E "s|127.0.0.1|0.0.0.0|g" $config_toml -sed -i -E "s|timeout_commit = \"5s\"|timeout_commit = \"${BLOCK_TIME}\"|g" $config_toml -sed -i -E "s|prometheus = false|prometheus = true|g" $config_toml - -sed -i -E "s|minimum-gas-prices = \".*\"|minimum-gas-prices = \"0${DENOM}\"|g" $app_toml -sed -i -E '/\[api\]/,/^enable = .*$/ s/^enable = .*$/enable = true/' $app_toml -sed -i -E 's|unsafe-cors = .*|unsafe-cors = true|g' $app_toml -sed -i -E 's|localhost|0.0.0.0|g' $app_toml - -sed -i -E "s|chain-id = \"\"|chain-id = \"${CHAIN_ID}\"|g" $client_toml -sed -i -E "s|keyring-backend = \"os\"|keyring-backend = \"test\"|g" $client_toml -sed -i -E "s|node = \".*\"|node = \"tcp://localhost:$RPC_PORT\"|g" $client_toml - -sed -i -E "s|\"stake\"|\"${DENOM}\"|g" $genesis_json -sed -i -E "s|\"aphoton\"|\"${DENOM}\"|g" $genesis_json # ethermint default - -mv ${SHARED_DIR}/validator_keys/val1.json ${CHAIN_HOME}/config/priv_validator_key.json diff --git a/integration-tests/validator.yaml b/integration-tests/validator.yaml index 570226460..9c43ed333 100644 --- a/integration-tests/validator.yaml +++ b/integration-tests/validator.yaml @@ -4,7 +4,7 @@ metadata: name: stride-validator namespace: integration spec: - replicas: 1 + replicas: 3 selector: matchLabels: app: stride-validator @@ -16,20 +16,49 @@ spec: spec: securityContext: fsGroup: 1000 - containers: - - name: stride-validator + initContainers: + - name: init image: gcr.io/stride-nodes/stride-test imagePullPolicy: Always command: - bash - "-c" - | - bash scripts/setup.sh - strided start --reject-config-defaults + POD_INDEX=${POD_NAME##*-} + if [[ "$POD_INDEX" == "0" ]]; then + bash scripts/init-chain.sh + fi + bash scripts/init-node.sh + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: state + mountPath: /home/stride/.stride + - name: shared + mountPath: /home/stride/shared + - name: scripts + mountPath: /home/stride/scripts + - name: config + mountPath: /home/stride/config + containers: + - name: stride-validator + image: gcr.io/stride-nodes/stride-test + imagePullPolicy: Always + command: ["strided", "start", "--reject-config-defaults"] ports: - name: rpc containerPort: 26657 protocol: TCP + - name: peer + containerPort: 26656 + protocol: TCP - name: api containerPort: 1317 protocol: TCP @@ -39,12 +68,6 @@ spec: volumeMounts: - name: state mountPath: /home/stride/.stride - - name: shared - mountPath: /home/stride/shared - - name: scripts - mountPath: /home/stride/scripts - - name: config - mountPath: /home/stride/config resources: limits: cpu: "1000m" @@ -52,45 +75,20 @@ spec: requests: cpu: "1000m" memory: "2Gi" - # readinessProbe: - # exec: - # command: - # - bash - # - "-c" - # - | - # set -e - - # BINARY=strided - # STRIDE_HOME=/home/stride/.stride - - # # If chain hasn't been initialized yet, exit immediately - # if [ ! -d $STRIDE_HOME/config ]; then - # echo "READINESS CHECK FAILED - Chain has not been initialized yet." - # exit 1 - # fi - - # # Check that the node is running - # if ! $($BINARY status &> /dev/null); then - # echo "READINESS CHECK FAILED - Chain is down" - # exit 1 - # fi - - # # Then check if the node is synced according to it's status query - # CATCHING_UP=$($BINARY status 2>&1 | jq ".SyncInfo.catching_up") - # if [[ "$CATCHING_UP" != "false" ]]; then - # echo "READINESS CHECK FAILED - Node is still syncing" - # exit 1 - # fi - # periodSeconds: 5 + livenessProbe: + exec: + command: ["bash", "scripts/liveness.sh"] + periodSeconds: 10 + initialDelaySeconds: 30 volumes: - name: state - emptyDir: + emptyDir: medium: "Memory" - name: shared - emptyDir: - medium: "Memory" + persistentVolumeClaim: + claimName: shared - name: scripts - configMap: + configMap: name: scripts - name: config configMap: @@ -102,15 +100,30 @@ metadata: name: stride-validator namespace: integration spec: + clusterIP: None selector: app: stride-validator ports: - port: 26657 name: rpc protocol: TCP + - port: 26656 + name: peer + protocol: TCP - port: 1317 name: api protocol: TCP - port: 9090 name: grpc protocol: TCP +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: shared +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi From d4cbe6cc3757702a3facefcc1f98e03505270e4e Mon Sep 17 00:00:00 2001 From: sampocs Date: Mon, 22 Jul 2024 18:52:36 -0500 Subject: [PATCH 04/24] parameterized scripts into config.sh --- integration-tests/scripts/config.sh | 35 +++++++++++++++++++++++ integration-tests/scripts/init-chain.sh | 35 +++++++---------------- integration-tests/scripts/init-node.sh | 37 +++++++------------------ integration-tests/validator.yaml | 3 ++ 4 files changed, 58 insertions(+), 52 deletions(-) create mode 100644 integration-tests/scripts/config.sh diff --git a/integration-tests/scripts/config.sh b/integration-tests/scripts/config.sh new file mode 100644 index 000000000..0b5ae9262 --- /dev/null +++ b/integration-tests/scripts/config.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +SCRIPTS_DIR=scripts +SHARED_DIR=shared +CONFIG_DIR=config + +VALIDATOR_KEYS_DIR=${SHARED_DIR}/validator-keys +NODE_KEYS_DIR=${SHARED_DIR}/node-keys +NODE_IDS_DIR=${SHARED_DIR}/node-ids +KEYS_FILE=${CONFIG_DIR}/keys.json + +POD_INDEX=${POD_NAME##*-} +VALIDATOR_INDEX=$((POD_INDEX+1)) +VALIDATOR_NAME=val${VALIDATOR_INDEX} + +PEER_PORT=26656 +RPC_PORT=26657 + +CHAIN_NAME=stride +BINARY=strided +DENOM=ustrd +MICRO_DENOM_UNITS=000000 +CHAIN_HOME=.stride +NUM_VALIDATORS=3 + +CHAIN_ID=${CHAIN_NAME}-test-1 +BLOCK_TIME=1s +VALIDATOR_BALANCE=10000000${MICRO_DENOM_UNITS} +VALIDATOR_STAKE=1000000${MICRO_DENOM_UNITS} + +STRIDE_DAY_EPOCH_DURATION="140s" +STRIDE_EPOCH_EPOCH_DURATION="35s" +MAX_DEPOSIT_PERIOD="30s" +VOTING_PERIOD="30s" +UNBONDING_TIME="240s" diff --git a/integration-tests/scripts/init-chain.sh b/integration-tests/scripts/init-chain.sh index ce59ca20b..50400e38c 100644 --- a/integration-tests/scripts/init-chain.sh +++ b/integration-tests/scripts/init-chain.sh @@ -1,25 +1,10 @@ #!/bin/bash set -e +source scripts/config.sh LOCAL_MODE=${1:-false} -CHAIN_NAME=stride -BINARY=strided -DENOM=ustrd -CHAIN_ID=${CHAIN_NAME}-test-1 -VALIDATOR_BALANCE=10000000 -MICRO_DENOM_UNITS=000000 -CHAIN_HOME=.stride -SHARED_DIR=shared -NUM_VALIDATORS=3 - -STRIDE_DAY_EPOCH_DURATION="140s" -STRIDE_EPOCH_EPOCH_DURATION="35s" -MAX_DEPOSIT_PERIOD="30s" -VOTING_PERIOD="30s" -UNBONDING_TIME="240s" - # If this is being run locally, don't overwrite the main chain folder if [[ "$LOCAL_MODE" == "true" ]]; then CHAIN_HOME=state @@ -54,7 +39,7 @@ add_validators() { validator_public_keys="" for (( i=1; i <= $NUM_VALIDATORS; i++ )); do # Extract the validator name and mnemonic from keys.json - validator_config=$(jq -r '.validators[$index]' --argjson index "$((i-1))" config/keys.json) + validator_config=$(jq -r '.validators[$index]' --argjson index "$((i-1))" ${KEYS_FILE}) name=$(echo $validator_config | jq -r '.name') mnemonic=$(echo $validator_config | jq -r '.mnemonic') @@ -63,7 +48,7 @@ add_validators() { address=$($BINARY keys show $name -a) # Add the genesis account - genesis_balance=${VALIDATOR_BALANCE}${MICRO_DENOM_UNITS}${DENOM} + genesis_balance=${VALIDATOR_BALANCE}${DENOM} $BINARY $chain_genesis_command add-genesis-account $address $genesis_balance # Save the node-id and validator private keys to the shared directory @@ -71,16 +56,16 @@ add_validators() { $BINARY init $CHAIN_NAME-$name --chain-id $CHAIN_ID --overwrite --home ${validator_home} &> /dev/null node_id=$($BINARY tendermint show-node-id --home ${validator_home}) - mkdir -p ${SHARED_DIR}/validator-keys - mkdir -p ${SHARED_DIR}/node-keys - mkdir -p ${SHARED_DIR}/node-ids + mkdir -p ${VALIDATOR_KEYS_DIR} + mkdir -p ${NODE_KEYS_DIR} + mkdir -p ${NODE_IDS_DIR} - mv ${validator_home}/config/priv_validator_key.json ${SHARED_DIR}/validator-keys/${name}.json - mv ${validator_home}/config/node_key.json ${SHARED_DIR}/node-keys/${name}.json - echo $node_id > ${SHARED_DIR}/node-ids/${name}.txt + mv ${validator_home}/config/priv_validator_key.json ${VALIDATOR_KEYS_DIR}/${name}.json + mv ${validator_home}/config/node_key.json ${NODE_KEYS_DIR}/${name}.json + echo $node_id > ${NODE_IDS_DIR}/${name}.txt # Save the comma separted public keys for the ICS genesis update - validator_public_keys+="$(jq -r '.pub_key.value' ${SHARED_DIR}/validator-keys/${name}.json)," + validator_public_keys+="$(jq -r '.pub_key.value' ${VALIDATOR_KEYS_DIR}/${name}.json)," done } diff --git a/integration-tests/scripts/init-node.sh b/integration-tests/scripts/init-node.sh index 4ef8d1098..3ccd46f6f 100644 --- a/integration-tests/scripts/init-node.sh +++ b/integration-tests/scripts/init-node.sh @@ -1,26 +1,7 @@ #!/bin/bash set -eu - -BINARY=strided -CHAIN_NAME=stride -CHAIN_HOME=.stride -DENOM=ustrd -BLOCK_TIME=1s -CHAIN_ID=${CHAIN_NAME}-test-1 -STAKE_TOKENS=1000000 -MICRO_DENOM_UNITS=000000 - -PEER_PORT=26656 -RPC_PORT=26657 -POD_INDEX=${POD_NAME##*-} -VALIDATOR_INDEX=$((POD_INDEX+1)) -VALIDATOR_NAME=val${VALIDATOR_INDEX} -SHARED_DIR=shared - -config_toml="${CHAIN_HOME}/config/config.toml" -client_toml="${CHAIN_HOME}/config/client.toml" -app_toml="${CHAIN_HOME}/config/app.toml" +source scripts/config.sh # Initialize the config directory and validator key if it's not the main node init_config() { @@ -29,7 +10,7 @@ init_config() { $BINARY init $moniker --chain-id $CHAIN_ID --overwrite $BINARY config keyring-backend test - validator_config=$(jq -r '.validators[$index]' --argjson index "$POD_INDEX" config/keys.json) + validator_config=$(jq -r '.validators[$index]' --argjson index "$POD_INDEX" ${KEYS_FILE}) mnemonic=$(echo $validator_config | jq -r '.mnemonic') name=$(echo $validator_config | jq -r '.name') @@ -39,6 +20,10 @@ init_config() { # Update config.toml, app.toml, and client.toml update_config() { + config_toml="${CHAIN_HOME}/config/config.toml" + client_toml="${CHAIN_HOME}/config/client.toml" + app_toml="${CHAIN_HOME}/config/app.toml" + echo "Updating config.toml..." sed -i -E "s|cors_allowed_origins = \[\]|cors_allowed_origins = [\"\*\"]|g" $config_toml sed -i -E "s|127.0.0.1|0.0.0.0|g" $config_toml @@ -60,10 +45,8 @@ update_config() { # Extract private keys and genesis download_shared() { echo "Retrieving private keys and genesis.json..." - ls -la ${SHARED_DIR} - ls -la ${SHARED_DIR}/validator-keys - cp ${SHARED_DIR}/validator-keys/val${VALIDATOR_INDEX}.json ${CHAIN_HOME}/config/priv_validator_key.json - cp ${SHARED_DIR}/node-keys/val${VALIDATOR_INDEX}.json ${CHAIN_HOME}/config/node_key.json + cp ${VALIDATOR_KEYS_DIR}/val${VALIDATOR_INDEX}.json ${CHAIN_HOME}/config/priv_validator_key.json + cp ${NODE_KEYS_DIR}/val${VALIDATOR_INDEX}.json ${CHAIN_HOME}/config/node_key.json cp ${SHARED_DIR}/genesis.json ${CHAIN_HOME}/config/genesis.json } @@ -75,7 +58,7 @@ add_peers() { sed -i -E "s|^persistent_peers = .*|persistent_peers = \"\"|g" $config_toml else # For the other nodes, add the main node as the persistent peer - main_node_id=$(cat ${SHARED_DIR}/node-ids/val1.txt) + main_node_id=$(cat ${NODE_IDS_DIR}/val1.txt) main_pod_id=${CHAIN_NAME}-validator-0 service=${CHAIN_NAME}-validator persistent_peer=${main_node_id}@${main_pod_id}.${service}.${NAMESPACE}.svc.cluster.local:${PEER_PORT} @@ -87,7 +70,7 @@ add_peers() { create_governor() { echo "Creating governor..." pub_key=$($BINARY tendermint show-validator) - $BINARY tx staking create-validator --amount ${STAKE_TOKENS}${MICRO_DENOM_UNITS}${DENOM} --from ${VALIDATOR_NAME} \ + $BINARY tx staking create-validator --amount ${VALIDATOR_STAKE}${DENOM} --from ${VALIDATOR_NAME} \ --pubkey=$pub_key --commission-rate="0.10" --commission-max-rate="0.20" \ --commission-max-change-rate="0.01" --min-self-delegation="1" -y } diff --git a/integration-tests/validator.yaml b/integration-tests/validator.yaml index 9c43ed333..0efe3d8a6 100644 --- a/integration-tests/validator.yaml +++ b/integration-tests/validator.yaml @@ -24,6 +24,9 @@ spec: - bash - "-c" - | + if [[ -d $STRIDE_HOME/config ]]; then + exit 0 + fi POD_INDEX=${POD_NAME##*-} if [[ "$POD_INDEX" == "0" ]]; then bash scripts/init-chain.sh From 6cba746a040f254728bd64d245deecb5337df3c0 Mon Sep 17 00:00:00 2001 From: sampocs Date: Fri, 26 Jul 2024 15:28:10 -0500 Subject: [PATCH 05/24] added create governors script --- integration-tests/scripts/config.sh | 2 +- integration-tests/scripts/create-governor.sh | 47 +++++++++++++++++++ integration-tests/scripts/init-chain.sh | 16 ++++--- integration-tests/scripts/init-node.sh | 28 ++++------- .../scripts/{liveness.sh => readiness.sh} | 7 +++ integration-tests/validator.yaml | 27 ++++++----- 6 files changed, 88 insertions(+), 39 deletions(-) create mode 100644 integration-tests/scripts/create-governor.sh rename integration-tests/scripts/{liveness.sh => readiness.sh} (68%) diff --git a/integration-tests/scripts/config.sh b/integration-tests/scripts/config.sh index 0b5ae9262..ee1d6908e 100644 --- a/integration-tests/scripts/config.sh +++ b/integration-tests/scripts/config.sh @@ -9,7 +9,7 @@ NODE_KEYS_DIR=${SHARED_DIR}/node-keys NODE_IDS_DIR=${SHARED_DIR}/node-ids KEYS_FILE=${CONFIG_DIR}/keys.json -POD_INDEX=${POD_NAME##*-} +POD_INDEX=${HOSTNAME##*-} VALIDATOR_INDEX=$((POD_INDEX+1)) VALIDATOR_NAME=val${VALIDATOR_INDEX} diff --git a/integration-tests/scripts/create-governor.sh b/integration-tests/scripts/create-governor.sh new file mode 100644 index 000000000..ae13056be --- /dev/null +++ b/integration-tests/scripts/create-governor.sh @@ -0,0 +1,47 @@ +# #!/bin/bash + +set -eu +source scripts/config.sh + +wait_for_startup() { + echo "Waiting for node to start..." + while ! (($BINARY status &> /dev/null) && [[ "$($BINARY status | jq -r '.SyncInfo.latest_block_height')" -gt "0" ]]); do + echo "Node still syncing..." + sleep 10 + done + echo "Node synced" +} + +add_keys() { + echo "Adding validator keys..." + + validator_config=$(jq -r '.validators[$index]' --argjson index "$POD_INDEX" ${KEYS_FILE}) + mnemonic=$(echo $validator_config | jq -r '.mnemonic') + name=$(echo $validator_config | jq -r '.name') + + if ! $BINARY keys show $name -a &> /dev/null ; then + echo "$mnemonic" | $BINARY keys add $name --recover + fi +} + +create_governor() { + echo "Creating governor..." + pub_key=$($BINARY tendermint show-validator) + $BINARY tx staking create-validator \ + --amount ${VALIDATOR_STAKE}${DENOM} \ + --pubkey=$pub_key \ + --commission-rate="0.10" \ + --commission-max-rate="0.20" \ + --commission-max-change-rate="0.01" \ + --min-self-delegation="1" \ + --from ${VALIDATOR_NAME} -y +} + +main() { + wait_for_startup + add_keys + create_governor + echo "Done" +} + +main >> governor.log 2>&1 & \ No newline at end of file diff --git a/integration-tests/scripts/init-chain.sh b/integration-tests/scripts/init-chain.sh index 50400e38c..74f69ed2f 100644 --- a/integration-tests/scripts/init-chain.sh +++ b/integration-tests/scripts/init-chain.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -e +set -eu source scripts/config.sh LOCAL_MODE=${1:-false} @@ -88,8 +88,12 @@ update_genesis() { cp $genesis_json ${SHARED_DIR}/genesis.json } -echo "Initializing chain..." -init_config -add_validators -update_genesis -echo "Done" +main() { + echo "Initializing chain..." + init_config + add_validators + update_genesis + echo "Done" +} + +main \ No newline at end of file diff --git a/integration-tests/scripts/init-node.sh b/integration-tests/scripts/init-node.sh index 3ccd46f6f..f20a0dd78 100644 --- a/integration-tests/scripts/init-node.sh +++ b/integration-tests/scripts/init-node.sh @@ -9,12 +9,6 @@ init_config() { moniker=${CHAIN_NAME}${VALIDATOR_INDEX} $BINARY init $moniker --chain-id $CHAIN_ID --overwrite $BINARY config keyring-backend test - - validator_config=$(jq -r '.validators[$index]' --argjson index "$POD_INDEX" ${KEYS_FILE}) - mnemonic=$(echo $validator_config | jq -r '.mnemonic') - name=$(echo $validator_config | jq -r '.name') - - # echo "$mnemonic" | $BINARY keys add $name --recover fi } @@ -66,19 +60,13 @@ add_peers() { fi } -# Create the governor -create_governor() { - echo "Creating governor..." - pub_key=$($BINARY tendermint show-validator) - $BINARY tx staking create-validator --amount ${VALIDATOR_STAKE}${DENOM} --from ${VALIDATOR_NAME} \ - --pubkey=$pub_key --commission-rate="0.10" --commission-max-rate="0.20" \ - --commission-max-change-rate="0.01" --min-self-delegation="1" -y +main() { + echo "Initializing node..." + init_config + update_config + download_shared + add_peers + echo "Done" } -echo "Initializing node..." -init_config -update_config -download_shared -add_peers -# create_governor -echo "Done" \ No newline at end of file +main \ No newline at end of file diff --git a/integration-tests/scripts/liveness.sh b/integration-tests/scripts/readiness.sh similarity index 68% rename from integration-tests/scripts/liveness.sh rename to integration-tests/scripts/readiness.sh index 7878ea741..91125c9d4 100644 --- a/integration-tests/scripts/liveness.sh +++ b/integration-tests/scripts/readiness.sh @@ -15,6 +15,13 @@ if ! $($BINARY status &> /dev/null); then exit 1 fi +# It's not possible for one node to start up by itself (without peers), +# so if we identify that the node is on block 0, we'll mark it as ready +# so the other nodes can start connecting +if [[ "$($BINARY status | jq -r '.SyncInfo.latest_block_height')" == "0" ]]; then + exit 0 +fi + # Then check if the node is synced according to it's status query CATCHING_UP=$($BINARY status 2>&1 | jq ".SyncInfo.catching_up") if [[ "$CATCHING_UP" != "false" ]]; then diff --git a/integration-tests/validator.yaml b/integration-tests/validator.yaml index 0efe3d8a6..52ecfe2df 100644 --- a/integration-tests/validator.yaml +++ b/integration-tests/validator.yaml @@ -27,16 +27,12 @@ spec: if [[ -d $STRIDE_HOME/config ]]; then exit 0 fi - POD_INDEX=${POD_NAME##*-} + POD_INDEX=${HOSTNAME##*-} if [[ "$POD_INDEX" == "0" ]]; then bash scripts/init-chain.sh fi bash scripts/init-node.sh env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - name: NAMESPACE valueFrom: fieldRef: @@ -55,6 +51,14 @@ spec: image: gcr.io/stride-nodes/stride-test imagePullPolicy: Always command: ["strided", "start", "--reject-config-defaults"] + lifecycle: + postStart: + exec: + command: ["bash", "scripts/create-governor.sh"] + readinessProbe: + exec: + command: ["bash", "scripts/readiness.sh"] + periodSeconds: 10 ports: - name: rpc containerPort: 26657 @@ -71,6 +75,10 @@ spec: volumeMounts: - name: state mountPath: /home/stride/.stride + - name: scripts + mountPath: /home/stride/scripts + - name: config + mountPath: /home/stride/config resources: limits: cpu: "1000m" @@ -78,15 +86,9 @@ spec: requests: cpu: "1000m" memory: "2Gi" - livenessProbe: - exec: - command: ["bash", "scripts/liveness.sh"] - periodSeconds: 10 - initialDelaySeconds: 30 volumes: - name: state - emptyDir: - medium: "Memory" + emptyDir: {} - name: shared persistentVolumeClaim: claimName: shared @@ -124,6 +126,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: shared + namespace: integration spec: accessModes: - ReadWriteOnce From 1a8b8f8cbd24b71abc340a2aa0f60c0d72fb6eaf Mon Sep 17 00:00:00 2001 From: sampocs Date: Fri, 26 Jul 2024 15:41:35 -0500 Subject: [PATCH 06/24] added hub dockerfile --- .../dockerfiles/Dockerfile.cosmos | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 integration-tests/dockerfiles/Dockerfile.cosmos diff --git a/integration-tests/dockerfiles/Dockerfile.cosmos b/integration-tests/dockerfiles/Dockerfile.cosmos new file mode 100644 index 000000000..763be9aef --- /dev/null +++ b/integration-tests/dockerfiles/Dockerfile.cosmos @@ -0,0 +1,34 @@ +FROM golang:1.22-alpine AS builder + +WORKDIR /opt + +RUN apk add --update curl make git libc-dev bash gcc linux-headers eudev-dev ca-certificates build-base git + +ENV REPO=https://github.com/cosmos/gaia +ENV COMMIT_HASH=v18.1.0 +ENV BINARY=gaiad + +RUN git clone ${REPO} chain \ + && cd chain \ + && git checkout $COMMIT_HASH + +WORKDIR /opt/chain + +RUN WASMVM_VERSION=$(cat go.mod | grep github.com/CosmWasm/wasmvm | awk '{print $2}') \ + && wget https://github.com/CosmWasm/wasmvm/releases/download/$WASMVM_VERSION/libwasmvm_muslc.$(uname -m).a \ + -O /lib/libwasmvm_muslc.a + +RUN CGO_ENABLED=1 BUILD_TAGS="muslc linkstatic" LINK_STATICALLY=true LEDGER_ENABLED=false make install + +FROM alpine:3.17 +COPY --from=builder /go/bin/$BINARY /usr/local/bin/ +RUN apk add bash vim \ + && addgroup -g 1000 validator \ + && adduser -S -h /home/validator -D validator -u 1000 -G validator + +USER 1000 +WORKDIR /home/validator + +EXPOSE 26657 26656 1317 9090 + +CMD ["$BINARY", "start"] From 23e4335a9f34e193fcb4ee7cdc0e2109548225e6 Mon Sep 17 00:00:00 2001 From: sampocs Date: Fri, 26 Jul 2024 17:05:05 -0500 Subject: [PATCH 07/24] checkpoint --- integration-tests/scripts/init-chain.sh | 67 +++++++++++++++++++------ 1 file changed, 53 insertions(+), 14 deletions(-) diff --git a/integration-tests/scripts/init-chain.sh b/integration-tests/scripts/init-chain.sh index 74f69ed2f..74c024a68 100644 --- a/integration-tests/scripts/init-chain.sh +++ b/integration-tests/scripts/init-chain.sh @@ -12,6 +12,11 @@ if [[ "$LOCAL_MODE" == "true" ]]; then BINARY="$BINARY --home $CHAIN_HOME" fi + +# check if the binary has genesis subcommand or not, if not, set CHAIN_GENESIS_CMD to empty +genesis_json=${CHAIN_HOME}/config/genesis.json +chain_genesis_command=$($BINARY 2>&1 | grep -q "genesis-related subcommands" && echo "genesis" || echo "") + # Helper to update a json attribute in-place jq_inplace() { jq_filter="$1" @@ -33,9 +38,6 @@ init_config() { add_validators() { echo "Adding validators..." - # check if the binary has genesis subcommand or not, if not, set CHAIN_GENESIS_CMD to empty - chain_genesis_command=$($BINARY 2>&1 | grep -q "genesis-related subcommands" && echo "genesis" || echo "") - validator_public_keys="" for (( i=1; i <= $NUM_VALIDATORS; i++ )); do # Extract the validator name and mnemonic from keys.json @@ -52,38 +54,69 @@ add_validators() { $BINARY $chain_genesis_command add-genesis-account $address $genesis_balance # Save the node-id and validator private keys to the shared directory - validator_home=/tmp/${CHAIN_NAME}-${name} + validator_home=/tmp/${CHAIN_NAME}-${name} && rm -rf $validator_home $BINARY init $CHAIN_NAME-$name --chain-id $CHAIN_ID --overwrite --home ${validator_home} &> /dev/null - node_id=$($BINARY tendermint show-node-id --home ${validator_home}) + $BINARY config keyring-backend test --home ${validator_home} + echo "$mnemonic" | $BINARY keys add $name --recover --home ${validator_home} mkdir -p ${VALIDATOR_KEYS_DIR} mkdir -p ${NODE_KEYS_DIR} mkdir -p ${NODE_IDS_DIR} - mv ${validator_home}/config/priv_validator_key.json ${VALIDATOR_KEYS_DIR}/${name}.json - mv ${validator_home}/config/node_key.json ${NODE_KEYS_DIR}/${name}.json + cp ${validator_home}/config/priv_validator_key.json ${VALIDATOR_KEYS_DIR}/${name}.json + cp ${validator_home}/config/node_key.json ${NODE_KEYS_DIR}/${name}.json + + node_id=$($BINARY tendermint show-node-id --home ${validator_home}) echo $node_id > ${NODE_IDS_DIR}/${name}.txt # Save the comma separted public keys for the ICS genesis update validator_public_keys+="$(jq -r '.pub_key.value' ${VALIDATOR_KEYS_DIR}/${name}.json)," + + # For non-stride nodes, collect genesis txs + if [[ "$CHAIN_NAME" != "STRIDE" ]]; then + cp $genesis_json ${validator_home}/config/genesis.json + $BINARY $chain_genesis_command gentx $name ${VALIDATOR_STAKE}${DENOM} --chain-id $CHAIN_ID --home ${validator_home} + mkdir -p ${CHAIN_HOME}/config/gentx + cp ${validator_home}/config/gentx/* ${CHAIN_HOME}/config/gentx/ + fi done + + if [[ "$CHAIN_NAME" != "STRIDE" ]]; then + $BINARY $chain_genesis_command collect-gentxs + fi } # Updates the genesis config with defaults -update_genesis() { - echo "Updating genesis.json" - - genesis_json=${CHAIN_HOME}/config/genesis.json +update_default_genesis() { + echo "Updating genesis.json with defaults..." sed -i -E "s|\"stake\"|\"${DENOM}\"|g" $genesis_json sed -i -E "s|\"aphoton\"|\"${DENOM}\"|g" $genesis_json # ethermint default + jq_inplace '.app_state.staking.params.unbonding_time |= "'$UNBONDING_TIME'"' $genesis_json + jq_inplace '.app_state.gov.params.max_deposit_period |= "'$DEPOSIT_PERIOD'"' $genesis_json + jq_inplace '.app_state.gov.params.voting_period |= "'$VOTING_PERIOD'"' $genesis_json + jq_inplace '.app_state.gov.params.expedited_voting_period |= "'$EXPEDITED_VOTING_PERIOD'"' $genesis_json +} + +# Genesis updates specific to stride +update_stride_genesis() { + echo "Updating genesis.json with stride configuration..." + jq_inplace '(.app_state.epochs.epochs[] | select(.identifier=="day") ).duration |= "'$STRIDE_DAY_EPOCH_DURATION'"' $genesis_json jq_inplace '(.app_state.epochs.epochs[] | select(.identifier=="stride_epoch") ).duration |= "'$STRIDE_EPOCH_EPOCH_DURATION'"' $genesis_json - jq_inplace '.app_state.gov.params.max_deposit_period |= "'$MAX_DEPOSIT_PERIOD'"' $genesis_json - jq_inplace '.app_state.gov.params.voting_period |= "'$VOTING_PERIOD'"' $genesis_json $BINARY add-consumer-section --validator-public-keys $validator_public_keys +} + +# Genesis updates specific to non-stride chains +update_host_genesis() { + echo "Updating genesis.json with host configuration..." +} + +# Moves the genesis file into the shared directory +save_genesis() { + echo "Saving genesis.json to shared directory..." cp $genesis_json ${SHARED_DIR}/genesis.json } @@ -92,7 +125,13 @@ main() { echo "Initializing chain..." init_config add_validators - update_genesis + update_default_genesis + if [[ "$CHAIN_NAME" == "stride" ]]; then + update_stride_genesis + else + update_host_genesis + fi + save_genesis echo "Done" } From 3ed8d12fb435ba35bb232d9c39737bc95f36304e Mon Sep 17 00:00:00 2001 From: sampocs Date: Fri, 26 Jul 2024 19:16:04 -0500 Subject: [PATCH 08/24] 3 nodes working with gaia --- integration-tests/Makefile | 17 ++- .../dockerfiles/Dockerfile.cosmos | 5 +- integration-tests/gaia-validator.yaml | 135 ++++++++++++++++++ integration-tests/scripts/config.sh | 28 +++- ...create-governor.sh => create-validator.sh} | 20 ++- integration-tests/scripts/init-chain.sh | 41 +++--- integration-tests/scripts/init-node.sh | 2 +- integration-tests/scripts/readiness.sh | 8 +- integration-tests/validator.yaml | 6 +- 9 files changed, 209 insertions(+), 53 deletions(-) create mode 100644 integration-tests/gaia-validator.yaml rename integration-tests/scripts/{create-governor.sh => create-validator.sh} (70%) diff --git a/integration-tests/Makefile b/integration-tests/Makefile index b570ebd26..11f428fb5 100644 --- a/integration-tests/Makefile +++ b/integration-tests/Makefile @@ -3,17 +3,20 @@ K8S_NAMESPACE=integration KUBECTL := $(shell which kubectl) DOCKER := $(shell which docker) -upload: docker-build docker-push - -docker-build: +build-stride: @echo "Building docker image: stride-test" @$(DOCKER) buildx build --platform linux/amd64 --tag stride-test .. @$(DOCKER) tag stride-test gcr.io/stride-nodes/stride-test - -docker-push: @echo "Pushing image to GCR" @$(DOCKER) push gcr.io/stride-nodes/stride-test +build-cosmos: + @echo "Building docker image" + @$(DOCKER) buildx build --platform linux/amd64 --tag cosmos-test -f dockerfiles/Dockerfile.cosmos . + @$(DOCKER) tag cosmos-test gcr.io/stride-nodes/cosmos-test + @echo "Pushing image to GCR" + @$(DOCKER) push gcr.io/stride-nodes/cosmos-test + update-scripts: @echo "Updating scripts configmap" @$(KUBECTL) delete configmap scripts -n $(K8S_NAMESPACE) --ignore-not-found=true @@ -25,7 +28,7 @@ update-config: @$(KUBECTL) create configmap config --from-file=config -n $(K8S_NAMESPACE) start: - @$(KUBECTL) apply -f validator.yaml + @$(KUBECTL) apply -f gaia-validator.yaml stop: - @$(KUBECTL) delete -f validator.yaml \ No newline at end of file + @$(KUBECTL) delete -f gaia-validator.yaml \ No newline at end of file diff --git a/integration-tests/dockerfiles/Dockerfile.cosmos b/integration-tests/dockerfiles/Dockerfile.cosmos index 763be9aef..b4ccbcb68 100644 --- a/integration-tests/dockerfiles/Dockerfile.cosmos +++ b/integration-tests/dockerfiles/Dockerfile.cosmos @@ -2,7 +2,7 @@ FROM golang:1.22-alpine AS builder WORKDIR /opt -RUN apk add --update curl make git libc-dev bash gcc linux-headers eudev-dev ca-certificates build-base git +RUN apk add --update curl make git libc-dev bash gcc linux-headers eudev-dev ca-certificates build-base git ENV REPO=https://github.com/cosmos/gaia ENV COMMIT_HASH=v18.1.0 @@ -22,7 +22,7 @@ RUN CGO_ENABLED=1 BUILD_TAGS="muslc linkstatic" LINK_STATICALLY=true LEDGER_ENAB FROM alpine:3.17 COPY --from=builder /go/bin/$BINARY /usr/local/bin/ -RUN apk add bash vim \ +RUN apk add bash vim sudo dasel jq \ && addgroup -g 1000 validator \ && adduser -S -h /home/validator -D validator -u 1000 -G validator @@ -31,4 +31,3 @@ WORKDIR /home/validator EXPOSE 26657 26656 1317 9090 -CMD ["$BINARY", "start"] diff --git a/integration-tests/gaia-validator.yaml b/integration-tests/gaia-validator.yaml new file mode 100644 index 000000000..9268989ac --- /dev/null +++ b/integration-tests/gaia-validator.yaml @@ -0,0 +1,135 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: cosmos-validator + namespace: integration +spec: + replicas: 3 + selector: + matchLabels: + app: cosmos-validator + serviceName: cosmos-validator + template: + metadata: + labels: + app: cosmos-validator + spec: + securityContext: + fsGroup: 1000 + initContainers: + - name: init + image: gcr.io/stride-nodes/cosmos-test + imagePullPolicy: Always + command: + - bash + - "-c" + - | + if [[ -d $CHAIN_HOME/config ]]; then + exit 0 + fi + POD_INDEX=${HOSTNAME##*-} + if [[ "$POD_INDEX" == "0" ]]; then + bash scripts/init-chain.sh + fi + bash scripts/init-node.sh + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: state + mountPath: /home/validator/.gaia + - name: shared + mountPath: /home/validator/shared + - name: scripts + mountPath: /home/validator/scripts + - name: config + mountPath: /home/validator/config + containers: + - name: cosmos-validator + image: gcr.io/stride-nodes/cosmos-test + imagePullPolicy: Always + command: ["gaiad", "start"] + lifecycle: + postStart: + exec: + command: ["bash", "scripts/create-validator.sh"] + readinessProbe: + exec: + command: ["bash", "scripts/readiness.sh"] + periodSeconds: 10 + ports: + - name: rpc + containerPort: 26657 + protocol: TCP + - name: peer + containerPort: 26656 + protocol: TCP + - name: api + containerPort: 1317 + protocol: TCP + - name: grpc + containerPort: 9090 + protocol: TCP + volumeMounts: + - name: state + mountPath: /home/validator/.gaia + - name: scripts + mountPath: /home/validator/scripts + - name: config + mountPath: /home/validator/config + resources: + limits: + cpu: "1000m" + memory: "2Gi" + requests: + cpu: "1000m" + memory: "2Gi" + volumes: + - name: state + emptyDir: {} + - name: shared + persistentVolumeClaim: + claimName: cosmos-shared + - name: scripts + configMap: + name: scripts + - name: config + configMap: + name: config +--- +apiVersion: v1 +kind: Service +metadata: + name: cosmos-validator + namespace: integration +spec: + clusterIP: None + selector: + app: cosmos-validator + ports: + - port: 26657 + name: rpc + protocol: TCP + - port: 26656 + name: peer + protocol: TCP + - port: 1317 + name: api + protocol: TCP + - port: 9090 + name: grpc + protocol: TCP +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: cosmos-shared + namespace: integration +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi diff --git a/integration-tests/scripts/config.sh b/integration-tests/scripts/config.sh index ee1d6908e..5558e9fca 100644 --- a/integration-tests/scripts/config.sh +++ b/integration-tests/scripts/config.sh @@ -1,5 +1,10 @@ #!/bin/bash +# Override the hostname only when in local mode +if [[ "$HOSTNAME" != *"validator"* ]]; then + HOSTNAME=validator-0 +fi + SCRIPTS_DIR=scripts SHARED_DIR=shared CONFIG_DIR=config @@ -16,11 +21,18 @@ VALIDATOR_NAME=val${VALIDATOR_INDEX} PEER_PORT=26656 RPC_PORT=26657 -CHAIN_NAME=stride -BINARY=strided -DENOM=ustrd +# CHAIN_NAME=stride +# CHAIN_HOME=${HOME}/.stride +# BINARY=strided +# DENOM=ustrd +# MICRO_DENOM_UNITS=000000 +# NUM_VALIDATORS=3 + +CHAIN_NAME=cosmos +CHAIN_HOME=.gaia +BINARY=gaiad +DENOM=uatom MICRO_DENOM_UNITS=000000 -CHAIN_HOME=.stride NUM_VALIDATORS=3 CHAIN_ID=${CHAIN_NAME}-test-1 @@ -28,8 +40,10 @@ BLOCK_TIME=1s VALIDATOR_BALANCE=10000000${MICRO_DENOM_UNITS} VALIDATOR_STAKE=1000000${MICRO_DENOM_UNITS} -STRIDE_DAY_EPOCH_DURATION="140s" -STRIDE_EPOCH_EPOCH_DURATION="35s" -MAX_DEPOSIT_PERIOD="30s" +DEPOSIT_PERIOD="30s" VOTING_PERIOD="30s" +EXPEDITED_VOTING_PERIOD="29s" UNBONDING_TIME="240s" + +STRIDE_DAY_EPOCH_DURATION="140s" +STRIDE_EPOCH_EPOCH_DURATION="35s" diff --git a/integration-tests/scripts/create-governor.sh b/integration-tests/scripts/create-validator.sh similarity index 70% rename from integration-tests/scripts/create-governor.sh rename to integration-tests/scripts/create-validator.sh index ae13056be..608259a12 100644 --- a/integration-tests/scripts/create-governor.sh +++ b/integration-tests/scripts/create-validator.sh @@ -24,8 +24,13 @@ add_keys() { fi } -create_governor() { - echo "Creating governor..." +create_validator() { + echo "Creating validator..." + min_self_delegation="" + if [[ $($BINARY tx staking create-validator --help | grep -c "min-self-delegation") -gt 0 ]]; then + min_self_delegation='--min-self-delegation=1000000' + fi + pub_key=$($BINARY tendermint show-validator) $BINARY tx staking create-validator \ --amount ${VALIDATOR_STAKE}${DENOM} \ @@ -33,15 +38,18 @@ create_governor() { --commission-rate="0.10" \ --commission-max-rate="0.20" \ --commission-max-change-rate="0.01" \ - --min-self-delegation="1" \ - --from ${VALIDATOR_NAME} -y + $min_self_delegation \ + --fees 300000$DENOM \ + --gas auto \ + --gas-adjustment 1.2 \ + --from ${VALIDATOR_NAME} -y } main() { wait_for_startup add_keys - create_governor + create_validator echo "Done" } -main >> governor.log 2>&1 & \ No newline at end of file +main >> validator.log 2>&1 & \ No newline at end of file diff --git a/integration-tests/scripts/init-chain.sh b/integration-tests/scripts/init-chain.sh index 74c024a68..b57d2dbf8 100644 --- a/integration-tests/scripts/init-chain.sh +++ b/integration-tests/scripts/init-chain.sh @@ -45,43 +45,40 @@ add_validators() { name=$(echo $validator_config | jq -r '.name') mnemonic=$(echo $validator_config | jq -r '.mnemonic') - # Add the key to the keyring - echo "$mnemonic" | $BINARY keys add $name --recover + # Add the key to the main keyring the the validator's sub-keyring + echo "$mnemonic" | $BINARY keys add $name --recover address=$($BINARY keys show $name -a) - # Add the genesis account + # Use a separate directory for the non-main nodes so we can generate unique validator keys + if [[ "$i" == "1" ]]; then + validator_home=${CHAIN_HOME} + else + validator_home=/tmp/${CHAIN_NAME}-${name} && rm -rf $validator_home + $BINARY init $name --chain-id $CHAIN_ID --overwrite --home ${validator_home} &> /dev/null + fi + + # Add the genesis account genesis_balance=${VALIDATOR_BALANCE}${DENOM} $BINARY $chain_genesis_command add-genesis-account $address $genesis_balance - # Save the node-id and validator private keys to the shared directory - validator_home=/tmp/${CHAIN_NAME}-${name} && rm -rf $validator_home - $BINARY init $CHAIN_NAME-$name --chain-id $CHAIN_ID --overwrite --home ${validator_home} &> /dev/null - $BINARY config keyring-backend test --home ${validator_home} - echo "$mnemonic" | $BINARY keys add $name --recover --home ${validator_home} - + # Save the node IDs and keys to the shared directory + mkdir -p ${NODE_IDS_DIR} mkdir -p ${VALIDATOR_KEYS_DIR} mkdir -p ${NODE_KEYS_DIR} - mkdir -p ${NODE_IDS_DIR} - cp ${validator_home}/config/priv_validator_key.json ${VALIDATOR_KEYS_DIR}/${name}.json - cp ${validator_home}/config/node_key.json ${NODE_KEYS_DIR}/${name}.json - node_id=$($BINARY tendermint show-node-id --home ${validator_home}) echo $node_id > ${NODE_IDS_DIR}/${name}.txt + cp ${validator_home}/config/priv_validator_key.json ${VALIDATOR_KEYS_DIR}/${name}.json + cp ${validator_home}/config/node_key.json ${NODE_KEYS_DIR}/${name}.json # Save the comma separted public keys for the ICS genesis update validator_public_keys+="$(jq -r '.pub_key.value' ${VALIDATOR_KEYS_DIR}/${name}.json)," - - # For non-stride nodes, collect genesis txs - if [[ "$CHAIN_NAME" != "STRIDE" ]]; then - cp $genesis_json ${validator_home}/config/genesis.json - $BINARY $chain_genesis_command gentx $name ${VALIDATOR_STAKE}${DENOM} --chain-id $CHAIN_ID --home ${validator_home} - mkdir -p ${CHAIN_HOME}/config/gentx - cp ${validator_home}/config/gentx/* ${CHAIN_HOME}/config/gentx/ - fi done - if [[ "$CHAIN_NAME" != "STRIDE" ]]; then + # For non-stride nodes, generate and collect the validator gentx (for the main node only) + # The other validators will be created after startup + if [[ "$CHAIN_NAME" != "stride" ]]; then + $BINARY $chain_genesis_command gentx val1 ${VALIDATOR_STAKE}${DENOM} --chain-id $CHAIN_ID $BINARY $chain_genesis_command collect-gentxs fi } diff --git a/integration-tests/scripts/init-node.sh b/integration-tests/scripts/init-node.sh index f20a0dd78..f5bc22942 100644 --- a/integration-tests/scripts/init-node.sh +++ b/integration-tests/scripts/init-node.sh @@ -69,4 +69,4 @@ main() { echo "Done" } -main \ No newline at end of file +main diff --git a/integration-tests/scripts/readiness.sh b/integration-tests/scripts/readiness.sh index 91125c9d4..74684d160 100644 --- a/integration-tests/scripts/readiness.sh +++ b/integration-tests/scripts/readiness.sh @@ -1,10 +1,10 @@ -set -e +#!/bin/bash -BINARY=strided -STRIDE_HOME=/home/stride/.stride +set -e +source scripts/config.sh # If chain hasn't been initialized yet, exit immediately -if [ ! -d $STRIDE_HOME/config ]; then +if [ ! -d $CHAIN_HOME/config ]; then echo "READINESS CHECK FAILED - Chain has not been initialized yet." exit 1 fi diff --git a/integration-tests/validator.yaml b/integration-tests/validator.yaml index 52ecfe2df..713290116 100644 --- a/integration-tests/validator.yaml +++ b/integration-tests/validator.yaml @@ -24,7 +24,7 @@ spec: - bash - "-c" - | - if [[ -d $STRIDE_HOME/config ]]; then + if [[ -d $CHAIN_HOME/config ]]; then exit 0 fi POD_INDEX=${HOSTNAME##*-} @@ -54,7 +54,7 @@ spec: lifecycle: postStart: exec: - command: ["bash", "scripts/create-governor.sh"] + command: ["bash", "scripts/create-validator.sh"] readinessProbe: exec: command: ["bash", "scripts/readiness.sh"] @@ -125,7 +125,7 @@ spec: apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: shared + name: stride-shared namespace: integration spec: accessModes: From a0133c963ab13103672781c858b4c08bb5fab811 Mon Sep 17 00:00:00 2001 From: sampocs Date: Fri, 2 Aug 2024 13:06:27 -0500 Subject: [PATCH 09/24] added api instead of volume for shared files --- .gitignore | 3 +- Dockerfile | 2 +- integration-tests/Makefile | 24 +++++++++- integration-tests/api.yaml | 43 ++++++++++++++++++ integration-tests/api/main.py | 45 +++++++++++++++++++ integration-tests/api/requirements.txt | 3 ++ integration-tests/dockerfiles/Dockerfile.api | 15 +++++++ integration-tests/scripts/config.sh | 40 ++++++++++------- integration-tests/scripts/init-chain.sh | 44 ++++++++++++------- integration-tests/scripts/init-node.sh | 28 ++++++++---- integration-tests/validator.yaml | 46 +++++++++++++------- 11 files changed, 235 insertions(+), 58 deletions(-) create mode 100644 integration-tests/api.yaml create mode 100644 integration-tests/api/main.py create mode 100644 integration-tests/api/requirements.txt create mode 100644 integration-tests/dockerfiles/Dockerfile.api diff --git a/.gitignore b/.gitignore index 433a4c1c5..5c7490749 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ vue/* !.vscode/settings.json .ipynb_checkpoints/* +__pycache__ integration-tests/state -integration-tests/shared \ No newline at end of file +integration-tests/storage \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 571a075de..320ce21b3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ RUN BUILD_TAGS=muslc LINK_STATICALLY=true make build FROM alpine:${RUNNER_IMAGE_VERSION} COPY --from=builder /opt/build/strided /usr/local/bin/strided -RUN apk add bash vim sudo dasel jq \ +RUN apk add bash vim sudo dasel jq curl \ && addgroup -g 1000 stride \ && adduser -S -h /home/stride -D stride -u 1000 -G stride diff --git a/integration-tests/Makefile b/integration-tests/Makefile index 11f428fb5..1e5e46a96 100644 --- a/integration-tests/Makefile +++ b/integration-tests/Makefile @@ -1,7 +1,25 @@ K8S_NAMESPACE=integration +VENV_NAME=integration +CONDA_BASE := $(shell conda info --base)/envs KUBECTL := $(shell which kubectl) DOCKER := $(shell which docker) +VENV_BIN := $(CONDA_BASE)/$(VENV_NAME)/bin +PYTHON := $(VENV_BIN)/python + +python-install: + conda create --name $(VENV_NAME) python=3.11 -y + $(PYTHON) -m pip install -r api/requirements.txt + +start-api: + @$(PYTHON) -m uvicorn api.main:app --proxy-headers + +build-api: + @echo "Building docker image: integration-test-api" + @$(DOCKER) buildx build --platform linux/amd64 --tag integration-test-api -f dockerfiles/Dockerfile.api api + @$(DOCKER) tag integration-test-api gcr.io/stride-nodes/integration-test-api + @echo "Pushing image to GCR" + @$(DOCKER) push gcr.io/stride-nodes/integration-test-api build-stride: @echo "Building docker image: stride-test" @@ -28,7 +46,9 @@ update-config: @$(KUBECTL) create configmap config --from-file=config -n $(K8S_NAMESPACE) start: - @$(KUBECTL) apply -f gaia-validator.yaml + @$(KUBECTL) apply -f api.yaml + @$(KUBECTL) apply -f validator.yaml stop: - @$(KUBECTL) delete -f gaia-validator.yaml \ No newline at end of file + @$(KUBECTL) delete -f api.yaml + @$(KUBECTL) delete -f validator.yaml \ No newline at end of file diff --git a/integration-tests/api.yaml b/integration-tests/api.yaml new file mode 100644 index 000000000..721987a72 --- /dev/null +++ b/integration-tests/api.yaml @@ -0,0 +1,43 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: api + namespace: integration +spec: + replicas: 1 + selector: + matchLabels: + app: api + template: + metadata: + labels: + app: api + spec: + securityContext: + fsGroup: 1000 + containers: + - name: api + image: gcr.io/stride-nodes/integration-test-api + imagePullPolicy: Always + ports: + - containerPort: 8000 + protocol: TCP + resources: + limits: + cpu: "400m" + memory: "8096M" + requests: + cpu: "200m" + memory: "4096M" +--- +apiVersion: v1 +kind: Service +metadata: + name: api + namespace: integration +spec: + selector: + app: api + ports: + - port: 8000 + protocol: TCP diff --git a/integration-tests/api/main.py b/integration-tests/api/main.py new file mode 100644 index 000000000..ef01137e5 --- /dev/null +++ b/integration-tests/api/main.py @@ -0,0 +1,45 @@ +from fastapi import FastAPI, File, HTTPException, UploadFile +from fastapi.responses import FileResponse +import os + +app = FastAPI() + +STORAGE_DIRECTORY = "storage" +os.makedirs(STORAGE_DIRECTORY, exist_ok=True) + + +@app.get("/status") +async def status() -> str: + """ + Health check + """ + return "ok" + + +@app.post("/upload/{file_name}") +@app.post("/upload/{path:path}/{file_name}") +async def upload_file(file_name: str, path: str = "", file: UploadFile = File(...)) -> dict: + """ + Allows uploading a file - stores it in the local file system + """ + parent = f"{STORAGE_DIRECTORY}/{path}" if path else STORAGE_DIRECTORY + os.makedirs(parent, exist_ok=True) + + with open(f"{parent}/{file_name}", "wb") as f: + f.write(file.file.read()) + + return {"info": f"file {file_name} saved"} + + +@app.get("/download/{file_name}") +@app.get("/download/{path:path}/{file_name}") +async def download_file(file_name: str, path: str = ""): + """ + Allows downloading a file from the local file system + """ + file_location = f"{STORAGE_DIRECTORY}/{path}/{file_name}" if path else f"{STORAGE_DIRECTORY}/{file_name}" + + if not os.path.exists(file_location): + return HTTPException(status_code=404, detail=f"file {file_location} not found") + + return FileResponse(file_location, media_type="application/octet-stream", filename=file_name) diff --git a/integration-tests/api/requirements.txt b/integration-tests/api/requirements.txt new file mode 100644 index 000000000..4733e582a --- /dev/null +++ b/integration-tests/api/requirements.txt @@ -0,0 +1,3 @@ +fastapi==0.103.2 +uvicorn==0.23.2 +python-multipart==0.0.9 \ No newline at end of file diff --git a/integration-tests/dockerfiles/Dockerfile.api b/integration-tests/dockerfiles/Dockerfile.api new file mode 100644 index 000000000..21c11cc98 --- /dev/null +++ b/integration-tests/dockerfiles/Dockerfile.api @@ -0,0 +1,15 @@ +FROM python:3.11-slim + +WORKDIR /app + +RUN apt-get update && apt-get install -y gcc g++ bash vim git curl tzdata \ + && adduser --system --home /home/python --disabled-password --disabled-login python -u 1000 \ + && pip install --upgrade pip + +COPY ./requirements.txt /app/requirements.txt + +RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt + +COPY main.py main.py + +CMD ["uvicorn", "main:app", "--proxy-headers", "--host", "0.0.0.0"] diff --git a/integration-tests/scripts/config.sh b/integration-tests/scripts/config.sh index 5558e9fca..66491e2e3 100644 --- a/integration-tests/scripts/config.sh +++ b/integration-tests/scripts/config.sh @@ -6,35 +6,36 @@ if [[ "$HOSTNAME" != *"validator"* ]]; then fi SCRIPTS_DIR=scripts -SHARED_DIR=shared CONFIG_DIR=config -VALIDATOR_KEYS_DIR=${SHARED_DIR}/validator-keys -NODE_KEYS_DIR=${SHARED_DIR}/node-keys -NODE_IDS_DIR=${SHARED_DIR}/node-ids +VALIDATOR_KEYS_DIR=validator-keys +NODE_KEYS_DIR=node-keys +NODE_IDS_DIR=node-ids KEYS_FILE=${CONFIG_DIR}/keys.json POD_INDEX=${HOSTNAME##*-} VALIDATOR_INDEX=$((POD_INDEX+1)) VALIDATOR_NAME=val${VALIDATOR_INDEX} +API_ENDPOINT=http://api.integration.svc:8000 + PEER_PORT=26656 RPC_PORT=26657 -# CHAIN_NAME=stride -# CHAIN_HOME=${HOME}/.stride -# BINARY=strided -# DENOM=ustrd -# MICRO_DENOM_UNITS=000000 -# NUM_VALIDATORS=3 - -CHAIN_NAME=cosmos -CHAIN_HOME=.gaia -BINARY=gaiad -DENOM=uatom +CHAIN_NAME=stride +CHAIN_HOME=${HOME}/.stride +BINARY=strided +DENOM=ustrd MICRO_DENOM_UNITS=000000 NUM_VALIDATORS=3 +# CHAIN_NAME=cosmos +# CHAIN_HOME=.gaia +# BINARY=gaiad +# DENOM=uatom +# MICRO_DENOM_UNITS=000000 +# NUM_VALIDATORS=3 + CHAIN_ID=${CHAIN_NAME}-test-1 BLOCK_TIME=1s VALIDATOR_BALANCE=10000000${MICRO_DENOM_UNITS} @@ -47,3 +48,12 @@ UNBONDING_TIME="240s" STRIDE_DAY_EPOCH_DURATION="140s" STRIDE_EPOCH_EPOCH_DURATION="35s" + +# Wait for API server to start +wait_for_api() { + api_endpoint="$1" + until [[ $(curl -o /dev/null -s -w "%{http_code}\n" "${api_endpoint}/status") -eq 200 ]]; do + echo "Waiting for API to start..." + sleep 2 + done +} diff --git a/integration-tests/scripts/init-chain.sh b/integration-tests/scripts/init-chain.sh index b57d2dbf8..1f4b461c2 100644 --- a/integration-tests/scripts/init-chain.sh +++ b/integration-tests/scripts/init-chain.sh @@ -10,8 +10,11 @@ if [[ "$LOCAL_MODE" == "true" ]]; then CHAIN_HOME=state rm -rf state BINARY="$BINARY --home $CHAIN_HOME" + API_ENDPOINT="http://localhost:8000" fi +# Wait for API server to start +wait_for_api $API_ENDPOINT # check if the binary has genesis subcommand or not, if not, set CHAIN_GENESIS_CMD to empty genesis_json=${CHAIN_HOME}/config/genesis.json @@ -32,9 +35,21 @@ init_config() { $BINARY config keyring-backend test } +# Helper to upload shared files to the API +upload_shared_file() { + file_path="$1" + saved_path="${2:-}" + file_name=$(basename $file_path) + + curl -s -X 'POST' "${API_ENDPOINT}/upload/${saved_path}" \ + -H 'accept: application/json' \ + -H 'Content-Type: multipart/form-data' \ + -F "file=@$file_path" && echo +} + # Adds each validator to the genesis file, and also saves down the public keys # which are needed for ICS -# Each validators public private key and node ID are saved in the shared directory +# Each validators public private key and node ID are saved in the API add_validators() { echo "Adding validators..." @@ -61,18 +76,14 @@ add_validators() { genesis_balance=${VALIDATOR_BALANCE}${DENOM} $BINARY $chain_genesis_command add-genesis-account $address $genesis_balance - # Save the node IDs and keys to the shared directory - mkdir -p ${NODE_IDS_DIR} - mkdir -p ${VALIDATOR_KEYS_DIR} - mkdir -p ${NODE_KEYS_DIR} - - node_id=$($BINARY tendermint show-node-id --home ${validator_home}) - echo $node_id > ${NODE_IDS_DIR}/${name}.txt - cp ${validator_home}/config/priv_validator_key.json ${VALIDATOR_KEYS_DIR}/${name}.json - cp ${validator_home}/config/node_key.json ${NODE_KEYS_DIR}/${name}.json + # Save the node IDs and keys to the API + $BINARY tendermint show-node-id --home ${validator_home} > node_id.txt + upload_shared_file node_id.txt ${NODE_IDS_DIR}/${name}.txt + upload_shared_file ${validator_home}/config/priv_validator_key.json ${VALIDATOR_KEYS_DIR}/${name}.json + upload_shared_file ${validator_home}/config/node_key.json ${NODE_KEYS_DIR}/${name}.json # Save the comma separted public keys for the ICS genesis update - validator_public_keys+="$(jq -r '.pub_key.value' ${VALIDATOR_KEYS_DIR}/${name}.json)," + validator_public_keys+="$(jq -r '.pub_key.value' ${validator_home}/config/priv_validator_key.json)," done # For non-stride nodes, generate and collect the validator gentx (for the main node only) @@ -93,7 +104,10 @@ update_default_genesis() { jq_inplace '.app_state.staking.params.unbonding_time |= "'$UNBONDING_TIME'"' $genesis_json jq_inplace '.app_state.gov.params.max_deposit_period |= "'$DEPOSIT_PERIOD'"' $genesis_json jq_inplace '.app_state.gov.params.voting_period |= "'$VOTING_PERIOD'"' $genesis_json - jq_inplace '.app_state.gov.params.expedited_voting_period |= "'$EXPEDITED_VOTING_PERIOD'"' $genesis_json + + if jq 'has(.app_state.gov.params.expedited_voting_period)' $genesis_json > /dev/null 2>&1; then + jq_inplace '.app_state.gov.params.expedited_voting_period |= "'$EXPEDITED_VOTING_PERIOD'"' $genesis_json + fi } # Genesis updates specific to stride @@ -111,11 +125,11 @@ update_host_genesis() { echo "Updating genesis.json with host configuration..." } -# Moves the genesis file into the shared directory +# Saves the genesis file in the API save_genesis() { - echo "Saving genesis.json to shared directory..." + echo "Saving genesis.json..." - cp $genesis_json ${SHARED_DIR}/genesis.json + upload_shared_file $genesis_json genesis.json } main() { diff --git a/integration-tests/scripts/init-node.sh b/integration-tests/scripts/init-node.sh index f5bc22942..45d5f2a3a 100644 --- a/integration-tests/scripts/init-node.sh +++ b/integration-tests/scripts/init-node.sh @@ -3,6 +3,9 @@ set -eu source scripts/config.sh +# Wait for API server to start +wait_for_api $API_ENDPOINT + # Initialize the config directory and validator key if it's not the main node init_config() { if [[ "$VALIDATOR_INDEX" != "1" ]]; then @@ -12,6 +15,18 @@ init_config() { fi } +# Helper function to download a file from the API +download_shared_file() { + stored_path="$1" + destination_path="$2" + + status_code=$(curl -s -o $destination_path -w "%{http_code}" "${API_ENDPOINT}/download/${stored_path}") + if [[ "$status_code" != "200" ]]; then + echo "ERROR - Failed to download $stored_path, status code ${status_code}" + exit 1 + fi +} + # Update config.toml, app.toml, and client.toml update_config() { config_toml="${CHAIN_HOME}/config/config.toml" @@ -34,14 +49,11 @@ update_config() { sed -i -E "s|chain-id = \"\"|chain-id = \"${CHAIN_ID}\"|g" $client_toml sed -i -E "s|keyring-backend = \"os\"|keyring-backend = \"test\"|g" $client_toml sed -i -E "s|node = \".*\"|node = \"tcp://localhost:${RPC_PORT}\"|g" $client_toml -} -# Extract private keys and genesis -download_shared() { echo "Retrieving private keys and genesis.json..." - cp ${VALIDATOR_KEYS_DIR}/val${VALIDATOR_INDEX}.json ${CHAIN_HOME}/config/priv_validator_key.json - cp ${NODE_KEYS_DIR}/val${VALIDATOR_INDEX}.json ${CHAIN_HOME}/config/node_key.json - cp ${SHARED_DIR}/genesis.json ${CHAIN_HOME}/config/genesis.json + download_shared_file ${VALIDATOR_KEYS_DIR}/val${VALIDATOR_INDEX}.json ${CHAIN_HOME}/config/priv_validator_key.json + download_shared_file ${NODE_KEYS_DIR}/val${VALIDATOR_INDEX}.json ${CHAIN_HOME}/config/node_key.json + download_shared_file genesis.json ${CHAIN_HOME}/config/genesis.json } # Update the persistent peers conditionally based on which node it is @@ -52,7 +64,8 @@ add_peers() { sed -i -E "s|^persistent_peers = .*|persistent_peers = \"\"|g" $config_toml else # For the other nodes, add the main node as the persistent peer - main_node_id=$(cat ${NODE_IDS_DIR}/val1.txt) + download_shared_file ${NODE_IDS_DIR}/val1.txt main_node_id.txt + main_node_id=$(cat main_node_id.txt) main_pod_id=${CHAIN_NAME}-validator-0 service=${CHAIN_NAME}-validator persistent_peer=${main_node_id}@${main_pod_id}.${service}.${NAMESPACE}.svc.cluster.local:${PEER_PORT} @@ -64,7 +77,6 @@ main() { echo "Initializing node..." init_config update_config - download_shared add_peers echo "Done" } diff --git a/integration-tests/validator.yaml b/integration-tests/validator.yaml index 713290116..40d964383 100644 --- a/integration-tests/validator.yaml +++ b/integration-tests/validator.yaml @@ -24,9 +24,6 @@ spec: - bash - "-c" - | - if [[ -d $CHAIN_HOME/config ]]; then - exit 0 - fi POD_INDEX=${HOSTNAME##*-} if [[ "$POD_INDEX" == "0" ]]; then bash scripts/init-chain.sh @@ -40,8 +37,6 @@ spec: volumeMounts: - name: state mountPath: /home/stride/.stride - - name: shared - mountPath: /home/stride/shared - name: scripts mountPath: /home/stride/scripts - name: config @@ -89,9 +84,6 @@ spec: volumes: - name: state emptyDir: {} - - name: shared - persistentVolumeClaim: - claimName: shared - name: scripts configMap: name: scripts @@ -122,14 +114,36 @@ spec: name: grpc protocol: TCP --- -apiVersion: v1 -kind: PersistentVolumeClaim +apiVersion: networking.k8s.io/v1 +kind: Ingress metadata: - name: stride-shared + name: stride-ingress namespace: integration + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/proxy-connect-timeout: "360" + nginx.ingress.kubernetes.io/proxy-send-timeout: "360" + nginx.ingress.kubernetes.io/proxy-read-timeout: "360" spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 50Gi + ingressClassName: nginx + rules: + - host: stride-api.internal.stridenet.co + http: + paths: + - pathType: Prefix + backend: + service: + name: stride-validator + port: + number: 1317 + path: /(.*) + - host: stride-rpc.internal.stridenet.co + http: + paths: + - pathType: Prefix + backend: + service: + name: stride-validator + port: + number: 26657 + path: /(.*) From 5a408dde3caaf8148120dbb908196bdaf06256b7 Mon Sep 17 00:00:00 2001 From: sampocs Date: Fri, 2 Aug 2024 13:14:33 -0500 Subject: [PATCH 10/24] moved chain config variables into yaml --- integration-tests/scripts/config.sh | 20 +++++++------------- integration-tests/validator.yaml | 15 ++++++++++++++- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/integration-tests/scripts/config.sh b/integration-tests/scripts/config.sh index 66491e2e3..b5c01e402 100644 --- a/integration-tests/scripts/config.sh +++ b/integration-tests/scripts/config.sh @@ -22,19 +22,13 @@ API_ENDPOINT=http://api.integration.svc:8000 PEER_PORT=26656 RPC_PORT=26657 -CHAIN_NAME=stride -CHAIN_HOME=${HOME}/.stride -BINARY=strided -DENOM=ustrd -MICRO_DENOM_UNITS=000000 -NUM_VALIDATORS=3 - -# CHAIN_NAME=cosmos -# CHAIN_HOME=.gaia -# BINARY=gaiad -# DENOM=uatom -# MICRO_DENOM_UNITS=000000 -# NUM_VALIDATORS=3 +# Redefined to confirm they're set +CHAIN_NAME=${CHAIN_NAME} +CHAIN_HOME=${CHAIN_HOME} +BINARY=${BINARY} +DENOM=${DENOM} +MICRO_DENOM_UNITS=${MICRO_DENOM_UNITS} +NUM_VALIDATORS=${NUM_VALIDATORS} CHAIN_ID=${CHAIN_NAME}-test-1 BLOCK_TIME=1s diff --git a/integration-tests/validator.yaml b/integration-tests/validator.yaml index 40d964383..32916cddb 100644 --- a/integration-tests/validator.yaml +++ b/integration-tests/validator.yaml @@ -1,3 +1,4 @@ +# Helm TODO: app name, images, home directories, start command apiVersion: apps/v1 kind: StatefulSet metadata: @@ -34,6 +35,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: CHAIN_NAME + value: stride + - name: CHAIN_HOME + value: /home/stride/.stride + - name: BINARY + value: strided + - name: DENOM + value: ustrd + - name: MICRO_DENOM_UNIT + value: "000000" + - name: NUM_VALIDATORS + value: "3" volumeMounts: - name: state mountPath: /home/stride/.stride @@ -42,7 +55,7 @@ spec: - name: config mountPath: /home/stride/config containers: - - name: stride-validator + - name: validator image: gcr.io/stride-nodes/stride-test imagePullPolicy: Always command: ["strided", "start", "--reject-config-defaults"] From 6c6e9a2f91056537fb87fd5638ae0c21a360f8f8 Mon Sep 17 00:00:00 2001 From: sampocs Date: Fri, 2 Aug 2024 13:28:40 -0500 Subject: [PATCH 11/24] confirmed it still works for gaia --- .../dockerfiles/Dockerfile.cosmos | 2 +- integration-tests/validator.yaml | 63 +++++++++++-------- 2 files changed, 39 insertions(+), 26 deletions(-) diff --git a/integration-tests/dockerfiles/Dockerfile.cosmos b/integration-tests/dockerfiles/Dockerfile.cosmos index b4ccbcb68..be9973a21 100644 --- a/integration-tests/dockerfiles/Dockerfile.cosmos +++ b/integration-tests/dockerfiles/Dockerfile.cosmos @@ -22,7 +22,7 @@ RUN CGO_ENABLED=1 BUILD_TAGS="muslc linkstatic" LINK_STATICALLY=true LEDGER_ENAB FROM alpine:3.17 COPY --from=builder /go/bin/$BINARY /usr/local/bin/ -RUN apk add bash vim sudo dasel jq \ +RUN apk add bash vim sudo dasel jq curl \ && addgroup -g 1000 validator \ && adduser -S -h /home/validator -D validator -u 1000 -G validator diff --git a/integration-tests/validator.yaml b/integration-tests/validator.yaml index 32916cddb..7f2c313ba 100644 --- a/integration-tests/validator.yaml +++ b/integration-tests/validator.yaml @@ -2,24 +2,24 @@ apiVersion: apps/v1 kind: StatefulSet metadata: - name: stride-validator + name: cosmos-validator namespace: integration spec: replicas: 3 selector: matchLabels: - app: stride-validator - serviceName: stride-validator + app: cosmos-validator + serviceName: cosmos-validator template: metadata: labels: - app: stride-validator + app: cosmos-validator spec: securityContext: fsGroup: 1000 initContainers: - name: init - image: gcr.io/stride-nodes/stride-test + image: gcr.io/stride-nodes/cosmos-test imagePullPolicy: Always command: - bash @@ -36,29 +36,29 @@ spec: fieldRef: fieldPath: metadata.namespace - name: CHAIN_NAME - value: stride + value: cosmos - name: CHAIN_HOME - value: /home/stride/.stride + value: /home/validator/.gaia - name: BINARY - value: strided + value: gaiad - name: DENOM - value: ustrd - - name: MICRO_DENOM_UNIT + value: uatom + - name: MICRO_DENOM_UNITS value: "000000" - name: NUM_VALIDATORS value: "3" volumeMounts: - name: state - mountPath: /home/stride/.stride + mountPath: /home/validator/.gaia - name: scripts - mountPath: /home/stride/scripts + mountPath: /home/validator/scripts - name: config - mountPath: /home/stride/config + mountPath: /home/validator/config containers: - name: validator - image: gcr.io/stride-nodes/stride-test + image: gcr.io/stride-nodes/cosmos-test imagePullPolicy: Always - command: ["strided", "start", "--reject-config-defaults"] + command: ["gaiad", "start"] lifecycle: postStart: exec: @@ -67,6 +67,19 @@ spec: exec: command: ["bash", "scripts/readiness.sh"] periodSeconds: 10 + env: + - name: CHAIN_NAME + value: cosmos + - name: CHAIN_HOME + value: /home/validator/.gaia + - name: BINARY + value: gaiad + - name: DENOM + value: uatom + - name: MICRO_DENOM_UNITS + value: "000000" + - name: NUM_VALIDATORS + value: "3" ports: - name: rpc containerPort: 26657 @@ -82,11 +95,11 @@ spec: protocol: TCP volumeMounts: - name: state - mountPath: /home/stride/.stride + mountPath: /home/validator/.gaia - name: scripts - mountPath: /home/stride/scripts + mountPath: /home/validator/scripts - name: config - mountPath: /home/stride/config + mountPath: /home/validator/config resources: limits: cpu: "1000m" @@ -107,12 +120,12 @@ spec: apiVersion: v1 kind: Service metadata: - name: stride-validator + name: cosmos-validator namespace: integration spec: clusterIP: None selector: - app: stride-validator + app: cosmos-validator ports: - port: 26657 name: rpc @@ -130,7 +143,7 @@ spec: apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: stride-ingress + name: cosmos-ingress namespace: integration annotations: nginx.ingress.kubernetes.io/rewrite-target: / @@ -140,23 +153,23 @@ metadata: spec: ingressClassName: nginx rules: - - host: stride-api.internal.stridenet.co + - host: cosmos-api.internal.stridenet.co http: paths: - pathType: Prefix backend: service: - name: stride-validator + name: cosmos-validator port: number: 1317 path: /(.*) - - host: stride-rpc.internal.stridenet.co + - host: cosmos-rpc.internal.stridenet.co http: paths: - pathType: Prefix backend: service: - name: stride-validator + name: cosmos-validator port: number: 26657 path: /(.*) From 03a02f784f413304b28a965bcd60c0174aa35a9e Mon Sep 17 00:00:00 2001 From: sampocs Date: Mon, 5 Aug 2024 12:43:41 -0500 Subject: [PATCH 12/24] renamed denom decimals --- integration-tests/gaia-validator.yaml | 135 -------------------------- integration-tests/scripts/config.sh | 7 +- integration-tests/validator.yaml | 4 +- 3 files changed, 6 insertions(+), 140 deletions(-) delete mode 100644 integration-tests/gaia-validator.yaml diff --git a/integration-tests/gaia-validator.yaml b/integration-tests/gaia-validator.yaml deleted file mode 100644 index 9268989ac..000000000 --- a/integration-tests/gaia-validator.yaml +++ /dev/null @@ -1,135 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: cosmos-validator - namespace: integration -spec: - replicas: 3 - selector: - matchLabels: - app: cosmos-validator - serviceName: cosmos-validator - template: - metadata: - labels: - app: cosmos-validator - spec: - securityContext: - fsGroup: 1000 - initContainers: - - name: init - image: gcr.io/stride-nodes/cosmos-test - imagePullPolicy: Always - command: - - bash - - "-c" - - | - if [[ -d $CHAIN_HOME/config ]]; then - exit 0 - fi - POD_INDEX=${HOSTNAME##*-} - if [[ "$POD_INDEX" == "0" ]]; then - bash scripts/init-chain.sh - fi - bash scripts/init-node.sh - env: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: state - mountPath: /home/validator/.gaia - - name: shared - mountPath: /home/validator/shared - - name: scripts - mountPath: /home/validator/scripts - - name: config - mountPath: /home/validator/config - containers: - - name: cosmos-validator - image: gcr.io/stride-nodes/cosmos-test - imagePullPolicy: Always - command: ["gaiad", "start"] - lifecycle: - postStart: - exec: - command: ["bash", "scripts/create-validator.sh"] - readinessProbe: - exec: - command: ["bash", "scripts/readiness.sh"] - periodSeconds: 10 - ports: - - name: rpc - containerPort: 26657 - protocol: TCP - - name: peer - containerPort: 26656 - protocol: TCP - - name: api - containerPort: 1317 - protocol: TCP - - name: grpc - containerPort: 9090 - protocol: TCP - volumeMounts: - - name: state - mountPath: /home/validator/.gaia - - name: scripts - mountPath: /home/validator/scripts - - name: config - mountPath: /home/validator/config - resources: - limits: - cpu: "1000m" - memory: "2Gi" - requests: - cpu: "1000m" - memory: "2Gi" - volumes: - - name: state - emptyDir: {} - - name: shared - persistentVolumeClaim: - claimName: cosmos-shared - - name: scripts - configMap: - name: scripts - - name: config - configMap: - name: config ---- -apiVersion: v1 -kind: Service -metadata: - name: cosmos-validator - namespace: integration -spec: - clusterIP: None - selector: - app: cosmos-validator - ports: - - port: 26657 - name: rpc - protocol: TCP - - port: 26656 - name: peer - protocol: TCP - - port: 1317 - name: api - protocol: TCP - - port: 9090 - name: grpc - protocol: TCP ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: cosmos-shared - namespace: integration -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 50Gi diff --git a/integration-tests/scripts/config.sh b/integration-tests/scripts/config.sh index b5c01e402..a6e884d4f 100644 --- a/integration-tests/scripts/config.sh +++ b/integration-tests/scripts/config.sh @@ -27,13 +27,14 @@ CHAIN_NAME=${CHAIN_NAME} CHAIN_HOME=${CHAIN_HOME} BINARY=${BINARY} DENOM=${DENOM} -MICRO_DENOM_UNITS=${MICRO_DENOM_UNITS} +DENOM_DECIMALS=${DENOM_DECIMALS} NUM_VALIDATORS=${NUM_VALIDATORS} +MICRO_DENOM_ZERO_PAD=$(printf "%${DENOM_DECIMALS}s" | tr ' ' "0") CHAIN_ID=${CHAIN_NAME}-test-1 BLOCK_TIME=1s -VALIDATOR_BALANCE=10000000${MICRO_DENOM_UNITS} -VALIDATOR_STAKE=1000000${MICRO_DENOM_UNITS} +VALIDATOR_BALANCE=10000000${MICRO_DENOM_ZERO_PAD} +VALIDATOR_STAKE=1000000${MICRO_DENOM_ZERO_PAD} DEPOSIT_PERIOD="30s" VOTING_PERIOD="30s" diff --git a/integration-tests/validator.yaml b/integration-tests/validator.yaml index 7f2c313ba..74424720f 100644 --- a/integration-tests/validator.yaml +++ b/integration-tests/validator.yaml @@ -76,8 +76,8 @@ spec: value: gaiad - name: DENOM value: uatom - - name: MICRO_DENOM_UNITS - value: "000000" + - name: DENOM_DECIMALS + value: "6" - name: NUM_VALIDATORS value: "3" ports: From 65784b94a9c8736203b83d2234c6600da19f5f6a Mon Sep 17 00:00:00 2001 From: sampocs Date: Mon, 5 Aug 2024 13:12:31 -0500 Subject: [PATCH 13/24] updated docker image paths --- integration-tests/Makefile | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/integration-tests/Makefile b/integration-tests/Makefile index 1e5e46a96..aecd1bc7a 100644 --- a/integration-tests/Makefile +++ b/integration-tests/Makefile @@ -15,25 +15,25 @@ start-api: @$(PYTHON) -m uvicorn api.main:app --proxy-headers build-api: - @echo "Building docker image: integration-test-api" - @$(DOCKER) buildx build --platform linux/amd64 --tag integration-test-api -f dockerfiles/Dockerfile.api api - @$(DOCKER) tag integration-test-api gcr.io/stride-nodes/integration-test-api + @echo "Building docker image: api" + @$(DOCKER) buildx build --platform linux/amd64 --tag api -f dockerfiles/Dockerfile.api api + @$(DOCKER) tag api gcr.io/stride-nodes/integration-tests/api:latest @echo "Pushing image to GCR" - @$(DOCKER) push gcr.io/stride-nodes/integration-test-api + @$(DOCKER) push gcr.io/stride-nodes/integration-tests/api:latest build-stride: - @echo "Building docker image: stride-test" - @$(DOCKER) buildx build --platform linux/amd64 --tag stride-test .. - @$(DOCKER) tag stride-test gcr.io/stride-nodes/stride-test + @echo "Building docker image: stride-validator" + @$(DOCKER) buildx build --platform linux/amd64 --tag stride-validator .. + @$(DOCKER) tag stride-validator gcr.io/stride-nodes/integration-tests/chains/stride:v23.0.0 @echo "Pushing image to GCR" - @$(DOCKER) push gcr.io/stride-nodes/stride-test + @$(DOCKER) push gcr.io/stride-nodes/integration-tests/chains/stride:v23.0.0 build-cosmos: @echo "Building docker image" - @$(DOCKER) buildx build --platform linux/amd64 --tag cosmos-test -f dockerfiles/Dockerfile.cosmos . - @$(DOCKER) tag cosmos-test gcr.io/stride-nodes/cosmos-test + @$(DOCKER) buildx build --platform linux/amd64 --tag cosmos-validator -f dockerfiles/Dockerfile.cosmos . + @$(DOCKER) tag cosmos-validator gcr.io/stride-nodes/integration-tests/chains/cosmoshub:v18.1.0 @echo "Pushing image to GCR" - @$(DOCKER) push gcr.io/stride-nodes/cosmos-test + @$(DOCKER) push gcr.io/stride-nodes/integration-tests/chains/cosmoshub:v18.1.0 update-scripts: @echo "Updating scripts configmap" From d4a95e5775fd33ee4aa176a66a7aaf6d7a9ca8da Mon Sep 17 00:00:00 2001 From: sampocs Date: Mon, 5 Aug 2024 13:39:10 -0500 Subject: [PATCH 14/24] wired up helm --- integration-tests/Makefile | 7 ++-- integration-tests/network/.helmignore | 23 ++++++++++++ integration-tests/network/Chart.yaml | 24 +++++++++++++ .../{ => network/templates}/api.yaml | 2 +- .../{ => network/templates}/validator.yaml | 35 ++++++++++--------- integration-tests/network/values.yaml | 1 + 6 files changed, 70 insertions(+), 22 deletions(-) create mode 100644 integration-tests/network/.helmignore create mode 100644 integration-tests/network/Chart.yaml rename integration-tests/{ => network/templates}/api.yaml (91%) rename integration-tests/{ => network/templates}/validator.yaml (81%) create mode 100644 integration-tests/network/values.yaml diff --git a/integration-tests/Makefile b/integration-tests/Makefile index aecd1bc7a..028851979 100644 --- a/integration-tests/Makefile +++ b/integration-tests/Makefile @@ -4,6 +4,7 @@ VENV_NAME=integration CONDA_BASE := $(shell conda info --base)/envs KUBECTL := $(shell which kubectl) DOCKER := $(shell which docker) +HELM := $(shell which helm) VENV_BIN := $(CONDA_BASE)/$(VENV_NAME)/bin PYTHON := $(VENV_BIN)/python @@ -46,9 +47,7 @@ update-config: @$(KUBECTL) create configmap config --from-file=config -n $(K8S_NAMESPACE) start: - @$(KUBECTL) apply -f api.yaml - @$(KUBECTL) apply -f validator.yaml + @$(HELM) install network network/ --values network/values.yaml stop: - @$(KUBECTL) delete -f api.yaml - @$(KUBECTL) delete -f validator.yaml \ No newline at end of file + @$(HELM) uninstall network \ No newline at end of file diff --git a/integration-tests/network/.helmignore b/integration-tests/network/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/integration-tests/network/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/integration-tests/network/Chart.yaml b/integration-tests/network/Chart.yaml new file mode 100644 index 000000000..f16160910 --- /dev/null +++ b/integration-tests/network/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: network +description: Network infrastructure for integration tests + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.0.1 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/integration-tests/api.yaml b/integration-tests/network/templates/api.yaml similarity index 91% rename from integration-tests/api.yaml rename to integration-tests/network/templates/api.yaml index 721987a72..012f18b9b 100644 --- a/integration-tests/api.yaml +++ b/integration-tests/network/templates/api.yaml @@ -17,7 +17,7 @@ spec: fsGroup: 1000 containers: - name: api - image: gcr.io/stride-nodes/integration-test-api + image: gcr.io/stride-nodes/integration-tests/api:latest imagePullPolicy: Always ports: - containerPort: 8000 diff --git a/integration-tests/validator.yaml b/integration-tests/network/templates/validator.yaml similarity index 81% rename from integration-tests/validator.yaml rename to integration-tests/network/templates/validator.yaml index 74424720f..2c1054665 100644 --- a/integration-tests/validator.yaml +++ b/integration-tests/network/templates/validator.yaml @@ -2,24 +2,24 @@ apiVersion: apps/v1 kind: StatefulSet metadata: - name: cosmos-validator + name: {{ .Values.chainName }}-validator namespace: integration spec: replicas: 3 selector: matchLabels: - app: cosmos-validator - serviceName: cosmos-validator + app: {{ .Values.chainName }}-validator + serviceName: {{ .Values.chainName }}-validator template: metadata: labels: - app: cosmos-validator + app: {{ .Values.chainName }}-validator spec: securityContext: fsGroup: 1000 initContainers: - name: init - image: gcr.io/stride-nodes/cosmos-test + image: gcr.io/stride-nodes/integration-tests/chains/cosmoshub:v18.1.0 imagePullPolicy: Always command: - bash @@ -36,15 +36,15 @@ spec: fieldRef: fieldPath: metadata.namespace - name: CHAIN_NAME - value: cosmos + value: {{ .Values.chainName }} - name: CHAIN_HOME value: /home/validator/.gaia - name: BINARY value: gaiad - name: DENOM value: uatom - - name: MICRO_DENOM_UNITS - value: "000000" + - name: DENOM_DECIMALS + value: "6" - name: NUM_VALIDATORS value: "3" volumeMounts: @@ -56,7 +56,7 @@ spec: mountPath: /home/validator/config containers: - name: validator - image: gcr.io/stride-nodes/cosmos-test + image: gcr.io/stride-nodes/integration-tests/chains/cosmoshub:v18.1.0 imagePullPolicy: Always command: ["gaiad", "start"] lifecycle: @@ -68,8 +68,9 @@ spec: command: ["bash", "scripts/readiness.sh"] periodSeconds: 10 env: + # TODO: Make it so that these aren't all required - name: CHAIN_NAME - value: cosmos + value: {{ .Values.chainName }} - name: CHAIN_HOME value: /home/validator/.gaia - name: BINARY @@ -120,12 +121,12 @@ spec: apiVersion: v1 kind: Service metadata: - name: cosmos-validator + name: {{ .Values.chainName }}-validator namespace: integration spec: clusterIP: None selector: - app: cosmos-validator + app: {{ .Values.chainName }}-validator ports: - port: 26657 name: rpc @@ -143,7 +144,7 @@ spec: apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: cosmos-ingress + name: {{ .Values.chainName }}-ingress namespace: integration annotations: nginx.ingress.kubernetes.io/rewrite-target: / @@ -153,23 +154,23 @@ metadata: spec: ingressClassName: nginx rules: - - host: cosmos-api.internal.stridenet.co + - host: {{ .Values.chainName }}-api.internal.stridenet.co http: paths: - pathType: Prefix backend: service: - name: cosmos-validator + name: {{ .Values.chainName }}-validator port: number: 1317 path: /(.*) - - host: cosmos-rpc.internal.stridenet.co + - host: {{ .Values.chainName }}-rpc.internal.stridenet.co http: paths: - pathType: Prefix backend: service: - name: cosmos-validator + name: {{ .Values.chainName }}-validator port: number: 26657 path: /(.*) diff --git a/integration-tests/network/values.yaml b/integration-tests/network/values.yaml new file mode 100644 index 000000000..384eb6dec --- /dev/null +++ b/integration-tests/network/values.yaml @@ -0,0 +1 @@ +chainName: cosmoshub \ No newline at end of file From 5acd92b01228775bf68001a389e407e24e547956 Mon Sep 17 00:00:00 2001 From: sampocs Date: Mon, 5 Aug 2024 22:12:50 -0500 Subject: [PATCH 15/24] rounded out templating --- integration-tests/Makefile | 9 ++- integration-tests/network/templates/api.yaml | 2 +- .../network/templates/validator.yaml | 71 ++++++++++--------- integration-tests/network/values.yaml | 13 +++- 4 files changed, 59 insertions(+), 36 deletions(-) diff --git a/integration-tests/Makefile b/integration-tests/Makefile index 028851979..f5667312a 100644 --- a/integration-tests/Makefile +++ b/integration-tests/Makefile @@ -8,6 +8,8 @@ HELM := $(shell which helm) VENV_BIN := $(CONDA_BASE)/$(VENV_NAME)/bin PYTHON := $(VENV_BIN)/python +HELM_CHART=network + python-install: conda create --name $(VENV_NAME) python=3.11 -y $(PYTHON) -m pip install -r api/requirements.txt @@ -47,7 +49,10 @@ update-config: @$(KUBECTL) create configmap config --from-file=config -n $(K8S_NAMESPACE) start: - @$(HELM) install network network/ --values network/values.yaml + @$(HELM) install $(HELM_CHART) $(HELM_CHART) --values $(HELM_CHART)/values.yaml stop: - @$(HELM) uninstall network \ No newline at end of file + @$(HELM) uninstall $(HELM_CHART) + +lint: + @$(HELM) lint $(HELM_CHART) \ No newline at end of file diff --git a/integration-tests/network/templates/api.yaml b/integration-tests/network/templates/api.yaml index 012f18b9b..99476febd 100644 --- a/integration-tests/network/templates/api.yaml +++ b/integration-tests/network/templates/api.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: api - namespace: integration + namespace: {{ .Values.namespace }} spec: replicas: 1 selector: diff --git a/integration-tests/network/templates/validator.yaml b/integration-tests/network/templates/validator.yaml index 2c1054665..0bb35b7d1 100644 --- a/integration-tests/network/templates/validator.yaml +++ b/integration-tests/network/templates/validator.yaml @@ -1,25 +1,30 @@ -# Helm TODO: app name, images, home directories, start command +{{- $namespace := .Values.namespace -}} +{{- $imageRepo := .Values.chainImagesRepo -}} +{{- range $chain := .Values.chains }} +{{- $appName := printf "%s-validator" $chain.name -}} +{{- $image := printf "%s/%s:%s" $imageRepo $chain.name $chain.version -}} +--- apiVersion: apps/v1 kind: StatefulSet metadata: - name: {{ .Values.chainName }}-validator - namespace: integration + name: {{ $appName }} + namespace: {{ $namespace }} spec: - replicas: 3 + replicas: {{ $chain.numValidators }} selector: matchLabels: - app: {{ .Values.chainName }}-validator - serviceName: {{ .Values.chainName }}-validator + app: {{ $appName }} + serviceName: {{ $appName }} template: metadata: labels: - app: {{ .Values.chainName }}-validator + app: {{ $appName }} spec: securityContext: fsGroup: 1000 initContainers: - name: init - image: gcr.io/stride-nodes/integration-tests/chains/cosmoshub:v18.1.0 + image: {{ $image }} imagePullPolicy: Always command: - bash @@ -36,27 +41,27 @@ spec: fieldRef: fieldPath: metadata.namespace - name: CHAIN_NAME - value: {{ .Values.chainName }} + value: {{ $chain.name }} - name: CHAIN_HOME - value: /home/validator/.gaia + value: {{ $chain.home }} - name: BINARY - value: gaiad + value: {{ $chain.binary }} - name: DENOM - value: uatom + value: {{ $chain.denom }} - name: DENOM_DECIMALS - value: "6" + value: "{{ $chain.decimals }}" - name: NUM_VALIDATORS - value: "3" + value: "{{ $chain.numValidators }}" volumeMounts: - name: state - mountPath: /home/validator/.gaia + mountPath: {{ $chain.home }} - name: scripts mountPath: /home/validator/scripts - name: config mountPath: /home/validator/config containers: - name: validator - image: gcr.io/stride-nodes/integration-tests/chains/cosmoshub:v18.1.0 + image: {{ $image }} imagePullPolicy: Always command: ["gaiad", "start"] lifecycle: @@ -70,17 +75,17 @@ spec: env: # TODO: Make it so that these aren't all required - name: CHAIN_NAME - value: {{ .Values.chainName }} + value: {{ $chain.name }} - name: CHAIN_HOME - value: /home/validator/.gaia + value: {{ $chain.home }} - name: BINARY - value: gaiad + value: {{ $chain.binary }} - name: DENOM - value: uatom + value: {{ $chain.denom }} - name: DENOM_DECIMALS - value: "6" + value: "{{ $chain.decimals }}" - name: NUM_VALIDATORS - value: "3" + value: "{{ $chain.numValidators }}" ports: - name: rpc containerPort: 26657 @@ -96,7 +101,7 @@ spec: protocol: TCP volumeMounts: - name: state - mountPath: /home/validator/.gaia + mountPath: {{ $chain.home }} - name: scripts mountPath: /home/validator/scripts - name: config @@ -121,12 +126,12 @@ spec: apiVersion: v1 kind: Service metadata: - name: {{ .Values.chainName }}-validator - namespace: integration + name: {{ $appName }} + namespace: {{ $namespace }} spec: clusterIP: None selector: - app: {{ .Values.chainName }}-validator + app: {{ $appName }} ports: - port: 26657 name: rpc @@ -144,8 +149,8 @@ spec: apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: {{ .Values.chainName }}-ingress - namespace: integration + name: {{ $appName }} + namespace: {{ $namespace }} annotations: nginx.ingress.kubernetes.io/rewrite-target: / nginx.ingress.kubernetes.io/proxy-connect-timeout: "360" @@ -154,23 +159,25 @@ metadata: spec: ingressClassName: nginx rules: - - host: {{ .Values.chainName }}-api.internal.stridenet.co + - host: {{ $chain.name }}-api.internal.stridenet.co http: paths: - pathType: Prefix backend: service: - name: {{ .Values.chainName }}-validator + name: {{ $appName }} port: number: 1317 path: /(.*) - - host: {{ .Values.chainName }}-rpc.internal.stridenet.co + - host: {{ $chain.name }}-rpc.internal.stridenet.co http: paths: - pathType: Prefix backend: service: - name: {{ .Values.chainName }}-validator + name: {{ $appName }} port: number: 26657 path: /(.*) +--- +{{- end }} \ No newline at end of file diff --git a/integration-tests/network/values.yaml b/integration-tests/network/values.yaml index 384eb6dec..401927bb9 100644 --- a/integration-tests/network/values.yaml +++ b/integration-tests/network/values.yaml @@ -1 +1,12 @@ -chainName: cosmoshub \ No newline at end of file +namespace: integration +chainImagesRepo: gcr.io/stride-nodes/integration-tests/chains + +chains: + - name: cosmoshub + binary: gaiad + version: v18.1.0 + numValidators: 3 + home: /home/validator/.gaia + denom: uatom + decimals: 6 + command: ["gaiad", "start"] From 6d71406f35e9b064c0c03de9bc8778bfc86be0df Mon Sep 17 00:00:00 2001 From: sampocs Date: Mon, 5 Aug 2024 22:34:41 -0500 Subject: [PATCH 16/24] separated out environment variables to template variable --- .../network/templates/validator.yaml | 55 ++++++++----------- 1 file changed, 24 insertions(+), 31 deletions(-) diff --git a/integration-tests/network/templates/validator.yaml b/integration-tests/network/templates/validator.yaml index 0bb35b7d1..81c72eb47 100644 --- a/integration-tests/network/templates/validator.yaml +++ b/integration-tests/network/templates/validator.yaml @@ -1,5 +1,25 @@ +{{- define "chain.env" -}} +- name: CHAIN_NAME + value: {{ .name }} +- name: CHAIN_HOME + value: {{ .home }} +- name: BINARY + value: {{ .binary }} +- name: DENOM + value: {{ .denom }} +- name: DENOM_DECIMALS + value: "{{ .decimals }}" +- name: NUM_VALIDATORS + value: "{{ .numValidators }}" +- name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +{{- end -}} + {{- $namespace := .Values.namespace -}} {{- $imageRepo := .Values.chainImagesRepo -}} + {{- range $chain := .Values.chains }} {{- $appName := printf "%s-validator" $chain.name -}} {{- $image := printf "%s/%s:%s" $imageRepo $chain.name $chain.version -}} @@ -35,23 +55,8 @@ spec: bash scripts/init-chain.sh fi bash scripts/init-node.sh - env: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: CHAIN_NAME - value: {{ $chain.name }} - - name: CHAIN_HOME - value: {{ $chain.home }} - - name: BINARY - value: {{ $chain.binary }} - - name: DENOM - value: {{ $chain.denom }} - - name: DENOM_DECIMALS - value: "{{ $chain.decimals }}" - - name: NUM_VALIDATORS - value: "{{ $chain.numValidators }}" + env: + {{- include "chain.env" $chain | nindent 10 }} volumeMounts: - name: state mountPath: {{ $chain.home }} @@ -72,20 +77,8 @@ spec: exec: command: ["bash", "scripts/readiness.sh"] periodSeconds: 10 - env: - # TODO: Make it so that these aren't all required - - name: CHAIN_NAME - value: {{ $chain.name }} - - name: CHAIN_HOME - value: {{ $chain.home }} - - name: BINARY - value: {{ $chain.binary }} - - name: DENOM - value: {{ $chain.denom }} - - name: DENOM_DECIMALS - value: "{{ $chain.decimals }}" - - name: NUM_VALIDATORS - value: "{{ $chain.numValidators }}" + env: + {{- include "chain.env" $chain | nindent 10 }} ports: - name: rpc containerPort: 26657 From 7273111c5e395323019bca81536147b77ac2c5fd Mon Sep 17 00:00:00 2001 From: sampocs Date: Mon, 5 Aug 2024 22:53:51 -0500 Subject: [PATCH 17/24] templated command --- .dockerignore | 3 ++- integration-tests/Makefile | 7 ++++--- integration-tests/dockerfiles/Dockerfile.cosmos | 1 - integration-tests/network/templates/validator.yaml | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.dockerignore b/.dockerignore index a264428f3..05059109e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,4 +5,5 @@ deps dockernet scripts genesis -testutil/localstride \ No newline at end of file +testutil/localstride +integration-tests \ No newline at end of file diff --git a/integration-tests/Makefile b/integration-tests/Makefile index f5667312a..f6b93ab58 100644 --- a/integration-tests/Makefile +++ b/integration-tests/Makefile @@ -26,10 +26,11 @@ build-api: build-stride: @echo "Building docker image: stride-validator" - @$(DOCKER) buildx build --platform linux/amd64 --tag stride-validator .. - @$(DOCKER) tag stride-validator gcr.io/stride-nodes/integration-tests/chains/stride:v23.0.0 + @$(DOCKER) buildx build --platform linux/amd64 --tag core:stride .. + @$(DOCKER) buildx build --platform linux/amd64 --tag stride-validator -f dockerfiles/Dockerfile.stride . + @$(DOCKER) tag stride-validator gcr.io/stride-nodes/integration-tests/chains/stride:latest @echo "Pushing image to GCR" - @$(DOCKER) push gcr.io/stride-nodes/integration-tests/chains/stride:v23.0.0 + @$(DOCKER) push gcr.io/stride-nodes/integration-tests/chains/stride:latest build-cosmos: @echo "Building docker image" diff --git a/integration-tests/dockerfiles/Dockerfile.cosmos b/integration-tests/dockerfiles/Dockerfile.cosmos index be9973a21..b88d9475f 100644 --- a/integration-tests/dockerfiles/Dockerfile.cosmos +++ b/integration-tests/dockerfiles/Dockerfile.cosmos @@ -30,4 +30,3 @@ USER 1000 WORKDIR /home/validator EXPOSE 26657 26656 1317 9090 - diff --git a/integration-tests/network/templates/validator.yaml b/integration-tests/network/templates/validator.yaml index 81c72eb47..9d2b029ad 100644 --- a/integration-tests/network/templates/validator.yaml +++ b/integration-tests/network/templates/validator.yaml @@ -68,7 +68,7 @@ spec: - name: validator image: {{ $image }} imagePullPolicy: Always - command: ["gaiad", "start"] + command: {{ toYaml $chain.command | nindent 10 }} lifecycle: postStart: exec: From d1ab337bdc1618839888d201bd43a5a23770fad3 Mon Sep 17 00:00:00 2001 From: sampocs Date: Mon, 5 Aug 2024 23:15:02 -0500 Subject: [PATCH 18/24] added stride chain --- integration-tests/dockerfiles/Dockerfile.stride | 12 ++++++++++++ integration-tests/network/templates/validator.yaml | 7 ++++--- integration-tests/network/values.yaml | 13 +++++++++++-- integration-tests/scripts/config.sh | 1 + integration-tests/scripts/init-chain.sh | 8 ++++---- integration-tests/scripts/init-node.sh | 8 ++++---- 6 files changed, 36 insertions(+), 13 deletions(-) create mode 100644 integration-tests/dockerfiles/Dockerfile.stride diff --git a/integration-tests/dockerfiles/Dockerfile.stride b/integration-tests/dockerfiles/Dockerfile.stride new file mode 100644 index 000000000..329b62bff --- /dev/null +++ b/integration-tests/dockerfiles/Dockerfile.stride @@ -0,0 +1,12 @@ +FROM alpine:3.17 + +RUN apk add bash vim sudo dasel jq curl \ + && addgroup -g 1000 validator \ + && adduser -S -h /home/validator -D validator -u 1000 -G validator + +COPY --from=core:stride --chown=stride:stride /usr/local/bin/strided /usr/local/bin/strided + +USER 1000 +WORKDIR /home/validator + +EXPOSE 26657 26656 1317 9090 \ No newline at end of file diff --git a/integration-tests/network/templates/validator.yaml b/integration-tests/network/templates/validator.yaml index 9d2b029ad..5b6aff00f 100644 --- a/integration-tests/network/templates/validator.yaml +++ b/integration-tests/network/templates/validator.yaml @@ -2,7 +2,7 @@ - name: CHAIN_NAME value: {{ .name }} - name: CHAIN_HOME - value: {{ .home }} + value: {{ printf "/home/validator/%s" .home }} - name: BINARY value: {{ .binary }} - name: DENOM @@ -22,6 +22,7 @@ {{- range $chain := .Values.chains }} {{- $appName := printf "%s-validator" $chain.name -}} +{{- $chainHomeDirectory := printf "/home/validator/%s" $chain.home -}} {{- $image := printf "%s/%s:%s" $imageRepo $chain.name $chain.version -}} --- apiVersion: apps/v1 @@ -59,7 +60,7 @@ spec: {{- include "chain.env" $chain | nindent 10 }} volumeMounts: - name: state - mountPath: {{ $chain.home }} + mountPath: {{ $chainHomeDirectory }} - name: scripts mountPath: /home/validator/scripts - name: config @@ -94,7 +95,7 @@ spec: protocol: TCP volumeMounts: - name: state - mountPath: {{ $chain.home }} + mountPath: {{ $chainHomeDirectory }} - name: scripts mountPath: /home/validator/scripts - name: config diff --git a/integration-tests/network/values.yaml b/integration-tests/network/values.yaml index 401927bb9..8bf230e8a 100644 --- a/integration-tests/network/values.yaml +++ b/integration-tests/network/values.yaml @@ -2,11 +2,20 @@ namespace: integration chainImagesRepo: gcr.io/stride-nodes/integration-tests/chains chains: + - name: stride + binary: strided + version: latest + numValidators: 3 + home: .stride + denom: ustrd + decimals: 6 + command: ["strided", "start", "--reject-config-defaults"] + - name: cosmoshub binary: gaiad version: v18.1.0 numValidators: 3 - home: /home/validator/.gaia + home: .gaia denom: uatom decimals: 6 - command: ["gaiad", "start"] + command: ["gaiad", "start"] \ No newline at end of file diff --git a/integration-tests/scripts/config.sh b/integration-tests/scripts/config.sh index a6e884d4f..c147de772 100644 --- a/integration-tests/scripts/config.sh +++ b/integration-tests/scripts/config.sh @@ -11,6 +11,7 @@ CONFIG_DIR=config VALIDATOR_KEYS_DIR=validator-keys NODE_KEYS_DIR=node-keys NODE_IDS_DIR=node-ids +GENESIS_DIR=genesis KEYS_FILE=${CONFIG_DIR}/keys.json POD_INDEX=${HOSTNAME##*-} diff --git a/integration-tests/scripts/init-chain.sh b/integration-tests/scripts/init-chain.sh index 1f4b461c2..ccfc6a24e 100644 --- a/integration-tests/scripts/init-chain.sh +++ b/integration-tests/scripts/init-chain.sh @@ -78,9 +78,9 @@ add_validators() { # Save the node IDs and keys to the API $BINARY tendermint show-node-id --home ${validator_home} > node_id.txt - upload_shared_file node_id.txt ${NODE_IDS_DIR}/${name}.txt - upload_shared_file ${validator_home}/config/priv_validator_key.json ${VALIDATOR_KEYS_DIR}/${name}.json - upload_shared_file ${validator_home}/config/node_key.json ${NODE_KEYS_DIR}/${name}.json + upload_shared_file node_id.txt ${NODE_IDS_DIR}/${CHAIN_NAME}/${name}.txt + upload_shared_file ${validator_home}/config/priv_validator_key.json ${VALIDATOR_KEYS_DIR}/${CHAIN_NAME}/${name}.json + upload_shared_file ${validator_home}/config/node_key.json ${NODE_KEYS_DIR}/${CHAIN_NAME}/${name}.json # Save the comma separted public keys for the ICS genesis update validator_public_keys+="$(jq -r '.pub_key.value' ${validator_home}/config/priv_validator_key.json)," @@ -129,7 +129,7 @@ update_host_genesis() { save_genesis() { echo "Saving genesis.json..." - upload_shared_file $genesis_json genesis.json + upload_shared_file $genesis_json ${GENESIS_DIR}/${CHAIN_NAME}/genesis.json } main() { diff --git a/integration-tests/scripts/init-node.sh b/integration-tests/scripts/init-node.sh index 45d5f2a3a..ba0dd1a9f 100644 --- a/integration-tests/scripts/init-node.sh +++ b/integration-tests/scripts/init-node.sh @@ -51,9 +51,9 @@ update_config() { sed -i -E "s|node = \".*\"|node = \"tcp://localhost:${RPC_PORT}\"|g" $client_toml echo "Retrieving private keys and genesis.json..." - download_shared_file ${VALIDATOR_KEYS_DIR}/val${VALIDATOR_INDEX}.json ${CHAIN_HOME}/config/priv_validator_key.json - download_shared_file ${NODE_KEYS_DIR}/val${VALIDATOR_INDEX}.json ${CHAIN_HOME}/config/node_key.json - download_shared_file genesis.json ${CHAIN_HOME}/config/genesis.json + download_shared_file ${VALIDATOR_KEYS_DIR}/${CHAIN_NAME}/val${VALIDATOR_INDEX}.json ${CHAIN_HOME}/config/priv_validator_key.json + download_shared_file ${NODE_KEYS_DIR}/${CHAIN_NAME}/val${VALIDATOR_INDEX}.json ${CHAIN_HOME}/config/node_key.json + download_shared_file ${GENESIS_DIR}/${CHAIN_NAME}/genesis.json ${CHAIN_HOME}/config/genesis.json } # Update the persistent peers conditionally based on which node it is @@ -64,7 +64,7 @@ add_peers() { sed -i -E "s|^persistent_peers = .*|persistent_peers = \"\"|g" $config_toml else # For the other nodes, add the main node as the persistent peer - download_shared_file ${NODE_IDS_DIR}/val1.txt main_node_id.txt + download_shared_file ${NODE_IDS_DIR}/${CHAIN_NAME}/val1.txt main_node_id.txt main_node_id=$(cat main_node_id.txt) main_pod_id=${CHAIN_NAME}-validator-0 service=${CHAIN_NAME}-validator From df72c537d10c4fc9b4f77aaca85c965cfa65f741 Mon Sep 17 00:00:00 2001 From: sampocs Date: Tue, 6 Aug 2024 10:32:34 -0500 Subject: [PATCH 19/24] added readme --- integration-tests/README.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 integration-tests/README.md diff --git a/integration-tests/README.md b/integration-tests/README.md new file mode 100644 index 000000000..391b352eb --- /dev/null +++ b/integration-tests/README.md @@ -0,0 +1,29 @@ +# Integration Tests + +This design for this integration test framework is heavily inspired by the Cosmology team's [starship](https://github.com/cosmology-tech/starship/tree/main). + +## Setup + +TODO + +## Motivation + +TODO + +## Network + +TODO + +## Testing Client + +TODO + +## Design Decisions + +### API Service to share files during chain setup + +In order to start the network as fast as possible, the chain should be initialized with ICS validators at genesis, rather than performing a switchover. However, in order to build the genesis file, the public keys must be gathered from each validator. This adds the constraint that keys must be consoldiated into a single process responsible for creating the genesis file. + +This can be achieved by having a master node creating the genesis.json and keys for each validator, and then having each validator download the files from the master node. Ideally this would be handled by a shared PVC across each validator; however, Kuberentes has a constraint where you cannot mount multiple pods onto the same volume. + +This led to the decision to use an API service to act as the intermediary that allows uploading and downloading of files. While at first glance, this smells of overengineering, the fastAPI implementation is actually quite simple (only marginally more code than creating and mounting a volume) and it improves the startup time dramatically since there's no need for the pods to wait for the volume to be mounted. Plus, it's likely that it can be leveraged in the future to help coordinate tasks across the different networks in the setup (e.g. it can store a registry of canonical IBC connections across chains). From 632dfd91d87a8594d11b671745c5625ff89eff65 Mon Sep 17 00:00:00 2001 From: sampocs Date: Tue, 6 Aug 2024 11:18:09 -0500 Subject: [PATCH 20/24] added namespace to start/stop command --- integration-tests/Makefile | 4 ++-- integration-tests/network/templates/api.yaml | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/integration-tests/Makefile b/integration-tests/Makefile index f6b93ab58..5aa438a3c 100644 --- a/integration-tests/Makefile +++ b/integration-tests/Makefile @@ -50,10 +50,10 @@ update-config: @$(KUBECTL) create configmap config --from-file=config -n $(K8S_NAMESPACE) start: - @$(HELM) install $(HELM_CHART) $(HELM_CHART) --values $(HELM_CHART)/values.yaml + @$(HELM) install $(HELM_CHART) $(HELM_CHART) --values $(HELM_CHART)/values.yaml -n $(K8S_NAMESPACE) stop: - @$(HELM) uninstall $(HELM_CHART) + @$(HELM) uninstall $(HELM_CHART) -n $(K8S_NAMESPACE) lint: @$(HELM) lint $(HELM_CHART) \ No newline at end of file diff --git a/integration-tests/network/templates/api.yaml b/integration-tests/network/templates/api.yaml index 99476febd..2d29bb005 100644 --- a/integration-tests/network/templates/api.yaml +++ b/integration-tests/network/templates/api.yaml @@ -13,8 +13,6 @@ spec: labels: app: api spec: - securityContext: - fsGroup: 1000 containers: - name: api image: gcr.io/stride-nodes/integration-tests/api:latest From 2b43b111ce49618e8d7fa69cad663215c08c0bd9 Mon Sep 17 00:00:00 2001 From: sampocs Date: Tue, 6 Aug 2024 16:17:29 -0500 Subject: [PATCH 21/24] moved configs and scripts to inside network --- integration-tests/{ => network}/config/keys.json | 0 integration-tests/{ => network}/scripts/config.sh | 0 integration-tests/{ => network}/scripts/create-validator.sh | 0 integration-tests/{ => network}/scripts/init-chain.sh | 0 integration-tests/{ => network}/scripts/init-node.sh | 0 integration-tests/{ => network}/scripts/readiness.sh | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename integration-tests/{ => network}/config/keys.json (100%) rename integration-tests/{ => network}/scripts/config.sh (100%) rename integration-tests/{ => network}/scripts/create-validator.sh (100%) rename integration-tests/{ => network}/scripts/init-chain.sh (100%) rename integration-tests/{ => network}/scripts/init-node.sh (100%) rename integration-tests/{ => network}/scripts/readiness.sh (100%) diff --git a/integration-tests/config/keys.json b/integration-tests/network/config/keys.json similarity index 100% rename from integration-tests/config/keys.json rename to integration-tests/network/config/keys.json diff --git a/integration-tests/scripts/config.sh b/integration-tests/network/scripts/config.sh similarity index 100% rename from integration-tests/scripts/config.sh rename to integration-tests/network/scripts/config.sh diff --git a/integration-tests/scripts/create-validator.sh b/integration-tests/network/scripts/create-validator.sh similarity index 100% rename from integration-tests/scripts/create-validator.sh rename to integration-tests/network/scripts/create-validator.sh diff --git a/integration-tests/scripts/init-chain.sh b/integration-tests/network/scripts/init-chain.sh similarity index 100% rename from integration-tests/scripts/init-chain.sh rename to integration-tests/network/scripts/init-chain.sh diff --git a/integration-tests/scripts/init-node.sh b/integration-tests/network/scripts/init-node.sh similarity index 100% rename from integration-tests/scripts/init-node.sh rename to integration-tests/network/scripts/init-node.sh diff --git a/integration-tests/scripts/readiness.sh b/integration-tests/network/scripts/readiness.sh similarity index 100% rename from integration-tests/scripts/readiness.sh rename to integration-tests/network/scripts/readiness.sh From fff2f701add0fe73de77fa5394c0f57d4b0a9bb8 Mon Sep 17 00:00:00 2001 From: sampocs Date: Tue, 6 Aug 2024 16:45:55 -0500 Subject: [PATCH 22/24] moved configs and scripts to inside helm --- integration-tests/Makefile | 12 +----------- integration-tests/network/templates/scripts.yaml | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 11 deletions(-) create mode 100644 integration-tests/network/templates/scripts.yaml diff --git a/integration-tests/Makefile b/integration-tests/Makefile index 5aa438a3c..202ef9c87 100644 --- a/integration-tests/Makefile +++ b/integration-tests/Makefile @@ -39,16 +39,6 @@ build-cosmos: @echo "Pushing image to GCR" @$(DOCKER) push gcr.io/stride-nodes/integration-tests/chains/cosmoshub:v18.1.0 -update-scripts: - @echo "Updating scripts configmap" - @$(KUBECTL) delete configmap scripts -n $(K8S_NAMESPACE) --ignore-not-found=true - @$(KUBECTL) create configmap scripts --from-file=scripts -n $(K8S_NAMESPACE) - -update-config: - @echo "Updating config configmap" - @$(KUBECTL) delete configmap config -n $(K8S_NAMESPACE) --ignore-not-found=true - @$(KUBECTL) create configmap config --from-file=config -n $(K8S_NAMESPACE) - start: @$(HELM) install $(HELM_CHART) $(HELM_CHART) --values $(HELM_CHART)/values.yaml -n $(K8S_NAMESPACE) @@ -56,4 +46,4 @@ stop: @$(HELM) uninstall $(HELM_CHART) -n $(K8S_NAMESPACE) lint: - @$(HELM) lint $(HELM_CHART) \ No newline at end of file + @$(HELM) lint $(HELM_CHART) \ No newline at end of file diff --git a/integration-tests/network/templates/scripts.yaml b/integration-tests/network/templates/scripts.yaml new file mode 100644 index 000000000..41e4e8974 --- /dev/null +++ b/integration-tests/network/templates/scripts.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: scripts + namespace: {{ .Values.namespace }} +data: +{{ (.Files.Glob "scripts/*").AsConfig | indent 2 }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: config + namespace: {{ .Values.namespace }} +data: +{{ (.Files.Glob "config/*").AsConfig | indent 2 }} \ No newline at end of file From 652267bc22617c4c14d62f8b24fddb46ec7607f4 Mon Sep 17 00:00:00 2001 From: sampocs Date: Tue, 6 Aug 2024 17:09:47 -0500 Subject: [PATCH 23/24] renamed config -> configs --- integration-tests/network/{config => configs}/keys.json | 0 integration-tests/network/templates/scripts.yaml | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename integration-tests/network/{config => configs}/keys.json (100%) diff --git a/integration-tests/network/config/keys.json b/integration-tests/network/configs/keys.json similarity index 100% rename from integration-tests/network/config/keys.json rename to integration-tests/network/configs/keys.json diff --git a/integration-tests/network/templates/scripts.yaml b/integration-tests/network/templates/scripts.yaml index 41e4e8974..268962f33 100644 --- a/integration-tests/network/templates/scripts.yaml +++ b/integration-tests/network/templates/scripts.yaml @@ -9,7 +9,7 @@ data: apiVersion: v1 kind: ConfigMap metadata: - name: config + name: configs namespace: {{ .Values.namespace }} data: -{{ (.Files.Glob "config/*").AsConfig | indent 2 }} \ No newline at end of file +{{ (.Files.Glob "configs/*").AsConfig | indent 2 }} \ No newline at end of file From 40e5a7d124d1c22b5b3e9df383bee43c5e540d00 Mon Sep 17 00:00:00 2001 From: sampocs Date: Wed, 7 Aug 2024 08:35:55 -0500 Subject: [PATCH 24/24] renamed config -> configs --- integration-tests/network/scripts/config.sh | 12 +++++++++++- integration-tests/network/templates/validator.yaml | 12 ++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/integration-tests/network/scripts/config.sh b/integration-tests/network/scripts/config.sh index c147de772..a3c8d2da8 100644 --- a/integration-tests/network/scripts/config.sh +++ b/integration-tests/network/scripts/config.sh @@ -6,7 +6,7 @@ if [[ "$HOSTNAME" != *"validator"* ]]; then fi SCRIPTS_DIR=scripts -CONFIG_DIR=config +CONFIG_DIR=configs VALIDATOR_KEYS_DIR=validator-keys NODE_KEYS_DIR=node-keys @@ -53,3 +53,13 @@ wait_for_api() { sleep 2 done } + +# Wait for node to start +wait_for_node() { + chain_name="$1" + rpc_endpoint="http://${chain_name}-validator.integration.svc:26657/status" + until [[ $(curl -o /dev/null -s $rpc_endpoint | jq '.result.sync_info.catching_up') == "false" ]]; do + echo "Waiting for $chain_name to start..." + sleep 2 + done +} \ No newline at end of file diff --git a/integration-tests/network/templates/validator.yaml b/integration-tests/network/templates/validator.yaml index 5b6aff00f..a278208ed 100644 --- a/integration-tests/network/templates/validator.yaml +++ b/integration-tests/network/templates/validator.yaml @@ -63,8 +63,8 @@ spec: mountPath: {{ $chainHomeDirectory }} - name: scripts mountPath: /home/validator/scripts - - name: config - mountPath: /home/validator/config + - name: configs + mountPath: /home/validator/configs containers: - name: validator image: {{ $image }} @@ -98,8 +98,8 @@ spec: mountPath: {{ $chainHomeDirectory }} - name: scripts mountPath: /home/validator/scripts - - name: config - mountPath: /home/validator/config + - name: configs + mountPath: /home/validator/configs resources: limits: cpu: "1000m" @@ -113,9 +113,9 @@ spec: - name: scripts configMap: name: scripts - - name: config + - name: configs configMap: - name: config + name: configs --- apiVersion: v1 kind: Service