diff --git a/.dockerignore b/.dockerignore index a264428f3..05059109e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,4 +5,5 @@ deps dockernet scripts genesis -testutil/localstride \ No newline at end of file +testutil/localstride +integration-tests \ No newline at end of file diff --git a/.gitignore b/.gitignore index 9cf83e83f..e732d8aa6 100644 --- a/.gitignore +++ b/.gitignore @@ -40,5 +40,8 @@ vue/* !.vscode/settings.json .ipynb_checkpoints/* +__pycache__ +node_modules -node_modules \ No newline at end of file +integration-tests/state +integration-tests/storage diff --git a/Dockerfile b/Dockerfile index 571a075de..320ce21b3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ RUN BUILD_TAGS=muslc LINK_STATICALLY=true make build FROM alpine:${RUNNER_IMAGE_VERSION} COPY --from=builder /opt/build/strided /usr/local/bin/strided -RUN apk add bash vim sudo dasel jq \ +RUN apk add bash vim sudo dasel jq curl \ && addgroup -g 1000 stride \ && adduser -S -h /home/stride -D stride -u 1000 -G stride diff --git a/cmd/consumer.go b/cmd/consumer.go index a7d79fa43..68235e819 100644 --- a/cmd/consumer.go +++ b/cmd/consumer.go @@ -1,13 +1,17 @@ package cmd import ( + "encoding/base64" "encoding/json" + "errors" "fmt" - "strconv" + "strings" errorsmod "cosmossdk.io/errors" types1 "github.com/cometbft/cometbft/abci/types" + "github.com/cometbft/cometbft/config" pvm "github.com/cometbft/cometbft/privval" + tmprotocrypto "github.com/cometbft/cometbft/proto/tendermint/crypto" tmtypes "github.com/cometbft/cometbft/types" "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/client/flags" @@ -22,48 +26,110 @@ import ( "github.com/Stride-Labs/stride/v23/testutil" ) +const ( + FlagValidatorPublicKeys = "validator-public-keys" + FlagValidatorHomeDirectories = "validator-home-directories" +) + +// Builds the list of validator Ed25519 pubkeys from a comma separate list of base64 encoded pubkeys +func buildPublicKeysFromString(publicKeysRaw string) (publicKeys []tmprotocrypto.PublicKey, err error) { + for _, publicKeyEncoded := range strings.Split(publicKeysRaw, ",") { + if publicKeyEncoded == "" { + continue + } + publicKeyBytes, err := base64.StdEncoding.DecodeString(publicKeyEncoded) + if err != nil { + return nil, errorsmod.Wrapf(err, "unable to decode public key") + } + publicKeys = append(publicKeys, tmprotocrypto.PublicKey{ + Sum: &tmprotocrypto.PublicKey_Ed25519{ + Ed25519: publicKeyBytes, + }, + }) + } + + return publicKeys, nil +} + +// Builds the list validator Ed25519 pubkeys from a comma separated list of validator home directories +func buildPublicKeysFromHomeDirectories(config *config.Config, homeDirectories string) (publicKeys []tmprotocrypto.PublicKey, err error) { + for _, homeDir := range strings.Split(homeDirectories, ",") { + if homeDir == "" { + continue + } + config.SetRoot(homeDir) + + privValidator := pvm.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()) + pk, err := privValidator.GetPubKey() + if err != nil { + return nil, err + } + sdkPublicKey, err := cryptocodec.FromTmPubKeyInterface(pk) + if err != nil { + return nil, err + } + tmProtoPublicKey, err := cryptocodec.ToTmProtoPublicKey(sdkPublicKey) + if err != nil { + return nil, err + } + publicKeys = append(publicKeys, tmProtoPublicKey) + } + + return publicKeys, nil +} + func AddConsumerSectionCmd(defaultNodeHome string) *cobra.Command { genesisMutator := NewDefaultGenesisIO() - txCmd := &cobra.Command{ - Use: "add-consumer-section [num_nodes]", - Args: cobra.ExactArgs(1), + cmd := &cobra.Command{ + Use: "add-consumer-section", + Args: cobra.ExactArgs(0), Short: "ONLY FOR TESTING PURPOSES! Modifies genesis so that chain can be started locally with one node.", SuggestionsMinimumDistance: 2, RunE: func(cmd *cobra.Command, args []string) error { - numNodes, err := strconv.Atoi(args[0]) + // We need to public keys for each validator - they can either be provided explicitly + // or indirectly by providing the validator home directories + publicKeysRaw, err := cmd.Flags().GetString(FlagValidatorPublicKeys) if err != nil { - return errorsmod.Wrap(err, "invalid number of nodes") - } else if numNodes == 0 { - return errorsmod.Wrap(nil, "num_nodes can not be zero") + return errorsmod.Wrapf(err, "unable to parse validator public key flag") + } + homeDirectoriesRaw, err := cmd.Flags().GetString(FlagValidatorHomeDirectories) + if err != nil { + return errorsmod.Wrapf(err, "unable to parse validator home directories flag") + } + if (publicKeysRaw == "" && homeDirectoriesRaw == "") || (publicKeysRaw != "" && homeDirectoriesRaw != "") { + return fmt.Errorf("must specified either --%s or --%s", FlagValidatorPublicKeys, FlagValidatorHomeDirectories) + } + + // Build up a list of the validator public keys + // If the public keys were passed directly, decode them and create the tm proto pub keys + // Otherwise, derrive them from the private keys in each validator's home directory + var tmPublicKeys []tmprotocrypto.PublicKey + if publicKeysRaw != "" { + tmPublicKeys, err = buildPublicKeysFromString(publicKeysRaw) + if err != nil { + return err + } + } else { + serverCtx := server.GetServerContextFromCmd(cmd) + config := serverCtx.Config + + tmPublicKeys, err = buildPublicKeysFromHomeDirectories(config, homeDirectoriesRaw) + if err != nil { + return err + } + } + + if len(tmPublicKeys) == 0 { + return errors.New("no valid public keys or validator home directories provided") } return genesisMutator.AlterConsumerModuleState(cmd, func(state *GenesisData, _ map[string]json.RawMessage) error { initialValset := []types1.ValidatorUpdate{} genesisState := testutil.CreateMinimalConsumerTestGenesis() - clientCtx := client.GetClientContextFromCmd(cmd) - serverCtx := server.GetServerContextFromCmd(cmd) - config := serverCtx.Config - homeDir := clientCtx.HomeDir - for i := 1; i <= numNodes; i++ { - homeDir = fmt.Sprintf("%s%d", homeDir[:len(homeDir)-1], i) - config.SetRoot(homeDir) - - privValidator := pvm.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()) - pk, err := privValidator.GetPubKey() - if err != nil { - return err - } - sdkPublicKey, err := cryptocodec.FromTmPubKeyInterface(pk) - if err != nil { - return err - } - tmProtoPublicKey, err := cryptocodec.ToTmProtoPublicKey(sdkPublicKey) - if err != nil { - return err - } - - initialValset = append(initialValset, types1.ValidatorUpdate{PubKey: tmProtoPublicKey, Power: 100}) + + for _, publicKey := range tmPublicKeys { + initialValset = append(initialValset, types1.ValidatorUpdate{PubKey: publicKey, Power: 100}) } vals, err := tmtypes.PB2TM.ValidatorUpdates(initialValset) @@ -80,10 +146,11 @@ func AddConsumerSectionCmd(defaultNodeHome string) *cobra.Command { }, } - txCmd.Flags().String(flags.FlagHome, defaultNodeHome, "The application home directory") - flags.AddQueryFlagsToCmd(txCmd) + cmd.Flags().String(flags.FlagHome, defaultNodeHome, "The application home directory") + cmd.Flags().String(FlagValidatorPublicKeys, "", "Comma separated, base64-encoded public keys for each validator") + cmd.Flags().String(FlagValidatorHomeDirectories, "", "Comma separated list of home directories for each validator") - return txCmd + return cmd } type GenesisMutator interface { diff --git a/dockernet/src/init_chain.sh b/dockernet/src/init_chain.sh index 104f52745..715406402 100644 --- a/dockernet/src/init_chain.sh +++ b/dockernet/src/init_chain.sh @@ -139,7 +139,12 @@ set_consumer_genesis() { genesis_config=$1 # add consumer genesis - $MAIN_CMD add-consumer-section $NUM_NODES + home_directories="" + for (( i=1; i <= $NUM_NODES; i++ )); do + home_directories+="${STATE}/stride${i}," + done + + $MAIN_CMD add-consumer-section --validator-home-directories $home_directories jq '.app_state.ccvconsumer.params.unbonding_period = $newVal' --arg newVal "$UNBONDING_TIME" $genesis_config > json.tmp && mv json.tmp $genesis_config } diff --git a/integration-tests/Makefile b/integration-tests/Makefile new file mode 100644 index 000000000..202ef9c87 --- /dev/null +++ b/integration-tests/Makefile @@ -0,0 +1,49 @@ +K8S_NAMESPACE=integration +VENV_NAME=integration + +CONDA_BASE := $(shell conda info --base)/envs +KUBECTL := $(shell which kubectl) +DOCKER := $(shell which docker) +HELM := $(shell which helm) +VENV_BIN := $(CONDA_BASE)/$(VENV_NAME)/bin +PYTHON := $(VENV_BIN)/python + +HELM_CHART=network + +python-install: + conda create --name $(VENV_NAME) python=3.11 -y + $(PYTHON) -m pip install -r api/requirements.txt + +start-api: + @$(PYTHON) -m uvicorn api.main:app --proxy-headers + +build-api: + @echo "Building docker image: api" + @$(DOCKER) buildx build --platform linux/amd64 --tag api -f dockerfiles/Dockerfile.api api + @$(DOCKER) tag api gcr.io/stride-nodes/integration-tests/api:latest + @echo "Pushing image to GCR" + @$(DOCKER) push gcr.io/stride-nodes/integration-tests/api:latest + +build-stride: + @echo "Building docker image: stride-validator" + @$(DOCKER) buildx build --platform linux/amd64 --tag core:stride .. + @$(DOCKER) buildx build --platform linux/amd64 --tag stride-validator -f dockerfiles/Dockerfile.stride . + @$(DOCKER) tag stride-validator gcr.io/stride-nodes/integration-tests/chains/stride:latest + @echo "Pushing image to GCR" + @$(DOCKER) push gcr.io/stride-nodes/integration-tests/chains/stride:latest + +build-cosmos: + @echo "Building docker image" + @$(DOCKER) buildx build --platform linux/amd64 --tag cosmos-validator -f dockerfiles/Dockerfile.cosmos . + @$(DOCKER) tag cosmos-validator gcr.io/stride-nodes/integration-tests/chains/cosmoshub:v18.1.0 + @echo "Pushing image to GCR" + @$(DOCKER) push gcr.io/stride-nodes/integration-tests/chains/cosmoshub:v18.1.0 + +start: + @$(HELM) install $(HELM_CHART) $(HELM_CHART) --values $(HELM_CHART)/values.yaml -n $(K8S_NAMESPACE) + +stop: + @$(HELM) uninstall $(HELM_CHART) -n $(K8S_NAMESPACE) + +lint: + @$(HELM) lint $(HELM_CHART) \ No newline at end of file diff --git a/integration-tests/README.md b/integration-tests/README.md new file mode 100644 index 000000000..391b352eb --- /dev/null +++ b/integration-tests/README.md @@ -0,0 +1,29 @@ +# Integration Tests + +This design for this integration test framework is heavily inspired by the Cosmology team's [starship](https://github.com/cosmology-tech/starship/tree/main). + +## Setup + +TODO + +## Motivation + +TODO + +## Network + +TODO + +## Testing Client + +TODO + +## Design Decisions + +### API Service to share files during chain setup + +In order to start the network as fast as possible, the chain should be initialized with ICS validators at genesis, rather than performing a switchover. However, in order to build the genesis file, the public keys must be gathered from each validator. This adds the constraint that keys must be consoldiated into a single process responsible for creating the genesis file. + +This can be achieved by having a master node creating the genesis.json and keys for each validator, and then having each validator download the files from the master node. Ideally this would be handled by a shared PVC across each validator; however, Kuberentes has a constraint where you cannot mount multiple pods onto the same volume. + +This led to the decision to use an API service to act as the intermediary that allows uploading and downloading of files. While at first glance, this smells of overengineering, the fastAPI implementation is actually quite simple (only marginally more code than creating and mounting a volume) and it improves the startup time dramatically since there's no need for the pods to wait for the volume to be mounted. Plus, it's likely that it can be leveraged in the future to help coordinate tasks across the different networks in the setup (e.g. it can store a registry of canonical IBC connections across chains). diff --git a/integration-tests/api/main.py b/integration-tests/api/main.py new file mode 100644 index 000000000..ef01137e5 --- /dev/null +++ b/integration-tests/api/main.py @@ -0,0 +1,45 @@ +from fastapi import FastAPI, File, HTTPException, UploadFile +from fastapi.responses import FileResponse +import os + +app = FastAPI() + +STORAGE_DIRECTORY = "storage" +os.makedirs(STORAGE_DIRECTORY, exist_ok=True) + + +@app.get("/status") +async def status() -> str: + """ + Health check + """ + return "ok" + + +@app.post("/upload/{file_name}") +@app.post("/upload/{path:path}/{file_name}") +async def upload_file(file_name: str, path: str = "", file: UploadFile = File(...)) -> dict: + """ + Allows uploading a file - stores it in the local file system + """ + parent = f"{STORAGE_DIRECTORY}/{path}" if path else STORAGE_DIRECTORY + os.makedirs(parent, exist_ok=True) + + with open(f"{parent}/{file_name}", "wb") as f: + f.write(file.file.read()) + + return {"info": f"file {file_name} saved"} + + +@app.get("/download/{file_name}") +@app.get("/download/{path:path}/{file_name}") +async def download_file(file_name: str, path: str = ""): + """ + Allows downloading a file from the local file system + """ + file_location = f"{STORAGE_DIRECTORY}/{path}/{file_name}" if path else f"{STORAGE_DIRECTORY}/{file_name}" + + if not os.path.exists(file_location): + return HTTPException(status_code=404, detail=f"file {file_location} not found") + + return FileResponse(file_location, media_type="application/octet-stream", filename=file_name) diff --git a/integration-tests/api/requirements.txt b/integration-tests/api/requirements.txt new file mode 100644 index 000000000..4733e582a --- /dev/null +++ b/integration-tests/api/requirements.txt @@ -0,0 +1,3 @@ +fastapi==0.103.2 +uvicorn==0.23.2 +python-multipart==0.0.9 \ No newline at end of file diff --git a/integration-tests/dockerfiles/Dockerfile.api b/integration-tests/dockerfiles/Dockerfile.api new file mode 100644 index 000000000..21c11cc98 --- /dev/null +++ b/integration-tests/dockerfiles/Dockerfile.api @@ -0,0 +1,15 @@ +FROM python:3.11-slim + +WORKDIR /app + +RUN apt-get update && apt-get install -y gcc g++ bash vim git curl tzdata \ + && adduser --system --home /home/python --disabled-password --disabled-login python -u 1000 \ + && pip install --upgrade pip + +COPY ./requirements.txt /app/requirements.txt + +RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt + +COPY main.py main.py + +CMD ["uvicorn", "main:app", "--proxy-headers", "--host", "0.0.0.0"] diff --git a/integration-tests/dockerfiles/Dockerfile.cosmos b/integration-tests/dockerfiles/Dockerfile.cosmos new file mode 100644 index 000000000..b88d9475f --- /dev/null +++ b/integration-tests/dockerfiles/Dockerfile.cosmos @@ -0,0 +1,32 @@ +FROM golang:1.22-alpine AS builder + +WORKDIR /opt + +RUN apk add --update curl make git libc-dev bash gcc linux-headers eudev-dev ca-certificates build-base git + +ENV REPO=https://github.com/cosmos/gaia +ENV COMMIT_HASH=v18.1.0 +ENV BINARY=gaiad + +RUN git clone ${REPO} chain \ + && cd chain \ + && git checkout $COMMIT_HASH + +WORKDIR /opt/chain + +RUN WASMVM_VERSION=$(cat go.mod | grep github.com/CosmWasm/wasmvm | awk '{print $2}') \ + && wget https://github.com/CosmWasm/wasmvm/releases/download/$WASMVM_VERSION/libwasmvm_muslc.$(uname -m).a \ + -O /lib/libwasmvm_muslc.a + +RUN CGO_ENABLED=1 BUILD_TAGS="muslc linkstatic" LINK_STATICALLY=true LEDGER_ENABLED=false make install + +FROM alpine:3.17 +COPY --from=builder /go/bin/$BINARY /usr/local/bin/ +RUN apk add bash vim sudo dasel jq curl \ + && addgroup -g 1000 validator \ + && adduser -S -h /home/validator -D validator -u 1000 -G validator + +USER 1000 +WORKDIR /home/validator + +EXPOSE 26657 26656 1317 9090 diff --git a/integration-tests/dockerfiles/Dockerfile.stride b/integration-tests/dockerfiles/Dockerfile.stride new file mode 100644 index 000000000..329b62bff --- /dev/null +++ b/integration-tests/dockerfiles/Dockerfile.stride @@ -0,0 +1,12 @@ +FROM alpine:3.17 + +RUN apk add bash vim sudo dasel jq curl \ + && addgroup -g 1000 validator \ + && adduser -S -h /home/validator -D validator -u 1000 -G validator + +COPY --from=core:stride --chown=stride:stride /usr/local/bin/strided /usr/local/bin/strided + +USER 1000 +WORKDIR /home/validator + +EXPOSE 26657 26656 1317 9090 \ No newline at end of file diff --git a/integration-tests/network/.helmignore b/integration-tests/network/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/integration-tests/network/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/integration-tests/network/Chart.yaml b/integration-tests/network/Chart.yaml new file mode 100644 index 000000000..f16160910 --- /dev/null +++ b/integration-tests/network/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: network +description: Network infrastructure for integration tests + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.0.1 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/integration-tests/network/configs/keys.json b/integration-tests/network/configs/keys.json new file mode 100644 index 000000000..33aace07f --- /dev/null +++ b/integration-tests/network/configs/keys.json @@ -0,0 +1,30 @@ +{ + "validators": [ + { + "name": "val1", + "mnemonic": "close soup mirror crew erode defy knock trigger gather eyebrow tent farm gym gloom base lemon sleep weekend rich forget diagram hurt prize fly" + }, + { + "name": "val2", + "mnemonic": "turkey miss hurry unable embark hospital kangaroo nuclear outside term toy fall buffalo book opinion such moral meadow wing olive camp sad metal banner" + }, + { + "name": "val3", + "mnemonic": "tenant neck ask season exist hill churn rice convince shock modify evidence armor track army street stay light program harvest now settle feed wheat" + }, + { + "name": "val4", + "mnemonic": "tail forward era width glory magnet knock shiver cup broken turkey upgrade cigar story agent lake transfer misery sustain fragile parrot also air document" + }, + { + "name": "val5", + "mnemonic": "crime lumber parrot enforce chimney turtle wing iron scissors jealous indicate peace empty game host protect juice submit motor cause second picture nuclear area" + } + ], + "faucet": [ + { + "name": "faucet", + "mnemonic": "chimney become stuff spoil resource supply picture divorce casual curve check web valid survey zebra various pet sphere timber friend faint blame mansion film" + } + ] + } diff --git a/integration-tests/network/scripts/config.sh b/integration-tests/network/scripts/config.sh new file mode 100644 index 000000000..a3c8d2da8 --- /dev/null +++ b/integration-tests/network/scripts/config.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +# Override the hostname only when in local mode +if [[ "$HOSTNAME" != *"validator"* ]]; then + HOSTNAME=validator-0 +fi + +SCRIPTS_DIR=scripts +CONFIG_DIR=configs + +VALIDATOR_KEYS_DIR=validator-keys +NODE_KEYS_DIR=node-keys +NODE_IDS_DIR=node-ids +GENESIS_DIR=genesis +KEYS_FILE=${CONFIG_DIR}/keys.json + +POD_INDEX=${HOSTNAME##*-} +VALIDATOR_INDEX=$((POD_INDEX+1)) +VALIDATOR_NAME=val${VALIDATOR_INDEX} + +API_ENDPOINT=http://api.integration.svc:8000 + +PEER_PORT=26656 +RPC_PORT=26657 + +# Redefined to confirm they're set +CHAIN_NAME=${CHAIN_NAME} +CHAIN_HOME=${CHAIN_HOME} +BINARY=${BINARY} +DENOM=${DENOM} +DENOM_DECIMALS=${DENOM_DECIMALS} +NUM_VALIDATORS=${NUM_VALIDATORS} + +MICRO_DENOM_ZERO_PAD=$(printf "%${DENOM_DECIMALS}s" | tr ' ' "0") +CHAIN_ID=${CHAIN_NAME}-test-1 +BLOCK_TIME=1s +VALIDATOR_BALANCE=10000000${MICRO_DENOM_ZERO_PAD} +VALIDATOR_STAKE=1000000${MICRO_DENOM_ZERO_PAD} + +DEPOSIT_PERIOD="30s" +VOTING_PERIOD="30s" +EXPEDITED_VOTING_PERIOD="29s" +UNBONDING_TIME="240s" + +STRIDE_DAY_EPOCH_DURATION="140s" +STRIDE_EPOCH_EPOCH_DURATION="35s" + +# Wait for API server to start +wait_for_api() { + api_endpoint="$1" + until [[ $(curl -o /dev/null -s -w "%{http_code}\n" "${api_endpoint}/status") -eq 200 ]]; do + echo "Waiting for API to start..." + sleep 2 + done +} + +# Wait for node to start +wait_for_node() { + chain_name="$1" + rpc_endpoint="http://${chain_name}-validator.integration.svc:26657/status" + until [[ $(curl -o /dev/null -s $rpc_endpoint | jq '.result.sync_info.catching_up') == "false" ]]; do + echo "Waiting for $chain_name to start..." + sleep 2 + done +} \ No newline at end of file diff --git a/integration-tests/network/scripts/create-validator.sh b/integration-tests/network/scripts/create-validator.sh new file mode 100644 index 000000000..608259a12 --- /dev/null +++ b/integration-tests/network/scripts/create-validator.sh @@ -0,0 +1,55 @@ +# #!/bin/bash + +set -eu +source scripts/config.sh + +wait_for_startup() { + echo "Waiting for node to start..." + while ! (($BINARY status &> /dev/null) && [[ "$($BINARY status | jq -r '.SyncInfo.latest_block_height')" -gt "0" ]]); do + echo "Node still syncing..." + sleep 10 + done + echo "Node synced" +} + +add_keys() { + echo "Adding validator keys..." + + validator_config=$(jq -r '.validators[$index]' --argjson index "$POD_INDEX" ${KEYS_FILE}) + mnemonic=$(echo $validator_config | jq -r '.mnemonic') + name=$(echo $validator_config | jq -r '.name') + + if ! $BINARY keys show $name -a &> /dev/null ; then + echo "$mnemonic" | $BINARY keys add $name --recover + fi +} + +create_validator() { + echo "Creating validator..." + min_self_delegation="" + if [[ $($BINARY tx staking create-validator --help | grep -c "min-self-delegation") -gt 0 ]]; then + min_self_delegation='--min-self-delegation=1000000' + fi + + pub_key=$($BINARY tendermint show-validator) + $BINARY tx staking create-validator \ + --amount ${VALIDATOR_STAKE}${DENOM} \ + --pubkey=$pub_key \ + --commission-rate="0.10" \ + --commission-max-rate="0.20" \ + --commission-max-change-rate="0.01" \ + $min_self_delegation \ + --fees 300000$DENOM \ + --gas auto \ + --gas-adjustment 1.2 \ + --from ${VALIDATOR_NAME} -y +} + +main() { + wait_for_startup + add_keys + create_validator + echo "Done" +} + +main >> validator.log 2>&1 & \ No newline at end of file diff --git a/integration-tests/network/scripts/init-chain.sh b/integration-tests/network/scripts/init-chain.sh new file mode 100644 index 000000000..ccfc6a24e --- /dev/null +++ b/integration-tests/network/scripts/init-chain.sh @@ -0,0 +1,149 @@ +#!/bin/bash + +set -eu +source scripts/config.sh + +LOCAL_MODE=${1:-false} + +# If this is being run locally, don't overwrite the main chain folder +if [[ "$LOCAL_MODE" == "true" ]]; then + CHAIN_HOME=state + rm -rf state + BINARY="$BINARY --home $CHAIN_HOME" + API_ENDPOINT="http://localhost:8000" +fi + +# Wait for API server to start +wait_for_api $API_ENDPOINT + +# check if the binary has genesis subcommand or not, if not, set CHAIN_GENESIS_CMD to empty +genesis_json=${CHAIN_HOME}/config/genesis.json +chain_genesis_command=$($BINARY 2>&1 | grep -q "genesis-related subcommands" && echo "genesis" || echo "") + +# Helper to update a json attribute in-place +jq_inplace() { + jq_filter="$1" + file="$2" + + jq "$jq_filter" "$file" > /tmp/$(basename $file) && mv /tmp/$(basename $file) ${file} +} + +# Initializes the chain config +init_config() { + moniker=${CHAIN_NAME}1 + $BINARY init $moniker --chain-id $CHAIN_ID --overwrite + $BINARY config keyring-backend test +} + +# Helper to upload shared files to the API +upload_shared_file() { + file_path="$1" + saved_path="${2:-}" + file_name=$(basename $file_path) + + curl -s -X 'POST' "${API_ENDPOINT}/upload/${saved_path}" \ + -H 'accept: application/json' \ + -H 'Content-Type: multipart/form-data' \ + -F "file=@$file_path" && echo +} + +# Adds each validator to the genesis file, and also saves down the public keys +# which are needed for ICS +# Each validators public private key and node ID are saved in the API +add_validators() { + echo "Adding validators..." + + validator_public_keys="" + for (( i=1; i <= $NUM_VALIDATORS; i++ )); do + # Extract the validator name and mnemonic from keys.json + validator_config=$(jq -r '.validators[$index]' --argjson index "$((i-1))" ${KEYS_FILE}) + name=$(echo $validator_config | jq -r '.name') + mnemonic=$(echo $validator_config | jq -r '.mnemonic') + + # Add the key to the main keyring the the validator's sub-keyring + echo "$mnemonic" | $BINARY keys add $name --recover + address=$($BINARY keys show $name -a) + + # Use a separate directory for the non-main nodes so we can generate unique validator keys + if [[ "$i" == "1" ]]; then + validator_home=${CHAIN_HOME} + else + validator_home=/tmp/${CHAIN_NAME}-${name} && rm -rf $validator_home + $BINARY init $name --chain-id $CHAIN_ID --overwrite --home ${validator_home} &> /dev/null + fi + + # Add the genesis account + genesis_balance=${VALIDATOR_BALANCE}${DENOM} + $BINARY $chain_genesis_command add-genesis-account $address $genesis_balance + + # Save the node IDs and keys to the API + $BINARY tendermint show-node-id --home ${validator_home} > node_id.txt + upload_shared_file node_id.txt ${NODE_IDS_DIR}/${CHAIN_NAME}/${name}.txt + upload_shared_file ${validator_home}/config/priv_validator_key.json ${VALIDATOR_KEYS_DIR}/${CHAIN_NAME}/${name}.json + upload_shared_file ${validator_home}/config/node_key.json ${NODE_KEYS_DIR}/${CHAIN_NAME}/${name}.json + + # Save the comma separted public keys for the ICS genesis update + validator_public_keys+="$(jq -r '.pub_key.value' ${validator_home}/config/priv_validator_key.json)," + done + + # For non-stride nodes, generate and collect the validator gentx (for the main node only) + # The other validators will be created after startup + if [[ "$CHAIN_NAME" != "stride" ]]; then + $BINARY $chain_genesis_command gentx val1 ${VALIDATOR_STAKE}${DENOM} --chain-id $CHAIN_ID + $BINARY $chain_genesis_command collect-gentxs + fi +} + +# Updates the genesis config with defaults +update_default_genesis() { + echo "Updating genesis.json with defaults..." + + sed -i -E "s|\"stake\"|\"${DENOM}\"|g" $genesis_json + sed -i -E "s|\"aphoton\"|\"${DENOM}\"|g" $genesis_json # ethermint default + + jq_inplace '.app_state.staking.params.unbonding_time |= "'$UNBONDING_TIME'"' $genesis_json + jq_inplace '.app_state.gov.params.max_deposit_period |= "'$DEPOSIT_PERIOD'"' $genesis_json + jq_inplace '.app_state.gov.params.voting_period |= "'$VOTING_PERIOD'"' $genesis_json + + if jq 'has(.app_state.gov.params.expedited_voting_period)' $genesis_json > /dev/null 2>&1; then + jq_inplace '.app_state.gov.params.expedited_voting_period |= "'$EXPEDITED_VOTING_PERIOD'"' $genesis_json + fi +} + +# Genesis updates specific to stride +update_stride_genesis() { + echo "Updating genesis.json with stride configuration..." + + jq_inplace '(.app_state.epochs.epochs[] | select(.identifier=="day") ).duration |= "'$STRIDE_DAY_EPOCH_DURATION'"' $genesis_json + jq_inplace '(.app_state.epochs.epochs[] | select(.identifier=="stride_epoch") ).duration |= "'$STRIDE_EPOCH_EPOCH_DURATION'"' $genesis_json + + $BINARY add-consumer-section --validator-public-keys $validator_public_keys +} + +# Genesis updates specific to non-stride chains +update_host_genesis() { + echo "Updating genesis.json with host configuration..." +} + +# Saves the genesis file in the API +save_genesis() { + echo "Saving genesis.json..." + + upload_shared_file $genesis_json ${GENESIS_DIR}/${CHAIN_NAME}/genesis.json +} + +main() { + echo "Initializing chain..." + init_config + add_validators + update_default_genesis + if [[ "$CHAIN_NAME" == "stride" ]]; then + update_stride_genesis + else + update_host_genesis + fi + save_genesis + echo "Done" +} + +main \ No newline at end of file diff --git a/integration-tests/network/scripts/init-node.sh b/integration-tests/network/scripts/init-node.sh new file mode 100644 index 000000000..ba0dd1a9f --- /dev/null +++ b/integration-tests/network/scripts/init-node.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +set -eu +source scripts/config.sh + +# Wait for API server to start +wait_for_api $API_ENDPOINT + +# Initialize the config directory and validator key if it's not the main node +init_config() { + if [[ "$VALIDATOR_INDEX" != "1" ]]; then + moniker=${CHAIN_NAME}${VALIDATOR_INDEX} + $BINARY init $moniker --chain-id $CHAIN_ID --overwrite + $BINARY config keyring-backend test + fi +} + +# Helper function to download a file from the API +download_shared_file() { + stored_path="$1" + destination_path="$2" + + status_code=$(curl -s -o $destination_path -w "%{http_code}" "${API_ENDPOINT}/download/${stored_path}") + if [[ "$status_code" != "200" ]]; then + echo "ERROR - Failed to download $stored_path, status code ${status_code}" + exit 1 + fi +} + +# Update config.toml, app.toml, and client.toml +update_config() { + config_toml="${CHAIN_HOME}/config/config.toml" + client_toml="${CHAIN_HOME}/config/client.toml" + app_toml="${CHAIN_HOME}/config/app.toml" + + echo "Updating config.toml..." + sed -i -E "s|cors_allowed_origins = \[\]|cors_allowed_origins = [\"\*\"]|g" $config_toml + sed -i -E "s|127.0.0.1|0.0.0.0|g" $config_toml + sed -i -E "s|timeout_commit = \"5s\"|timeout_commit = \"${BLOCK_TIME}\"|g" $config_toml + sed -i -E "s|prometheus = false|prometheus = true|g" $config_toml + + echo "Updating app.toml..." + sed -i -E "s|minimum-gas-prices = \".*\"|minimum-gas-prices = \"0${DENOM}\"|g" $app_toml + sed -i -E '/\[api\]/,/^enable = .*$/ s/^enable = .*$/enable = true/' $app_toml + sed -i -E 's|unsafe-cors = .*|unsafe-cors = true|g' $app_toml + sed -i -E 's|localhost|0.0.0.0|g' $app_toml + + echo "Updating client.toml" + sed -i -E "s|chain-id = \"\"|chain-id = \"${CHAIN_ID}\"|g" $client_toml + sed -i -E "s|keyring-backend = \"os\"|keyring-backend = \"test\"|g" $client_toml + sed -i -E "s|node = \".*\"|node = \"tcp://localhost:${RPC_PORT}\"|g" $client_toml + + echo "Retrieving private keys and genesis.json..." + download_shared_file ${VALIDATOR_KEYS_DIR}/${CHAIN_NAME}/val${VALIDATOR_INDEX}.json ${CHAIN_HOME}/config/priv_validator_key.json + download_shared_file ${NODE_KEYS_DIR}/${CHAIN_NAME}/val${VALIDATOR_INDEX}.json ${CHAIN_HOME}/config/node_key.json + download_shared_file ${GENESIS_DIR}/${CHAIN_NAME}/genesis.json ${CHAIN_HOME}/config/genesis.json +} + +# Update the persistent peers conditionally based on which node it is +add_peers() { + echo "Setting peers..." + if [[ "$VALIDATOR_INDEX" == "1" ]]; then + # For the main node, wipe out the persistent peers that are incorrectly generated + sed -i -E "s|^persistent_peers = .*|persistent_peers = \"\"|g" $config_toml + else + # For the other nodes, add the main node as the persistent peer + download_shared_file ${NODE_IDS_DIR}/${CHAIN_NAME}/val1.txt main_node_id.txt + main_node_id=$(cat main_node_id.txt) + main_pod_id=${CHAIN_NAME}-validator-0 + service=${CHAIN_NAME}-validator + persistent_peer=${main_node_id}@${main_pod_id}.${service}.${NAMESPACE}.svc.cluster.local:${PEER_PORT} + sed -i -E "s|^persistent_peers = .*|persistent_peers = \"${persistent_peer}\"|g" $config_toml + fi +} + +main() { + echo "Initializing node..." + init_config + update_config + add_peers + echo "Done" +} + +main diff --git a/integration-tests/network/scripts/readiness.sh b/integration-tests/network/scripts/readiness.sh new file mode 100644 index 000000000..74684d160 --- /dev/null +++ b/integration-tests/network/scripts/readiness.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -e +source scripts/config.sh + +# If chain hasn't been initialized yet, exit immediately +if [ ! -d $CHAIN_HOME/config ]; then + echo "READINESS CHECK FAILED - Chain has not been initialized yet." + exit 1 +fi + +# Check that the node is running +if ! $($BINARY status &> /dev/null); then + echo "READINESS CHECK FAILED - Chain is down" + exit 1 +fi + +# It's not possible for one node to start up by itself (without peers), +# so if we identify that the node is on block 0, we'll mark it as ready +# so the other nodes can start connecting +if [[ "$($BINARY status | jq -r '.SyncInfo.latest_block_height')" == "0" ]]; then + exit 0 +fi + +# Then check if the node is synced according to it's status query +CATCHING_UP=$($BINARY status 2>&1 | jq ".SyncInfo.catching_up") +if [[ "$CATCHING_UP" != "false" ]]; then + echo "READINESS CHECK FAILED - Node is still syncing" + exit 1 +fi \ No newline at end of file diff --git a/integration-tests/network/templates/api.yaml b/integration-tests/network/templates/api.yaml new file mode 100644 index 000000000..2d29bb005 --- /dev/null +++ b/integration-tests/network/templates/api.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: api + namespace: {{ .Values.namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app: api + template: + metadata: + labels: + app: api + spec: + containers: + - name: api + image: gcr.io/stride-nodes/integration-tests/api:latest + imagePullPolicy: Always + ports: + - containerPort: 8000 + protocol: TCP + resources: + limits: + cpu: "400m" + memory: "8096M" + requests: + cpu: "200m" + memory: "4096M" +--- +apiVersion: v1 +kind: Service +metadata: + name: api + namespace: integration +spec: + selector: + app: api + ports: + - port: 8000 + protocol: TCP diff --git a/integration-tests/network/templates/scripts.yaml b/integration-tests/network/templates/scripts.yaml new file mode 100644 index 000000000..268962f33 --- /dev/null +++ b/integration-tests/network/templates/scripts.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: scripts + namespace: {{ .Values.namespace }} +data: +{{ (.Files.Glob "scripts/*").AsConfig | indent 2 }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: configs + namespace: {{ .Values.namespace }} +data: +{{ (.Files.Glob "configs/*").AsConfig | indent 2 }} \ No newline at end of file diff --git a/integration-tests/network/templates/validator.yaml b/integration-tests/network/templates/validator.yaml new file mode 100644 index 000000000..a278208ed --- /dev/null +++ b/integration-tests/network/templates/validator.yaml @@ -0,0 +1,177 @@ +{{- define "chain.env" -}} +- name: CHAIN_NAME + value: {{ .name }} +- name: CHAIN_HOME + value: {{ printf "/home/validator/%s" .home }} +- name: BINARY + value: {{ .binary }} +- name: DENOM + value: {{ .denom }} +- name: DENOM_DECIMALS + value: "{{ .decimals }}" +- name: NUM_VALIDATORS + value: "{{ .numValidators }}" +- name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +{{- end -}} + +{{- $namespace := .Values.namespace -}} +{{- $imageRepo := .Values.chainImagesRepo -}} + +{{- range $chain := .Values.chains }} +{{- $appName := printf "%s-validator" $chain.name -}} +{{- $chainHomeDirectory := printf "/home/validator/%s" $chain.home -}} +{{- $image := printf "%s/%s:%s" $imageRepo $chain.name $chain.version -}} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ $appName }} + namespace: {{ $namespace }} +spec: + replicas: {{ $chain.numValidators }} + selector: + matchLabels: + app: {{ $appName }} + serviceName: {{ $appName }} + template: + metadata: + labels: + app: {{ $appName }} + spec: + securityContext: + fsGroup: 1000 + initContainers: + - name: init + image: {{ $image }} + imagePullPolicy: Always + command: + - bash + - "-c" + - | + POD_INDEX=${HOSTNAME##*-} + if [[ "$POD_INDEX" == "0" ]]; then + bash scripts/init-chain.sh + fi + bash scripts/init-node.sh + env: + {{- include "chain.env" $chain | nindent 10 }} + volumeMounts: + - name: state + mountPath: {{ $chainHomeDirectory }} + - name: scripts + mountPath: /home/validator/scripts + - name: configs + mountPath: /home/validator/configs + containers: + - name: validator + image: {{ $image }} + imagePullPolicy: Always + command: {{ toYaml $chain.command | nindent 10 }} + lifecycle: + postStart: + exec: + command: ["bash", "scripts/create-validator.sh"] + readinessProbe: + exec: + command: ["bash", "scripts/readiness.sh"] + periodSeconds: 10 + env: + {{- include "chain.env" $chain | nindent 10 }} + ports: + - name: rpc + containerPort: 26657 + protocol: TCP + - name: peer + containerPort: 26656 + protocol: TCP + - name: api + containerPort: 1317 + protocol: TCP + - name: grpc + containerPort: 9090 + protocol: TCP + volumeMounts: + - name: state + mountPath: {{ $chainHomeDirectory }} + - name: scripts + mountPath: /home/validator/scripts + - name: configs + mountPath: /home/validator/configs + resources: + limits: + cpu: "1000m" + memory: "2Gi" + requests: + cpu: "1000m" + memory: "2Gi" + volumes: + - name: state + emptyDir: {} + - name: scripts + configMap: + name: scripts + - name: configs + configMap: + name: configs +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $appName }} + namespace: {{ $namespace }} +spec: + clusterIP: None + selector: + app: {{ $appName }} + ports: + - port: 26657 + name: rpc + protocol: TCP + - port: 26656 + name: peer + protocol: TCP + - port: 1317 + name: api + protocol: TCP + - port: 9090 + name: grpc + protocol: TCP +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ $appName }} + namespace: {{ $namespace }} + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/proxy-connect-timeout: "360" + nginx.ingress.kubernetes.io/proxy-send-timeout: "360" + nginx.ingress.kubernetes.io/proxy-read-timeout: "360" +spec: + ingressClassName: nginx + rules: + - host: {{ $chain.name }}-api.internal.stridenet.co + http: + paths: + - pathType: Prefix + backend: + service: + name: {{ $appName }} + port: + number: 1317 + path: /(.*) + - host: {{ $chain.name }}-rpc.internal.stridenet.co + http: + paths: + - pathType: Prefix + backend: + service: + name: {{ $appName }} + port: + number: 26657 + path: /(.*) +--- +{{- end }} \ No newline at end of file diff --git a/integration-tests/network/values.yaml b/integration-tests/network/values.yaml new file mode 100644 index 000000000..8bf230e8a --- /dev/null +++ b/integration-tests/network/values.yaml @@ -0,0 +1,21 @@ +namespace: integration +chainImagesRepo: gcr.io/stride-nodes/integration-tests/chains + +chains: + - name: stride + binary: strided + version: latest + numValidators: 3 + home: .stride + denom: ustrd + decimals: 6 + command: ["strided", "start", "--reject-config-defaults"] + + - name: cosmoshub + binary: gaiad + version: v18.1.0 + numValidators: 3 + home: .gaia + denom: uatom + decimals: 6 + command: ["gaiad", "start"] \ No newline at end of file diff --git a/scripts/start_local_node.sh b/scripts/start_local_node.sh index fa9ae2a12..827bb9a0d 100644 --- a/scripts/start_local_node.sh +++ b/scripts/start_local_node.sh @@ -49,11 +49,7 @@ jq "del(.app_state.interchain_accounts)" $genesis_json > json.tmp && mv json.tmp interchain_accts=$(cat dockernet/config/ica_controller.json) jq ".app_state += $interchain_accts" $genesis_json > json.tmp && mv json.tmp $genesis_json -# hack since add-comsumer-section is built for dockernet -rm -rf ~/.stride-loca1 -cp -r ${STRIDE_HOME} ~/.stride-loca1 - -$STRIDED add-consumer-section 1 +$STRIDED add-consumer-section --validator-home-directories $STRIDE_HOME jq '.app_state.ccvconsumer.params.unbonding_period = $newVal' --arg newVal "$UNBONDING_TIME" $genesis_json > json.tmp && mv json.tmp $genesis_json rm -rf ~/.stride-loca1