integration test initial network setup (#1256)

.dockerignore

@@ -5,4 +5,5 @@ deps
 dockernet
 scripts
 genesis
-testutil/localstride
+testutil/localstride
+integration-tests

.gitignore

@@ -40,5 +40,8 @@ vue/*
 !.vscode/settings.json
 
 .ipynb_checkpoints/*
+__pycache__
+node_modules
 
-node_modules
+integration-tests/state
+integration-tests/storage

Dockerfile

@@ -27,7 +27,7 @@ RUN BUILD_TAGS=muslc LINK_STATICALLY=true make build
 FROM alpine:${RUNNER_IMAGE_VERSION}
 
 COPY --from=builder /opt/build/strided /usr/local/bin/strided
-RUN apk add bash vim sudo dasel jq \
+RUN apk add bash vim sudo dasel jq curl \
     && addgroup -g 1000 stride \
     && adduser -S -h /home/stride -D stride -u 1000 -G stride 
 

cmd/consumer.go

@@ -1,13 +1,17 @@
 package cmd
 
 import (
+	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
-	"strconv"
+	"strings"
 
 	errorsmod "cosmossdk.io/errors"
 	types1 "github.com/cometbft/cometbft/abci/types"
+	"github.com/cometbft/cometbft/config"
 	pvm "github.com/cometbft/cometbft/privval"
+	tmprotocrypto "github.com/cometbft/cometbft/proto/tendermint/crypto"
 	tmtypes "github.com/cometbft/cometbft/types"
 	"github.com/cosmos/cosmos-sdk/client"
 	"github.com/cosmos/cosmos-sdk/client/flags"
@@ -22,48 +26,110 @@ import (
 	"github.com/Stride-Labs/stride/v23/testutil"
 )
 
+const (
+	FlagValidatorPublicKeys      = "validator-public-keys"
+	FlagValidatorHomeDirectories = "validator-home-directories"
+)
+
+// Builds the list of validator Ed25519 pubkeys from a comma separate list of base64 encoded pubkeys
+func buildPublicKeysFromString(publicKeysRaw string) (publicKeys []tmprotocrypto.PublicKey, err error) {
+	for _, publicKeyEncoded := range strings.Split(publicKeysRaw, ",") {
+		if publicKeyEncoded == "" {
+			continue
+		}
+		publicKeyBytes, err := base64.StdEncoding.DecodeString(publicKeyEncoded)
+		if err != nil {
+			return nil, errorsmod.Wrapf(err, "unable to decode public key")
+		}
+		publicKeys = append(publicKeys, tmprotocrypto.PublicKey{
+			Sum: &tmprotocrypto.PublicKey_Ed25519{
+				Ed25519: publicKeyBytes,
+			},
+		})
+	}
+
+	return publicKeys, nil
+}
+
+// Builds the list validator Ed25519 pubkeys from a comma separated list of validator home directories
+func buildPublicKeysFromHomeDirectories(config *config.Config, homeDirectories string) (publicKeys []tmprotocrypto.PublicKey, err error) {
+	for _, homeDir := range strings.Split(homeDirectories, ",") {
+		if homeDir == "" {
+			continue
+		}
+		config.SetRoot(homeDir)
+
+		privValidator := pvm.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile())
+		pk, err := privValidator.GetPubKey()
+		if err != nil {
+			return nil, err
+		}
+		sdkPublicKey, err := cryptocodec.FromTmPubKeyInterface(pk)
+		if err != nil {
+			return nil, err
+		}
+		tmProtoPublicKey, err := cryptocodec.ToTmProtoPublicKey(sdkPublicKey)
+		if err != nil {
+			return nil, err
+		}
+		publicKeys = append(publicKeys, tmProtoPublicKey)
+	}
+
+	return publicKeys, nil
+}
+
 func AddConsumerSectionCmd(defaultNodeHome string) *cobra.Command {
 	genesisMutator := NewDefaultGenesisIO()
 
-	txCmd := &cobra.Command{
-		Use:                        "add-consumer-section [num_nodes]",
-		Args:                       cobra.ExactArgs(1),
+	cmd := &cobra.Command{
+		Use:                        "add-consumer-section",
+		Args:                       cobra.ExactArgs(0),
 		Short:                      "ONLY FOR TESTING PURPOSES! Modifies genesis so that chain can be started locally with one node.",
 		SuggestionsMinimumDistance: 2,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			numNodes, err := strconv.Atoi(args[0])
+			// We need to public keys for each validator - they can either be provided explicitly
+			// or indirectly by providing the validator home directories
+			publicKeysRaw, err := cmd.Flags().GetString(FlagValidatorPublicKeys)
 			if err != nil {
-				return errorsmod.Wrap(err, "invalid number of nodes")
-			} else if numNodes == 0 {
-				return errorsmod.Wrap(nil, "num_nodes can not be zero")
+				return errorsmod.Wrapf(err, "unable to parse validator public key flag")
+			}
+			homeDirectoriesRaw, err := cmd.Flags().GetString(FlagValidatorHomeDirectories)
+			if err != nil {
+				return errorsmod.Wrapf(err, "unable to parse validator home directories flag")
+			}
+			if (publicKeysRaw == "" && homeDirectoriesRaw == "") || (publicKeysRaw != "" && homeDirectoriesRaw != "") {
+				return fmt.Errorf("must specified either --%s or --%s", FlagValidatorPublicKeys, FlagValidatorHomeDirectories)
+			}
+
+			// Build up a list of the validator public keys
+			// If the public keys were passed directly, decode them and create the tm proto pub keys
+			// Otherwise, derrive them from the private keys in each validator's home directory
+			var tmPublicKeys []tmprotocrypto.PublicKey
+			if publicKeysRaw != "" {
+				tmPublicKeys, err = buildPublicKeysFromString(publicKeysRaw)
+				if err != nil {
+					return err
+				}
+			} else {
+				serverCtx := server.GetServerContextFromCmd(cmd)
+				config := serverCtx.Config
+
+				tmPublicKeys, err = buildPublicKeysFromHomeDirectories(config, homeDirectoriesRaw)
+				if err != nil {
+					return err
+				}
+			}
+
+			if len(tmPublicKeys) == 0 {
+				return errors.New("no valid public keys or validator home directories provided")
 			}
 
 			return genesisMutator.AlterConsumerModuleState(cmd, func(state *GenesisData, _ map[string]json.RawMessage) error {
 				initialValset := []types1.ValidatorUpdate{}
 				genesisState := testutil.CreateMinimalConsumerTestGenesis()
-				clientCtx := client.GetClientContextFromCmd(cmd)
-				serverCtx := server.GetServerContextFromCmd(cmd)
-				config := serverCtx.Config
-				homeDir := clientCtx.HomeDir
-				for i := 1; i <= numNodes; i++ {
-					homeDir = fmt.Sprintf("%s%d", homeDir[:len(homeDir)-1], i)
-					config.SetRoot(homeDir)
-
-					privValidator := pvm.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile())
-					pk, err := privValidator.GetPubKey()
-					if err != nil {
-						return err
-					}
-					sdkPublicKey, err := cryptocodec.FromTmPubKeyInterface(pk)
-					if err != nil {
-						return err
-					}
-					tmProtoPublicKey, err := cryptocodec.ToTmProtoPublicKey(sdkPublicKey)
-					if err != nil {
-						return err
-					}
-
-					initialValset = append(initialValset, types1.ValidatorUpdate{PubKey: tmProtoPublicKey, Power: 100})
+
+				for _, publicKey := range tmPublicKeys {
+					initialValset = append(initialValset, types1.ValidatorUpdate{PubKey: publicKey, Power: 100})
 				}
 
 				vals, err := tmtypes.PB2TM.ValidatorUpdates(initialValset)
@@ -80,10 +146,11 @@ func AddConsumerSectionCmd(defaultNodeHome string) *cobra.Command {
 		},
 	}
 
-	txCmd.Flags().String(flags.FlagHome, defaultNodeHome, "The application home directory")
-	flags.AddQueryFlagsToCmd(txCmd)
+	cmd.Flags().String(flags.FlagHome, defaultNodeHome, "The application home directory")
+	cmd.Flags().String(FlagValidatorPublicKeys, "", "Comma separated, base64-encoded public keys for each validator")
+	cmd.Flags().String(FlagValidatorHomeDirectories, "", "Comma separated list of home directories for each validator")
 
-	return txCmd
+	return cmd
 }
 
 type GenesisMutator interface {

dockernet/src/init_chain.sh

@@ -139,7 +139,12 @@ set_consumer_genesis() {
     genesis_config=$1
 
     # add consumer genesis
-    $MAIN_CMD add-consumer-section $NUM_NODES
+    home_directories=""
+    for (( i=1; i <= $NUM_NODES; i++ )); do
+        home_directories+="${STATE}/stride${i},"
+    done
+
+    $MAIN_CMD add-consumer-section --validator-home-directories $home_directories
     jq '.app_state.ccvconsumer.params.unbonding_period = $newVal' --arg newVal "$UNBONDING_TIME" $genesis_config > json.tmp && mv json.tmp $genesis_config
 }
 

integration-tests/Makefile

@@ -0,0 +1,49 @@
+K8S_NAMESPACE=integration
+VENV_NAME=integration
+
+CONDA_BASE := $(shell conda info --base)/envs
+KUBECTL := $(shell which kubectl)
+DOCKER := $(shell which docker)
+HELM := $(shell which helm)
+VENV_BIN := $(CONDA_BASE)/$(VENV_NAME)/bin
+PYTHON := $(VENV_BIN)/python
+
+HELM_CHART=network
+
+python-install:
+	conda create --name $(VENV_NAME) python=3.11 -y
+	$(PYTHON) -m pip install -r api/requirements.txt
+
+start-api:
+	@$(PYTHON) -m uvicorn api.main:app --proxy-headers 
+
+build-api:
+	@echo "Building docker image: api"
+	@$(DOCKER) buildx build --platform linux/amd64 --tag api -f dockerfiles/Dockerfile.api api
+	@$(DOCKER) tag api gcr.io/stride-nodes/integration-tests/api:latest
+	@echo "Pushing image to GCR"
+	@$(DOCKER) push gcr.io/stride-nodes/integration-tests/api:latest
+
+build-stride:
+	@echo "Building docker image: stride-validator"
+	@$(DOCKER) buildx build --platform linux/amd64 --tag core:stride ..
+	@$(DOCKER) buildx build --platform linux/amd64 --tag stride-validator -f dockerfiles/Dockerfile.stride .
+	@$(DOCKER) tag stride-validator gcr.io/stride-nodes/integration-tests/chains/stride:latest
+	@echo "Pushing image to GCR"
+	@$(DOCKER) push gcr.io/stride-nodes/integration-tests/chains/stride:latest
+
+build-cosmos:
+	@echo "Building docker image"
+	@$(DOCKER) buildx build --platform linux/amd64 --tag cosmos-validator -f dockerfiles/Dockerfile.cosmos .
+	@$(DOCKER) tag cosmos-validator gcr.io/stride-nodes/integration-tests/chains/cosmoshub:v18.1.0
+	@echo "Pushing image to GCR"
+	@$(DOCKER) push gcr.io/stride-nodes/integration-tests/chains/cosmoshub:v18.1.0
+
+start:
+	@$(HELM) install $(HELM_CHART) $(HELM_CHART) --values $(HELM_CHART)/values.yaml -n $(K8S_NAMESPACE)
+
+stop:
+	@$(HELM) uninstall $(HELM_CHART) -n $(K8S_NAMESPACE)
+
+lint: 
+	@$(HELM) lint $(HELM_CHART) 

integration-tests/README.md

@@ -0,0 +1,29 @@
+# Integration Tests
+
+This design for this integration test framework is heavily inspired by the Cosmology team's [starship](https://github.com/cosmology-tech/starship/tree/main).
+
+## Setup
+
+TODO
+
+## Motivation
+
+TODO
+
+## Network
+
+TODO
+
+## Testing Client
+
+TODO
+
+## Design Decisions
+
+### API Service to share files during chain setup
+
+In order to start the network as fast as possible, the chain should be initialized with ICS validators at genesis, rather than performing a switchover. However, in order to build the genesis file, the public keys must be gathered from each validator. This adds the constraint that keys must be consoldiated into a single process responsible for creating the genesis file.
+
+This can be achieved by having a master node creating the genesis.json and keys for each validator, and then having each validator download the files from the master node. Ideally this would be handled by a shared PVC across each validator; however, Kuberentes has a constraint where you cannot mount multiple pods onto the same volume.
+
+This led to the decision to use an API service to act as the intermediary that allows uploading and downloading of files. While at first glance, this smells of overengineering, the fastAPI implementation is actually quite simple (only marginally more code than creating and mounting a volume) and it improves the startup time dramatically since there's no need for the pods to wait for the volume to be mounted. Plus, it's likely that it can be leveraged in the future to help coordinate tasks across the different networks in the setup (e.g. it can store a registry of canonical IBC connections across chains).

integration-tests/api/main.py

@@ -0,0 +1,45 @@
+from fastapi import FastAPI, File, HTTPException, UploadFile
+from fastapi.responses import FileResponse
+import os
+
+app = FastAPI()
+
+STORAGE_DIRECTORY = "storage"
+os.makedirs(STORAGE_DIRECTORY, exist_ok=True)
+
+
+@app.get("/status")
+async def status() -> str:
+    """
+    Health check
+    """
+    return "ok"
+
+
+@app.post("/upload/{file_name}")
+@app.post("/upload/{path:path}/{file_name}")
+async def upload_file(file_name: str, path: str = "", file: UploadFile = File(...)) -> dict:
+    """
+    Allows uploading a file - stores it in the local file system
+    """
+    parent = f"{STORAGE_DIRECTORY}/{path}" if path else STORAGE_DIRECTORY
+    os.makedirs(parent, exist_ok=True)
+
+    with open(f"{parent}/{file_name}", "wb") as f:
+        f.write(file.file.read())
+
+    return {"info": f"file {file_name} saved"}
+
+
+@app.get("/download/{file_name}")
+@app.get("/download/{path:path}/{file_name}")
+async def download_file(file_name: str, path: str = ""):
+    """
+    Allows downloading a file from the local file system
+    """
+    file_location = f"{STORAGE_DIRECTORY}/{path}/{file_name}" if path else f"{STORAGE_DIRECTORY}/{file_name}"
+
+    if not os.path.exists(file_location):
+        return HTTPException(status_code=404, detail=f"file {file_location} not found")
+
+    return FileResponse(file_location, media_type="application/octet-stream", filename=file_name)

integration-tests/api/requirements.txt

@@ -0,0 +1,3 @@
+fastapi==0.103.2
+uvicorn==0.23.2
+python-multipart==0.0.9

integration-tests/dockerfiles/Dockerfile.api

@@ -0,0 +1,15 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y gcc g++ bash vim git curl tzdata \
+    && adduser --system --home /home/python --disabled-password --disabled-login python -u 1000 \
+    && pip install --upgrade pip
+
+COPY ./requirements.txt /app/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt
+
+COPY main.py main.py
+
+CMD ["uvicorn", "main:app", "--proxy-headers", "--host", "0.0.0.0"]

integration-tests/dockerfiles/Dockerfile.cosmos

@@ -0,0 +1,32 @@
+FROM golang:1.22-alpine AS builder
+
+WORKDIR /opt
+
+RUN apk add --update curl make git libc-dev bash gcc linux-headers eudev-dev ca-certificates build-base git 
+
+ENV REPO=https://github.com/cosmos/gaia
+ENV COMMIT_HASH=v18.1.0
+ENV BINARY=gaiad
+
+RUN git clone ${REPO} chain \
+    && cd chain \
+    && git checkout $COMMIT_HASH 
+
+WORKDIR /opt/chain
+
+RUN WASMVM_VERSION=$(cat go.mod | grep github.com/CosmWasm/wasmvm | awk '{print $2}') \
+    && wget https://github.com/CosmWasm/wasmvm/releases/download/$WASMVM_VERSION/libwasmvm_muslc.$(uname -m).a \
+    -O /lib/libwasmvm_muslc.a
+
+RUN CGO_ENABLED=1 BUILD_TAGS="muslc linkstatic" LINK_STATICALLY=true LEDGER_ENABLED=false make install
+
+FROM alpine:3.17
+COPY --from=builder /go/bin/$BINARY /usr/local/bin/
+RUN apk add bash vim sudo dasel jq curl \
+    && addgroup -g 1000 validator \
+    && adduser -S -h /home/validator -D validator -u 1000 -G validator
+
+USER 1000
+WORKDIR /home/validator
+
+EXPOSE 26657 26656 1317 9090