integration test initial network setup

.dockerignore

@@ -5,4 +5,5 @@ deps
 dockernet
 scripts
 genesis
-testutil/localstride
+testutil/localstride
+integration-tests

.gitignore

@@ -40,5 +40,8 @@ vue/*
 !.vscode/settings.json
 
 .ipynb_checkpoints/*
+__pycache__
+node_modules
 
-node_modules
+integration-tests/state
+integration-tests/storage

Dockerfile

@@ -27,7 +27,7 @@ RUN BUILD_TAGS=muslc LINK_STATICALLY=true make build
 FROM alpine:${RUNNER_IMAGE_VERSION}
 
 COPY --from=builder /opt/build/strided /usr/local/bin/strided
-RUN apk add bash vim sudo dasel jq \
+RUN apk add bash vim sudo dasel jq curl \
     && addgroup -g 1000 stride \
     && adduser -S -h /home/stride -D stride -u 1000 -G stride 
 

cmd/consumer.go

@@ -1,13 +1,17 @@
 package cmd
 
 import (
+	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
-	"strconv"
+	"strings"
 
 	errorsmod "cosmossdk.io/errors"
 	types1 "github.com/cometbft/cometbft/abci/types"
+	"github.com/cometbft/cometbft/config"
 	pvm "github.com/cometbft/cometbft/privval"
+	tmprotocrypto "github.com/cometbft/cometbft/proto/tendermint/crypto"
 	tmtypes "github.com/cometbft/cometbft/types"
 	"github.com/cosmos/cosmos-sdk/client"
 	"github.com/cosmos/cosmos-sdk/client/flags"
@@ -22,48 +26,110 @@ import (
 	"github.com/Stride-Labs/stride/v23/testutil"
 )
 
+const (
+	FlagValidatorPublicKeys      = "validator-public-keys"
+	FlagValidatorHomeDirectories = "validator-home-directories"
+)
+
+// Builds the list of validator Ed25519 pubkeys from a comma separate list of base64 encoded pubkeys
+func buildPublicKeysFromString(publicKeysRaw string) (publicKeys []tmprotocrypto.PublicKey, err error) {
+	for _, publicKeyEncoded := range strings.Split(publicKeysRaw, ",") {
+		if publicKeyEncoded == "" {
+			continue
+		}
+		publicKeyBytes, err := base64.StdEncoding.DecodeString(publicKeyEncoded)
+		if err != nil {
+			return nil, errorsmod.Wrapf(err, "unable to decode public key")
+		}
+		publicKeys = append(publicKeys, tmprotocrypto.PublicKey{
+			Sum: &tmprotocrypto.PublicKey_Ed25519{
+				Ed25519: publicKeyBytes,
+			},
+		})
+	}
+
+	return publicKeys, nil
+}
+
+// Builds the list validator Ed25519 pubkeys from a comma separated list of validator home directories
+func buildPublicKeysFromHomeDirectories(config *config.Config, homeDirectories string) (publicKeys []tmprotocrypto.PublicKey, err error) {
+	for _, homeDir := range strings.Split(homeDirectories, ",") {
+		if homeDir == "" {
+			continue
+		}
+		config.SetRoot(homeDir)
+
+		privValidator := pvm.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile())
+		pk, err := privValidator.GetPubKey()
+		if err != nil {
+			return nil, err
+		}
+		sdkPublicKey, err := cryptocodec.FromTmPubKeyInterface(pk)
+		if err != nil {
+			return nil, err
+		}
+		tmProtoPublicKey, err := cryptocodec.ToTmProtoPublicKey(sdkPublicKey)
+		if err != nil {
+			return nil, err
+		}
+		publicKeys = append(publicKeys, tmProtoPublicKey)
+	}
+
+	return publicKeys, nil
+}
+
 func AddConsumerSectionCmd(defaultNodeHome string) *cobra.Command {
 	genesisMutator := NewDefaultGenesisIO()
 
-	txCmd := &cobra.Command{
-		Use:                        "add-consumer-section [num_nodes]",
-		Args:                       cobra.ExactArgs(1),
+	cmd := &cobra.Command{
+		Use:                        "add-consumer-section",
+		Args:                       cobra.ExactArgs(0),
 		Short:                      "ONLY FOR TESTING PURPOSES! Modifies genesis so that chain can be started locally with one node.",
 		SuggestionsMinimumDistance: 2,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			numNodes, err := strconv.Atoi(args[0])
+			// We need to public keys for each validator - they can either be provided explicitly
+			// or indirectly by providing the validator home directories
+			publicKeysRaw, err := cmd.Flags().GetString(FlagValidatorPublicKeys)
 			if err != nil {
-				return errorsmod.Wrap(err, "invalid number of nodes")
-			} else if numNodes == 0 {
-				return errorsmod.Wrap(nil, "num_nodes can not be zero")
+				return errorsmod.Wrapf(err, "unable to parse validator public key flag")
+			}
+			homeDirectoriesRaw, err := cmd.Flags().GetString(FlagValidatorHomeDirectories)
+			if err != nil {
+				return errorsmod.Wrapf(err, "unable to parse validator home directories flag")
+			}
+			if (publicKeysRaw == "" && homeDirectoriesRaw == "") || (publicKeysRaw != "" && homeDirectoriesRaw != "") {
+				return fmt.Errorf("must specified either --%s or --%s", FlagValidatorPublicKeys, FlagValidatorHomeDirectories)
+			}
+
+			// Build up a list of the validator public keys
+			// If the public keys were passed directly, decode them and create the tm proto pub keys
+			// Otherwise, derrive them from the private keys in each validator's home directory
+			var tmPublicKeys []tmprotocrypto.PublicKey
+			if publicKeysRaw != "" {
+				tmPublicKeys, err = buildPublicKeysFromString(publicKeysRaw)
+				if err != nil {
+					return err
+				}
+			} else {
+				serverCtx := server.GetServerContextFromCmd(cmd)
+				config := serverCtx.Config
+
+				tmPublicKeys, err = buildPublicKeysFromHomeDirectories(config, homeDirectoriesRaw)
+				if err != nil {
+					return err
+				}
+			}
+
+			if len(tmPublicKeys) == 0 {
+				return errors.New("no valid public keys or validator home directories provided")
 			}
 
 			return genesisMutator.AlterConsumerModuleState(cmd, func(state *GenesisData, _ map[string]json.RawMessage) error {
 				initialValset := []types1.ValidatorUpdate{}
 				genesisState := testutil.CreateMinimalConsumerTestGenesis()
-				clientCtx := client.GetClientContextFromCmd(cmd)
-				serverCtx := server.GetServerContextFromCmd(cmd)
-				config := serverCtx.Config
-				homeDir := clientCtx.HomeDir
-				for i := 1; i <= numNodes; i++ {
-					homeDir = fmt.Sprintf("%s%d", homeDir[:len(homeDir)-1], i)
-					config.SetRoot(homeDir)
-
-					privValidator := pvm.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile())
-					pk, err := privValidator.GetPubKey()
-					if err != nil {
-						return err
-					}
-					sdkPublicKey, err := cryptocodec.FromTmPubKeyInterface(pk)
-					if err != nil {
-						return err
-					}
-					tmProtoPublicKey, err := cryptocodec.ToTmProtoPublicKey(sdkPublicKey)
-					if err != nil {
-						return err
-					}
-
-					initialValset = append(initialValset, types1.ValidatorUpdate{PubKey: tmProtoPublicKey, Power: 100})
+
+				for _, publicKey := range tmPublicKeys {
+					initialValset = append(initialValset, types1.ValidatorUpdate{PubKey: publicKey, Power: 100})
 				}
 
 				vals, err := tmtypes.PB2TM.ValidatorUpdates(initialValset)
@@ -80,10 +146,11 @@ func AddConsumerSectionCmd(defaultNodeHome string) *cobra.Command {
 		},
 	}
 
-	txCmd.Flags().String(flags.FlagHome, defaultNodeHome, "The application home directory")
-	flags.AddQueryFlagsToCmd(txCmd)
+	cmd.Flags().String(flags.FlagHome, defaultNodeHome, "The application home directory")
+	cmd.Flags().String(FlagValidatorPublicKeys, "", "Comma separated, base64-encoded public keys for each validator")
+	cmd.Flags().String(FlagValidatorHomeDirectories, "", "Comma separated list of home directories for each validator")
 
-	return txCmd
+	return cmd
 }
 
 type GenesisMutator interface {

dockernet/src/init_chain.sh

@@ -139,7 +139,12 @@ set_consumer_genesis() {
     genesis_config=$1
 
     # add consumer genesis
-    $MAIN_CMD add-consumer-section $NUM_NODES
+    home_directories=""
+    for (( i=1; i <= $NUM_NODES; i++ )); do
+        home_directories+="${STATE}/stride${i},"
+    done
+
+    $MAIN_CMD add-consumer-section --validator-home-directories $home_directories
     jq '.app_state.ccvconsumer.params.unbonding_period = $newVal' --arg newVal "$UNBONDING_TIME" $genesis_config > json.tmp && mv json.tmp $genesis_config
 }
 

integration-tests/Makefile

@@ -0,0 +1,59 @@
+K8S_NAMESPACE=integration
+VENV_NAME=integration
+
+CONDA_BASE := $(shell conda info --base)/envs
+KUBECTL := $(shell which kubectl)
+DOCKER := $(shell which docker)
+HELM := $(shell which helm)
+VENV_BIN := $(CONDA_BASE)/$(VENV_NAME)/bin
+PYTHON := $(VENV_BIN)/python
+
+HELM_CHART=network
+
+python-install:
+	conda create --name $(VENV_NAME) python=3.11 -y
+	$(PYTHON) -m pip install -r api/requirements.txt
+
+start-api:
+	@$(PYTHON) -m uvicorn api.main:app --proxy-headers 
+
+build-api:
+	@echo "Building docker image: api"
+	@$(DOCKER) buildx build --platform linux/amd64 --tag api -f dockerfiles/Dockerfile.api api
+	@$(DOCKER) tag api gcr.io/stride-nodes/integration-tests/api:latest
+	@echo "Pushing image to GCR"
+	@$(DOCKER) push gcr.io/stride-nodes/integration-tests/api:latest
+
+build-stride:
+	@echo "Building docker image: stride-validator"
+	@$(DOCKER) buildx build --platform linux/amd64 --tag core:stride ..
+	@$(DOCKER) buildx build --platform linux/amd64 --tag stride-validator -f dockerfiles/Dockerfile.stride .
+	@$(DOCKER) tag stride-validator gcr.io/stride-nodes/integration-tests/chains/stride:latest
+	@echo "Pushing image to GCR"
+	@$(DOCKER) push gcr.io/stride-nodes/integration-tests/chains/stride:latest
+
+build-cosmos:
+	@echo "Building docker image"
+	@$(DOCKER) buildx build --platform linux/amd64 --tag cosmos-validator -f dockerfiles/Dockerfile.cosmos .
+	@$(DOCKER) tag cosmos-validator gcr.io/stride-nodes/integration-tests/chains/cosmoshub:v18.1.0
+	@echo "Pushing image to GCR"
+	@$(DOCKER) push gcr.io/stride-nodes/integration-tests/chains/cosmoshub:v18.1.0
+
+update-scripts:
+	@echo "Updating scripts configmap"
+	@$(KUBECTL) delete configmap scripts -n $(K8S_NAMESPACE) --ignore-not-found=true
+	@$(KUBECTL) create configmap scripts --from-file=scripts -n $(K8S_NAMESPACE)
+
+update-config:
+	@echo "Updating config configmap"
+	@$(KUBECTL) delete configmap config -n $(K8S_NAMESPACE) --ignore-not-found=true
+	@$(KUBECTL) create configmap config --from-file=config -n $(K8S_NAMESPACE)
+
+start:
+	@$(HELM) install $(HELM_CHART) $(HELM_CHART) --values $(HELM_CHART)/values.yaml -n $(K8S_NAMESPACE)
+
+stop:
+	@$(HELM) uninstall $(HELM_CHART) -n $(K8S_NAMESPACE)
+
+lint: 
+	@$(HELM) lint $(HELM_CHART)

integration-tests/README.md

@@ -0,0 +1,29 @@
+# Integration Tests
+
+This design for this integration test framework is heavily inspired by the Cosmology team's [starship](https://github.com/cosmology-tech/starship/tree/main).
+
+## Setup
+
+TODO
+
+## Motivation
+
+TODO
+
+## Network
+
+TODO
+
+## Testing Client
+
+TODO
+
+## Design Decisions
+
+### API Service to share files during chain setup
+
+In order to start the network as fast as possible, the chain should be initialized with ICS validators at genesis, rather than performing a switchover. However, in order to build the genesis file, the public keys must be gathered from each validator. This adds the constraint that keys must be consoldiated into a single process responsible for creating the genesis file.
+
+This can be achieved by having a master node creating the genesis.json and keys for each validator, and then having each validator download the files from the master node. Ideally this would be handled by a shared PVC across each validator; however, Kuberentes has a constraint where you cannot mount multiple pods onto the same volume.
+
+This led to the decision to use an API service to act as the intermediary that allows uploading and downloading of files. While at first glance, this smells of overengineering, the fastAPI implementation is actually quite simple (only marginally more code than creating and mounting a volume) and it improves the startup time dramatically since there's no need for the pods to wait for the volume to be mounted. Plus, it's likely that it can be leveraged in the future to help coordinate tasks across the different networks in the setup (e.g. it can store a registry of canonical IBC connections across chains).

integration-tests/api/main.py

@@ -0,0 +1,45 @@
+from fastapi import FastAPI, File, HTTPException, UploadFile
+from fastapi.responses import FileResponse
+import os
+
+app = FastAPI()
+
+STORAGE_DIRECTORY = "storage"
+os.makedirs(STORAGE_DIRECTORY, exist_ok=True)
+
+
+@app.get("/status")
+async def status() -> str:
+    """
+    Health check
+    """
+    return "ok"
+
+
+@app.post("/upload/{file_name}")
+@app.post("/upload/{path:path}/{file_name}")
+async def upload_file(file_name: str, path: str = "", file: UploadFile = File(...)) -> dict:
+    """
+    Allows uploading a file - stores it in the local file system
+    """
+    parent = f"{STORAGE_DIRECTORY}/{path}" if path else STORAGE_DIRECTORY
+    os.makedirs(parent, exist_ok=True)
+
+    with open(f"{parent}/{file_name}", "wb") as f:
+        f.write(file.file.read())
+
+    return {"info": f"file {file_name} saved"}
+
+
+@app.get("/download/{file_name}")
+@app.get("/download/{path:path}/{file_name}")
+async def download_file(file_name: str, path: str = ""):
+    """
+    Allows downloading a file from the local file system
+    """
+    file_location = f"{STORAGE_DIRECTORY}/{path}/{file_name}" if path else f"{STORAGE_DIRECTORY}/{file_name}"
+
+    if not os.path.exists(file_location):
+        return HTTPException(status_code=404, detail=f"file {file_location} not found")
+
+    return FileResponse(file_location, media_type="application/octet-stream", filename=file_name)

integration-tests/api/requirements.txt

@@ -0,0 +1,3 @@
+fastapi==0.103.2
+uvicorn==0.23.2
+python-multipart==0.0.9

integration-tests/config/keys.json

@@ -0,0 +1,30 @@
+{
+    "validators": [
+      {
+        "name": "val1",
+        "mnemonic": "close soup mirror crew erode defy knock trigger gather eyebrow tent farm gym gloom base lemon sleep weekend rich forget diagram hurt prize fly"
+      },
+      {
+        "name": "val2",
+        "mnemonic": "turkey miss hurry unable embark hospital kangaroo nuclear outside term toy fall buffalo book opinion such moral meadow wing olive camp sad metal banner"
+      },
+      {
+        "name": "val3",
+        "mnemonic": "tenant neck ask season exist hill churn rice convince shock modify evidence armor track army street stay light program harvest now settle feed wheat"
+      },
+      {
+        "name": "val4",
+        "mnemonic": "tail forward era width glory magnet knock shiver cup broken turkey upgrade cigar story agent lake transfer misery sustain fragile parrot also air document"
+      },
+      {
+        "name": "val5",
+        "mnemonic": "crime lumber parrot enforce chimney turtle wing iron scissors jealous indicate peace empty game host protect juice submit motor cause second picture nuclear area"
+      }
+    ],
+    "faucet": [
+      {
+        "name": "faucet",
+        "mnemonic": "chimney become stuff spoil resource supply picture divorce casual curve check web valid survey zebra various pet sphere timber friend faint blame mansion film"
+      }
+    ]
+  }