Skip to content

Commit

Permalink
[antithesis] Add test setup for xsvm
Browse files Browse the repository at this point in the history
  • Loading branch information
marun committed May 2, 2024
1 parent ef81f95 commit e83887b
Show file tree
Hide file tree
Showing 21 changed files with 1,171 additions and 85 deletions.
9 changes: 7 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -265,8 +265,8 @@ jobs:
- name: Check image build
shell: bash
run: bash -x scripts/tests.build_image.sh
test_build_antithesis_avalanchego_image:
name: Antithesis avalanchego build
test_build_antithesis_images:
name: Build Antithesis images
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
Expand All @@ -275,6 +275,11 @@ jobs:
run: bash -x scripts/tests.build_antithesis_images.sh
env:
TEST_SETUP: avalanchego
- name: Check image build for xsvm test setup
shell: bash
run: bash -x scripts/tests.build_antithesis_images.sh
env:
TEST_SETUP: xsvm
govulncheck:
runs-on: ubuntu-latest
name: govulncheck
Expand Down
7 changes: 7 additions & 0 deletions .github/workflows/publish_antithesis_images.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,10 @@ jobs:
IMAGE_PREFIX: ${{ env.REGISTRY }}/${{ env.REPOSITORY }}
TAG: latest
TEST_SETUP: avalanchego

- name: Build images for xsvm test setup
run: bash -x ./scripts/build_antithesis_images.sh
env:
IMAGE_PREFIX: ${{ env.REGISTRY }}/${{ env.REPOSITORY }}
TAG: latest
TEST_SETUP: xsvm
32 changes: 26 additions & 6 deletions scripts/build_antithesis_images.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,9 @@ set -euo pipefail
# Builds docker images for antithesis testing.

# e.g.,
# ./scripts/build_antithesis_images.sh # Build local images
# IMAGE_PREFIX=<registry>/<repo> TAG=latest ./scripts/build_antithesis_images.sh # Specify a prefix to enable image push and use a specific tag
# TEST_SETUP=avalanchego ./scripts/build_antithesis_images.sh # Build local images for avalanchego
# TEST_SETUP=xsvm ./scripts/build_antithesis_images.sh # Build local images for xsvm
# TEST_SETUP=xsvm IMAGE_PREFIX=<registry>/<repo> TAG=latest ./scripts/build_antithesis_images.sh # Specify a prefix to enable image push and use a specific tag

# Directory above this script
AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd )
Expand All @@ -28,11 +29,14 @@ GO_VERSION="$(go list -m -f '{{.GoVersion}}')"
function build_images {
local test_setup=$1
local uninstrumented_node_dockerfile=$2
local node_only=${3:-}

# Define image names
local base_image_name="antithesis-${test_setup}"
local avalanchego_node_image_name="antithesis-avalanchego-node:${TAG}"
if [[ -n "${IMAGE_PREFIX}" ]]; then
base_image_name="${IMAGE_PREFIX}/${base_image_name}"
avalanchego_node_image_name="${IMAGE_PREFIX}/${avalanchego_node_image_name}"
fi
local node_image_name="${base_image_name}-node:${TAG}"
local workload_image_name="${base_image_name}-workload:${TAG}"
Expand All @@ -49,18 +53,34 @@ function build_images {
fi

# Define default build command
local docker_cmd="docker buildx build --build-arg GO_VERSION=${GO_VERSION}"
local docker_cmd="docker buildx build --build-arg GO_VERSION=${GO_VERSION} --build-arg NODE_IMAGE=${node_image_name}"

if [[ "${test_setup}" == "xsvm" ]]; then
# The xsvm node image is build on the avalanchego node image
docker_cmd="${docker_cmd} --build-arg AVALANCHEGO_NODE_IMAGE=${avalanchego_node_image_name}"
fi

# Build node image first to allow the config and workload image builds to use it.
${docker_cmd} -t "${node_image_name}" -f "${node_dockerfile}" "${AVALANCHE_PATH}"
${docker_cmd} --build-arg NODE_IMAGE="${node_image_name}" -t "${workload_image_name}" -f "${base_dockerfile}.workload" "${AVALANCHE_PATH}"
${docker_cmd} --build-arg IMAGE_TAG="${TAG}" -t "${config_image_name}" -f "${base_dockerfile}.config" "${AVALANCHE_PATH}"

if [[ -z "${node_only}" || "${node_only}" == "0" ]]; then
# Skip building the config and workload images. Supports building the avalanchego
# node image as the base image for the xsvm node image.
${docker_cmd} --build-arg IMAGE_TAG="${TAG}" -t "${config_image_name}" -f "${base_dockerfile}.config" "${AVALANCHE_PATH}"
${docker_cmd} -t "${workload_image_name}" -f "${base_dockerfile}.workload" "${AVALANCHE_PATH}"
fi
}

TEST_SETUP="${TEST_SETUP:-}"
if [[ "${TEST_SETUP}" == "avalanchego" ]]; then
build_images avalanchego "${AVALANCHE_PATH}/Dockerfile"
elif [[ "${TEST_SETUP}" == "xsvm" ]]; then
# Only build the node image to use as the base for the xsvm image
NODE_ONLY=1
build_images avalanchego "${AVALANCHE_PATH}/Dockerfile" "${NODE_ONLY}"

build_images xsvm "${AVALANCHE_PATH}/vms/example/xsvm/Dockerfile"
else
echo "TEST_SETUP must be set. Valid values are 'avalanchego'"
echo "TEST_SETUP must be set. Valid values are 'avalanchego' or 'xsvm'"
exit 255
fi
11 changes: 11 additions & 0 deletions scripts/build_antithesis_xsvm_workload.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
#!/usr/bin/env bash

set -euo pipefail

# Directory above this script
AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd )
# Load the constants
source "$AVALANCHE_PATH"/scripts/constants.sh

echo "Building Workload..."
go build -o "$AVALANCHE_PATH/build/antithesis-xsvm-workload" "$AVALANCHE_PATH/tests/antithesis/xsvm/"*.go
11 changes: 6 additions & 5 deletions tests/antithesis/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,12 @@ enables discovery and reproduction of anomalous behavior.

## Package details

| Filename | Purpose |
|:-------------|:----------------------------------------------------------------------------------|
| compose.go | Enables generation of Docker Compose project files for antithesis testing. |
| avalanchego/ | Contains resources supporting antithesis testing of avalanchego's primary chains. |

| Filename | Purpose |
|:-------------|:--------------------------------------------------------------------------------|
| compose.go | Generates Docker Compose project files and volume paths for antithesis testing. |
| initdb.go | Initializes db state for subnet testing |
| avalanchego/ | Defines an antithesis test setup for avalanchego's primary chains. |
| xsvm/ | Defines an antithesis test setup for the xsvm VM. |

## Instrumentation

Expand Down
3 changes: 3 additions & 0 deletions tests/antithesis/avalanchego/Dockerfile.node
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@ RUN mkdir -p /symbols
COPY --from=builder /avalanchego_instrumented/symbols /symbols
COPY --from=builder /opt/antithesis/lib/libvoidstar.so /usr/lib/libvoidstar.so

# Use the same path as the uninstrumented node image for consistency
WORKDIR /avalanchego/build

# Copy the executable into the container
COPY --from=builder /avalanchego_instrumented/customer/build/avalanchego ./avalanchego

Expand Down
57 changes: 43 additions & 14 deletions tests/antithesis/compose.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,13 +104,34 @@ func newComposeProject(network *tmpnet.Network, nodeImageName string, workloadIm
config.StakingSignerKeyContentKey: signerKey,
}

nodeName := "avalanche"
serviceName := getServiceName(i)

volumes := []types.ServiceVolumeConfig{
{
Type: types.VolumeTypeBind,
Source: fmt.Sprintf("./volumes/%s/logs", serviceName),
Target: "/root/.avalanchego/logs",
},
}

trackSubnets, err := node.Flags.GetStringVal(config.TrackSubnetsKey)
if err != nil {
return nil, err
}
if len(trackSubnets) > 0 {
env[config.TrackSubnetsKey] = trackSubnets
// DB volume will need to initialized with the subnet
volumes = append(volumes, types.ServiceVolumeConfig{
Type: types.VolumeTypeBind,
Source: fmt.Sprintf("./volumes/%s/db", serviceName),
Target: "/root/.avalanchego/db",
})
}

if i == 0 {
nodeName += "-bootstrap-node"
bootstrapIP = address + ":9651"
bootstrapIDs = node.NodeID.String()
} else {
nodeName = fmt.Sprintf("%s-node-%d", nodeName, i+1)
env[config.BootstrapIPsKey] = bootstrapIP
env[config.BootstrapIDsKey] = bootstrapIDs
}
Expand All @@ -120,18 +141,12 @@ func newComposeProject(network *tmpnet.Network, nodeImageName string, workloadIm
env = keyMapToEnvVarMap(env)

services[i+1] = types.ServiceConfig{
Name: nodeName,
ContainerName: nodeName,
Hostname: nodeName,
Name: serviceName,
ContainerName: serviceName,
Hostname: serviceName,
Image: nodeImageName,
Volumes: []types.ServiceVolumeConfig{
{
Type: types.VolumeTypeBind,
Source: fmt.Sprintf("./volumes/%s/logs", nodeName),
Target: "/root/.avalanchego/logs",
},
},
Environment: env.ToMappingWithEquals(),
Volumes: volumes,
Environment: env.ToMappingWithEquals(),
Networks: map[string]*types.ServiceNetworkConfig{
networkName: {
Ipv4Address: address,
Expand All @@ -146,6 +161,9 @@ func newComposeProject(network *tmpnet.Network, nodeImageName string, workloadIm
workloadEnv := types.Mapping{
"AVAWL_URIS": strings.Join(uris, " "),
}
if len(network.Subnets) > 0 && len(network.Subnets[0].Chains) > 0 {
workloadEnv["AVAWL_CHAIN_ID"] = network.Subnets[0].Chains[0].ChainID.String()
}

workloadName := "workload"
services[0] = types.ServiceConfig{
Expand Down Expand Up @@ -191,3 +209,14 @@ func keyMapToEnvVarMap(keyMap types.Mapping) types.Mapping {
}
return envVarMap
}

// Retrieve the service name for a node at the given index. Common to
// GenerateComposeConfig and InitDBVolumes to ensure consistency
// between db volumes configuration and volume paths.
func getServiceName(index int) string {
baseName := "avalanche"
if index == 0 {
return baseName + "-bootstrap-node"
}
return fmt.Sprintf("%s-node-%d", baseName, index+1)
}
59 changes: 59 additions & 0 deletions tests/antithesis/initdb.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
// Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package antithesis

import (
"context"
"fmt"
"os"
"os/exec"
"path/filepath"
"time"

"github.com/ava-labs/avalanchego/tests/fixture/tmpnet"
"github.com/ava-labs/avalanchego/utils/perms"
)

// Initialize the db volumes for a docker-compose configuration. The returned network will be updated with
// subnet and chain IDs and the target path will be configured with volumes for each node containing the
// initialized db state.
func InitDBVolumes(network *tmpnet.Network, avalancheGoPath string, pluginDir string, targetPath string) error {
ctx, cancel := context.WithTimeout(context.Background(), time.Minute*2)
defer cancel()
if err := tmpnet.StartNewNetwork(
ctx,
os.Stdout,
network,
"",
avalancheGoPath,
pluginDir,
0, // nodeCount is 0 because nodes should already be configured for subnets
); err != nil {
return fmt.Errorf("failed to start network: %w", err)
}
// Since the goal is to initialize the DB, we can stop the network after it has been started successfully
if err := network.Stop(ctx); err != nil {
return fmt.Errorf("failed to stop network: %w", err)
}

absPath, err := filepath.Abs(targetPath)
if err != nil {
return fmt.Errorf("failed to convert target path to absolute path: %w", err)
}

// Create volumes in the target path and copy the db state from the nodes
for i, node := range network.Nodes {
sourcePath := filepath.Join(node.GetDataDir(), "db")
destPath := filepath.Join(absPath, "volumes", getServiceName(i))
if err := os.MkdirAll(destPath, perms.ReadWriteExecute); err != nil {
return fmt.Errorf("failed to create volume path %q: %w", destPath, err)
}
cmd := exec.Command("cp", "-r", sourcePath, destPath)
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to copy db state from %q to %q: %w", sourcePath, destPath, err)
}
}

return nil
}
39 changes: 39 additions & 0 deletions tests/antithesis/xsvm/Dockerfile.config
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
# The version is supplied as a build argument rather than hard-coded
# to minimize the cost of version changes.
ARG GO_VERSION

# NODE_IMAGE needs to identify an existing xsvm node image and should include the tag
ARG NODE_IMAGE

FROM $NODE_IMAGE AS xsvm_node

# ============= Compilation Stage ================
FROM golang:$GO_VERSION-bullseye AS builder

WORKDIR /build
# Copy and download avalanche dependencies using go mod
COPY go.mod .
COPY go.sum .
RUN go mod download

# Copy the code into the container
COPY . .

# IMAGE_TAG should be set to the tag for the images in the generated docker compose file.
ARG IMAGE_TAG=latest

# Copy the avalanchego binary and plugin from the node image
RUN mkdir -p ./build/plugins
COPY --from=xsvm_node /avalanchego/build/avalanchego ./build
COPY --from=xsvm_node /root/.avalanchego/plugins/* ./build/plugins/

# Generate docker compose configuration
RUN AVALANCHEGO_PATH=./build/avalanchego AVALANCHEGO_PLUGIN_DIR=./build/plugins\
TARGET_PATH=./build IMAGE_TAG="$IMAGE_TAG" go run ./tests/antithesis/xsvm/gencomposeconfig

# ============= Cleanup Stage ================
FROM scratch AS execution

# Copy the docker compose file and volumes into the container
COPY --from=builder /build/build/docker-compose.yml /docker-compose.yml
COPY --from=builder /build/build/volumes /volumes
61 changes: 61 additions & 0 deletions tests/antithesis/xsvm/Dockerfile.node
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
# The version is supplied as a build argument rather than hard-coded
# to minimize the cost of version changes.
ARG GO_VERSION

# AVALANCHEGO_NODE_IMAGE needs to identify an existing avalanchego node image and should include the tag
ARG AVALANCHEGO_NODE_IMAGE

# Antithesis: Getting the Antithesis golang instrumentation library
FROM docker.io/antithesishq/go-instrumentor AS instrumentor

# ============= Compilation Stage ================
FROM golang:$GO_VERSION-bullseye AS builder

WORKDIR /build
# Copy and download avalanche dependencies using go mod
COPY go.mod .
COPY go.sum .
RUN go mod download

# Copy the code into the container
COPY . .

# Keep the commit hash to easily verify the exact version that is running
RUN git rev-parse HEAD > ./commit_hash.txt

# Copy the instrumentor and supporting files to their correct locations
COPY --from=instrumentor /opt/antithesis /opt/antithesis
COPY --from=instrumentor /opt/antithesis/lib /lib

# Create the destination output directory for the instrumented code
RUN mkdir -p /avalanchego_instrumented

# Park the .git file in a safe location
RUN mkdir -p /opt/tmp/
RUN cp -r .git /opt/tmp/

# Instrument avalanchego
RUN /opt/antithesis/bin/goinstrumentor \
-stderrthreshold=INFO \
-antithesis /opt/antithesis/instrumentation \
. \
/avalanchego_instrumented

WORKDIR /avalanchego_instrumented/customer
RUN go mod download
RUN ln -s /opt/tmp/.git .git

# Build xsvm VM
RUN ./scripts/build_xsvm.sh

# ============= Cleanup Stage ================
FROM $AVALANCHEGO_NODE_IMAGE AS execution

# The commit hash and antithesis dependencies should be part of the base image.

# Copy the executable into the container
RUN mkdir -p /root/.avalanchego/plugins
COPY --from=builder /avalanchego_instrumented/customer/build/xsvm \
/root/.avalanchego/plugins/v3m4wPxaHpvGr8qfMeyK6PRW3idZrPHmYcMTt7oXdK47yurVH

# The node image's entrypoint will be reused.
Loading

0 comments on commit e83887b

Please sign in to comment.