release-22.1: roachtest: declare encryption support as part of TestSpec

build/teamcity-nightly-roachtest.sh

@@ -27,7 +27,7 @@ build/builder/mkrelease.sh amd64-linux-gnu build bin/workload bin/roachtest \
 source $root/build/teamcity/util/roachtest_util.sh
 
 build/teamcity-roachtest-invoke.sh \
-  --encrypt=random \
+  --metamorphic-encryption-probability=0.5 \
   --cloud="${CLOUD}" \
   --count="${COUNT-1}" \
   --parallelism="${PARALLELISM}" \

build/teamcity-weekly-roachtest.sh

@@ -57,7 +57,7 @@ timeout -s INT $((7800*60)) bin/roachtest run \
   --workload "$PWD/bin/workload" \
   --artifacts "$artifacts" \
   --parallelism 5 \
-  --encrypt=random \
+  --metamorphic-encryption-probability=0.5 \
   --teamcity || exit_status=$?
 
 if [[ ${exit_status} -eq 10 ]]; then

build/teamcity/cockroach/nightlies/roachtest_nightly_impl.sh

@@ -16,7 +16,7 @@ artifacts=/artifacts
 source $root/build/teamcity/util/roachtest_util.sh
 
 build/teamcity-roachtest-invoke.sh \
-  --encrypt=random \
+  --metamorphic-encryption-probability=0.5 \
   --cloud="${CLOUD}" \
   --count="${COUNT-1}" \
   --parallelism="${PARALLELISM}" \

build/teamcity/cockroach/nightlies/roachtest_weekly_impl.sh

@@ -52,7 +52,7 @@ timeout -s INT $((7800*60)) bin/roachtest run \
   --artifacts "/artifacts/${artifacts_subdir}" \
   --artifacts-literal="${artifacts}" \
   --parallelism 5 \
-  --encrypt=random \
+  --metamorphic-encryption-probability=0.5 \
   --teamcity || exit_status=$?
 
 if [[ ${exit_status} -eq 10 ]]; then

pkg/cmd/roachtest/BUILD.bazel

@@ -34,6 +34,7 @@ go_library(
         "//pkg/util/ctxgroup",
         "//pkg/util/log",
         "//pkg/util/quotapool",
+        "//pkg/util/randutil",
         "//pkg/util/stop",
         "//pkg/util/syncutil",
         "//pkg/util/timeutil",

pkg/cmd/roachtest/cluster.go

@@ -18,7 +18,6 @@ import (
 	"io"
 	"io/fs"
 	"io/ioutil"
-	"math/rand"
 	"net"
 	"net/url"
 	"os"
@@ -62,10 +61,18 @@ var (
 	// alias for `--cloud=local` and remove this variable.
 	local bool
 
-	cockroach                 string
-	libraryFilePaths          []string
-	cloud                                  = spec.GCE
-	encrypt                   encryptValue = "false"
+	cockroach        string
+	libraryFilePaths []string
+	cloud            = spec.GCE
+	// encryptionProbability controls when encryption-at-rest is enabled
+	// in a cluster for tests that have opted-in to metamorphic
+	// encryption (EncryptionMetamorphic).
+	//
+	// Tests that have opted-in to metamorphic encryption will run with
+	// encryption enabled by default (probability 1). In order to run
+	// them with encryption disabled (perhaps to reproduce a test
+	// failure), roachtest can be invoked with --metamorphic-encryption-probability=0
+	encryptionProbability     float64
 	instanceType              string
 	localSSDArg               bool
 	workload                  string
@@ -87,39 +94,9 @@ var (
 	disableIssue     bool
 )
 
-type encryptValue string
-
-func (v *encryptValue) String() string {
-	return string(*v)
-}
-
-func (v *encryptValue) Set(s string) error {
-	if s == "random" {
-		*v = encryptValue(s)
-		return nil
-	}
-	t, err := strconv.ParseBool(s)
-	if err != nil {
-		return err
-	}
-	*v = encryptValue(fmt.Sprint(t))
-	return nil
-}
-
-func (v *encryptValue) asBool() bool {
-	if *v == "random" {
-		return rand.Intn(2) == 0
-	}
-	t, err := strconv.ParseBool(string(*v))
-	if err != nil {
-		return false
-	}
-	return t
-}
-
-func (v *encryptValue) Type() string {
-	return "string"
-}
+const (
+	defaultEncryptionProbability = 1
+)
 
 type errBinaryOrLibraryNotFound struct {
 	binary string

pkg/cmd/roachtest/main.go

@@ -132,9 +132,10 @@ func main() {
 		&cockroach, "cockroach", "", "path to cockroach binary to use")
 	rootCmd.PersistentFlags().StringVar(
 		&workload, "workload", "", "path to workload binary to use")
-	f := rootCmd.PersistentFlags().VarPF(
-		&encrypt, "encrypt", "", "start cluster with encryption at rest turned on")
-	f.NoOptDefVal = "true"
+	rootCmd.PersistentFlags().Float64Var(
+		&encryptionProbability, "metamorphic-encryption-probability", defaultEncryptionProbability,
+		"probability that clusters will be created with encryption-at-rest enabled "+
+			"for tests that support metamorphic encryption (default 1.0)")
 
 	rootCmd.AddCommand(&cobra.Command{
 		Use:   `version`,

pkg/cmd/roachtest/option/options.go

@@ -19,8 +19,7 @@ import (
 type StartOpts struct {
 	RoachprodOpts install.StartOpts
 	RoachtestOpts struct {
-		Worker      bool
-		DontEncrypt bool
+		Worker bool
 	}
 }
 

pkg/cmd/roachtest/registry/BUILD.bazel

@@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
 go_library(
     name = "registry",
     srcs = [
+        "encryption.go",
         "filter.go",
         "owners.go",
         "registry_interface.go",

pkg/cmd/roachtest/registry/encryption.go

@@ -0,0 +1,48 @@
+// Copyright 2022 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package registry
+
+import "fmt"
+
+// EncryptionSupport encodes the relationship of a test with
+// encryption-at-rest. Tests can either opt-in to metamorphic
+// encryption, or require that encryption is always on or always off
+// (default).
+type EncryptionSupport int
+
+func (es EncryptionSupport) String() string {
+	switch es {
+	case EncryptionAlwaysEnabled:
+		return "always-enabled"
+	case EncryptionAlwaysDisabled:
+		return "always-disabled"
+	case EncryptionMetamorphic:
+		return "metamorphic"
+	default:
+		return fmt.Sprintf("unknown-%d", es)
+	}
+}
+
+const (
+	// EncryptionAlwaysDisabled indicates that the test requires
+	// encryption to be disabled. The test will only run on clusters
+	// with encryption disabled.
+	EncryptionAlwaysDisabled = EncryptionSupport(iota)
+	// EncryptionAlwaysEnabled indicates that the test requires
+	// encryption to be enabled. The test will only run on clusters
+	// with encryption enabled.
+	EncryptionAlwaysEnabled
+	// EncryptionMetamorphic indicates that a test opted-in to
+	// metamorphic encryption. Whether the test runs on a cluster with
+	// encryption enabled depends on the probability passed to
+	// --metamorphic-encryption-probability.
+	EncryptionMetamorphic
+)

pkg/cmd/roachtest/registry/test_spec.go

@@ -67,10 +67,12 @@ type TestSpec struct {
 	// in the environment.
 	RequiresLicense bool
 
-	// EncryptAtRandom specifies that even when roachtest is invoked without the
-	// `--encrypt` flag, clusters handed to this test will randomly have
-	// encryption-at-rest enabled.
-	EncryptAtRandom bool
+	// EncryptionSupport encodes to what extent tests supports
+	// encryption-at-rest. See the EncryptionSupport type for details.
+	// Encryption support is opt-in -- i.e., if the TestSpec does not
+	// pass a value to this field, it will be assumed that the test
+	// cannot be run with encryption enabled.
+	EncryptionSupport EncryptionSupport
 
 	// Run is the test function.
 	Run func(ctx context.Context, t test.Test, c cluster.Cluster)

pkg/cmd/roachtest/test_runner.go

@@ -40,6 +40,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/ctxgroup"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/quotapool"
+	"github.com/cockroachdb/cockroach/pkg/util/randutil"
 	"github.com/cockroachdb/cockroach/pkg/util/stop"
 	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
 	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
@@ -453,6 +454,9 @@ func (r *testRunner) runWorker(
 			l.PrintfCtx(ctx, "Worker exiting with canceled ctx. Not destroying cluster.")
 		}
 	}()
+
+	prng, _ := randutil.NewPseudoRand()
+
 	// Loop until there's no more work in the pool, we get interrupted, or an
 	// error occurs.
 	for {
@@ -598,13 +602,17 @@ func (r *testRunner) runWorker(
 			c.status("running test")
 			c.setTest(t)
 
-			// Populate encryption at rest from the --encrypt flag.
-			encAtRest := encrypt.asBool()
-			if encrypt.String() == "random" && !t.Spec().(*registry.TestSpec).EncryptAtRandom {
-				// In random mode, enable enc-at-rest only if tests opted into it.
-				encAtRest = false
+			switch t.Spec().(*registry.TestSpec).EncryptionSupport {
+			case registry.EncryptionAlwaysEnabled:
+				c.encAtRest = true
+			case registry.EncryptionAlwaysDisabled:
+				c.encAtRest = false
+			case registry.EncryptionMetamorphic:
+				// when tests opted-in to metamorphic testing, encryption will
+				// be enabled according to the probability passed to
+				// --metamorphic-encryption-probability
+				c.encAtRest = prng.Float64() < encryptionProbability
 			}
-			c.encAtRest = encAtRest
 
 			wStatus.SetCluster(c)
 			wStatus.SetTest(t, testToRun)

pkg/cmd/roachtest/test_test.go

@@ -188,21 +188,21 @@ func TestRunnerRun(t *testing.T) {
 }
 
 func TestRunnerEncryptionAtRest(t *testing.T) {
-	// Verify that if a test opts into EncryptAtRandom, it will (eventually) get
-	// a cluster that has encryption at rest enabled.
+	// Verify that if a test opts into EncryptionMetamorphic, it will
+	// (eventually) get a cluster that has encryption at rest enabled.
 	{
-		prev := encrypt.String()
-		require.NoError(t, encrypt.Set("random")) // --encrypt=random
+		prevProb := encryptionProbability
+		encryptionProbability = 0.5 // --metamorphic-encrypt-probability=0.5
 		defer func() {
-			require.NoError(t, encrypt.Set(prev))
+			encryptionProbability = prevProb
 		}()
 	}
 	r := mkReg(t)
 	var sawEncrypted int32 // atomic
 	r.Add(registry.TestSpec{
-		Name:            "enc-random",
-		Owner:           OwnerUnitTest,
-		EncryptAtRandom: true,
+		Name:              "enc-random",
+		Owner:             OwnerUnitTest,
+		EncryptionSupport: registry.EncryptionMetamorphic,
 		Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
 			encAtRest := c.(*clusterImpl).encAtRest
 			t.L().Printf("encryption-at-rest=%t", encAtRest)
@@ -221,9 +221,8 @@ func TestRunnerEncryptionAtRest(t *testing.T) {
 			rt.copt, testOpts{}, rt.lopt, nil, // clusterAllocator
 		))
 		if atomic.LoadInt32(&sawEncrypted) == 0 {
-			// NB: since it's a 50% chance, hitting this reliably over 10k trials
-			// has probability (0.5)^10000 which is for all intents and purposes
-			// one, even taking any stressing we might ever do into account.
+			// NB: since it's a 50% chance, the probability of *not* hitting
+			// this branch over 10k runs is 1 - (0.5)^10000 which is essentially 1.
 			continue
 		}
 		t.Logf("done after %d iterations", i+1)

pkg/cmd/roachtest/tests/acceptance.go

@@ -21,22 +21,23 @@ import (
 
 func registerAcceptance(r registry.Registry) {
 	testCases := map[registry.Owner][]struct {
-		name            string
-		fn              func(ctx context.Context, t test.Test, c cluster.Cluster)
-		skip            string
-		minVersion      string
-		numNodes        int
-		timeout         time.Duration
-		encryptAtRandom bool
+		name              string
+		fn                func(ctx context.Context, t test.Test, c cluster.Cluster)
+		skip              string
+		minVersion        string
+		numNodes          int
+		timeout           time.Duration
+		encryptionSupport registry.EncryptionSupport
 	}{
 		registry.OwnerKV: {
 			{name: "decommission-self", fn: runDecommissionSelf},
 			{name: "event-log", fn: runEventLog},
 			{name: "gossip/peerings", fn: runGossipPeerings},
 			{name: "gossip/restart", fn: runGossipRestart},
 			{
-				name: "gossip/restart-node-one",
-				fn:   runGossipRestartNodeOne,
+				name:              "gossip/restart-node-one",
+				fn:                runGossipRestartNodeOne,
+				encryptionSupport: registry.EncryptionAlwaysDisabled,
 			},
 			{name: "gossip/locality-address", fn: runCheckLocalityIPAddress},
 			{
@@ -47,8 +48,8 @@ func registerAcceptance(r registry.Registry) {
 			{name: "reset-quorum", fn: runResetQuorum, numNodes: 8},
 			{
 				name: "many-splits", fn: runManySplits,
-				minVersion:      "v19.2.0", // SQL syntax unsupported on 19.1.x
-				encryptAtRandom: true,
+				minVersion:        "v19.2.0", // SQL syntax unsupported on 19.1.x
+				encryptionSupport: registry.EncryptionMetamorphic,
 			},
 			{
 				name: "version-upgrade",
@@ -102,7 +103,7 @@ func registerAcceptance(r registry.Registry) {
 			if tc.timeout != 0 {
 				spec.Timeout = tc.timeout
 			}
-			spec.EncryptAtRandom = tc.encryptAtRandom
+			spec.EncryptionSupport = tc.encryptionSupport
 			spec.Run = func(ctx context.Context, t test.Test, c cluster.Cluster) {
 				tc.fn(ctx, t, c)
 			}