Use a consistent zone on EBS restores (#5084)

pkg/blockstorage/awsebs/zone.go

@@ -2,6 +2,9 @@ package awsebs
 
 import (
 	"context"
+	"hash/fnv"
+	"sort"
+	"strings"
 
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
@@ -48,12 +51,26 @@ func zoneFromKnownNodeZones(ctx context.Context, region string, sourceZone strin
 		}
 	}
 	// If any nodes are available, return an arbitrary one.
-	// This is relatively random based on go's map iteration.
+	return consistentZone(sourceZone, nzs)
+}
+
+func consistentZone(sourceZone string, nzs map[string]struct{}) (string, error) {
+	if len(nzs) == 0 {
+		return "", errors.New("could not restore volume: no zone found")
+	}
+	s := make([]string, 0, len(nzs))
 	for nz := range nzs {
-		return nz, nil
+		s = append(s, nz)
+	}
+	sort.Slice(s, func(i, j int) bool {
+		return strings.Compare(s[i], s[j]) < 0
+	})
+	h := fnv.New32()
+	if _, err := h.Write([]byte(sourceZone)); err != nil {
+		return "", errors.Errorf("failed to hash source zone %s: %s", sourceZone, err.Error())
 	}
-	// Unreachable
-	return "", nil
+	i := int(h.Sum32()) % len(nzs)
+	return s[i], nil
 }
 
 func zoneWithUnknownNodeZones(ctx context.Context, region string, sourceZone string) (string, error) {

pkg/blockstorage/awsebs/zone_test.go

@@ -47,3 +47,40 @@ func (s ZoneSuite) TestZoneWithUnknownNodeZones(c *C) {
 		}
 	}
 }
+
+func (s ZoneSuite) TestConsistentZone(c *C) {
+	// We don't care what the answer is as long as it's consistent.
+	for _, tc := range []struct {
+		sourceZone string
+		nzs        map[string]struct{}
+		out        string
+	}{
+		{
+			sourceZone: "",
+			nzs: map[string]struct{}{
+				"zone1": struct{}{},
+			},
+			out: "zone1",
+		},
+		{
+			sourceZone: "",
+			nzs: map[string]struct{}{
+				"zone1": struct{}{},
+				"zone2": struct{}{},
+			},
+			out: "zone2",
+		},
+		{
+			sourceZone: "from1",
+			nzs: map[string]struct{}{
+				"zone1": struct{}{},
+				"zone2": struct{}{},
+			},
+			out: "zone1",
+		},
+	} {
+		out, err := consistentZone(tc.sourceZone, tc.nzs)
+		c.Assert(err, IsNil)
+		c.Assert(out, Equals, tc.out)
+	}
+}