Fix a somewhat buggy testUpdateManyNoConflicts (as metadata is update…

…d old metadata is no longer current.

Also rename a variable to be more clear, and make invalid arguments to the integration/testdb scripts fail faster.

Signed-off-by: Ying Li <ying.li@docker.com>

buildscripts/dbtests.sh

@@ -13,6 +13,10 @@ case ${db} in
     dbContainerOpts="--name rethinkdb_tests rdb-01 --bind all --driver-tls-key /tls/key.pem --driver-tls-cert /tls/cert.pem"
     DBURL="rethinkdb_tests"
     ;;
+  *)
+    echo "Usage: $0 (mysql|rethink)"
+    exit 1
+    ;;
 esac
 
 composeFile="development.${db}.yml"

buildscripts/integrationtest.sh

@@ -1,6 +1,19 @@
 #!/usr/bin/env bash
 
 db="$1"
+case ${db} in
+  mysql*)
+    db="mysql"
+    ;;
+  rethink*)
+    db="rethink"
+    ;;
+  *)
+    echo "Usage: $0 (mysql|rethink)"
+    exit 1
+    ;;
+esac
+
 composeFile="development.${db}.yml"
 project=integration
 

server/storage/memory.go

@@ -4,6 +4,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -20,19 +21,29 @@ type ver struct {
 	createupdate time.Time
 }
 
+// we want to keep these sorted by version so that it's in increasing version
+// order
+type verList []ver
+
+func (k verList) Len() int      { return len(k) }
+func (k verList) Swap(i, j int) { k[i], k[j] = k[j], k[i] }
+func (k verList) Less(i, j int) bool {
+	return k[i].version < k[j].version
+}
+
 // MemStorage is really just designed for dev and testing. It is very
 // inefficient in many scenarios
 type MemStorage struct {
 	lock      sync.Mutex
-	tufMeta   map[string][]*ver
+	tufMeta   map[string]verList
 	keys      map[string]map[string]*key
 	checksums map[string]map[string]ver
 }
 
 // NewMemStorage instantiates a memStorage instance
 func NewMemStorage() *MemStorage {
 	return &MemStorage{
-		tufMeta:   make(map[string][]*ver),
+		tufMeta:   make(map[string]verList),
 		keys:      make(map[string]map[string]*key),
 		checksums: make(map[string]map[string]ver),
 	}
@@ -51,7 +62,7 @@ func (st *MemStorage) UpdateCurrent(gun string, update MetaUpdate) error {
 		}
 	}
 	version := ver{version: update.Version, data: update.Data, createupdate: time.Now()}
-	st.tufMeta[id] = append(st.tufMeta[id], &version)
+	st.tufMeta[id] = append(st.tufMeta[id], version)
 	checksumBytes := sha256.Sum256(update.Data)
 	checksum := hex.EncodeToString(checksumBytes[:])
 
@@ -97,7 +108,8 @@ func (st *MemStorage) UpdateMany(gun string, updates []MetaUpdate) error {
 		id := entryKey(gun, u.Role)
 
 		version := ver{version: u.Version, data: u.Data, createupdate: time.Now()}
-		st.tufMeta[id] = append(st.tufMeta[id], &version)
+		st.tufMeta[id] = append(st.tufMeta[id], version)
+		sort.Sort(st.tufMeta[id]) // ensure that it's sorted
 		checksumBytes := sha256.Sum256(u.Data)
 		checksum := hex.EncodeToString(checksumBytes[:])
 

server/storage/memory_test.go

@@ -10,17 +10,18 @@ import (
 )
 
 func assertExpectedMemoryTUFMeta(t *testing.T, expected []StoredTUFMeta, s *MemStorage) {
-	counter := make(map[string]int)
 	for _, tufObj := range expected {
 		k := entryKey(tufObj.Gun, tufObj.Role)
-		gun, ok := s.tufMeta[k]
-		require.True(t, len(gun) >= counter[k])
-		v := gun[counter[k]]
-		require.True(t, ok, "Did not find gun in store")
-		require.Equal(t, tufObj.Version, v.version, "Version mismatch. Expected %d, found %d",
-			tufObj.Version, v.version)
+		versionList, ok := s.tufMeta[k]
+		require.True(t, ok, "Did not find this gun+role in store")
+		byVersion := make(map[int]ver)
+		for _, v := range versionList {
+			byVersion[v.version] = v
+		}
+
+		v, ok := byVersion[tufObj.Version]
+		require.True(t, ok, "Did not find version %d in store", tufObj.Version)
 		require.Equal(t, tufObj.Data, v.data, "Data was incorrect")
-		counter[k]++
 	}
 }
 

server/storage/storage_test.go

@@ -18,7 +18,10 @@ type StoredTUFMeta struct {
 	Version int
 }
 
-func SampleCustomTUFObj(role, gun string, tufdata []byte, version int) StoredTUFMeta {
+func SampleCustomTUFObj(gun, role string, version int, tufdata []byte) StoredTUFMeta {
+	if tufdata == nil {
+		tufdata = []byte(fmt.Sprintf("%s_%s_%d", gun, role, version))
+	}
 	checksum := sha256.Sum256(tufdata)
 	hexChecksum := hex.EncodeToString(checksum[:])
 	return StoredTUFMeta{
@@ -30,24 +33,26 @@ func SampleCustomTUFObj(role, gun string, tufdata []byte, version int) StoredTUF
 	}
 }
 
-func SampleCustomUpdate(role string, tufdata []byte, version int) MetaUpdate {
+func MakeUpdate(tufObj StoredTUFMeta) MetaUpdate {
 	return MetaUpdate{
-		Role:    role,
-		Version: version,
-		Data:    tufdata,
+		Role:    tufObj.Role,
+		Version: tufObj.Version,
+		Data:    tufObj.Data,
 	}
 }
 
-func assertExpectedTUFMetaInStore(t *testing.T, s MetaStore, expected []StoredTUFMeta) {
+func assertExpectedTUFMetaInStore(t *testing.T, s MetaStore, expected []StoredTUFMeta, current bool) {
 	for _, tufObj := range expected {
-		_, tufdata, err := s.GetCurrent(tufObj.Gun, tufObj.Role)
-		require.NoError(t, err)
-		require.Equal(t, tufObj.Data, tufdata)
+		if current {
+			_, tufdata, err := s.GetCurrent(tufObj.Gun, tufObj.Role)
+			require.NoError(t, err)
+			require.Equal(t, tufObj.Data, tufdata)
+		}
 
 		checksumBytes := sha256.Sum256(tufObj.Data)
 		checksum := hex.EncodeToString(checksumBytes[:])
 
-		_, tufdata, err = s.GetChecksum(tufObj.Gun, tufObj.Role, checksum)
+		_, tufdata, err := s.GetChecksum(tufObj.Gun, tufObj.Role, checksum)
 		require.NoError(t, err)
 		require.Equal(t, tufObj.Data, tufdata)
 	}
@@ -60,79 +65,92 @@ func testUpdateCurrentEmptyStore(t *testing.T, s MetaStore) []StoredTUFMeta {
 	for _, role := range append(data.BaseRoles, "targets/a") {
 		for _, gun := range []string{"gun1", "gun2"} {
 			// Adding a new TUF file should succeed
-			tufdata := []byte(role + gun)
-			require.NoError(t, s.UpdateCurrent(gun, SampleCustomUpdate(role, tufdata, 1)))
-			expected = append(expected, SampleCustomTUFObj(role, gun, tufdata, 1))
+			tufObj := SampleCustomTUFObj(gun, role, 1, nil)
+			require.NoError(t, s.UpdateCurrent(tufObj.Gun, MakeUpdate(tufObj)))
+			expected = append(expected, tufObj)
 		}
 	}
 
-	assertExpectedTUFMetaInStore(t, s, expected)
+	assertExpectedTUFMetaInStore(t, s, expected, true)
 	return expected
 }
 
 // UpdateCurrent will successfully add a new (higher) version of an existing TUF file,
 // but will return an error if there is an older version of a TUF file.
 func testUpdateCurrentVersionCheck(t *testing.T, s MetaStore) []StoredTUFMeta {
-	role, gun, tufdata := data.CanonicalRootRole, "testGUN", []byte("1")
+	role, gun := data.CanonicalRootRole, "testGUN"
+
+	expected := []StoredTUFMeta{
+		SampleCustomTUFObj(gun, role, 1, nil),
+		SampleCustomTUFObj(gun, role, 2, nil),
+		SampleCustomTUFObj(gun, role, 4, nil),
+	}
 
 	// starting meta is version 1
-	require.NoError(t, s.UpdateCurrent(gun, SampleCustomUpdate(role, tufdata, 1)))
+	require.NoError(t, s.UpdateCurrent(gun, MakeUpdate(expected[0])))
 
 	// inserting meta version immediately above it and skipping ahead will succeed
-	require.NoError(t, s.UpdateCurrent(gun, SampleCustomUpdate(role, tufdata, 2)))
-	require.NoError(t, s.UpdateCurrent(gun, SampleCustomUpdate(role, tufdata, 4)))
+	require.NoError(t, s.UpdateCurrent(gun, MakeUpdate(expected[1])))
+	require.NoError(t, s.UpdateCurrent(gun, MakeUpdate(expected[2])))
 
 	// Inserting a version that already exists, or that is lower than the current version, will fail
 	for _, version := range []int{3, 4} {
-		err := s.UpdateCurrent(gun, SampleCustomUpdate(role, tufdata, version))
+		tufObj := SampleCustomTUFObj(gun, role, version, nil)
+		err := s.UpdateCurrent(gun, MakeUpdate(tufObj))
 		require.Error(t, err, "Error should not be nil")
 		require.IsType(t, &ErrOldVersion{}, err,
 			"Expected ErrOldVersion error type, got: %v", err)
 	}
 
-	expected := []StoredTUFMeta{
-		SampleCustomTUFObj(role, gun, tufdata, 1),
-		SampleCustomTUFObj(role, gun, tufdata, 2),
-		SampleCustomTUFObj(role, gun, tufdata, 4),
-	}
-	assertExpectedTUFMetaInStore(t, s, expected)
+	assertExpectedTUFMetaInStore(t, s, expected[:2], false)
+	assertExpectedTUFMetaInStore(t, s, expected[2:], true)
 	return expected
 }
 
 // UpdateMany succeeds if the updates do not conflict with each other or with what's
 // already in the DB
 func testUpdateManyNoConflicts(t *testing.T, s MetaStore) []StoredTUFMeta {
-	gun, tufdata := "testGUN", []byte("many")
-	expected := make([]StoredTUFMeta, 4)
+	gun := "testGUN"
+	firstBatch := make([]StoredTUFMeta, 4)
 	updates := make([]MetaUpdate, 4)
 	for i, role := range data.BaseRoles {
-		expected[i] = SampleCustomTUFObj(role, gun, tufdata, 1)
-		updates[i] = SampleCustomUpdate(role, tufdata, 1)
+		firstBatch[i] = SampleCustomTUFObj(gun, role, 1, nil)
+		updates[i] = MakeUpdate(firstBatch[i])
 	}
 
 	require.NoError(t, s.UpdateMany(gun, updates))
+	assertExpectedTUFMetaInStore(t, s, firstBatch, true)
 
+	secondBatch := make([]StoredTUFMeta, 4)
 	// no conflicts with what's in DB or with itself
 	for i, role := range data.BaseRoles {
-		expected = append(expected, SampleCustomTUFObj(role, gun, tufdata, 2))
-		updates[i] = SampleCustomUpdate(role, tufdata, 2)
+		secondBatch[i] = SampleCustomTUFObj(gun, role, 2, nil)
+		updates[i] = MakeUpdate(secondBatch[i])
 	}
 
 	require.NoError(t, s.UpdateMany(gun, updates))
+	// the first batch is still there, but are no longer the current ones
+	assertExpectedTUFMetaInStore(t, s, firstBatch, false)
+	assertExpectedTUFMetaInStore(t, s, secondBatch, true)
 
 	// and no conflicts if the same role and gun but different version is included
 	// in the same update.  Even if they're out of order.
+	thirdBatch := make([]StoredTUFMeta, 2)
+	role := data.CanonicalRootRole
 	updates = updates[:2]
 	for i, version := range []int{4, 3} {
-		role := data.CanonicalRootRole
-		expected = append(expected, SampleCustomTUFObj(role, gun, tufdata, version))
-		updates[i] = SampleCustomUpdate(role, tufdata, version)
+		thirdBatch[i] = SampleCustomTUFObj(gun, role, version, nil)
+		updates[i] = MakeUpdate(thirdBatch[i])
 	}
 
 	require.NoError(t, s.UpdateMany(gun, updates))
 
-	assertExpectedTUFMetaInStore(t, s, expected)
-	return expected
+	// all the other data is still there, but are no longer the current ones
+	assertExpectedTUFMetaInStore(t, s, append(firstBatch, secondBatch...), false)
+	assertExpectedTUFMetaInStore(t, s, thirdBatch[:1], true)
+	assertExpectedTUFMetaInStore(t, s, thirdBatch[1:], false)
+
+	return append(append(firstBatch, secondBatch...), thirdBatch...)
 }
 
 // UpdateMany does not insert any rows (or at least rolls them back) if there
@@ -142,9 +160,8 @@ func testUpdateManyConflictRollback(t *testing.T, s MetaStore) []StoredTUFMeta {
 	successBatch := make([]StoredTUFMeta, 4)
 	updates := make([]MetaUpdate, 4)
 	for i, role := range data.BaseRoles {
-		tufdata := []byte(gun + "_" + role + "_1")
-		successBatch[i] = SampleCustomTUFObj(role, gun, tufdata, 1)
-		updates[i] = SampleCustomUpdate(role, tufdata, 1)
+		successBatch[i] = SampleCustomTUFObj(gun, role, 1, nil)
+		updates[i] = MakeUpdate(successBatch[i])
 	}
 
 	require.NoError(t, s.UpdateMany(gun, updates))
@@ -157,22 +174,22 @@ func testUpdateManyConflictRollback(t *testing.T, s MetaStore) []StoredTUFMeta {
 			version = 1
 		}
 		tufdata := []byte(fmt.Sprintf("%s_%s_%d_bad", gun, role, version))
-		badBatch[i] = SampleCustomTUFObj(role, gun, tufdata, version)
-		updates[i] = SampleCustomUpdate(role, tufdata, version)
+		badBatch[i] = SampleCustomTUFObj(gun, role, version, tufdata)
+		updates[i] = MakeUpdate(badBatch[i])
 	}
 
 	err := s.UpdateMany(gun, updates)
 	require.Error(t, err)
 	require.IsType(t, &ErrOldVersion{}, err)
 
 	// self-conflicting, in that it's a duplicate, but otherwise no DB conflicts
-	duplicate := SampleCustomTUFObj(data.CanonicalTimestampRole, gun, []byte("duplicate"), 3)
-	duplicateUpdate := SampleCustomUpdate(duplicate.Role, duplicate.Data, duplicate.Version)
+	duplicate := SampleCustomTUFObj(gun, data.CanonicalTimestampRole, 3, []byte("duplicate"))
+	duplicateUpdate := MakeUpdate(duplicate)
 	err = s.UpdateMany(gun, []MetaUpdate{duplicateUpdate, duplicateUpdate})
 	require.Error(t, err)
 	require.IsType(t, &ErrOldVersion{}, err)
 
-	assertExpectedTUFMetaInStore(t, s, successBatch)
+	assertExpectedTUFMetaInStore(t, s, successBatch, true)
 
 	for _, tufObj := range append(badBatch, duplicate) {
 		checksumBytes := sha256.Sum256(tufObj.Data)
@@ -188,7 +205,7 @@ func testUpdateManyConflictRollback(t *testing.T, s MetaStore) []StoredTUFMeta {
 
 // Delete will remove all TUF metadata, all versions, associated with a gun
 func testDeleteSuccess(t *testing.T, s MetaStore) {
-	tufdata, gun := []byte("hello"), "testGUN"
+	gun := "testGUN"
 	// If there is nothing in the DB, delete is a no-op success
 	require.NoError(t, s.Delete(gun))
 
@@ -197,8 +214,9 @@ func testDeleteSuccess(t *testing.T, s MetaStore) {
 	updates := make([]MetaUpdate, 0, 10)
 	for _, role := range append(data.BaseRoles, "targets/a") {
 		for version := 1; version < 3; version++ {
-			unexpected = append(unexpected, SampleCustomTUFObj(role, gun, tufdata, version))
-			updates = append(updates, SampleCustomUpdate(role, tufdata, version))
+			tufObj := SampleCustomTUFObj(gun, role, version, nil)
+			unexpected = append(unexpected, tufObj)
+			updates = append(updates, MakeUpdate(tufObj))
 		}
 	}
 	require.NoError(t, s.UpdateMany(gun, updates))