diff --git a/buildscripts/dbtests.sh b/buildscripts/dbtests.sh index 263ca6360..de4c1d1c3 100755 --- a/buildscripts/dbtests.sh +++ b/buildscripts/dbtests.sh @@ -13,6 +13,10 @@ case ${db} in dbContainerOpts="--name rethinkdb_tests rdb-01 --bind all --driver-tls-key /tls/key.pem --driver-tls-cert /tls/cert.pem" DBURL="rethinkdb_tests" ;; + *) + echo "Usage: $0 (mysql|rethink)" + exit 1 + ;; esac composeFile="development.${db}.yml" diff --git a/buildscripts/integrationtest.sh b/buildscripts/integrationtest.sh index 9e7c575c3..18642f860 100755 --- a/buildscripts/integrationtest.sh +++ b/buildscripts/integrationtest.sh @@ -1,6 +1,19 @@ #!/usr/bin/env bash db="$1" +case ${db} in + mysql*) + db="mysql" + ;; + rethink*) + db="rethink" + ;; + *) + echo "Usage: $0 (mysql|rethink)" + exit 1 + ;; +esac + composeFile="development.${db}.yml" project=integration diff --git a/server/storage/memory.go b/server/storage/memory.go index afe4ea36e..3d1fc6d8d 100644 --- a/server/storage/memory.go +++ b/server/storage/memory.go @@ -4,6 +4,7 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "sort" "strings" "sync" "time" @@ -20,11 +21,21 @@ type ver struct { createupdate time.Time } +// we want to keep these sorted by version so that it's in increasing version +// order +type verList []ver + +func (k verList) Len() int { return len(k) } +func (k verList) Swap(i, j int) { k[i], k[j] = k[j], k[i] } +func (k verList) Less(i, j int) bool { + return k[i].version < k[j].version +} + // MemStorage is really just designed for dev and testing. It is very // inefficient in many scenarios type MemStorage struct { lock sync.Mutex - tufMeta map[string][]*ver + tufMeta map[string]verList keys map[string]map[string]*key checksums map[string]map[string]ver } @@ -32,7 +43,7 @@ type MemStorage struct { // NewMemStorage instantiates a memStorage instance func NewMemStorage() *MemStorage { return &MemStorage{ - tufMeta: make(map[string][]*ver), + tufMeta: make(map[string]verList), keys: make(map[string]map[string]*key), checksums: make(map[string]map[string]ver), } @@ -51,7 +62,7 @@ func (st *MemStorage) UpdateCurrent(gun string, update MetaUpdate) error { } } version := ver{version: update.Version, data: update.Data, createupdate: time.Now()} - st.tufMeta[id] = append(st.tufMeta[id], &version) + st.tufMeta[id] = append(st.tufMeta[id], version) checksumBytes := sha256.Sum256(update.Data) checksum := hex.EncodeToString(checksumBytes[:]) @@ -97,7 +108,8 @@ func (st *MemStorage) UpdateMany(gun string, updates []MetaUpdate) error { id := entryKey(gun, u.Role) version := ver{version: u.Version, data: u.Data, createupdate: time.Now()} - st.tufMeta[id] = append(st.tufMeta[id], &version) + st.tufMeta[id] = append(st.tufMeta[id], version) + sort.Sort(st.tufMeta[id]) // ensure that it's sorted checksumBytes := sha256.Sum256(u.Data) checksum := hex.EncodeToString(checksumBytes[:]) diff --git a/server/storage/memory_test.go b/server/storage/memory_test.go index 94cd80145..47c54f396 100644 --- a/server/storage/memory_test.go +++ b/server/storage/memory_test.go @@ -10,17 +10,18 @@ import ( ) func assertExpectedMemoryTUFMeta(t *testing.T, expected []StoredTUFMeta, s *MemStorage) { - counter := make(map[string]int) for _, tufObj := range expected { k := entryKey(tufObj.Gun, tufObj.Role) - gun, ok := s.tufMeta[k] - require.True(t, len(gun) >= counter[k]) - v := gun[counter[k]] - require.True(t, ok, "Did not find gun in store") - require.Equal(t, tufObj.Version, v.version, "Version mismatch. Expected %d, found %d", - tufObj.Version, v.version) + versionList, ok := s.tufMeta[k] + require.True(t, ok, "Did not find this gun+role in store") + byVersion := make(map[int]ver) + for _, v := range versionList { + byVersion[v.version] = v + } + + v, ok := byVersion[tufObj.Version] + require.True(t, ok, "Did not find version %d in store", tufObj.Version) require.Equal(t, tufObj.Data, v.data, "Data was incorrect") - counter[k]++ } } diff --git a/server/storage/storage_test.go b/server/storage/storage_test.go index 32b8dff3c..c2245bf30 100644 --- a/server/storage/storage_test.go +++ b/server/storage/storage_test.go @@ -18,7 +18,10 @@ type StoredTUFMeta struct { Version int } -func SampleCustomTUFObj(role, gun string, tufdata []byte, version int) StoredTUFMeta { +func SampleCustomTUFObj(gun, role string, version int, tufdata []byte) StoredTUFMeta { + if tufdata == nil { + tufdata = []byte(fmt.Sprintf("%s_%s_%d", gun, role, version)) + } checksum := sha256.Sum256(tufdata) hexChecksum := hex.EncodeToString(checksum[:]) return StoredTUFMeta{ @@ -30,24 +33,26 @@ func SampleCustomTUFObj(role, gun string, tufdata []byte, version int) StoredTUF } } -func SampleCustomUpdate(role string, tufdata []byte, version int) MetaUpdate { +func MakeUpdate(tufObj StoredTUFMeta) MetaUpdate { return MetaUpdate{ - Role: role, - Version: version, - Data: tufdata, + Role: tufObj.Role, + Version: tufObj.Version, + Data: tufObj.Data, } } -func assertExpectedTUFMetaInStore(t *testing.T, s MetaStore, expected []StoredTUFMeta) { +func assertExpectedTUFMetaInStore(t *testing.T, s MetaStore, expected []StoredTUFMeta, current bool) { for _, tufObj := range expected { - _, tufdata, err := s.GetCurrent(tufObj.Gun, tufObj.Role) - require.NoError(t, err) - require.Equal(t, tufObj.Data, tufdata) + if current { + _, tufdata, err := s.GetCurrent(tufObj.Gun, tufObj.Role) + require.NoError(t, err) + require.Equal(t, tufObj.Data, tufdata) + } checksumBytes := sha256.Sum256(tufObj.Data) checksum := hex.EncodeToString(checksumBytes[:]) - _, tufdata, err = s.GetChecksum(tufObj.Gun, tufObj.Role, checksum) + _, tufdata, err := s.GetChecksum(tufObj.Gun, tufObj.Role, checksum) require.NoError(t, err) require.Equal(t, tufObj.Data, tufdata) } @@ -60,79 +65,92 @@ func testUpdateCurrentEmptyStore(t *testing.T, s MetaStore) []StoredTUFMeta { for _, role := range append(data.BaseRoles, "targets/a") { for _, gun := range []string{"gun1", "gun2"} { // Adding a new TUF file should succeed - tufdata := []byte(role + gun) - require.NoError(t, s.UpdateCurrent(gun, SampleCustomUpdate(role, tufdata, 1))) - expected = append(expected, SampleCustomTUFObj(role, gun, tufdata, 1)) + tufObj := SampleCustomTUFObj(gun, role, 1, nil) + require.NoError(t, s.UpdateCurrent(tufObj.Gun, MakeUpdate(tufObj))) + expected = append(expected, tufObj) } } - assertExpectedTUFMetaInStore(t, s, expected) + assertExpectedTUFMetaInStore(t, s, expected, true) return expected } // UpdateCurrent will successfully add a new (higher) version of an existing TUF file, // but will return an error if there is an older version of a TUF file. func testUpdateCurrentVersionCheck(t *testing.T, s MetaStore) []StoredTUFMeta { - role, gun, tufdata := data.CanonicalRootRole, "testGUN", []byte("1") + role, gun := data.CanonicalRootRole, "testGUN" + + expected := []StoredTUFMeta{ + SampleCustomTUFObj(gun, role, 1, nil), + SampleCustomTUFObj(gun, role, 2, nil), + SampleCustomTUFObj(gun, role, 4, nil), + } // starting meta is version 1 - require.NoError(t, s.UpdateCurrent(gun, SampleCustomUpdate(role, tufdata, 1))) + require.NoError(t, s.UpdateCurrent(gun, MakeUpdate(expected[0]))) // inserting meta version immediately above it and skipping ahead will succeed - require.NoError(t, s.UpdateCurrent(gun, SampleCustomUpdate(role, tufdata, 2))) - require.NoError(t, s.UpdateCurrent(gun, SampleCustomUpdate(role, tufdata, 4))) + require.NoError(t, s.UpdateCurrent(gun, MakeUpdate(expected[1]))) + require.NoError(t, s.UpdateCurrent(gun, MakeUpdate(expected[2]))) // Inserting a version that already exists, or that is lower than the current version, will fail for _, version := range []int{3, 4} { - err := s.UpdateCurrent(gun, SampleCustomUpdate(role, tufdata, version)) + tufObj := SampleCustomTUFObj(gun, role, version, nil) + err := s.UpdateCurrent(gun, MakeUpdate(tufObj)) require.Error(t, err, "Error should not be nil") require.IsType(t, &ErrOldVersion{}, err, "Expected ErrOldVersion error type, got: %v", err) } - expected := []StoredTUFMeta{ - SampleCustomTUFObj(role, gun, tufdata, 1), - SampleCustomTUFObj(role, gun, tufdata, 2), - SampleCustomTUFObj(role, gun, tufdata, 4), - } - assertExpectedTUFMetaInStore(t, s, expected) + assertExpectedTUFMetaInStore(t, s, expected[:2], false) + assertExpectedTUFMetaInStore(t, s, expected[2:], true) return expected } // UpdateMany succeeds if the updates do not conflict with each other or with what's // already in the DB func testUpdateManyNoConflicts(t *testing.T, s MetaStore) []StoredTUFMeta { - gun, tufdata := "testGUN", []byte("many") - expected := make([]StoredTUFMeta, 4) + gun := "testGUN" + firstBatch := make([]StoredTUFMeta, 4) updates := make([]MetaUpdate, 4) for i, role := range data.BaseRoles { - expected[i] = SampleCustomTUFObj(role, gun, tufdata, 1) - updates[i] = SampleCustomUpdate(role, tufdata, 1) + firstBatch[i] = SampleCustomTUFObj(gun, role, 1, nil) + updates[i] = MakeUpdate(firstBatch[i]) } require.NoError(t, s.UpdateMany(gun, updates)) + assertExpectedTUFMetaInStore(t, s, firstBatch, true) + secondBatch := make([]StoredTUFMeta, 4) // no conflicts with what's in DB or with itself for i, role := range data.BaseRoles { - expected = append(expected, SampleCustomTUFObj(role, gun, tufdata, 2)) - updates[i] = SampleCustomUpdate(role, tufdata, 2) + secondBatch[i] = SampleCustomTUFObj(gun, role, 2, nil) + updates[i] = MakeUpdate(secondBatch[i]) } require.NoError(t, s.UpdateMany(gun, updates)) + // the first batch is still there, but are no longer the current ones + assertExpectedTUFMetaInStore(t, s, firstBatch, false) + assertExpectedTUFMetaInStore(t, s, secondBatch, true) // and no conflicts if the same role and gun but different version is included // in the same update. Even if they're out of order. + thirdBatch := make([]StoredTUFMeta, 2) + role := data.CanonicalRootRole updates = updates[:2] for i, version := range []int{4, 3} { - role := data.CanonicalRootRole - expected = append(expected, SampleCustomTUFObj(role, gun, tufdata, version)) - updates[i] = SampleCustomUpdate(role, tufdata, version) + thirdBatch[i] = SampleCustomTUFObj(gun, role, version, nil) + updates[i] = MakeUpdate(thirdBatch[i]) } require.NoError(t, s.UpdateMany(gun, updates)) - assertExpectedTUFMetaInStore(t, s, expected) - return expected + // all the other data is still there, but are no longer the current ones + assertExpectedTUFMetaInStore(t, s, append(firstBatch, secondBatch...), false) + assertExpectedTUFMetaInStore(t, s, thirdBatch[:1], true) + assertExpectedTUFMetaInStore(t, s, thirdBatch[1:], false) + + return append(append(firstBatch, secondBatch...), thirdBatch...) } // UpdateMany does not insert any rows (or at least rolls them back) if there @@ -142,9 +160,8 @@ func testUpdateManyConflictRollback(t *testing.T, s MetaStore) []StoredTUFMeta { successBatch := make([]StoredTUFMeta, 4) updates := make([]MetaUpdate, 4) for i, role := range data.BaseRoles { - tufdata := []byte(gun + "_" + role + "_1") - successBatch[i] = SampleCustomTUFObj(role, gun, tufdata, 1) - updates[i] = SampleCustomUpdate(role, tufdata, 1) + successBatch[i] = SampleCustomTUFObj(gun, role, 1, nil) + updates[i] = MakeUpdate(successBatch[i]) } require.NoError(t, s.UpdateMany(gun, updates)) @@ -157,8 +174,8 @@ func testUpdateManyConflictRollback(t *testing.T, s MetaStore) []StoredTUFMeta { version = 1 } tufdata := []byte(fmt.Sprintf("%s_%s_%d_bad", gun, role, version)) - badBatch[i] = SampleCustomTUFObj(role, gun, tufdata, version) - updates[i] = SampleCustomUpdate(role, tufdata, version) + badBatch[i] = SampleCustomTUFObj(gun, role, version, tufdata) + updates[i] = MakeUpdate(badBatch[i]) } err := s.UpdateMany(gun, updates) @@ -166,13 +183,13 @@ func testUpdateManyConflictRollback(t *testing.T, s MetaStore) []StoredTUFMeta { require.IsType(t, &ErrOldVersion{}, err) // self-conflicting, in that it's a duplicate, but otherwise no DB conflicts - duplicate := SampleCustomTUFObj(data.CanonicalTimestampRole, gun, []byte("duplicate"), 3) - duplicateUpdate := SampleCustomUpdate(duplicate.Role, duplicate.Data, duplicate.Version) + duplicate := SampleCustomTUFObj(gun, data.CanonicalTimestampRole, 3, []byte("duplicate")) + duplicateUpdate := MakeUpdate(duplicate) err = s.UpdateMany(gun, []MetaUpdate{duplicateUpdate, duplicateUpdate}) require.Error(t, err) require.IsType(t, &ErrOldVersion{}, err) - assertExpectedTUFMetaInStore(t, s, successBatch) + assertExpectedTUFMetaInStore(t, s, successBatch, true) for _, tufObj := range append(badBatch, duplicate) { checksumBytes := sha256.Sum256(tufObj.Data) @@ -188,7 +205,7 @@ func testUpdateManyConflictRollback(t *testing.T, s MetaStore) []StoredTUFMeta { // Delete will remove all TUF metadata, all versions, associated with a gun func testDeleteSuccess(t *testing.T, s MetaStore) { - tufdata, gun := []byte("hello"), "testGUN" + gun := "testGUN" // If there is nothing in the DB, delete is a no-op success require.NoError(t, s.Delete(gun)) @@ -197,8 +214,9 @@ func testDeleteSuccess(t *testing.T, s MetaStore) { updates := make([]MetaUpdate, 0, 10) for _, role := range append(data.BaseRoles, "targets/a") { for version := 1; version < 3; version++ { - unexpected = append(unexpected, SampleCustomTUFObj(role, gun, tufdata, version)) - updates = append(updates, SampleCustomUpdate(role, tufdata, version)) + tufObj := SampleCustomTUFObj(gun, role, version, nil) + unexpected = append(unexpected, tufObj) + updates = append(updates, MakeUpdate(tufObj)) } } require.NoError(t, s.UpdateMany(gun, updates))