Merge pull request #825 from endophage/reimplement_import_export

#806 Part 3: Reimplement import export

.gitignore

@@ -1,3 +1,4 @@
+/.vscode
 /cmd/notary-server/notary-server
 /cmd/notary-server/local.config.json
 /cmd/notary-signer/local.config.json
@@ -8,4 +9,5 @@ cross
 *.swp
 .idea
 *.iml
+*.test
 coverage.out

client/backwards_compatibility_test.go

@@ -13,9 +13,9 @@ import (
 	"time"
 
 	"github.com/docker/notary/passphrase"
+	store "github.com/docker/notary/storage"
 	"github.com/docker/notary/trustpinning"
 	"github.com/docker/notary/tuf/data"
-	"github.com/docker/notary/tuf/store"
 	"github.com/stretchr/testify/require"
 )
 
@@ -101,7 +101,7 @@ func Test0Dot1RepoFormat(t *testing.T) {
 
 	// delete the timestamp metadata, since the server will ignore the uploaded
 	// one and try to create a new one from scratch, which will be the wrong version
-	require.NoError(t, repo.fileStore.RemoveMeta(data.CanonicalTimestampRole))
+	require.NoError(t, repo.fileStore.Remove(data.CanonicalTimestampRole))
 
 	// rotate the timestamp key, since the server doesn't have that one
 	err = repo.RotateKey(data.CanonicalTimestampRole, true)

client/client.go

@@ -16,13 +16,13 @@ import (
 	"github.com/docker/notary"
 	"github.com/docker/notary/client/changelist"
 	"github.com/docker/notary/cryptoservice"
+	store "github.com/docker/notary/storage"
 	"github.com/docker/notary/trustmanager"
 	"github.com/docker/notary/trustpinning"
 	"github.com/docker/notary/tuf"
 	tufclient "github.com/docker/notary/tuf/client"
 	"github.com/docker/notary/tuf/data"
 	"github.com/docker/notary/tuf/signed"
-	"github.com/docker/notary/tuf/store"
 	"github.com/docker/notary/tuf/utils"
 )
 
@@ -159,7 +159,7 @@ func rootCertKey(gun string, privKey data.PrivateKey) (data.PublicKey, error) {
 		return nil, err
 	}
 
-	x509PublicKey := trustmanager.CertToKey(cert)
+	x509PublicKey := utils.CertToKey(cert)
 	if x509PublicKey == nil {
 		return nil, fmt.Errorf(
 			"cannot use regenerated certificate: format %s", cert.PublicKeyAlgorithm)
@@ -676,7 +676,7 @@ func (r *NotaryRepository) publish(cl changelist.Changelist) error {
 		return err
 	}
 
-	return remote.SetMultiMeta(updatedFiles)
+	return remote.SetMulti(updatedFiles)
 }
 
 // bootstrapRepo loads the repository from the local file system (i.e.
@@ -690,7 +690,7 @@ func (r *NotaryRepository) bootstrapRepo() error {
 	logrus.Debugf("Loading trusted collection.")
 
 	for _, role := range data.BaseRoles {
-		jsonBytes, err := r.fileStore.GetMeta(role, store.NoSizeLimit)
+		jsonBytes, err := r.fileStore.GetSized(role, store.NoSizeLimit)
 		if err != nil {
 			if _, ok := err.(store.ErrMetaNotFound); ok &&
 				// server snapshots are supported, and server timestamp management
@@ -722,7 +722,7 @@ func (r *NotaryRepository) saveMetadata(ignoreSnapshot bool) error {
 	if err != nil {
 		return err
 	}
-	err = r.fileStore.SetMeta(data.CanonicalRootRole, rootJSON)
+	err = r.fileStore.Set(data.CanonicalRootRole, rootJSON)
 	if err != nil {
 		return err
 	}
@@ -743,7 +743,7 @@ func (r *NotaryRepository) saveMetadata(ignoreSnapshot bool) error {
 	for role, blob := range targetsToSave {
 		parentDir := filepath.Dir(role)
 		os.MkdirAll(parentDir, 0755)
-		r.fileStore.SetMeta(role, blob)
+		r.fileStore.Set(role, blob)
 	}
 
 	if ignoreSnapshot {
@@ -755,7 +755,7 @@ func (r *NotaryRepository) saveMetadata(ignoreSnapshot bool) error {
 		return err
 	}
 
-	return r.fileStore.SetMeta(data.CanonicalSnapshotRole, snapshotJSON)
+	return r.fileStore.Set(data.CanonicalSnapshotRole, snapshotJSON)
 }
 
 // returns a properly constructed ErrRepositoryNotExist error based on this
@@ -825,7 +825,7 @@ func (r *NotaryRepository) bootstrapClient(checkInitialized bool) (*tufclient.Cl
 	// during update which will cause us to download a new root and perform a rotation.
 	// If we have an old root, and it's valid, then we overwrite the newBuilder to be one
 	// preloaded with the old root or one which uses the old root for trust bootstrapping.
-	if rootJSON, err := r.fileStore.GetMeta(data.CanonicalRootRole, store.NoSizeLimit); err == nil {
+	if rootJSON, err := r.fileStore.GetSized(data.CanonicalRootRole, store.NoSizeLimit); err == nil {
 		// if we can't load the cached root, fail hard because that is how we pin trust
 		if err := oldBuilder.Load(data.CanonicalRootRole, rootJSON, minVersion, true); err != nil {
 			return nil, err
@@ -852,7 +852,7 @@ func (r *NotaryRepository) bootstrapClient(checkInitialized bool) (*tufclient.Cl
 
 		// if remote store successfully set up, try and get root from remote
 		// We don't have any local data to determine the size of root, so try the maximum (though it is restricted at 100MB)
-		tmpJSON, err := remote.GetMeta(data.CanonicalRootRole, store.NoSizeLimit)
+		tmpJSON, err := remote.GetSized(data.CanonicalRootRole, store.NoSizeLimit)
 		if err != nil {
 			// we didn't have a root in cache and were unable to load one from
 			// the server. Nothing we can do but error.
@@ -865,7 +865,7 @@ func (r *NotaryRepository) bootstrapClient(checkInitialized bool) (*tufclient.Cl
 				return nil, err
 			}
 
-			err = r.fileStore.SetMeta(data.CanonicalRootRole, tmpJSON)
+			err = r.fileStore.Set(data.CanonicalRootRole, tmpJSON)
 			if err != nil {
 				// if we can't write cache we should still continue, just log error
 				logrus.Errorf("could not save root to cache: %s", err.Error())

client/client_test.go

@@ -29,11 +29,11 @@ import (
 	"github.com/docker/notary/passphrase"
 	"github.com/docker/notary/server"
 	"github.com/docker/notary/server/storage"
+	store "github.com/docker/notary/storage"
 	"github.com/docker/notary/trustmanager"
 	"github.com/docker/notary/trustpinning"
 	"github.com/docker/notary/tuf/data"
 	"github.com/docker/notary/tuf/signed"
-	"github.com/docker/notary/tuf/store"
 	"github.com/docker/notary/tuf/utils"
 	"github.com/docker/notary/tuf/validation"
 )
@@ -102,7 +102,7 @@ func simpleTestServer(t *testing.T, roles ...string) (
 	mux := http.NewServeMux()
 
 	for _, role := range roles {
-		key, err := trustmanager.GenerateECDSAKey(rand.Reader)
+		key, err := utils.GenerateECDSAKey(rand.Reader)
 		require.NoError(t, err)
 
 		keys[role] = key
@@ -1539,7 +1539,7 @@ func testValidateRootKey(t *testing.T, rootType string) {
 	for _, keyid := range keyids {
 		key, ok := decodedRoot.Keys[keyid]
 		require.True(t, ok, "key id not found in keys")
-		_, err := trustmanager.LoadCertFromPEM(key.Public())
+		_, err := utils.LoadCertFromPEM(key.Public())
 		require.NoError(t, err, "key is not a valid cert")
 	}
 }
@@ -1957,7 +1957,7 @@ func testPublishBadMetadata(t *testing.T, roleName string, repo *NotaryRepositor
 	addTarget(t, repo, "v1", "../fixtures/intermediate-ca.crt")
 
 	// readable, but corrupt file
-	repo.fileStore.SetMeta(roleName, []byte("this isn't JSON"))
+	repo.fileStore.Set(roleName, []byte("this isn't JSON"))
 	err := repo.Publish()
 	if succeeds {
 		require.NoError(t, err)
@@ -2054,7 +2054,7 @@ func createKey(t *testing.T, repo *NotaryRepository, role string, x509 bool) dat
 			privKey, role, start, start.AddDate(1, 0, 0),
 		)
 		require.NoError(t, err)
-		return data.NewECDSAx509PublicKey(trustmanager.CertToPEM(cert))
+		return data.NewECDSAx509PublicKey(utils.CertToPEM(cert))
 	}
 	return key
 }
@@ -2170,7 +2170,7 @@ func testPublishTargetsDelegationScopeFailIfNoKeys(t *testing.T, clearCache bool
 
 	// generate a key that isn't in the cryptoservice, so we can't sign this
 	// one
-	aPrivKey, err := trustmanager.GenerateECDSAKey(rand.Reader)
+	aPrivKey, err := utils.GenerateECDSAKey(rand.Reader)
 	require.NoError(t, err, "error generating key that is not in our cryptoservice")
 	aPubKey := data.PublicKeyFromPrivate(aPrivKey)
 
@@ -3341,19 +3341,19 @@ func TestDeleteRemoteRepo(t *testing.T) {
 	// Try connecting to the remote store directly and make sure that no metadata exists for this gun
 	remoteStore, err := getRemoteStore(repo.baseURL, repo.gun, repo.roundTrip)
 	require.NoError(t, err)
-	meta, err := remoteStore.GetMeta(data.CanonicalRootRole, store.NoSizeLimit)
+	meta, err := remoteStore.GetSized(data.CanonicalRootRole, store.NoSizeLimit)
 	require.Error(t, err)
 	require.IsType(t, store.ErrMetaNotFound{}, err)
 	require.Nil(t, meta)
-	meta, err = remoteStore.GetMeta(data.CanonicalTargetsRole, store.NoSizeLimit)
+	meta, err = remoteStore.GetSized(data.CanonicalTargetsRole, store.NoSizeLimit)
 	require.Error(t, err)
 	require.IsType(t, store.ErrMetaNotFound{}, err)
 	require.Nil(t, meta)
-	meta, err = remoteStore.GetMeta(data.CanonicalSnapshotRole, store.NoSizeLimit)
+	meta, err = remoteStore.GetSized(data.CanonicalSnapshotRole, store.NoSizeLimit)
 	require.Error(t, err)
 	require.IsType(t, store.ErrMetaNotFound{}, err)
 	require.Nil(t, meta)
-	meta, err = remoteStore.GetMeta(data.CanonicalTimestampRole, store.NoSizeLimit)
+	meta, err = remoteStore.GetSized(data.CanonicalTimestampRole, store.NoSizeLimit)
 	require.Error(t, err)
 	require.IsType(t, store.ErrMetaNotFound{}, err)
 	require.Nil(t, meta)
@@ -3364,16 +3364,16 @@ func TestDeleteRemoteRepo(t *testing.T) {
 	requireRepoHasExpectedMetadata(t, longLivingRepo, data.CanonicalSnapshotRole, true)
 	remoteStore, err = getRemoteStore(longLivingRepo.baseURL, longLivingRepo.gun, longLivingRepo.roundTrip)
 	require.NoError(t, err)
-	meta, err = remoteStore.GetMeta(data.CanonicalRootRole, store.NoSizeLimit)
+	meta, err = remoteStore.GetSized(data.CanonicalRootRole, store.NoSizeLimit)
 	require.NoError(t, err)
 	require.NotNil(t, meta)
-	meta, err = remoteStore.GetMeta(data.CanonicalTargetsRole, store.NoSizeLimit)
+	meta, err = remoteStore.GetSized(data.CanonicalTargetsRole, store.NoSizeLimit)
 	require.NoError(t, err)
 	require.NotNil(t, meta)
-	meta, err = remoteStore.GetMeta(data.CanonicalSnapshotRole, store.NoSizeLimit)
+	meta, err = remoteStore.GetSized(data.CanonicalSnapshotRole, store.NoSizeLimit)
 	require.NoError(t, err)
 	require.NotNil(t, meta)
-	meta, err = remoteStore.GetMeta(data.CanonicalTimestampRole, store.NoSizeLimit)
+	meta, err = remoteStore.GetSized(data.CanonicalTimestampRole, store.NoSizeLimit)
 	require.NoError(t, err)
 	require.NotNil(t, meta)
 

client/client_update_test.go

@@ -17,10 +17,10 @@ import (
 	"github.com/docker/go/canonical/json"
 	"github.com/docker/notary"
 	"github.com/docker/notary/passphrase"
+	store "github.com/docker/notary/storage"
 	"github.com/docker/notary/trustpinning"
 	"github.com/docker/notary/tuf/data"
 	"github.com/docker/notary/tuf/signed"
-	"github.com/docker/notary/tuf/store"
 	"github.com/docker/notary/tuf/testutils"
 	"github.com/gorilla/mux"
 	"github.com/stretchr/testify/require"
@@ -66,7 +66,7 @@ func readOnlyServer(t *testing.T, cache store.MetadataStore, notFoundStatus int,
 	m := mux.NewRouter()
 	handler := func(w http.ResponseWriter, r *http.Request) {
 		vars := mux.Vars(r)
-		metaBytes, err := cache.GetMeta(vars["role"], store.NoSizeLimit)
+		metaBytes, err := cache.GetSized(vars["role"], store.NoSizeLimit)
 		if _, ok := err.(store.ErrMetaNotFound); ok {
 			w.WriteHeader(notFoundStatus)
 		} else {
@@ -84,11 +84,11 @@ type unwritableStore struct {
 	roleToNotWrite string
 }
 
-func (u *unwritableStore) SetMeta(role string, serverMeta []byte) error {
+func (u *unwritableStore) Set(role string, serverMeta []byte) error {
 	if role == u.roleToNotWrite {
 		return fmt.Errorf("Non-writable")
 	}
-	return u.MetadataStore.SetMeta(role, serverMeta)
+	return u.MetadataStore.Set(role, serverMeta)
 }
 
 // Update can succeed even if we cannot write any metadata to the repo (assuming
@@ -111,7 +111,7 @@ func TestUpdateSucceedsEvenIfCannotWriteNewRepo(t *testing.T) {
 		require.NoError(t, err)
 
 		for r, expected := range serverMeta {
-			actual, err := repo.fileStore.GetMeta(r, store.NoSizeLimit)
+			actual, err := repo.fileStore.GetSized(r, store.NoSizeLimit)
 			if r == role {
 				require.Error(t, err)
 				require.IsType(t, store.ErrMetaNotFound{}, err,
@@ -158,7 +158,7 @@ func TestUpdateSucceedsEvenIfCannotWriteExistingRepo(t *testing.T) {
 			require.NoError(t, err)
 
 			for r, expected := range serverMeta {
-				actual, err := repo.fileStore.GetMeta(r, store.NoSizeLimit)
+				actual, err := repo.fileStore.GetSized(r, store.NoSizeLimit)
 				require.NoError(t, err, "problem getting repo metadata for %s", r)
 				if role == r {
 					require.False(t, bytes.Equal(expected, actual),
@@ -244,12 +244,12 @@ func TestUpdateReplacesCorruptOrMissingMetadata(t *testing.T) {
 					require.Error(t, err, "%s for %s: expected to error when bootstrapping root", text, role)
 					// revert our original metadata
 					for role := range origMeta {
-						require.NoError(t, repo.fileStore.SetMeta(role, origMeta[role]))
+						require.NoError(t, repo.fileStore.Set(role, origMeta[role]))
 					}
 				} else {
 					require.NoError(t, err)
 					for r, expected := range serverMeta {
-						actual, err := repo.fileStore.GetMeta(r, store.NoSizeLimit)
+						actual, err := repo.fileStore.GetSized(r, store.NoSizeLimit)
 						require.NoError(t, err, "problem getting repo metadata for %s", role)
 						require.True(t, bytes.Equal(expected, actual),
 							"%s for %s: expected to recover after update", text, role)
@@ -298,7 +298,7 @@ func TestUpdateFailsIfServerRootKeyChangedWithoutMultiSign(t *testing.T) {
 		text, messItUp := expt.desc, expt.swizzle
 		for _, forWrite := range []bool{true, false} {
 			require.NoError(t, messItUp(repoSwizzler, data.CanonicalRootRole), "could not fuzz root (%s)", text)
-			messedUpMeta, err := repo.fileStore.GetMeta(data.CanonicalRootRole, store.NoSizeLimit)
+			messedUpMeta, err := repo.fileStore.GetSized(data.CanonicalRootRole, store.NoSizeLimit)
 
 			if _, ok := err.(store.ErrMetaNotFound); ok { // one of the ways to mess up is to delete metadata
 
@@ -307,7 +307,7 @@ func TestUpdateFailsIfServerRootKeyChangedWithoutMultiSign(t *testing.T) {
 				require.NoError(t, err)
 				// revert our original metadata
 				for role := range origMeta {
-					require.NoError(t, repo.fileStore.SetMeta(role, origMeta[role]))
+					require.NoError(t, repo.fileStore.Set(role, origMeta[role]))
 				}
 			} else {
 
@@ -321,7 +321,7 @@ func TestUpdateFailsIfServerRootKeyChangedWithoutMultiSign(t *testing.T) {
 				// same because it has failed to update.
 				for role, expected := range origMeta {
 					if role != data.CanonicalTimestampRole && role != data.CanonicalSnapshotRole {
-						actual, err := repo.fileStore.GetMeta(role, store.NoSizeLimit)
+						actual, err := repo.fileStore.GetSized(role, store.NoSizeLimit)
 						require.NoError(t, err, "problem getting repo metadata for %s", role)
 
 						if role == data.CanonicalRootRole {
@@ -336,7 +336,7 @@ func TestUpdateFailsIfServerRootKeyChangedWithoutMultiSign(t *testing.T) {
 
 			// revert our original root metadata
 			require.NoError(t,
-				repo.fileStore.SetMeta(data.CanonicalRootRole, origMeta[data.CanonicalRootRole]))
+				repo.fileStore.Set(data.CanonicalRootRole, origMeta[data.CanonicalRootRole]))
 		}
 	}
 }
@@ -967,7 +967,7 @@ func waysToMessUpServerNonRootPerRole(t *testing.T) map[string][]swizzleExpectat
 					keyIDs = append(keyIDs, k)
 				}
 				// add the keys from root too
-				rootMeta, err := s.MetadataCache.GetMeta(data.CanonicalRootRole, store.NoSizeLimit)
+				rootMeta, err := s.MetadataCache.GetSized(data.CanonicalRootRole, store.NoSizeLimit)
 				require.NoError(t, err)
 
 				signedRoot := &data.SignedRoot{}
@@ -1349,7 +1349,7 @@ func signSerializeAndUpdateRoot(t *testing.T, signedRoot data.SignedRoot,
 	require.NoError(t, signed.Sign(serverSwizzler.CryptoService, signedObj, keys, len(keys), nil))
 	rootBytes, err := json.Marshal(signedObj)
 	require.NoError(t, err)
-	require.NoError(t, serverSwizzler.MetadataCache.SetMeta(data.CanonicalRootRole, rootBytes))
+	require.NoError(t, serverSwizzler.MetadataCache.Set(data.CanonicalRootRole, rootBytes))
 
 	// update the hashes on both snapshot and timestamp
 	require.NoError(t, serverSwizzler.UpdateSnapshotHashes())
@@ -1374,7 +1374,7 @@ func TestValidateRootRotationWithOldRole(t *testing.T) {
 	// --- key is saved, but doesn't matter at all for rotation if we're already on
 	// --- the root metadata with the 3 keys)
 
-	rootBytes, err := serverSwizzler.MetadataCache.GetMeta(data.CanonicalRootRole, store.NoSizeLimit)
+	rootBytes, err := serverSwizzler.MetadataCache.GetSized(data.CanonicalRootRole, store.NoSizeLimit)
 	require.NoError(t, err)
 	signedRoot := data.SignedRoot{}
 	require.NoError(t, json.Unmarshal(rootBytes, &signedRoot))
@@ -1626,7 +1626,7 @@ func TestRootOnDiskTrustPinning(t *testing.T) {
 	defer os.RemoveAll(repo.baseDir)
 	repo.trustPinning = restrictiveTrustPinning
 	// put root on disk
-	require.NoError(t, repo.fileStore.SetMeta(data.CanonicalRootRole, meta[data.CanonicalRootRole]))
+	require.NoError(t, repo.fileStore.Set(data.CanonicalRootRole, meta[data.CanonicalRootRole]))
 
 	require.NoError(t, repo.Update(false))
 }

client/delegations.go

@@ -8,8 +8,8 @@ import (
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/notary"
 	"github.com/docker/notary/client/changelist"
+	store "github.com/docker/notary/storage"
 	"github.com/docker/notary/tuf/data"
-	"github.com/docker/notary/tuf/store"
 	"github.com/docker/notary/tuf/utils"
 )
 

client/helpers.go

@@ -9,9 +9,9 @@ import (
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/notary/client/changelist"
-	tuf "github.com/docker/notary/tuf"
+	store "github.com/docker/notary/storage"
+	"github.com/docker/notary/tuf"
 	"github.com/docker/notary/tuf/data"
-	"github.com/docker/notary/tuf/store"
 	"github.com/docker/notary/tuf/utils"
 )