diff --git a/Makefile b/Makefile index 7a0401c49..34234baf2 100644 --- a/Makefile +++ b/Makefile @@ -42,11 +42,11 @@ COVERPROFILE?=$(COVERDIR)/cover.out COVERMODE=count PKGS ?= $(shell go list -tags "${NOTARY_BUILDTAGS}" ./... | grep -v /vendor/ | tr '\n' ' ') -.PHONY: clean all fmt vet lint build test binaries cross cover docker-images notary-dockerfile +.PHONY: clean all lint build test binaries cross cover docker-images notary-dockerfile .DELETE_ON_ERROR: cover .DEFAULT: default -all: AUTHORS clean fmt vet fmt lint build test binaries +all: AUTHORS clean lint build test binaries AUTHORS: .git/HEAD git log --format='%aN <%aE>' | sort -fu > $@ @@ -90,41 +90,26 @@ ${PREFIX}/bin/static/notary: @go build -tags ${NOTARY_BUILDTAGS} -o $@ ${GO_LDFLAGS_STATIC} ./cmd/notary endif -vet: - @echo "+ $@" + +# run all lint functionality - excludes Godep directory, vendoring, binaries, python tests, and git files +lint: + @echo "+ $@: golint, go vet, go fmt, misspell, ineffassign" + # golint + @test -z "$(shell find . -type f -name "*.go" -not -path "./vendor/*" -not -name "*.pb.*" -exec golint {} \; | tee /dev/stderr)" + # gofmt + @test -z "$$(gofmt -s -l .| grep -v .pb. | grep -v vendor/ | tee /dev/stderr)" + # govet ifeq ($(shell uname -s), Darwin) @test -z "$(shell find . -iname *test*.go | grep -v _test.go | grep -v vendor | xargs echo "This file should end with '_test':" | tee /dev/stderr)" else @test -z "$(shell find . -iname *test*.go | grep -v _test.go | grep -v vendor | xargs -r echo "This file should end with '_test':" | tee /dev/stderr)" endif @test -z "$$(go tool vet -printf=false . 2>&1 | grep -v vendor/ | tee /dev/stderr)" - -fmt: - @echo "+ $@" - @test -z "$$(gofmt -s -l .| grep -v .pb. | grep -v vendor/ | tee /dev/stderr)" - -lint: - @echo "+ $@" - @test -z "$(shell find . -type f -name "*.go" -not -path "./vendor/*" -not -name "*.pb.*" -exec golint {} \; | tee /dev/stderr)" - -# Requires that the following: -# go get -u github.com/client9/misspell/cmd/misspell -# -# be run first - -# misspell target, don't include Godeps, binaries, python tests, or git files -misspell: - @echo "+ $@" + # misspell - requires that the following be run first: + # go get -u github.com/client9/misspell/cmd/misspell @test -z "$$(find . -name '*' | grep -v vendor/ | grep -v bin/ | grep -v misc/ | grep -v .git/ | xargs misspell | tee /dev/stderr)" - -# Requires that the following: -# go get -u github.com/gordonklaus/ineffassign -# -# be run first - -# ineffassign target, don't include Godeps, binaries, python tests, or git files -ineffassign: - @echo "+ $@" + # ineffassign - requires that the following be run first: + # go get -u github.com/gordonklaus/ineffassign @test -z "$(shell find . -type f -name "*.go" -not -path "./vendor/*" -not -name "*.pb.*" -exec ineffassign {} \; | tee /dev/stderr)" build: @@ -140,16 +125,13 @@ test: @echo go test -tags "${NOTARY_BUILDTAGS}" $(TESTOPTS) $(PKGS) -test-full: TESTOPTS = -test-full: vet lint - @echo Note: when testing with a yubikey plugged in, make sure to include 'TESTOPTS="-p 1"' - @echo "+ $@" - @echo - go test -tags "${NOTARY_BUILDTAGS}" $(TESTOPTS) -v $(PKGS) - integration: TESTDB = mysql integration: - buildscripts/integrationtest.sh development.$(TESTDB).yml + buildscripts/integrationtest.sh $(TESTDB) + +testdb: TESTDB = mysql +testdb: + buildscripts/dbtests.sh $(TESTDB) protos: @protoc --go_out=plugins=grpc:. proto/*.proto diff --git a/buildscripts/circle_parallelism.sh b/buildscripts/circle_parallelism.sh index abd61806b..cc07fdc2d 100755 --- a/buildscripts/circle_parallelism.sh +++ b/buildscripts/circle_parallelism.sh @@ -6,10 +6,12 @@ case $CIRCLE_NODE_INDEX in ;; 1) docker run --rm -e NOTARY_BUILDTAGS=none --env-file buildscripts/env.list --user notary notary_client bash -c "make ci && codecov" ;; -2) SKIPENVCHECK=1 make TESTDB=mysql integration +2) SKIPENVCHECK=1 make TESTDB=mysql testdb + SKIPENVCHECK=1 make TESTDB=mysql integration ;; -3) SKIPENVCHECK=1 make TESTDB=rethink integration +3) SKIPENVCHECK=1 make TESTDB=rethink testdb + SKIPENVCHECK=1 make TESTDB=rethink integration ;; -4) docker run --rm -e NOTARY_BUILDTAGS=pkcs11 notary_client make vet lint fmt misspell ineffassign +4) docker run --rm -e NOTARY_BUILDTAGS=pkcs11 notary_client make lint ;; esac diff --git a/buildscripts/dbtests.sh b/buildscripts/dbtests.sh new file mode 100755 index 000000000..de4c1d1c3 --- /dev/null +++ b/buildscripts/dbtests.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +db="$1" + +case ${db} in + mysql*) + db="mysql" + dbContainerOpts="--name mysql_tests mysql mysqld --innodb_file_per_table" + DBURL="server@tcp(mysql_tests:3306)/notaryserver?parseTime=True" + ;; + rethink*) + db="rethink" + dbContainerOpts="--name rethinkdb_tests rdb-01 --bind all --driver-tls-key /tls/key.pem --driver-tls-cert /tls/cert.pem" + DBURL="rethinkdb_tests" + ;; + *) + echo "Usage: $0 (mysql|rethink)" + exit 1 + ;; +esac + +composeFile="development.${db}.yml" +project=dbtests + +function cleanup { + rm -f bin/notary + docker-compose -p "${project}_${db}" -f "${composeFile}" kill + # if we're in CircleCI, we cannot remove any containers + if [[ -z "${CIRCLECI}" ]]; then + docker-compose -p "${project}_${db}" -f "${composeFile}" down -v --remove-orphans + fi +} + +clientCmd="make test" +if [[ -z "${CIRCLECI}" ]]; then + BUILDOPTS="--force-rm" +else + clientCmd="make ci && codecov" +fi + +set -e +set -x + +cleanup + +docker-compose -p "${project}_${db}" -f ${composeFile} build ${BUILDOPTS} client + +trap cleanup SIGINT SIGTERM EXIT + +# run the unit tests that require a DB + +docker-compose -p "${project}_${db}" -f "${composeFile}" run --no-deps -d ${dbContainerOpts} +docker-compose -p "${project}_${db}" -f "${composeFile}" run --no-deps \ + -e NOTARY_BUILDTAGS="${db}db" -e DBURL="${DBURL}" \ + -e PKGS="github.com/docker/notary/server/storage" \ + client bash -c "${clientCmd}" diff --git a/buildscripts/env.list b/buildscripts/env.list index ee8fcbe7a..9c8bc43e7 100644 --- a/buildscripts/env.list +++ b/buildscripts/env.list @@ -38,3 +38,5 @@ WORKSPACE # TRAVIS_REPO_SLUG # TRAVIS_COMMIT # TRAVIS_BUILD_DIR + +SKIPENVCHECK=1 diff --git a/buildscripts/integrationtest.sh b/buildscripts/integrationtest.sh index 6b6e8d840..18642f860 100755 --- a/buildscripts/integrationtest.sh +++ b/buildscripts/integrationtest.sh @@ -1,13 +1,28 @@ #!/usr/bin/env bash -composeFile="$1" +db="$1" +case ${db} in + mysql*) + db="mysql" + ;; + rethink*) + db="rethink" + ;; + *) + echo "Usage: $0 (mysql|rethink)" + exit 1 + ;; +esac + +composeFile="development.${db}.yml" +project=integration function cleanup { rm -f bin/notary - docker-compose -f $composeFile kill + docker-compose -p "${project}_${db}" -f ${composeFile} kill # if we're in CircleCI, we cannot remove any containers if [[ -z "${CIRCLECI}" ]]; then - docker-compose -f $composeFile down -v --remove-orphans + docker-compose -p "${project}_${db}" -f ${composeFile} down -v --remove-orphans fi } @@ -32,8 +47,9 @@ set -x cleanup -docker-compose -f $composeFile config -docker-compose -f $composeFile build ${BUILDOPTS} --pull | tee -docker-compose -f $composeFile up --abort-on-container-exit +docker-compose -p "${project}_${db}" -f ${composeFile} config +docker-compose -p "${project}_${db}" -f ${composeFile} build ${BUILDOPTS} --pull | tee trap cleanupAndExit SIGINT SIGTERM EXIT + +docker-compose -p "${project}_${db}" -f ${composeFile} up --abort-on-container-exit diff --git a/development.mysql.yml b/development.mysql.yml index 87350b7c8..19e289453 100644 --- a/development.mysql.yml +++ b/development.mysql.yml @@ -42,16 +42,15 @@ services: build: context: . dockerfile: Dockerfile + env_file: buildscripts/env.list command: buildscripts/testclient.sh volumes: - ./test_output:/test_output networks: + - mdb - srv depends_on: - server -volumes: - notary_data: - external: false networks: mdb: external: false diff --git a/development.rethink.yml b/development.rethink.yml index cb170f911..e4d2f3a9d 100644 --- a/development.rethink.yml +++ b/development.rethink.yml @@ -87,13 +87,14 @@ services: - rdb-02 - rdb-03 client: + build: + context: . + dockerfile: Dockerfile volumes: - ./test_output:/test_output networks: - rdb - build: - context: . - dockerfile: Dockerfile + env_file: buildscripts/env.list links: - server:notary-server command: buildscripts/testclient.sh diff --git a/server/storage/database_test.go b/server/storage/database_test.go deleted file mode 100644 index 2c3657a58..000000000 --- a/server/storage/database_test.go +++ /dev/null @@ -1,520 +0,0 @@ -package storage - -import ( - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" - "io/ioutil" - "os" - "testing" - "time" - - "github.com/docker/notary/tuf/data" - "github.com/jinzhu/gorm" - _ "github.com/mattn/go-sqlite3" - "github.com/stretchr/testify/require" -) - -// SampleTUF returns a sample TUFFile with the given Version (ID will have -// to be set independently) -func SampleTUF(version int) TUFFile { - return SampleCustomTUF(data.CanonicalRootRole, "testGUN", []byte("1"), version) -} - -func SampleCustomTUF(role, gun string, data []byte, version int) TUFFile { - checksum := sha256.Sum256(data) - hexChecksum := hex.EncodeToString(checksum[:]) - return TUFFile{ - Gun: gun, - Role: role, - Version: version, - Sha256: hexChecksum, - Data: data, - } -} - -func SampleUpdate(version int) MetaUpdate { - return MetaUpdate{ - Role: "root", - Version: version, - Data: []byte("1"), - } -} - -// SetUpSQLite creates a sqlite database for testing -func SetUpSQLite(t *testing.T, dbDir string) (*gorm.DB, *SQLStorage) { - dbStore, err := NewSQLStorage("sqlite3", dbDir+"test_db") - require.NoError(t, err) - - // Create the DB tables - err = CreateTUFTable(dbStore.DB) - require.NoError(t, err) - - err = CreateKeyTable(dbStore.DB) - require.NoError(t, err) - - // verify that the tables are empty - var count int - for _, model := range [2]interface{}{&TUFFile{}, &Key{}} { - query := dbStore.DB.Model(model).Count(&count) - require.NoError(t, query.Error) - require.Equal(t, 0, count) - } - return &dbStore.DB, dbStore -} - -// TestSQLUpdateCurrent asserts that UpdateCurrent will add a new TUF file -// if no previous version existed. -func TestSQLUpdateCurrentNew(t *testing.T) { - tempBaseDir, err := ioutil.TempDir("", "notary-test-") - gormDB, dbStore := SetUpSQLite(t, tempBaseDir) - defer os.RemoveAll(tempBaseDir) - - // Adding a new TUF file should succeed - err = dbStore.UpdateCurrent("testGUN", SampleUpdate(0)) - require.NoError(t, err, "Creating a row in an empty DB failed.") - - // There should just be one row - var rows []TUFFile - query := gormDB.Select("ID, Gun, Role, Version, Sha256, Data").Find(&rows) - require.NoError(t, query.Error) - - expected := SampleTUF(0) - expected.ID = 1 - require.Equal(t, []TUFFile{expected}, rows) -} - -// TestSQLUpdateCurrentNewVersion asserts that UpdateCurrent will add a -// new (higher) version of an existing TUF file -func TestSQLUpdateCurrentNewVersion(t *testing.T) { - tempBaseDir, err := ioutil.TempDir("", "notary-test-") - gormDB, dbStore := SetUpSQLite(t, tempBaseDir) - defer os.RemoveAll(tempBaseDir) - - // insert row - oldVersion := SampleTUF(0) - query := gormDB.Create(&oldVersion) - require.NoError(t, query.Error, "Creating a row in an empty DB failed.") - - // UpdateCurrent with a newer version should succeed - update := SampleUpdate(2) - err = dbStore.UpdateCurrent("testGUN", update) - require.NoError(t, err, "Creating a row in an empty DB failed.") - - // There should just be one row - var rows []TUFFile - query = gormDB.Select("ID, Gun, Role, Version, Sha256, Data").Find(&rows) - require.NoError(t, query.Error) - - oldVersion.Model = gorm.Model{ID: 1} - expected := SampleTUF(2) - expected.Model = gorm.Model{ID: 2} - require.Equal(t, []TUFFile{oldVersion, expected}, rows) -} - -// TestSQLUpdateCurrentOldVersionError asserts that an error is raised if -// trying to update to an older version of a TUF file. -func TestSQLUpdateCurrentOldVersionError(t *testing.T) { - tempBaseDir, err := ioutil.TempDir("", "notary-test-") - gormDB, dbStore := SetUpSQLite(t, tempBaseDir) - defer os.RemoveAll(tempBaseDir) - - // insert row - newVersion := SampleTUF(3) - query := gormDB.Create(&newVersion) - require.NoError(t, query.Error, "Creating a row in an empty DB failed.") - - // UpdateCurrent should fail due to the version being lower than the - // previous row - err = dbStore.UpdateCurrent("testGUN", SampleUpdate(0)) - require.Error(t, err, "Error should not be nil") - require.IsType(t, &ErrOldVersion{}, err, - "Expected ErrOldVersion error type, got: %v", err) - - // There should just be one row - var rows []TUFFile - query = gormDB.Select("ID, Gun, Role, Version, Sha256, Data").Find(&rows) - require.NoError(t, query.Error) - - newVersion.Model = gorm.Model{ID: 1} - require.Equal(t, []TUFFile{newVersion}, rows) - - dbStore.DB.Close() -} - -// TestSQLUpdateMany asserts that inserting multiple updates succeeds if the -// updates do not conflict with each. -func TestSQLUpdateMany(t *testing.T) { - tempBaseDir, err := ioutil.TempDir("", "notary-test-") - gormDB, dbStore := SetUpSQLite(t, tempBaseDir) - defer os.RemoveAll(tempBaseDir) - - err = dbStore.UpdateMany("testGUN", []MetaUpdate{ - SampleUpdate(0), - { - Role: "targets", - Version: 1, - Data: []byte("2"), - }, - SampleUpdate(2), - }) - require.NoError(t, err, "UpdateMany errored unexpectedly: %v", err) - - gorm1 := SampleTUF(0) - gorm1.ID = 1 - data := []byte("2") - checksum := sha256.Sum256(data) - hexChecksum := hex.EncodeToString(checksum[:]) - gorm2 := TUFFile{ - Model: gorm.Model{ID: 2}, Gun: "testGUN", Role: "targets", - Version: 1, Sha256: hexChecksum, Data: data} - gorm3 := SampleTUF(2) - gorm3.ID = 3 - expected := []TUFFile{gorm1, gorm2, gorm3} - - var rows []TUFFile - query := gormDB.Select("ID, Gun, Role, Version, Sha256, Data").Find(&rows) - require.NoError(t, query.Error) - require.Equal(t, expected, rows) - - dbStore.DB.Close() -} - -// TestSQLUpdateManyVersionOrder asserts that inserting updates with -// non-monotonic versions still succeeds. -func TestSQLUpdateManyVersionOrder(t *testing.T) { - tempBaseDir, err := ioutil.TempDir("", "notary-test-") - gormDB, dbStore := SetUpSQLite(t, tempBaseDir) - defer os.RemoveAll(tempBaseDir) - - err = dbStore.UpdateMany( - "testGUN", []MetaUpdate{SampleUpdate(2), SampleUpdate(0)}) - require.NoError(t, err) - - // the whole transaction should have rolled back, so there should be - // no entries. - gorm1 := SampleTUF(2) - gorm1.ID = 1 - gorm2 := SampleTUF(0) - gorm2.ID = 2 - - var rows []TUFFile - query := gormDB.Select("ID, Gun, Role, Version, Sha256, Data").Find(&rows) - require.NoError(t, query.Error) - require.Equal(t, []TUFFile{gorm1, gorm2}, rows) - - dbStore.DB.Close() -} - -// TestSQLUpdateManyDuplicateRollback asserts that inserting duplicate -// updates fails. -func TestSQLUpdateManyDuplicateRollback(t *testing.T) { - tempBaseDir, err := ioutil.TempDir("", "notary-test-") - gormDB, dbStore := SetUpSQLite(t, tempBaseDir) - defer os.RemoveAll(tempBaseDir) - - update := SampleUpdate(0) - err = dbStore.UpdateMany("testGUN", []MetaUpdate{update, update}) - require.Error( - t, err, "There should be an error updating the same data twice.") - require.IsType(t, &ErrOldVersion{}, err, - "UpdateMany returned wrong error type") - - // the whole transaction should have rolled back, so there should be - // no entries. - var count int - query := gormDB.Model(&TUFFile{}).Count(&count) - require.NoError(t, query.Error) - require.Equal(t, 0, count) - - dbStore.DB.Close() -} - -func TestSQLGetCurrent(t *testing.T) { - tempBaseDir, err := ioutil.TempDir("", "notary-test-") - gormDB, dbStore := SetUpSQLite(t, tempBaseDir) - defer os.RemoveAll(tempBaseDir) - - _, byt, err := dbStore.GetCurrent("testGUN", "root") - require.Nil(t, byt) - require.Error(t, err, "There should be an error Getting an empty table") - require.IsType(t, ErrNotFound{}, err, "Should get a not found error") - - tuf := SampleTUF(0) - query := gormDB.Create(&tuf) - require.NoError(t, query.Error, "Creating a row in an empty DB failed.") - - cDate, byt, err := dbStore.GetCurrent("testGUN", "root") - require.NoError(t, err, "There should not be any errors getting.") - require.Equal(t, []byte("1"), byt, "Returned data was incorrect") - // the update date was sometime wthin the last minute - fmt.Println(cDate) - require.True(t, cDate.After(time.Now().Add(-1*time.Minute))) - require.True(t, cDate.Before(time.Now().Add(5*time.Second))) - - dbStore.DB.Close() -} - -func TestSQLDelete(t *testing.T) { - tempBaseDir, err := ioutil.TempDir("", "notary-test-") - gormDB, dbStore := SetUpSQLite(t, tempBaseDir) - defer os.RemoveAll(tempBaseDir) - - tuf := SampleTUF(0) - query := gormDB.Create(&tuf) - require.NoError(t, query.Error, "Creating a row in an empty DB failed.") - - err = dbStore.Delete("testGUN") - require.NoError(t, err, "There should not be any errors deleting.") - - // verify deletion - var count int - query = gormDB.Model(&TUFFile{}).Count(&count) - require.NoError(t, query.Error) - require.Equal(t, 0, count) - - dbStore.DB.Close() -} - -func TestSQLGetKeyNoKey(t *testing.T) { - tempBaseDir, err := ioutil.TempDir("", "notary-test-") - gormDB, dbStore := SetUpSQLite(t, tempBaseDir) - defer os.RemoveAll(tempBaseDir) - - cipher, public, err := dbStore.GetKey("testGUN", data.CanonicalTimestampRole) - require.Equal(t, "", cipher) - require.Nil(t, public) - require.IsType(t, &ErrNoKey{}, err, - "Expected ErrNoKey from GetKey") - - query := gormDB.Create(&Key{ - Gun: "testGUN", - Role: data.CanonicalTimestampRole, - Cipher: "testCipher", - Public: []byte("1"), - }) - require.NoError( - t, query.Error, "Inserting timestamp into empty DB should succeed") - - cipher, public, err = dbStore.GetKey("testGUN", data.CanonicalTimestampRole) - require.NoError(t, err) - require.Equal(t, "testCipher", cipher, - "Returned cipher was incorrect") - require.Equal(t, []byte("1"), public, "Returned pubkey was incorrect") -} - -func TestSQLSetKeyExists(t *testing.T) { - tempBaseDir, err := ioutil.TempDir("", "notary-test-") - gormDB, dbStore := SetUpSQLite(t, tempBaseDir) - defer os.RemoveAll(tempBaseDir) - - err = dbStore.SetKey("testGUN", data.CanonicalTimestampRole, "testCipher", []byte("1")) - require.NoError(t, err, "Inserting timestamp into empty DB should succeed") - - err = dbStore.SetKey("testGUN", data.CanonicalTimestampRole, "testCipher", []byte("1")) - require.Error(t, err) - require.IsType(t, &ErrKeyExists{}, err, - "Expected ErrKeyExists from SetKey") - - var rows []Key - query := gormDB.Select("ID, Gun, Cipher, Public").Find(&rows) - require.NoError(t, query.Error) - - expected := Key{Gun: "testGUN", Cipher: "testCipher", - Public: []byte("1")} - expected.Model = gorm.Model{ID: 1} - - require.Equal(t, []Key{expected}, rows) - - dbStore.DB.Close() -} - -func TestSQLSetKeyMultipleRoles(t *testing.T) { - tempBaseDir, err := ioutil.TempDir("", "notary-test-") - gormDB, dbStore := SetUpSQLite(t, tempBaseDir) - defer os.RemoveAll(tempBaseDir) - - err = dbStore.SetKey("testGUN", data.CanonicalTimestampRole, "testCipher", []byte("1")) - require.NoError(t, err, "Inserting timestamp into empty DB should succeed") - - err = dbStore.SetKey("testGUN", data.CanonicalSnapshotRole, "testCipher", []byte("1")) - require.NoError(t, err, "Inserting snapshot key into DB with timestamp key should succeed") - - var rows []Key - query := gormDB.Select("ID, Gun, Role, Cipher, Public").Find(&rows) - require.NoError(t, query.Error) - - expectedTS := Key{Gun: "testGUN", Role: "timestamp", Cipher: "testCipher", - Public: []byte("1")} - expectedTS.Model = gorm.Model{ID: 1} - - expectedSN := Key{Gun: "testGUN", Role: "snapshot", Cipher: "testCipher", - Public: []byte("1")} - expectedSN.Model = gorm.Model{ID: 2} - - require.Equal(t, []Key{expectedTS, expectedSN}, rows) - - dbStore.DB.Close() -} - -func TestSQLSetKeyMultipleGuns(t *testing.T) { - tempBaseDir, err := ioutil.TempDir("", "notary-test-") - gormDB, dbStore := SetUpSQLite(t, tempBaseDir) - defer os.RemoveAll(tempBaseDir) - - err = dbStore.SetKey("testGUN", data.CanonicalTimestampRole, "testCipher", []byte("1")) - require.NoError(t, err, "Inserting timestamp into empty DB should succeed") - - err = dbStore.SetKey("testAnotherGUN", data.CanonicalTimestampRole, "testCipher", []byte("1")) - require.NoError(t, err, "Inserting snapshot key into DB with timestamp key should succeed") - - var rows []Key - query := gormDB.Select("ID, Gun, Role, Cipher, Public").Find(&rows) - require.NoError(t, query.Error) - - expected1 := Key{Gun: "testGUN", Role: "timestamp", Cipher: "testCipher", - Public: []byte("1")} - expected1.Model = gorm.Model{ID: 1} - - expected2 := Key{Gun: "testAnotherGUN", Role: "timestamp", Cipher: "testCipher", - Public: []byte("1")} - expected2.Model = gorm.Model{ID: 2} - - require.Equal(t, []Key{expected1, expected2}, rows) - - dbStore.DB.Close() -} - -func TestSQLSetKeySameRoleGun(t *testing.T) { - tempBaseDir, err := ioutil.TempDir("", "notary-test-") - _, dbStore := SetUpSQLite(t, tempBaseDir) - defer os.RemoveAll(tempBaseDir) - - err = dbStore.SetKey("testGUN", data.CanonicalTimestampRole, "testCipher", []byte("1")) - require.NoError(t, err, "Inserting timestamp into empty DB should succeed") - - err = dbStore.SetKey("testGUN", data.CanonicalTimestampRole, "testCipher", []byte("2")) - require.Error(t, err) - require.IsType(t, &ErrKeyExists{}, err, - "Expected ErrKeyExists from SetKey") - - dbStore.DB.Close() -} - -// TestDBCheckHealthTableMissing asserts that the health check fails if one or -// both the tables are missing. -func TestDBCheckHealthTableMissing(t *testing.T) { - tempBaseDir, err := ioutil.TempDir("", "notary-test-") - _, dbStore := SetUpSQLite(t, tempBaseDir) - defer os.RemoveAll(tempBaseDir) - - dbStore.DropTable(&TUFFile{}) - dbStore.DropTable(&Key{}) - - // No tables, health check fails - err = dbStore.CheckHealth() - require.Error(t, err, "Cannot access table:") - - // only one table existing causes health check to fail - CreateTUFTable(dbStore.DB) - err = dbStore.CheckHealth() - require.Error(t, err, "Cannot access table:") - dbStore.DropTable(&TUFFile{}) - - CreateKeyTable(dbStore.DB) - err = dbStore.CheckHealth() - require.Error(t, err, "Cannot access table:") -} - -// TestDBCheckHealthDBCOnnection asserts that if the DB is not connectable, the -// health check fails. -func TestDBCheckHealthDBConnectionFail(t *testing.T) { - tempBaseDir, err := ioutil.TempDir("", "notary-test-") - _, dbStore := SetUpSQLite(t, tempBaseDir) - defer os.RemoveAll(tempBaseDir) - - err = dbStore.Close() - require.NoError(t, err) - - err = dbStore.CheckHealth() - require.Error(t, err, "Cannot access table:") -} - -// TestDBCheckHealthSuceeds asserts that if the DB is connectable and both -// tables exist, the health check succeeds. -func TestDBCheckHealthSucceeds(t *testing.T) { - tempBaseDir, err := ioutil.TempDir("", "notary-test-") - _, dbStore := SetUpSQLite(t, tempBaseDir) - defer os.RemoveAll(tempBaseDir) - - err = dbStore.CheckHealth() - require.NoError(t, err) -} - -func TestDBGetChecksum(t *testing.T) { - tempBaseDir, err := ioutil.TempDir("", "notary-test-") - _, store := SetUpSQLite(t, tempBaseDir) - defer os.RemoveAll(tempBaseDir) - - ts := data.SignedTimestamp{ - Signatures: make([]data.Signature, 0), - Signed: data.Timestamp{ - SignedCommon: data.SignedCommon{ - Type: data.TUFTypes[data.CanonicalTimestampRole], - Version: 1, - Expires: data.DefaultExpires(data.CanonicalTimestampRole), - }, - }, - } - j, err := json.Marshal(&ts) - require.NoError(t, err) - update := MetaUpdate{ - Role: data.CanonicalTimestampRole, - Version: 1, - Data: j, - } - checksumBytes := sha256.Sum256(j) - checksum := hex.EncodeToString(checksumBytes[:]) - - store.UpdateCurrent("gun", update) - - // create and add a newer timestamp. We're going to try and get the one - // created above by checksum - ts = data.SignedTimestamp{ - Signatures: make([]data.Signature, 0), - Signed: data.Timestamp{ - SignedCommon: data.SignedCommon{ - Type: data.TUFTypes[data.CanonicalTimestampRole], - Version: 2, - Expires: data.DefaultExpires(data.CanonicalTimestampRole), - }, - }, - } - newJ, err := json.Marshal(&ts) - require.NoError(t, err) - update = MetaUpdate{ - Role: data.CanonicalTimestampRole, - Version: 2, - Data: newJ, - } - - store.UpdateCurrent("gun", update) - - cDate, data, err := store.GetChecksum("gun", data.CanonicalTimestampRole, checksum) - require.NoError(t, err) - require.EqualValues(t, j, data) - // the creation date was sometime wthin the last minute - require.True(t, cDate.After(time.Now().Add(-1*time.Minute))) - require.True(t, cDate.Before(time.Now().Add(5*time.Second))) -} - -func TestDBGetChecksumNotFound(t *testing.T) { - tempBaseDir, err := ioutil.TempDir("", "notary-test-") - _, store := SetUpSQLite(t, tempBaseDir) - defer os.RemoveAll(tempBaseDir) - - _, _, err = store.GetChecksum("gun", data.CanonicalTimestampRole, "12345") - require.Error(t, err) - require.IsType(t, ErrNotFound{}, err) -} diff --git a/server/storage/memory.go b/server/storage/memory.go index 253dc8a91..3d1fc6d8d 100644 --- a/server/storage/memory.go +++ b/server/storage/memory.go @@ -4,6 +4,7 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "sort" "strings" "sync" "time" @@ -20,11 +21,21 @@ type ver struct { createupdate time.Time } +// we want to keep these sorted by version so that it's in increasing version +// order +type verList []ver + +func (k verList) Len() int { return len(k) } +func (k verList) Swap(i, j int) { k[i], k[j] = k[j], k[i] } +func (k verList) Less(i, j int) bool { + return k[i].version < k[j].version +} + // MemStorage is really just designed for dev and testing. It is very // inefficient in many scenarios type MemStorage struct { lock sync.Mutex - tufMeta map[string][]*ver + tufMeta map[string]verList keys map[string]map[string]*key checksums map[string]map[string]ver } @@ -32,7 +43,7 @@ type MemStorage struct { // NewMemStorage instantiates a memStorage instance func NewMemStorage() *MemStorage { return &MemStorage{ - tufMeta: make(map[string][]*ver), + tufMeta: make(map[string]verList), keys: make(map[string]map[string]*key), checksums: make(map[string]map[string]ver), } @@ -51,7 +62,7 @@ func (st *MemStorage) UpdateCurrent(gun string, update MetaUpdate) error { } } version := ver{version: update.Version, data: update.Data, createupdate: time.Now()} - st.tufMeta[id] = append(st.tufMeta[id], &version) + st.tufMeta[id] = append(st.tufMeta[id], version) checksumBytes := sha256.Sum256(update.Data) checksum := hex.EncodeToString(checksumBytes[:]) @@ -65,10 +76,48 @@ func (st *MemStorage) UpdateCurrent(gun string, update MetaUpdate) error { // UpdateMany updates multiple TUF records func (st *MemStorage) UpdateMany(gun string, updates []MetaUpdate) error { + st.lock.Lock() + defer st.lock.Unlock() + + versioner := make(map[string]map[int]struct{}) + constant := struct{}{} + + // ensure that we only update in one transaction + for _, u := range updates { + id := entryKey(gun, u.Role) + + // prevent duplicate versions of the same role + if _, ok := versioner[u.Role][u.Version]; ok { + return &ErrOldVersion{} + } + if _, ok := versioner[u.Role]; !ok { + versioner[u.Role] = make(map[int]struct{}) + } + versioner[u.Role][u.Version] = constant + + if space, ok := st.tufMeta[id]; ok { + for _, v := range space { + if v.version >= u.Version { + return &ErrOldVersion{} + } + } + } + } + for _, u := range updates { - if err := st.UpdateCurrent(gun, u); err != nil { - return err + id := entryKey(gun, u.Role) + + version := ver{version: u.Version, data: u.Data, createupdate: time.Now()} + st.tufMeta[id] = append(st.tufMeta[id], version) + sort.Sort(st.tufMeta[id]) // ensure that it's sorted + checksumBytes := sha256.Sum256(u.Data) + checksum := hex.EncodeToString(checksumBytes[:]) + + _, ok := st.checksums[gun] + if !ok { + st.checksums[gun] = make(map[string]ver) } + st.checksums[gun][checksum] = version } return nil } diff --git a/server/storage/memory_test.go b/server/storage/memory_test.go index 7de7db124..47c54f396 100644 --- a/server/storage/memory_test.go +++ b/server/storage/memory_test.go @@ -1,3 +1,5 @@ +// +build !mysqldb,!rethinkdb + package storage import ( @@ -7,38 +9,59 @@ import ( "github.com/stretchr/testify/require" ) -func TestUpdateCurrent(t *testing.T) { - s := NewMemStorage() - s.UpdateCurrent("gun", MetaUpdate{"role", 1, []byte("test")}) +func assertExpectedMemoryTUFMeta(t *testing.T, expected []StoredTUFMeta, s *MemStorage) { + for _, tufObj := range expected { + k := entryKey(tufObj.Gun, tufObj.Role) + versionList, ok := s.tufMeta[k] + require.True(t, ok, "Did not find this gun+role in store") + byVersion := make(map[int]ver) + for _, v := range versionList { + byVersion[v.version] = v + } + + v, ok := byVersion[tufObj.Version] + require.True(t, ok, "Did not find version %d in store", tufObj.Version) + require.Equal(t, tufObj.Data, v.data, "Data was incorrect") + } +} - k := entryKey("gun", "role") - gun, ok := s.tufMeta[k] - v := gun[0] - require.True(t, ok, "Did not find gun in store") - require.Equal(t, 1, v.version, "Version mismatch. Expected 1, found %d", v.version) - require.Equal(t, []byte("test"), v.data, "Data was incorrect") +// UpdateCurrent should succeed if there was no previous metadata of the same +// gun and role. They should be gettable. +func TestMemoryUpdateCurrentEmpty(t *testing.T) { + s := NewMemStorage() + expected := testUpdateCurrentEmptyStore(t, s) + assertExpectedMemoryTUFMeta(t, expected, s) } -func TestUpdateMany(t *testing.T) { +// UpdateCurrent will successfully add a new (higher) version of an existing TUF file, +// but will return an error if there is an older version of a TUF file. +func TestMemoryUpdateCurrentVersionCheck(t *testing.T) { s := NewMemStorage() - require.NoError(t, s.UpdateMany("gun", []MetaUpdate{ - {"role1", 1, []byte("test1")}, - {"role2", 1, []byte("test2")}, - })) + expected := testUpdateCurrentVersionCheck(t, s) + assertExpectedMemoryTUFMeta(t, expected, s) +} - _, d, err := s.GetCurrent("gun", "role1") - require.Nil(t, err, "Expected error to be nil") - require.Equal(t, []byte("test1"), d, "Data was incorrect") +// UpdateMany succeeds if the updates do not conflict with each other or with what's +// already in the DB +func TestMemoryUpdateManyNoConflicts(t *testing.T) { + s := NewMemStorage() + expected := testUpdateManyNoConflicts(t, s) + assertExpectedMemoryTUFMeta(t, expected, s) +} - _, d, err = s.GetCurrent("gun", "role2") - require.Nil(t, err, "Expected error to be nil") - require.Equal(t, []byte("test2"), d, "Data was incorrect") +// UpdateMany does not insert any rows (or at least rolls them back) if there +// are any conflicts. +func TestMemoryUpdateManyConflictRollback(t *testing.T) { + s := NewMemStorage() + expected := testUpdateManyConflictRollback(t, s) + assertExpectedMemoryTUFMeta(t, expected, s) +} - // updating even one with an equal version fails - require.IsType(t, &ErrOldVersion{}, s.UpdateMany("gun", []MetaUpdate{ - {"role1", 1, []byte("test1")}, - {"role2", 2, []byte("test2")}, - })) +// Delete will remove all TUF metadata, all versions, associated with a gun +func TestMemoryDeleteSuccess(t *testing.T) { + s := NewMemStorage() + testDeleteSuccess(t, s) + assertExpectedMemoryTUFMeta(t, nil, s) } func TestGetCurrent(t *testing.T) { @@ -53,16 +76,6 @@ func TestGetCurrent(t *testing.T) { require.Equal(t, []byte("test"), d, "Data was incorrect") } -func TestDelete(t *testing.T) { - s := NewMemStorage() - s.UpdateCurrent("gun", MetaUpdate{"role", 1, []byte("test")}) - s.Delete("gun") - - k := entryKey("gun", "role") - _, ok := s.tufMeta[k] - require.False(t, ok, "Found gun in store, should have been deleted") -} - func TestGetTimestampKey(t *testing.T) { s := NewMemStorage() diff --git a/server/storage/mysql_test.go b/server/storage/mysql_test.go new file mode 100644 index 000000000..fc366db80 --- /dev/null +++ b/server/storage/mysql_test.go @@ -0,0 +1,52 @@ +// +build mysqldb + +// Initializes a MySQL DB for testing purposes + +package storage + +import ( + "os" + "testing" + "time" + + "github.com/Sirupsen/logrus" + _ "github.com/go-sql-driver/mysql" + "github.com/jinzhu/gorm" + "github.com/stretchr/testify/require" +) + +func init() { + // Get the MYSQL connection string from an environment variable + dburl := os.Getenv("DBURL") + if dburl == "" { + logrus.Fatal("MYSQL environment variable not set") + } + + for i := 0; i < 30; i++ { + gormDB, err := gorm.Open("mysql", dburl) + if err == nil { + err := gormDB.DB().Ping() + if err == nil { + break + } + } + if i == 29 { + logrus.Fatalf("Unable to connect to %s after 60 seconds", dburl) + } + time.Sleep(2) + } + + sqldbSetup = func(t *testing.T) (*SQLStorage, func()) { + var cleanup = func() { + gormDB, err := gorm.Open("mysql", dburl) + require.NoError(t, err) + + // drop all tables, if they exist + gormDB.DropTable(&TUFFile{}) + gormDB.DropTable(&Key{}) + } + cleanup() + dbStore := SetupSQLDB(t, "mysql", dburl) + return dbStore, cleanup + } +} diff --git a/server/storage/rethink_realdb_test.go b/server/storage/rethink_realdb_test.go new file mode 100644 index 000000000..8dc37c514 --- /dev/null +++ b/server/storage/rethink_realdb_test.go @@ -0,0 +1,119 @@ +// +build rethinkdb + +// Uses a real RethinkDB connection testing purposes + +package storage + +import ( + "os" + "testing" + + "github.com/docker/go-connections/tlsconfig" + "github.com/docker/notary/storage/rethinkdb" + "github.com/docker/notary/tuf/data" + "github.com/stretchr/testify/require" + "gopkg.in/dancannon/gorethink.v2" +) + +var tlsOpts = tlsconfig.Options{InsecureSkipVerify: true} + +func rethinkSessionSetup(t *testing.T) (*gorethink.Session, string) { + // Get the Rethink connection string from an environment variable + rethinkSource := os.Getenv("DBURL") + require.NotEqual(t, "", rethinkSource) + + sess, err := rethinkdb.AdminConnection(tlsOpts, rethinkSource) + require.NoError(t, err) + + return sess, rethinkSource +} + +func rethinkDBSetup(t *testing.T) (RethinkDB, func()) { + session, _ := rethinkSessionSetup(t) + dbName := "testdb" + var cleanup = func() { gorethink.DBDrop(dbName).Exec(session) } + + cleanup() + require.NoError(t, rethinkdb.SetupDB(session, dbName, []rethinkdb.Table{ + TUFFilesRethinkTable, + PubKeysRethinkTable, + })) + return NewRethinkDBStorage(dbName, "", "", session), cleanup +} + +func TestBootstrapSetsUsernamePassword(t *testing.T) { + adminSession, source := rethinkSessionSetup(t) + dbname, username, password := "testdb", "testuser", "testpassword" + otherDB, otherUser, otherPass := "otherdb", "otheruser", "otherpassword" + + // create a separate user with access to a different DB + require.NoError(t, rethinkdb.SetupDB(adminSession, otherDB, nil)) + require.NoError(t, rethinkdb.CreateAndGrantDBUser(adminSession, otherDB, otherUser, otherPass)) + + // Bootstrap + s := NewRethinkDBStorage(dbname, username, password, adminSession) + require.NoError(t, s.Bootstrap()) + + // A user with an invalid password cannot connect to rethink DB at all + _, err := rethinkdb.UserConnection(tlsOpts, source, username, "wrongpass") + require.Error(t, err) + + // the other user cannot access rethink + userSession, err := rethinkdb.UserConnection(tlsOpts, source, otherUser, otherPass) + s = NewRethinkDBStorage(dbname, otherUser, otherPass, userSession) + _, _, err = s.GetCurrent("gun", data.CanonicalRootRole) + require.Error(t, err) + require.IsType(t, gorethink.RQLRuntimeError{}, err) + + // our user can access the DB though + userSession, err = rethinkdb.UserConnection(tlsOpts, source, username, password) + s = NewRethinkDBStorage(dbname, username, password, userSession) + _, _, err = s.GetCurrent("gun", data.CanonicalRootRole) + require.Error(t, err) + require.IsType(t, ErrNotFound{}, err) +} + +// UpdateCurrent will add a new TUF file if no previous version of that gun and role existed. +func TestRethinkUpdateCurrentEmpty(t *testing.T) { + dbStore, cleanup := rethinkDBSetup(t) + defer cleanup() + + testUpdateCurrentEmptyStore(t, dbStore) +} + +// UpdateCurrent will add a new TUF file if the version is higher than previous, but fail +// if the version is equal to or less than the current, whether or not that previous +// version exists +func TestRethinkUpdateCurrentVersionCheck(t *testing.T) { + t.Skip("Currently rethink only errors if the previous version exists - it doesn't check for strictly increasing") + dbStore, cleanup := rethinkDBSetup(t) + defer cleanup() + + testUpdateCurrentVersionCheck(t, dbStore) +} + +// UpdateMany succeeds if the updates do not conflict with each other or with what's +// already in the DB +func TestRethinkUpdateManyNoConflicts(t *testing.T) { + dbStore, cleanup := rethinkDBSetup(t) + defer cleanup() + + testUpdateManyNoConflicts(t, dbStore) +} + +// UpdateMany does not insert any rows (or at least rolls them back) if there +// are any conflicts. +func TestRethinkUpdateManyConflictRollback(t *testing.T) { + dbStore, cleanup := rethinkDBSetup(t) + defer cleanup() + + testUpdateManyConflictRollback(t, dbStore) +} + +// Delete will remove all TUF metadata, all versions, associated with a gun +func TestRethinkDeleteSuccess(t *testing.T) { + dbStore, cleanup := rethinkDBSetup(t) + defer cleanup() + + testDeleteSuccess(t, dbStore) +} diff --git a/server/storage/rethinkdb.go b/server/storage/rethinkdb.go index 357cf1d3b..225a23979 100644 --- a/server/storage/rethinkdb.go +++ b/server/storage/rethinkdb.go @@ -8,6 +8,7 @@ import ( "sort" "time" + "github.com/Sirupsen/logrus" "github.com/docker/notary/storage/rethinkdb" "github.com/docker/notary/tuf/data" "github.com/docker/notary/tuf/utils" @@ -234,7 +235,11 @@ func (rdb RethinkDB) UpdateMany(gun string, updates []MetaUpdate) error { for _, up := range updates { if err := rdb.UpdateCurrentWithTSChecksum(gun, tsChecksum, up); err != nil { // roll back with best-effort deletion, and then error out - rdb.deleteByTSChecksum(tsChecksum) + rollbackErr := rdb.deleteByTSChecksum(tsChecksum) + if rollbackErr != nil { + logrus.Errorf("Unable to rollback DB conflict - items with timestamp_checksum %s: %v", + tsChecksum, rollbackErr) + } return err } } @@ -289,7 +294,7 @@ func (rdb RethinkDB) GetChecksum(gun, role, checksum string) (created *time.Time // error if no metadata exists for the given GUN. func (rdb RethinkDB) Delete(gun string) error { _, err := gorethink.DB(rdb.dbName).Table(RDBTUFFile{}.TableName()).GetAllByIndex( - "gun", []string{gun}, + "gun", gun, ).Delete().RunWrite(rdb.sess) if err != nil { return fmt.Errorf("unable to delete %s from database: %s", gun, err.Error()) @@ -301,7 +306,7 @@ func (rdb RethinkDB) Delete(gun string) error { // from a call to rethinkdb's UpdateMany func (rdb RethinkDB) deleteByTSChecksum(tsChecksum string) error { _, err := gorethink.DB(rdb.dbName).Table(RDBTUFFile{}.TableName()).GetAllByIndex( - "timestamp_checksum", []string{tsChecksum}, + "timestamp_checksum", tsChecksum, ).Delete().RunWrite(rdb.sess) if err != nil { return fmt.Errorf("unable to delete timestamp checksum data: %s from database: %s", tsChecksum, err.Error()) diff --git a/server/storage/models.go b/server/storage/sql_models.go similarity index 100% rename from server/storage/models.go rename to server/storage/sql_models.go diff --git a/server/storage/database.go b/server/storage/sqldb.go similarity index 100% rename from server/storage/database.go rename to server/storage/sqldb.go diff --git a/server/storage/sqldb_test.go b/server/storage/sqldb_test.go new file mode 100644 index 000000000..290a2eb1a --- /dev/null +++ b/server/storage/sqldb_test.go @@ -0,0 +1,408 @@ +package storage + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "testing" + "time" + + "github.com/docker/notary/tuf/data" + "github.com/jinzhu/gorm" + "github.com/stretchr/testify/require" +) + +func SetupSQLDB(t *testing.T, dbtype, dburl string) *SQLStorage { + dbStore, err := NewSQLStorage(dbtype, dburl) + require.NoError(t, err) + + // Create the DB tables + err = CreateTUFTable(dbStore.DB) + require.NoError(t, err) + + err = CreateKeyTable(dbStore.DB) + require.NoError(t, err) + + // verify that the tables are empty + var count int + for _, model := range [2]interface{}{&TUFFile{}, &Key{}} { + query := dbStore.DB.Model(model).Count(&count) + require.NoError(t, query.Error) + require.Equal(t, 0, count) + } + return dbStore +} + +type sqldbSetupFunc func(*testing.T) (*SQLStorage, func()) + +var sqldbSetup sqldbSetupFunc + +// SampleTUF returns a sample TUFFile with the given Version (ID will have +// to be set independently) +func SampleTUF(version int) TUFFile { + return SampleCustomTUF(data.CanonicalRootRole, "testGUN", []byte("1"), version) +} + +func SampleCustomTUF(role, gun string, data []byte, version int) TUFFile { + checksum := sha256.Sum256(data) + hexChecksum := hex.EncodeToString(checksum[:]) + return TUFFile{ + Gun: gun, + Role: role, + Version: version, + Sha256: hexChecksum, + Data: data, + } +} + +func SampleUpdate(version int) MetaUpdate { + return MetaUpdate{ + Role: "root", + Version: version, + Data: []byte("1"), + } +} + +func assertExpectedGormTUFMeta(t *testing.T, expected []StoredTUFMeta, gormDB gorm.DB) { + expectedGorm := make([]TUFFile, len(expected)) + for i, tufObj := range expected { + expectedGorm[i] = TUFFile{ + Model: gorm.Model{ID: uint(i + 1)}, + Gun: tufObj.Gun, + Role: tufObj.Role, + Version: tufObj.Version, + Sha256: tufObj.Sha256, + Data: tufObj.Data, + } + } + + // There should just be one row + var rows []TUFFile + query := gormDB.Select("id, gun, role, version, sha256, data").Find(&rows) + require.NoError(t, query.Error) + // to avoid issues with nil vs zero len list + if len(expectedGorm) == 0 { + require.Len(t, rows, 0) + } else { + require.Equal(t, expectedGorm, rows) + } +} + +// TestSQLUpdateCurrent asserts that UpdateCurrent will add a new TUF file +// if no previous version of that gun and role existed. +func TestSQLUpdateCurrentEmpty(t *testing.T) { + dbStore, cleanup := sqldbSetup(t) + defer cleanup() + + expected := testUpdateCurrentEmptyStore(t, dbStore) + assertExpectedGormTUFMeta(t, expected, dbStore.DB) + + dbStore.DB.Close() +} + +// TestSQLUpdateCurrentNewVersion asserts that UpdateCurrent will add a +// new (higher) version of an existing TUF file, and that an error is raised if +// trying to update to an older version of a TUF file. +func TestSQLUpdateCurrentNewVersion(t *testing.T) { + dbStore, cleanup := sqldbSetup(t) + defer cleanup() + + expected := testUpdateCurrentVersionCheck(t, dbStore) + assertExpectedGormTUFMeta(t, expected, dbStore.DB) + + dbStore.DB.Close() +} + +// TestSQLUpdateManyNoConflicts asserts that inserting multiple updates succeeds if the +// updates do not conflict with each other or with the DB, even if there are +// 2 versions of the same role/gun in a non-monotonic order. +func TestSQLUpdateManyNoConflicts(t *testing.T) { + dbStore, cleanup := sqldbSetup(t) + defer cleanup() + + expected := testUpdateManyNoConflicts(t, dbStore) + assertExpectedGormTUFMeta(t, expected, dbStore.DB) + + dbStore.DB.Close() +} + +// TestSQLUpdateManyConflictRollback asserts that no data ends up in the DB if there is +// a conflict +func TestSQLUpdateManyConflictRollback(t *testing.T) { + dbStore, cleanup := sqldbSetup(t) + defer cleanup() + + expected := testUpdateManyConflictRollback(t, dbStore) + assertExpectedGormTUFMeta(t, expected, dbStore.DB) + + dbStore.DB.Close() +} + +// TestSQLDelete asserts that Delete will remove all TUF metadata, all versions, +// associated with a gun +func TestSQLDelete(t *testing.T) { + dbStore, cleanup := sqldbSetup(t) + defer cleanup() + + testDeleteSuccess(t, dbStore) + assertExpectedGormTUFMeta(t, nil, dbStore.DB) + + dbStore.DB.Close() +} + +func TestSQLGetCurrent(t *testing.T) { + dbStore, cleanup := sqldbSetup(t) + defer cleanup() + + _, byt, err := dbStore.GetCurrent("testGUN", "root") + require.Nil(t, byt) + require.Error(t, err, "There should be an error Getting an empty table") + require.IsType(t, ErrNotFound{}, err, "Should get a not found error") + + tuf := SampleTUF(0) + query := dbStore.DB.Create(&tuf) + require.NoError(t, query.Error, "Creating a row in an empty DB failed.") + + cDate, byt, err := dbStore.GetCurrent("testGUN", "root") + require.NoError(t, err, "There should not be any errors getting.") + require.Equal(t, []byte("1"), byt, "Returned data was incorrect") + // the update date was sometime wthin the last minute + fmt.Println(cDate) + require.True(t, cDate.After(time.Now().Add(-1*time.Minute))) + require.True(t, cDate.Before(time.Now().Add(5*time.Second))) + + dbStore.DB.Close() +} + +func TestSQLGetKeyNoKey(t *testing.T) { + dbStore, cleanup := sqldbSetup(t) + defer cleanup() + + cipher, public, err := dbStore.GetKey("testGUN", data.CanonicalTimestampRole) + require.Equal(t, "", cipher) + require.Nil(t, public) + require.IsType(t, &ErrNoKey{}, err, + "Expected ErrNoKey from GetKey") + + query := dbStore.DB.Create(&Key{ + Gun: "testGUN", + Role: data.CanonicalTimestampRole, + Cipher: "testCipher", + Public: []byte("1"), + }) + require.NoError( + t, query.Error, "Inserting timestamp into empty DB should succeed") + + cipher, public, err = dbStore.GetKey("testGUN", data.CanonicalTimestampRole) + require.NoError(t, err) + require.Equal(t, "testCipher", cipher, + "Returned cipher was incorrect") + require.Equal(t, []byte("1"), public, "Returned pubkey was incorrect") +} + +func TestSQLSetKeyExists(t *testing.T) { + dbStore, cleanup := sqldbSetup(t) + defer cleanup() + + err := dbStore.SetKey("testGUN", data.CanonicalTimestampRole, "testCipher", []byte("1")) + require.NoError(t, err, "Inserting timestamp into empty DB should succeed") + + err = dbStore.SetKey("testGUN", data.CanonicalTimestampRole, "testCipher", []byte("1")) + require.Error(t, err) + require.IsType(t, &ErrKeyExists{}, err, + "Expected ErrKeyExists from SetKey") + + var rows []Key + query := dbStore.DB.Select("id, gun, cipher, public").Find(&rows) + require.NoError(t, query.Error) + + expected := Key{Gun: "testGUN", Cipher: "testCipher", + Public: []byte("1")} + expected.Model = gorm.Model{ID: 1} + + require.Equal(t, []Key{expected}, rows) + + dbStore.DB.Close() +} + +func TestSQLSetKeyMultipleRoles(t *testing.T) { + dbStore, cleanup := sqldbSetup(t) + defer cleanup() + + err := dbStore.SetKey("testGUN", data.CanonicalTimestampRole, "testCipher", []byte("1")) + require.NoError(t, err, "Inserting timestamp into empty DB should succeed") + + err = dbStore.SetKey("testGUN", data.CanonicalSnapshotRole, "testCipher", []byte("1")) + require.NoError(t, err, "Inserting snapshot key into DB with timestamp key should succeed") + + var rows []Key + query := dbStore.DB.Select("id, gun, role, cipher, public").Find(&rows) + require.NoError(t, query.Error) + + expectedTS := Key{Gun: "testGUN", Role: "timestamp", Cipher: "testCipher", + Public: []byte("1")} + expectedTS.Model = gorm.Model{ID: 1} + + expectedSN := Key{Gun: "testGUN", Role: "snapshot", Cipher: "testCipher", + Public: []byte("1")} + expectedSN.Model = gorm.Model{ID: 2} + + require.Equal(t, []Key{expectedTS, expectedSN}, rows) + + dbStore.DB.Close() +} + +func TestSQLSetKeyMultipleGuns(t *testing.T) { + dbStore, cleanup := sqldbSetup(t) + defer cleanup() + + err := dbStore.SetKey("testGUN", data.CanonicalTimestampRole, "testCipher", []byte("1")) + require.NoError(t, err, "Inserting timestamp into empty DB should succeed") + + err = dbStore.SetKey("testAnotherGUN", data.CanonicalTimestampRole, "testCipher", []byte("1")) + require.NoError(t, err, "Inserting snapshot key into DB with timestamp key should succeed") + + var rows []Key + query := dbStore.DB.Select("id, gun, role, cipher, public").Find(&rows) + require.NoError(t, query.Error) + + expected1 := Key{Gun: "testGUN", Role: "timestamp", Cipher: "testCipher", + Public: []byte("1")} + expected1.Model = gorm.Model{ID: 1} + + expected2 := Key{Gun: "testAnotherGUN", Role: "timestamp", Cipher: "testCipher", + Public: []byte("1")} + expected2.Model = gorm.Model{ID: 2} + + require.Equal(t, []Key{expected1, expected2}, rows) + + dbStore.DB.Close() +} + +func TestSQLSetKeySameRoleGun(t *testing.T) { + dbStore, cleanup := sqldbSetup(t) + defer cleanup() + + err := dbStore.SetKey("testGUN", data.CanonicalTimestampRole, "testCipher", []byte("1")) + require.NoError(t, err, "Inserting timestamp into empty DB should succeed") + + err = dbStore.SetKey("testGUN", data.CanonicalTimestampRole, "testCipher", []byte("2")) + require.Error(t, err) + require.IsType(t, &ErrKeyExists{}, err, + "Expected ErrKeyExists from SetKey") + + dbStore.DB.Close() +} + +// TestDBCheckHealthTableMissing asserts that the health check fails if one or +// both the tables are missing. +func TestDBCheckHealthTableMissing(t *testing.T) { + dbStore, cleanup := sqldbSetup(t) + defer cleanup() + + dbStore.DropTable(&TUFFile{}) + dbStore.DropTable(&Key{}) + + // No tables, health check fails + err := dbStore.CheckHealth() + require.Error(t, err, "Cannot access table:") + + // only one table existing causes health check to fail + CreateTUFTable(dbStore.DB) + err = dbStore.CheckHealth() + require.Error(t, err, "Cannot access table:") + dbStore.DropTable(&TUFFile{}) + + CreateKeyTable(dbStore.DB) + err = dbStore.CheckHealth() + require.Error(t, err, "Cannot access table:") +} + +// TestDBCheckHealthDBCOnnection asserts that if the DB is not connectable, the +// health check fails. +func TestDBCheckHealthDBConnectionFail(t *testing.T) { + dbStore, cleanup := sqldbSetup(t) + defer cleanup() + + err := dbStore.Close() + require.NoError(t, err) + + err = dbStore.CheckHealth() + require.Error(t, err, "Cannot access table:") +} + +// TestDBCheckHealthSuceeds asserts that if the DB is connectable and both +// tables exist, the health check succeeds. +func TestDBCheckHealthSucceeds(t *testing.T) { + dbStore, cleanup := sqldbSetup(t) + defer cleanup() + + err := dbStore.CheckHealth() + require.NoError(t, err) +} + +func TestDBGetChecksum(t *testing.T) { + dbStore, cleanup := sqldbSetup(t) + defer cleanup() + + ts := data.SignedTimestamp{ + Signatures: make([]data.Signature, 0), + Signed: data.Timestamp{ + SignedCommon: data.SignedCommon{ + Type: data.TUFTypes[data.CanonicalTimestampRole], + Version: 1, + Expires: data.DefaultExpires(data.CanonicalTimestampRole), + }, + }, + } + j, err := json.Marshal(&ts) + require.NoError(t, err) + update := MetaUpdate{ + Role: data.CanonicalTimestampRole, + Version: 1, + Data: j, + } + checksumBytes := sha256.Sum256(j) + checksum := hex.EncodeToString(checksumBytes[:]) + + dbStore.UpdateCurrent("gun", update) + + // create and add a newer timestamp. We're going to try and get the one + // created above by checksum + ts = data.SignedTimestamp{ + Signatures: make([]data.Signature, 0), + Signed: data.Timestamp{ + SignedCommon: data.SignedCommon{ + Type: data.TUFTypes[data.CanonicalTimestampRole], + Version: 2, + Expires: data.DefaultExpires(data.CanonicalTimestampRole), + }, + }, + } + newJ, err := json.Marshal(&ts) + require.NoError(t, err) + update = MetaUpdate{ + Role: data.CanonicalTimestampRole, + Version: 2, + Data: newJ, + } + + dbStore.UpdateCurrent("gun", update) + + cDate, data, err := dbStore.GetChecksum("gun", data.CanonicalTimestampRole, checksum) + require.NoError(t, err) + require.EqualValues(t, j, data) + // the creation date was sometime wthin the last minute + require.True(t, cDate.After(time.Now().Add(-1*time.Minute))) + require.True(t, cDate.Before(time.Now().Add(5*time.Second))) +} + +func TestDBGetChecksumNotFound(t *testing.T) { + dbStore, cleanup := sqldbSetup(t) + defer cleanup() + + _, _, err := dbStore.GetChecksum("gun", data.CanonicalTimestampRole, "12345") + require.Error(t, err) + require.IsType(t, ErrNotFound{}, err) +} diff --git a/server/storage/sqlite_test.go b/server/storage/sqlite_test.go new file mode 100644 index 000000000..fada11dc3 --- /dev/null +++ b/server/storage/sqlite_test.go @@ -0,0 +1,27 @@ +// +build !mysqldb + +// Initializes an SQLlite DBs for testing purposes + +package storage + +import ( + "io/ioutil" + "os" + "testing" + + _ "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/require" +) + +func sqlite3Setup(t *testing.T) (*SQLStorage, func()) { + tempBaseDir, err := ioutil.TempDir("", "notary-test-") + require.NoError(t, err) + + dbStore := SetupSQLDB(t, "sqlite3", tempBaseDir+"test_db") + var cleanup = func() { os.RemoveAll(tempBaseDir) } + return dbStore, cleanup +} + +func init() { + sqldbSetup = sqlite3Setup +} diff --git a/server/storage/storage_test.go b/server/storage/storage_test.go new file mode 100644 index 000000000..c2245bf30 --- /dev/null +++ b/server/storage/storage_test.go @@ -0,0 +1,236 @@ +package storage + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "testing" + + "github.com/docker/notary/tuf/data" + "github.com/stretchr/testify/require" +) + +type StoredTUFMeta struct { + Gun string + Role string + Sha256 string + Data []byte + Version int +} + +func SampleCustomTUFObj(gun, role string, version int, tufdata []byte) StoredTUFMeta { + if tufdata == nil { + tufdata = []byte(fmt.Sprintf("%s_%s_%d", gun, role, version)) + } + checksum := sha256.Sum256(tufdata) + hexChecksum := hex.EncodeToString(checksum[:]) + return StoredTUFMeta{ + Gun: gun, + Role: role, + Version: version, + Sha256: hexChecksum, + Data: tufdata, + } +} + +func MakeUpdate(tufObj StoredTUFMeta) MetaUpdate { + return MetaUpdate{ + Role: tufObj.Role, + Version: tufObj.Version, + Data: tufObj.Data, + } +} + +func assertExpectedTUFMetaInStore(t *testing.T, s MetaStore, expected []StoredTUFMeta, current bool) { + for _, tufObj := range expected { + if current { + _, tufdata, err := s.GetCurrent(tufObj.Gun, tufObj.Role) + require.NoError(t, err) + require.Equal(t, tufObj.Data, tufdata) + } + + checksumBytes := sha256.Sum256(tufObj.Data) + checksum := hex.EncodeToString(checksumBytes[:]) + + _, tufdata, err := s.GetChecksum(tufObj.Gun, tufObj.Role, checksum) + require.NoError(t, err) + require.Equal(t, tufObj.Data, tufdata) + } +} + +// UpdateCurrent should succeed if there was no previous metadata of the same +// gun and role. They should be gettable. +func testUpdateCurrentEmptyStore(t *testing.T, s MetaStore) []StoredTUFMeta { + expected := make([]StoredTUFMeta, 0, 10) + for _, role := range append(data.BaseRoles, "targets/a") { + for _, gun := range []string{"gun1", "gun2"} { + // Adding a new TUF file should succeed + tufObj := SampleCustomTUFObj(gun, role, 1, nil) + require.NoError(t, s.UpdateCurrent(tufObj.Gun, MakeUpdate(tufObj))) + expected = append(expected, tufObj) + } + } + + assertExpectedTUFMetaInStore(t, s, expected, true) + return expected +} + +// UpdateCurrent will successfully add a new (higher) version of an existing TUF file, +// but will return an error if there is an older version of a TUF file. +func testUpdateCurrentVersionCheck(t *testing.T, s MetaStore) []StoredTUFMeta { + role, gun := data.CanonicalRootRole, "testGUN" + + expected := []StoredTUFMeta{ + SampleCustomTUFObj(gun, role, 1, nil), + SampleCustomTUFObj(gun, role, 2, nil), + SampleCustomTUFObj(gun, role, 4, nil), + } + + // starting meta is version 1 + require.NoError(t, s.UpdateCurrent(gun, MakeUpdate(expected[0]))) + + // inserting meta version immediately above it and skipping ahead will succeed + require.NoError(t, s.UpdateCurrent(gun, MakeUpdate(expected[1]))) + require.NoError(t, s.UpdateCurrent(gun, MakeUpdate(expected[2]))) + + // Inserting a version that already exists, or that is lower than the current version, will fail + for _, version := range []int{3, 4} { + tufObj := SampleCustomTUFObj(gun, role, version, nil) + err := s.UpdateCurrent(gun, MakeUpdate(tufObj)) + require.Error(t, err, "Error should not be nil") + require.IsType(t, &ErrOldVersion{}, err, + "Expected ErrOldVersion error type, got: %v", err) + } + + assertExpectedTUFMetaInStore(t, s, expected[:2], false) + assertExpectedTUFMetaInStore(t, s, expected[2:], true) + return expected +} + +// UpdateMany succeeds if the updates do not conflict with each other or with what's +// already in the DB +func testUpdateManyNoConflicts(t *testing.T, s MetaStore) []StoredTUFMeta { + gun := "testGUN" + firstBatch := make([]StoredTUFMeta, 4) + updates := make([]MetaUpdate, 4) + for i, role := range data.BaseRoles { + firstBatch[i] = SampleCustomTUFObj(gun, role, 1, nil) + updates[i] = MakeUpdate(firstBatch[i]) + } + + require.NoError(t, s.UpdateMany(gun, updates)) + assertExpectedTUFMetaInStore(t, s, firstBatch, true) + + secondBatch := make([]StoredTUFMeta, 4) + // no conflicts with what's in DB or with itself + for i, role := range data.BaseRoles { + secondBatch[i] = SampleCustomTUFObj(gun, role, 2, nil) + updates[i] = MakeUpdate(secondBatch[i]) + } + + require.NoError(t, s.UpdateMany(gun, updates)) + // the first batch is still there, but are no longer the current ones + assertExpectedTUFMetaInStore(t, s, firstBatch, false) + assertExpectedTUFMetaInStore(t, s, secondBatch, true) + + // and no conflicts if the same role and gun but different version is included + // in the same update. Even if they're out of order. + thirdBatch := make([]StoredTUFMeta, 2) + role := data.CanonicalRootRole + updates = updates[:2] + for i, version := range []int{4, 3} { + thirdBatch[i] = SampleCustomTUFObj(gun, role, version, nil) + updates[i] = MakeUpdate(thirdBatch[i]) + } + + require.NoError(t, s.UpdateMany(gun, updates)) + + // all the other data is still there, but are no longer the current ones + assertExpectedTUFMetaInStore(t, s, append(firstBatch, secondBatch...), false) + assertExpectedTUFMetaInStore(t, s, thirdBatch[:1], true) + assertExpectedTUFMetaInStore(t, s, thirdBatch[1:], false) + + return append(append(firstBatch, secondBatch...), thirdBatch...) +} + +// UpdateMany does not insert any rows (or at least rolls them back) if there +// are any conflicts. +func testUpdateManyConflictRollback(t *testing.T, s MetaStore) []StoredTUFMeta { + gun := "testGUN" + successBatch := make([]StoredTUFMeta, 4) + updates := make([]MetaUpdate, 4) + for i, role := range data.BaseRoles { + successBatch[i] = SampleCustomTUFObj(gun, role, 1, nil) + updates[i] = MakeUpdate(successBatch[i]) + } + + require.NoError(t, s.UpdateMany(gun, updates)) + + // conflicts with what's in DB + badBatch := make([]StoredTUFMeta, 4) + for i, role := range data.BaseRoles { + version := 2 + if role == data.CanonicalTargetsRole { + version = 1 + } + tufdata := []byte(fmt.Sprintf("%s_%s_%d_bad", gun, role, version)) + badBatch[i] = SampleCustomTUFObj(gun, role, version, tufdata) + updates[i] = MakeUpdate(badBatch[i]) + } + + err := s.UpdateMany(gun, updates) + require.Error(t, err) + require.IsType(t, &ErrOldVersion{}, err) + + // self-conflicting, in that it's a duplicate, but otherwise no DB conflicts + duplicate := SampleCustomTUFObj(gun, data.CanonicalTimestampRole, 3, []byte("duplicate")) + duplicateUpdate := MakeUpdate(duplicate) + err = s.UpdateMany(gun, []MetaUpdate{duplicateUpdate, duplicateUpdate}) + require.Error(t, err) + require.IsType(t, &ErrOldVersion{}, err) + + assertExpectedTUFMetaInStore(t, s, successBatch, true) + + for _, tufObj := range append(badBatch, duplicate) { + checksumBytes := sha256.Sum256(tufObj.Data) + checksum := hex.EncodeToString(checksumBytes[:]) + + _, _, err = s.GetChecksum(tufObj.Gun, tufObj.Role, checksum) + require.Error(t, err) + require.IsType(t, ErrNotFound{}, err) + } + + return successBatch +} + +// Delete will remove all TUF metadata, all versions, associated with a gun +func testDeleteSuccess(t *testing.T, s MetaStore) { + gun := "testGUN" + // If there is nothing in the DB, delete is a no-op success + require.NoError(t, s.Delete(gun)) + + // If there is data in the DB, all versions are deleted + unexpected := make([]StoredTUFMeta, 0, 10) + updates := make([]MetaUpdate, 0, 10) + for _, role := range append(data.BaseRoles, "targets/a") { + for version := 1; version < 3; version++ { + tufObj := SampleCustomTUFObj(gun, role, version, nil) + unexpected = append(unexpected, tufObj) + updates = append(updates, MakeUpdate(tufObj)) + } + } + require.NoError(t, s.UpdateMany(gun, updates)) + require.NoError(t, s.Delete(gun)) + + for _, tufObj := range unexpected { + _, _, err := s.GetCurrent(tufObj.Gun, tufObj.Role) + require.IsType(t, ErrNotFound{}, err) + + checksumBytes := sha256.Sum256(tufObj.Data) + checksum := hex.EncodeToString(checksumBytes[:]) + + _, _, err = s.GetChecksum(tufObj.Gun, tufObj.Role, checksum) + require.Error(t, err) + require.IsType(t, ErrNotFound{}, err) + } +} diff --git a/signer/keydbstore/keydbstore.go b/signer/keydbstore/sql_keydbstore.go similarity index 100% rename from signer/keydbstore/keydbstore.go rename to signer/keydbstore/sql_keydbstore.go diff --git a/signer/keydbstore/keydbstore_test.go b/signer/keydbstore/sql_keydbstore_test.go similarity index 100% rename from signer/keydbstore/keydbstore_test.go rename to signer/keydbstore/sql_keydbstore_test.go