From 4a9a862716bf5595a4439e1f13bfabd27c59a8ca Mon Sep 17 00:00:00 2001 From: Wilson Ding Date: Fri, 5 May 2017 17:26:58 -0500 Subject: [PATCH 01/21] Fixes #1148 incorrect markdown formatting Signed-off-by: Wilson Ding --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c650e30050..64d2326f9b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing to notary -## Before reporting an issue... +## Before reporting an issue... ### If your problem is with... @@ -26,7 +26,7 @@ By following these simple rules you will get better and faster feedback on your - search the bugtracker for an already reported issue -### If you found an issue that describes your problem: +### If you found an issue that describes your problem: - please read other user comments first, and confirm this is the same issue: a given error condition might be indicative of different problems - you may also find a workaround in the comments - please refrain from adding "same thing here" or "+1" comments From ef07f200876b6c467250d2a324232c1aa3ee522c Mon Sep 17 00:00:00 2001 From: Ashwini Oruganti Date: Mon, 8 May 2017 11:29:25 -0700 Subject: [PATCH 02/21] Upgrade and pin mattes/migrate to the v3.0.0 tag Signed-off-by: Ashwini Oruganti --- server.Dockerfile | 4 ++-- signer.Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server.Dockerfile b/server.Dockerfile index 64606231c1..0aa3cfa57f 100644 --- a/server.Dockerfile +++ b/server.Dockerfile @@ -3,8 +3,8 @@ MAINTAINER David Lawrence "david.lawrence@docker.com" RUN apk add --update git gcc libc-dev && rm -rf /var/cache/apk/* -# Pin to the specific v1 version -RUN git clone -b v1 https://github.com/mattes/migrate.git /go/src/github.com/mattes/migrate/ && \ +# Pin to the specific v3.0.0 version +RUN git clone -b v3.0.0 https://github.com/mattes/migrate.git /go/src/github.com/mattes/migrate/ && \ go get github.com/mattes/migrate && \ go build -tags 'mysql' -o /usr/local/bin/migrate github.com/mattes/migrate diff --git a/signer.Dockerfile b/signer.Dockerfile index 77e20342bb..bf4afa1886 100644 --- a/signer.Dockerfile +++ b/signer.Dockerfile @@ -3,8 +3,8 @@ MAINTAINER David Lawrence "david.lawrence@docker.com" RUN apk add --update git gcc libc-dev && rm -rf /var/cache/apk/* -# Pin to the specific v1 version -RUN git clone -b v1 https://github.com/mattes/migrate.git /go/src/github.com/mattes/migrate/ && \ +# Pin to the specific v3.0.0 version +RUN git clone -b v3.0.0 https://github.com/mattes/migrate.git /go/src/github.com/mattes/migrate/ && \ go get github.com/mattes/migrate && \ go build -tags 'mysql' -o /usr/local/bin/migrate github.com/mattes/migrate From acb275a6c50cb174bdec83907b6571549e511232 Mon Sep 17 00:00:00 2001 From: David Lawrence Date: Mon, 8 May 2017 14:42:51 -0700 Subject: [PATCH 03/21] attempting to fix migration Signed-off-by: David Lawrence (github: endophage) --- migrations/migrate.sh | 14 +++++++------- server.Dockerfile | 4 +--- signer.Dockerfile | 4 +--- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/migrations/migrate.sh b/migrations/migrate.sh index 928b3b93e4..8c3b0ecbbe 100755 --- a/migrations/migrate.sh +++ b/migrations/migrate.sh @@ -10,7 +10,7 @@ case $SERVICE_NAME in MIGRATIONS_PATH=${MIGRATIONS_PATH:-migrations/server/mysql} DB_URL=${DB_URL:-mysql://server@tcp(mysql:3306)/notaryserver} # have to poll for DB to come up - until migrate -path=$MIGRATIONS_PATH -url=$DB_URL version > /dev/null + until migrate -path=$MIGRATIONS_PATH -database=$DB_URL version > /dev/null do iter=$(( iter+1 )) if [[ $iter -gt 30 ]]; then @@ -20,8 +20,8 @@ case $SERVICE_NAME in echo "waiting for $DB_URL to come up." sleep 1 done - pre=$(migrate -path=$MIGRATIONS_PATH -url="${DB_URL}" version) - if migrate -path=$MIGRATIONS_PATH -url="${DB_URL}" up ; then + pre=$(migrate -path=$MIGRATIONS_PATH -database="${DB_URL}" version) + if migrate -path=$MIGRATIONS_PATH -database="${DB_URL}" up ; then post=$(migrate -path=$MIGRATIONS_PATH -url="${DB_URL}" version) if [ "$pre" != "$post" ]; then echo "notaryserver database migrated to latest version" @@ -37,7 +37,7 @@ case $SERVICE_NAME in MIGRATIONS_PATH=${MIGRATIONS_PATH:-migrations/signer/mysql} DB_URL=${DB_URL:-mysql://signer@tcp(mysql:3306)/notarysigner} # have to poll for DB to come up - until migrate -path=$MIGRATIONS_PATH -url=$DB_URL up version > /dev/null + until migrate -path=$MIGRATIONS_PATH -database=$DB_URL up version > /dev/null do iter=$(( iter+1 )) if [[ $iter -gt 30 ]]; then @@ -47,9 +47,9 @@ case $SERVICE_NAME in echo "waiting for $DB_URL to come up." sleep 1 done - pre=$(migrate -path=$MIGRATIONS_PATH -url="${DB_URL}" version) - if migrate -path=$MIGRATIONS_PATH -url="${DB_URL}" up ; then - post=$(migrate -path=$MIGRATIONS_PATH -url="${DB_URL}" version) + pre=$(migrate -path=$MIGRATIONS_PATH -database="${DB_URL}" version) + if migrate -path=$MIGRATIONS_PATH -database="${DB_URL}" up ; then + post=$(migrate -path=$MIGRATIONS_PATH -database="${DB_URL}" version) if [ "$pre" != "$post" ]; then echo "notarysigner database migrated to latest version" else diff --git a/server.Dockerfile b/server.Dockerfile index 0aa3cfa57f..92135bebe8 100644 --- a/server.Dockerfile +++ b/server.Dockerfile @@ -4,9 +4,7 @@ MAINTAINER David Lawrence "david.lawrence@docker.com" RUN apk add --update git gcc libc-dev && rm -rf /var/cache/apk/* # Pin to the specific v3.0.0 version -RUN git clone -b v3.0.0 https://github.com/mattes/migrate.git /go/src/github.com/mattes/migrate/ && \ - go get github.com/mattes/migrate && \ - go build -tags 'mysql' -o /usr/local/bin/migrate github.com/mattes/migrate +RUN go get -tags 'mysql postgres file' github.com/mattes/migrate/cli && mv /go/bin/cli /go/bin/migrate ENV NOTARYPKG github.com/docker/notary diff --git a/signer.Dockerfile b/signer.Dockerfile index bf4afa1886..da4963b4b5 100644 --- a/signer.Dockerfile +++ b/signer.Dockerfile @@ -4,9 +4,7 @@ MAINTAINER David Lawrence "david.lawrence@docker.com" RUN apk add --update git gcc libc-dev && rm -rf /var/cache/apk/* # Pin to the specific v3.0.0 version -RUN git clone -b v3.0.0 https://github.com/mattes/migrate.git /go/src/github.com/mattes/migrate/ && \ - go get github.com/mattes/migrate && \ - go build -tags 'mysql' -o /usr/local/bin/migrate github.com/mattes/migrate +RUN go get -tags 'mysql postgres file' github.com/mattes/migrate/cli && mv /go/bin/cli /go/bin/migrate ENV NOTARYPKG github.com/docker/notary From abdd22478b4c06c17ca22e3eb493a102ecba0a3a Mon Sep 17 00:00:00 2001 From: Ashwini Oruganti Date: Mon, 8 May 2017 16:00:20 -0700 Subject: [PATCH 04/21] Migrate tool now returns an actual error value` Signed-off-by: Ashwini Oruganti --- migrations/migrate.sh | 30 ++++-------------------------- 1 file changed, 4 insertions(+), 26 deletions(-) diff --git a/migrations/migrate.sh b/migrations/migrate.sh index 8c3b0ecbbe..9fd988dcba 100755 --- a/migrations/migrate.sh +++ b/migrations/migrate.sh @@ -10,7 +10,7 @@ case $SERVICE_NAME in MIGRATIONS_PATH=${MIGRATIONS_PATH:-migrations/server/mysql} DB_URL=${DB_URL:-mysql://server@tcp(mysql:3306)/notaryserver} # have to poll for DB to come up - until migrate -path=$MIGRATIONS_PATH -database=$DB_URL version > /dev/null + until migrate -path=$MIGRATIONS_PATH -database="${DB_URL}" up do iter=$(( iter+1 )) if [[ $iter -gt 30 ]]; then @@ -20,24 +20,13 @@ case $SERVICE_NAME in echo "waiting for $DB_URL to come up." sleep 1 done - pre=$(migrate -path=$MIGRATIONS_PATH -database="${DB_URL}" version) - if migrate -path=$MIGRATIONS_PATH -database="${DB_URL}" up ; then - post=$(migrate -path=$MIGRATIONS_PATH -url="${DB_URL}" version) - if [ "$pre" != "$post" ]; then - echo "notaryserver database migrated to latest version" - else - echo "notaryserver database already at latest version" - fi - else - echo "notaryserver database migration failed" - exit 1 - fi + echo "notaryserver database migrated to latest version" ;; notary_signer) MIGRATIONS_PATH=${MIGRATIONS_PATH:-migrations/signer/mysql} DB_URL=${DB_URL:-mysql://signer@tcp(mysql:3306)/notarysigner} # have to poll for DB to come up - until migrate -path=$MIGRATIONS_PATH -database=$DB_URL up version > /dev/null + until migrate -path=$MIGRATIONS_PATH -database="${DB_URL}" up do iter=$(( iter+1 )) if [[ $iter -gt 30 ]]; then @@ -47,17 +36,6 @@ case $SERVICE_NAME in echo "waiting for $DB_URL to come up." sleep 1 done - pre=$(migrate -path=$MIGRATIONS_PATH -database="${DB_URL}" version) - if migrate -path=$MIGRATIONS_PATH -database="${DB_URL}" up ; then - post=$(migrate -path=$MIGRATIONS_PATH -database="${DB_URL}" version) - if [ "$pre" != "$post" ]; then - echo "notarysigner database migrated to latest version" - else - echo "notarysigner database already at latest version" - fi - else - echo "notarysigner database migration failed" - exit 1 - fi + echo "notarysigner database migrated to latest version" ;; esac From caf4d9ef5cb10140764bb14389347459d265729d Mon Sep 17 00:00:00 2001 From: Ashwini Oruganti Date: Wed, 3 May 2017 14:11:06 -0700 Subject: [PATCH 05/21] Expose target custom data in the NotaryRepository API - Update Target definition and NewTarget to store custom data - tufAdd and tufAddByHash now take a filename as a flag, open and read the file, and pass the custom data bytes to NewTarget. - Client APIs now understand Target.Custom - Test that client lists and gets targets with custom data - getTargetCustom should return a nil RawMessage, not a []byte - Test tufAdd with target custom data - Trigger a usage error for the tufLookup command in a test Signed-off-by: Ashwini Oruganti --- client/client.go | 38 ++++++++--- client/client_test.go | 60 +++++++++++++++-- cmd/notary/integration_test.go | 116 ++++++++++++++++++++++++++++++++- cmd/notary/tuf.go | 37 ++++++++++- 4 files changed, 235 insertions(+), 16 deletions(-) diff --git a/client/client.go b/client/client.go index 55a5ba4b75..39228cff84 100644 --- a/client/client.go +++ b/client/client.go @@ -13,6 +13,7 @@ import ( "time" "github.com/Sirupsen/logrus" + canonicaljson "github.com/docker/go/canonical/json" "github.com/docker/notary" "github.com/docker/notary/client/changelist" "github.com/docker/notary/cryptoservice" @@ -128,9 +129,10 @@ func (r *NotaryRepository) GetGUN() data.GUN { // Target represents a simplified version of the data TUF operates on, so external // applications don't have to depend on TUF data types. type Target struct { - Name string // the name of the target - Hashes data.Hashes // the hash of the target - Length int64 // the size in bytes of the target + Name string // the name of the target + Hashes data.Hashes // the hash of the target + Length int64 // the size in bytes of the target + Custom canonicaljson.RawMessage // the custom data provided to describe the file at TARGETPATH } // TargetWithRole represents a Target that exists in a particular role - this is @@ -141,7 +143,7 @@ type TargetWithRole struct { } // NewTarget is a helper method that returns a Target -func NewTarget(targetName string, targetPath string) (*Target, error) { +func NewTarget(targetName, targetPath string, targetCustom canonicaljson.RawMessage) (*Target, error) { b, err := ioutil.ReadFile(targetPath) if err != nil { return nil, err @@ -152,7 +154,7 @@ func NewTarget(targetName string, targetPath string) (*Target, error) { return nil, err } - return &Target{Name: targetName, Hashes: meta.Hashes, Length: meta.Length}, nil + return &Target{Name: targetName, Hashes: meta.Hashes, Length: meta.Length, Custom: targetCustom}, nil } func rootCertKey(gun data.GUN, privKey data.PrivateKey) (data.PublicKey, error) { @@ -360,7 +362,14 @@ func (r *NotaryRepository) AddTarget(target *Target, roles ...data.RoleName) err } logrus.Debugf("Adding target \"%s\" with sha256 \"%x\" and size %d bytes.\n", target.Name, target.Hashes["sha256"], target.Length) - meta := data.FileMeta{Length: target.Length, Hashes: target.Hashes} + var customData *canonicaljson.RawMessage + if target.Custom != nil { + customData = new(canonicaljson.RawMessage) + if err := customData.UnmarshalJSON(target.Custom); err != nil { + return err + } + } + meta := data.FileMeta{Length: target.Length, Hashes: target.Hashes, Custom: customData} metaJSON, err := json.Marshal(meta) if err != nil { return err @@ -412,11 +421,16 @@ func (r *NotaryRepository) ListTargets(roles ...data.RoleName) ([]*TargetWithRol if _, ok := targets[targetName]; ok || !validRole.CheckPaths(targetName) { continue } + var custom canonicaljson.RawMessage + if targetMeta.Custom != nil { + custom = *targetMeta.Custom + } targets[targetName] = &TargetWithRole{ Target: Target{ Name: targetName, Hashes: targetMeta.Hashes, Length: targetMeta.Length, + Custom: custom, }, Role: validRole.Name, } @@ -472,7 +486,11 @@ func (r *NotaryRepository) GetTargetByName(name string, roles ...data.RoleName) } // Check that we didn't error, and that we assigned to our target if err := r.tufRepo.WalkTargets(name, role, getTargetVisitorFunc, skipRoles...); err == nil && foundTarget { - return &TargetWithRole{Target: Target{Name: name, Hashes: resultMeta.Hashes, Length: resultMeta.Length}, Role: resultRoleName}, nil + var custom canonicaljson.RawMessage + if resultMeta.Custom != nil { + custom = *resultMeta.Custom + } + return &TargetWithRole{Target: Target{Name: name, Hashes: resultMeta.Hashes, Length: resultMeta.Length, Custom: custom}, Role: resultRoleName}, nil } } return nil, fmt.Errorf("No trust data for %s", name) @@ -514,9 +532,13 @@ func (r *NotaryRepository) GetAllTargetMetadataByName(name string) ([]TargetSign } for targetName, resultMeta := range targetMetaToAdd { + var custom canonicaljson.RawMessage + if resultMeta.Custom != nil { + custom = *resultMeta.Custom + } targetInfo := TargetSignedStruct{ Role: validRole, - Target: Target{Name: targetName, Hashes: resultMeta.Hashes, Length: resultMeta.Length}, + Target: Target{Name: targetName, Hashes: resultMeta.Hashes, Length: resultMeta.Length, Custom: custom}, Signatures: tgt.Signatures, } targetInfoList = append(targetInfoList, targetInfo) diff --git a/client/client_test.go b/client/client_test.go index 67cbbbd399..3526657f6b 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -631,7 +631,13 @@ func testInitRepoPasswordInvalid(t *testing.T, rootType string) { func addTarget(t *testing.T, repo *NotaryRepository, targetName, targetFile string, roles ...data.RoleName) *Target { - target, err := NewTarget(targetName, targetFile) + var targetCustom json.RawMessage + return addTargetWithCustom(t, repo, targetName, targetFile, targetCustom, roles...) +} + +func addTargetWithCustom(t *testing.T, repo *NotaryRepository, targetName, + targetFile string, targetCustom json.RawMessage, roles ...data.RoleName) *Target { + target, err := NewTarget(targetName, targetFile, targetCustom) require.NoError(t, err, "error creating target") err = repo.AddTarget(target, roles...) require.NoError(t, err, "error adding target") @@ -815,7 +821,8 @@ func testAddTargetToSpecifiedInvalidRoles(t *testing.T, clearCache bool) { } for _, invalidRole := range invalidRoles { - target, err := NewTarget("latest", "../fixtures/intermediate-ca.crt") + var targetCustom json.RawMessage + target, err := NewTarget("latest", "../fixtures/intermediate-ca.crt", targetCustom) require.NoError(t, err, "error creating target") err = repo.AddTarget(target, data.CanonicalTargetsRole, invalidRole) @@ -877,7 +884,8 @@ func TestAddTargetWithInvalidTarget(t *testing.T) { repo, _ := initializeRepo(t, data.ECDSAKey, "docker.com/notary", ts.URL, false) defer os.RemoveAll(repo.baseDir) - target, err := NewTarget("latest", "../fixtures/intermediate-ca.crt") + var targetCustom json.RawMessage + target, err := NewTarget("latest", "../fixtures/intermediate-ca.crt", targetCustom) require.NoError(t, err, "error creating target") // Clear the hashes @@ -889,7 +897,8 @@ func TestAddTargetWithInvalidTarget(t *testing.T) { // to be propagated. func TestAddTargetErrorWritingChanges(t *testing.T) { testErrorWritingChangefiles(t, func(repo *NotaryRepository) error { - target, err := NewTarget("latest", "../fixtures/intermediate-ca.crt") + var targetCustom json.RawMessage + target, err := NewTarget("latest", "../fixtures/intermediate-ca.crt", targetCustom) require.NoError(t, err, "error creating target") return repo.AddTarget(target, data.CanonicalTargetsRole) }) @@ -1019,6 +1028,7 @@ func TestRemoveTargetErrorWritingChanges(t *testing.T) { func TestListTarget(t *testing.T) { testListEmptyTargets(t, data.ECDSAKey) testListTarget(t, data.ECDSAKey) + testListTargetWithCustom(t, data.ECDSAKey) testListTargetWithDelegates(t, data.ECDSAKey) if !testing.Short() { testListEmptyTargets(t, data.RSAKey) @@ -1238,6 +1248,48 @@ func testListTarget(t *testing.T, rootType string) { require.True(t, reflect.DeepEqual(*currentTarget, newCurrentTarget.Target), "current target does not match") } +func testListTargetWithCustom(t *testing.T, rootType string) { + ts, mux, keys := simpleTestServer(t) + defer ts.Close() + + repo, _ := initializeRepo(t, rootType, "docker.com/notary", ts.URL, false) + defer os.RemoveAll(repo.baseDir) + + // tests need to manually bootstrap timestamp as client doesn't generate it + err := repo.tufRepo.InitTimestamp() + require.NoError(t, err, "error creating repository: %s", err) + + var targetCustom json.RawMessage + err = json.Unmarshal([]byte("\"Lorem ipsum dolor sit\""), &targetCustom) + require.NoError(t, err) + latestTarget := addTargetWithCustom(t, repo, "latest", "../fixtures/intermediate-ca.crt", targetCustom) + require.Equal(t, targetCustom, latestTarget.Custom, "Target created does not contain the expected custom data") + + // Apply the changelist. Normally, this would be done by Publish + + // load the changelist for this repo + cl, err := changelist.NewFileChangelist( + filepath.Join(repo.baseDir, "tuf", filepath.FromSlash(repo.gun.String()), "changelist")) + require.NoError(t, err, "could not open changelist") + + // apply the changelist to the repo + err = applyChangelist(repo.tufRepo, nil, cl) + require.NoError(t, err, "could not apply changelist") + + fakeServerData(t, repo, mux, keys) + + targets, err := repo.ListTargets(data.CanonicalTargetsRole) + require.NoError(t, err) + + require.True(t, reflect.DeepEqual(*latestTarget, targets[0].Target), "latest target does not match") + + // Also test GetTargetByName for a target with custom data + newLatestTarget, err := repo.GetTargetByName("latest") + require.NoError(t, err) + require.Equal(t, data.CanonicalTargetsRole, newLatestTarget.Role) + require.True(t, reflect.DeepEqual(*latestTarget, newLatestTarget.Target), "latest target does not match") +} + func testListTargetWithDelegates(t *testing.T, rootType string) { ts, mux, keys := simpleTestServer(t) defer ts.Close() diff --git a/cmd/notary/integration_test.go b/cmd/notary/integration_test.go index fc3af68435..3111e0d8d9 100644 --- a/cmd/notary/integration_test.go +++ b/cmd/notary/integration_test.go @@ -23,8 +23,11 @@ import ( "testing" "time" + "encoding/json" + "github.com/Sirupsen/logrus" ctxu "github.com/docker/distribution/context" + canonicaljson "github.com/docker/go/canonical/json" "github.com/docker/notary" "github.com/docker/notary/client" "github.com/docker/notary/cryptoservice" @@ -198,8 +201,9 @@ func TestClientTUFInteraction(t *testing.T) { defer os.Remove(tempFile.Name()) var ( - output string - target = "sdgkadga" + output string + target = "sdgkadga" + target2 = "foobar" ) // -- tests -- @@ -251,6 +255,62 @@ func TestClientTUFInteraction(t *testing.T) { output, err = runCommand(t, tempDir, "-s", server.URL, "list", "gun") require.NoError(t, err) require.False(t, strings.Contains(string(output), target)) + + // Test a target with custom data. + tempFileForTargetCustom, err := ioutil.TempFile("", "targetCustom") + require.NoError(t, err) + var customData canonicaljson.RawMessage + err = canonicaljson.Unmarshal([]byte("\"Lorem ipsum dolor sit amet, consectetur adipiscing elit\""), &customData) + require.NoError(t, err) + _, err = tempFileForTargetCustom.Write(customData) + require.NoError(t, err) + tempFileForTargetCustom.Close() + defer os.Remove(tempFileForTargetCustom.Name()) + + // add a target + _, err = runCommand(t, tempDir, "add", "gun", target2, tempFile.Name(), "--custom", tempFileForTargetCustom.Name()) + require.NoError(t, err) + + // check status - see target + output, err = runCommand(t, tempDir, "status", "gun") + require.NoError(t, err) + require.Contains(t, output, target2) + + // publish repo + _, err = runCommand(t, tempDir, "-s", server.URL, "publish", "gun") + require.NoError(t, err) + + // check status - no targets + output, err = runCommand(t, tempDir, "status", "gun") + require.NoError(t, err) + require.False(t, strings.Contains(string(output), target2)) + + // list repo - see target + output, err = runCommand(t, tempDir, "-s", server.URL, "list", "gun") + require.NoError(t, err) + require.Contains(t, output, target2) + + // Check the file this was written to to inspect metadata + cache, err := nstorage.NewFileStore( + filepath.Join(tempDir, "tuf", filepath.FromSlash("gun"), "metadata"), + "json", + ) + require.NoError(t, err) + rawTargets, err := cache.Get("targets") + require.NoError(t, err) + parsedTargets := data.SignedTargets{} + err = json.Unmarshal(rawTargets, &parsedTargets) + require.NoError(t, err) + require.Equal(t, *parsedTargets.Signed.Targets[target2].Custom, customData) + + // trigger a lookup error with < 2 args + _, err = runCommand(t, tempDir, "-s", server.URL, "lookup", "gun") + require.Error(t, err) + + // lookup target and repo - see target + output, err = runCommand(t, tempDir, "-s", server.URL, "lookup", "gun", target2) + require.NoError(t, err) + require.Contains(t, output, target2) } func TestClientDeleteTUFInteraction(t *testing.T) { @@ -422,6 +482,7 @@ func TestClientTUFAddByHashInteraction(t *testing.T) { target1 = "sdgkadga" target2 = "asdfasdf" target3 = "qwerty" + target4 = "foobar" ) // -- tests -- @@ -541,6 +602,57 @@ func TestClientTUFAddByHashInteraction(t *testing.T) { // publish repo _, err = runCommand(t, tempDir, "-s", server.URL, "publish", "gun") require.NoError(t, err) + + tempFile, err := ioutil.TempFile("", "targetCustom") + require.NoError(t, err) + var customData canonicaljson.RawMessage + err = canonicaljson.Unmarshal([]byte("\"Lorem ipsum dolor sit amet, consectetur adipiscing elit\""), &customData) + require.NoError(t, err) + _, err = tempFile.Write(customData) + require.NoError(t, err) + tempFile.Close() + defer os.Remove(tempFile.Name()) + + // add a target by sha512 and custom data + _, err = runCommand(t, tempDir, "addhash", "gun", target4, "3", "--sha512", targetSha512Hex, "--custom", tempFile.Name()) + require.NoError(t, err) + + // check status - see target + output, err = runCommand(t, tempDir, "status", "gun") + require.NoError(t, err) + require.Contains(t, output, target4) + + // publish repo + _, err = runCommand(t, tempDir, "-s", server.URL, "publish", "gun") + require.NoError(t, err) + + // check status - no targets + output, err = runCommand(t, tempDir, "status", "gun") + require.NoError(t, err) + require.False(t, strings.Contains(string(output), target4)) + + // list repo - see target + output, err = runCommand(t, tempDir, "-s", server.URL, "list", "gun") + require.NoError(t, err) + require.Contains(t, output, target4) + + // Check the file this was written to to inspect metadata + cache, err := nstorage.NewFileStore( + filepath.Join(tempDir, "tuf", filepath.FromSlash("gun"), "metadata"), + "json", + ) + require.NoError(t, err) + rawTargets, err := cache.Get("targets") + require.NoError(t, err) + parsedTargets := data.SignedTargets{} + err = json.Unmarshal(rawTargets, &parsedTargets) + require.NoError(t, err) + require.Equal(t, *parsedTargets.Signed.Targets[target4].Custom, customData) + + // lookup target and repo - see target + output, err = runCommand(t, tempDir, "-s", server.URL, "lookup", "gun", target4) + require.NoError(t, err) + require.Contains(t, output, target4) } // Initialize repo and test delegations commands by adding, listing, and removing delegations diff --git a/cmd/notary/tuf.go b/cmd/notary/tuf.go index c6dcf88b6e..7c8f8fccc4 100644 --- a/cmd/notary/tuf.go +++ b/cmd/notary/tuf.go @@ -21,6 +21,7 @@ import ( "github.com/docker/distribution/registry/client/auth/challenge" "github.com/docker/distribution/registry/client/transport" "github.com/docker/go-connections/tlsconfig" + "github.com/docker/go/canonical/json" "github.com/docker/notary" notaryclient "github.com/docker/notary/client" "github.com/docker/notary/cryptoservice" @@ -116,6 +117,7 @@ type tufCommander struct { sha256 string sha512 string rootKey string + custom string input string output string @@ -155,6 +157,7 @@ func (t *tufCommander) AddToCommand(cmd *cobra.Command) { cmdTUFAdd := cmdTUFAddTemplate.ToCommand(t.tufAdd) cmdTUFAdd.Flags().StringSliceVarP(&t.roles, "roles", "r", nil, "Delegation roles to add this target to") cmdTUFAdd.Flags().BoolVarP(&t.autoPublish, "publish", "p", false, htAutoPublish) + cmdTUFAdd.Flags().StringVar(&t.custom, "custom", "", "Name of the file containing custom data for this target") cmd.AddCommand(cmdTUFAdd) cmdTUFRemove := cmdTUFRemoveTemplate.ToCommand(t.tufRemove) @@ -167,6 +170,7 @@ func (t *tufCommander) AddToCommand(cmd *cobra.Command) { cmdTUFAddHash.Flags().StringVar(&t.sha256, notary.SHA256, "", "hex encoded sha256 of the target to add") cmdTUFAddHash.Flags().StringVar(&t.sha512, notary.SHA512, "", "hex encoded sha512 of the target to add") cmdTUFAddHash.Flags().BoolVarP(&t.autoPublish, "publish", "p", false, htAutoPublish) + cmdTUFAddHash.Flags().StringVar(&t.custom, "custom", "", "Name of the file containing custom data for this target") cmd.AddCommand(cmdTUFAddHash) cmdTUFVerify := cmdTUFVerifyTemplate.ToCommand(t.tufVerify) @@ -249,6 +253,21 @@ func getTargetHashes(t *tufCommander) (data.Hashes, error) { return targetHash, nil } +// Open and read a file containing the targetCustom data +func getTargetCustom(targetCustomFilename string) (json.RawMessage, error) { + var nilCustom json.RawMessage + targetCustomFile, err := os.Open(targetCustomFilename) + if err != nil { + return nilCustom, err + } + defer targetCustomFile.Close() + targetCustom, err := ioutil.ReadAll(targetCustomFile) + if err != nil { + return nilCustom, err + } + return targetCustom, nil +} + func (t *tufCommander) tufAddByHash(cmd *cobra.Command, args []string) error { if len(args) < 3 || t.sha256 == "" && t.sha512 == "" { cmd.Usage() @@ -262,6 +281,13 @@ func (t *tufCommander) tufAddByHash(cmd *cobra.Command, args []string) error { gun := data.GUN(args[0]) targetName := args[1] targetSize := args[2] + var targetCustom []byte + if t.custom != "" { + targetCustom, err = getTargetCustom(t.custom) + if err != nil { + return err + } + } targetInt64Len, err := strconv.ParseInt(targetSize, 0, 64) if err != nil { @@ -287,7 +313,7 @@ func (t *tufCommander) tufAddByHash(cmd *cobra.Command, args []string) error { } // Manually construct the target with the given byte size and hashes - target := ¬aryclient.Target{Name: targetName, Hashes: targetHashes, Length: targetInt64Len} + target := ¬aryclient.Target{Name: targetName, Hashes: targetHashes, Length: targetInt64Len, Custom: targetCustom} roleNames := data.NewRoleList(t.roles) @@ -321,6 +347,13 @@ func (t *tufCommander) tufAdd(cmd *cobra.Command, args []string) error { gun := data.GUN(args[0]) targetName := args[1] targetPath := args[2] + var targetCustom []byte + if t.custom != "" { + targetCustom, err = getTargetCustom(t.custom) + if err != nil { + return err + } + } trustPin, err := getTrustPinning(config) if err != nil { @@ -335,7 +368,7 @@ func (t *tufCommander) tufAdd(cmd *cobra.Command, args []string) error { return err } - target, err := notaryclient.NewTarget(targetName, targetPath) + target, err := notaryclient.NewTarget(targetName, targetPath, targetCustom) if err != nil { return err } From 4229ecb74f8dc239f700db164d9125cd7d9aa5e4 Mon Sep 17 00:00:00 2001 From: Ashwini Oruganti Date: Thu, 4 May 2017 17:13:43 -0700 Subject: [PATCH 06/21] Fold testListTargetWithCustom into testListTarget Signed-off-by: Ashwini Oruganti --- client/client_test.go | 50 ++++++------------------------------------- 1 file changed, 6 insertions(+), 44 deletions(-) diff --git a/client/client_test.go b/client/client_test.go index 3526657f6b..99aa216edc 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -1028,7 +1028,6 @@ func TestRemoveTargetErrorWritingChanges(t *testing.T) { func TestListTarget(t *testing.T) { testListEmptyTargets(t, data.ECDSAKey) testListTarget(t, data.ECDSAKey) - testListTargetWithCustom(t, data.ECDSAKey) testListTargetWithDelegates(t, data.ECDSAKey) if !testing.Short() { testListEmptyTargets(t, data.RSAKey) @@ -1202,8 +1201,13 @@ func testListTarget(t *testing.T, rootType string) { // tests need to manually bootstrap timestamp as client doesn't generate it err := repo.tufRepo.InitTimestamp() require.NoError(t, err, "error creating repository: %s", err) + var targetCustom json.RawMessage + err = json.Unmarshal([]byte("\"Lorem ipsum dolor sit\""), &targetCustom) + require.NoError(t, err) + + latestTarget := addTargetWithCustom(t, repo, "latest", "../fixtures/intermediate-ca.crt", targetCustom) + require.Equal(t, targetCustom, latestTarget.Custom, "Target created does not contain the expected custom data") - latestTarget := addTarget(t, repo, "latest", "../fixtures/intermediate-ca.crt") currentTarget := addTarget(t, repo, "current", "../fixtures/intermediate-ca.crt") // Apply the changelist. Normally, this would be done by Publish @@ -1248,48 +1252,6 @@ func testListTarget(t *testing.T, rootType string) { require.True(t, reflect.DeepEqual(*currentTarget, newCurrentTarget.Target), "current target does not match") } -func testListTargetWithCustom(t *testing.T, rootType string) { - ts, mux, keys := simpleTestServer(t) - defer ts.Close() - - repo, _ := initializeRepo(t, rootType, "docker.com/notary", ts.URL, false) - defer os.RemoveAll(repo.baseDir) - - // tests need to manually bootstrap timestamp as client doesn't generate it - err := repo.tufRepo.InitTimestamp() - require.NoError(t, err, "error creating repository: %s", err) - - var targetCustom json.RawMessage - err = json.Unmarshal([]byte("\"Lorem ipsum dolor sit\""), &targetCustom) - require.NoError(t, err) - latestTarget := addTargetWithCustom(t, repo, "latest", "../fixtures/intermediate-ca.crt", targetCustom) - require.Equal(t, targetCustom, latestTarget.Custom, "Target created does not contain the expected custom data") - - // Apply the changelist. Normally, this would be done by Publish - - // load the changelist for this repo - cl, err := changelist.NewFileChangelist( - filepath.Join(repo.baseDir, "tuf", filepath.FromSlash(repo.gun.String()), "changelist")) - require.NoError(t, err, "could not open changelist") - - // apply the changelist to the repo - err = applyChangelist(repo.tufRepo, nil, cl) - require.NoError(t, err, "could not apply changelist") - - fakeServerData(t, repo, mux, keys) - - targets, err := repo.ListTargets(data.CanonicalTargetsRole) - require.NoError(t, err) - - require.True(t, reflect.DeepEqual(*latestTarget, targets[0].Target), "latest target does not match") - - // Also test GetTargetByName for a target with custom data - newLatestTarget, err := repo.GetTargetByName("latest") - require.NoError(t, err) - require.Equal(t, data.CanonicalTargetsRole, newLatestTarget.Role) - require.True(t, reflect.DeepEqual(*latestTarget, newLatestTarget.Target), "latest target does not match") -} - func testListTargetWithDelegates(t *testing.T, rootType string) { ts, mux, keys := simpleTestServer(t) defer ts.Close() From dc47345236e83abeb494e9f53b8a4ceca03505ac Mon Sep 17 00:00:00 2001 From: Ashwini Oruganti Date: Thu, 4 May 2017 17:29:09 -0700 Subject: [PATCH 07/21] Use ioutil.ReadFile (thanks for the tip, @cyli!) Signed-off-by: Ashwini Oruganti --- cmd/notary/tuf.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/cmd/notary/tuf.go b/cmd/notary/tuf.go index 7c8f8fccc4..d6210bfff2 100644 --- a/cmd/notary/tuf.go +++ b/cmd/notary/tuf.go @@ -256,12 +256,7 @@ func getTargetHashes(t *tufCommander) (data.Hashes, error) { // Open and read a file containing the targetCustom data func getTargetCustom(targetCustomFilename string) (json.RawMessage, error) { var nilCustom json.RawMessage - targetCustomFile, err := os.Open(targetCustomFilename) - if err != nil { - return nilCustom, err - } - defer targetCustomFile.Close() - targetCustom, err := ioutil.ReadAll(targetCustomFile) + targetCustom, err := ioutil.ReadFile(targetCustomFilename) if err != nil { return nilCustom, err } From 4b651cfb61897b16ac4baacccbfe4c7aa68ba996 Mon Sep 17 00:00:00 2001 From: Ashwini Oruganti Date: Fri, 5 May 2017 11:16:37 -0700 Subject: [PATCH 08/21] Make Target.Custom a pointer of json.RawMessage Signed-off-by: Ashwini Oruganti --- client/client.go | 31 ++++++------------------------- client/client_test.go | 17 +++++++++-------- cmd/notary/integration_test.go | 4 ++-- cmd/notary/tuf.go | 13 +++++++------ 4 files changed, 24 insertions(+), 41 deletions(-) diff --git a/client/client.go b/client/client.go index 39228cff84..a9b29213bd 100644 --- a/client/client.go +++ b/client/client.go @@ -132,7 +132,7 @@ type Target struct { Name string // the name of the target Hashes data.Hashes // the hash of the target Length int64 // the size in bytes of the target - Custom canonicaljson.RawMessage // the custom data provided to describe the file at TARGETPATH + Custom *canonicaljson.RawMessage // the custom data provided to describe the file at TARGETPATH } // TargetWithRole represents a Target that exists in a particular role - this is @@ -143,7 +143,7 @@ type TargetWithRole struct { } // NewTarget is a helper method that returns a Target -func NewTarget(targetName, targetPath string, targetCustom canonicaljson.RawMessage) (*Target, error) { +func NewTarget(targetName, targetPath string, targetCustom *canonicaljson.RawMessage) (*Target, error) { b, err := ioutil.ReadFile(targetPath) if err != nil { return nil, err @@ -362,14 +362,7 @@ func (r *NotaryRepository) AddTarget(target *Target, roles ...data.RoleName) err } logrus.Debugf("Adding target \"%s\" with sha256 \"%x\" and size %d bytes.\n", target.Name, target.Hashes["sha256"], target.Length) - var customData *canonicaljson.RawMessage - if target.Custom != nil { - customData = new(canonicaljson.RawMessage) - if err := customData.UnmarshalJSON(target.Custom); err != nil { - return err - } - } - meta := data.FileMeta{Length: target.Length, Hashes: target.Hashes, Custom: customData} + meta := data.FileMeta{Length: target.Length, Hashes: target.Hashes, Custom: target.Custom} metaJSON, err := json.Marshal(meta) if err != nil { return err @@ -421,16 +414,12 @@ func (r *NotaryRepository) ListTargets(roles ...data.RoleName) ([]*TargetWithRol if _, ok := targets[targetName]; ok || !validRole.CheckPaths(targetName) { continue } - var custom canonicaljson.RawMessage - if targetMeta.Custom != nil { - custom = *targetMeta.Custom - } targets[targetName] = &TargetWithRole{ Target: Target{ Name: targetName, Hashes: targetMeta.Hashes, Length: targetMeta.Length, - Custom: custom, + Custom: targetMeta.Custom, }, Role: validRole.Name, } @@ -486,11 +475,7 @@ func (r *NotaryRepository) GetTargetByName(name string, roles ...data.RoleName) } // Check that we didn't error, and that we assigned to our target if err := r.tufRepo.WalkTargets(name, role, getTargetVisitorFunc, skipRoles...); err == nil && foundTarget { - var custom canonicaljson.RawMessage - if resultMeta.Custom != nil { - custom = *resultMeta.Custom - } - return &TargetWithRole{Target: Target{Name: name, Hashes: resultMeta.Hashes, Length: resultMeta.Length, Custom: custom}, Role: resultRoleName}, nil + return &TargetWithRole{Target: Target{Name: name, Hashes: resultMeta.Hashes, Length: resultMeta.Length, Custom: resultMeta.Custom}, Role: resultRoleName}, nil } } return nil, fmt.Errorf("No trust data for %s", name) @@ -532,13 +517,9 @@ func (r *NotaryRepository) GetAllTargetMetadataByName(name string) ([]TargetSign } for targetName, resultMeta := range targetMetaToAdd { - var custom canonicaljson.RawMessage - if resultMeta.Custom != nil { - custom = *resultMeta.Custom - } targetInfo := TargetSignedStruct{ Role: validRole, - Target: Target{Name: targetName, Hashes: resultMeta.Hashes, Length: resultMeta.Length, Custom: custom}, + Target: Target{Name: targetName, Hashes: resultMeta.Hashes, Length: resultMeta.Length, Custom: resultMeta.Custom}, Signatures: tgt.Signatures, } targetInfoList = append(targetInfoList, targetInfo) diff --git a/client/client_test.go b/client/client_test.go index 99aa216edc..54741612b1 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -631,12 +631,12 @@ func testInitRepoPasswordInvalid(t *testing.T, rootType string) { func addTarget(t *testing.T, repo *NotaryRepository, targetName, targetFile string, roles ...data.RoleName) *Target { - var targetCustom json.RawMessage + var targetCustom *json.RawMessage return addTargetWithCustom(t, repo, targetName, targetFile, targetCustom, roles...) } func addTargetWithCustom(t *testing.T, repo *NotaryRepository, targetName, - targetFile string, targetCustom json.RawMessage, roles ...data.RoleName) *Target { + targetFile string, targetCustom *json.RawMessage, roles ...data.RoleName) *Target { target, err := NewTarget(targetName, targetFile, targetCustom) require.NoError(t, err, "error creating target") err = repo.AddTarget(target, roles...) @@ -821,7 +821,7 @@ func testAddTargetToSpecifiedInvalidRoles(t *testing.T, clearCache bool) { } for _, invalidRole := range invalidRoles { - var targetCustom json.RawMessage + var targetCustom *json.RawMessage target, err := NewTarget("latest", "../fixtures/intermediate-ca.crt", targetCustom) require.NoError(t, err, "error creating target") @@ -884,7 +884,7 @@ func TestAddTargetWithInvalidTarget(t *testing.T) { repo, _ := initializeRepo(t, data.ECDSAKey, "docker.com/notary", ts.URL, false) defer os.RemoveAll(repo.baseDir) - var targetCustom json.RawMessage + var targetCustom *json.RawMessage target, err := NewTarget("latest", "../fixtures/intermediate-ca.crt", targetCustom) require.NoError(t, err, "error creating target") @@ -897,7 +897,7 @@ func TestAddTargetWithInvalidTarget(t *testing.T) { // to be propagated. func TestAddTargetErrorWritingChanges(t *testing.T) { testErrorWritingChangefiles(t, func(repo *NotaryRepository) error { - var targetCustom json.RawMessage + var targetCustom *json.RawMessage target, err := NewTarget("latest", "../fixtures/intermediate-ca.crt", targetCustom) require.NoError(t, err, "error creating target") return repo.AddTarget(target, data.CanonicalTargetsRole) @@ -1202,11 +1202,12 @@ func testListTarget(t *testing.T, rootType string) { err := repo.tufRepo.InitTimestamp() require.NoError(t, err, "error creating repository: %s", err) var targetCustom json.RawMessage - err = json.Unmarshal([]byte("\"Lorem ipsum dolor sit\""), &targetCustom) + rawTargetCustom := []byte("\"Lorem ipsum dolor sit\"") + err = json.Unmarshal(rawTargetCustom, &targetCustom) require.NoError(t, err) - latestTarget := addTargetWithCustom(t, repo, "latest", "../fixtures/intermediate-ca.crt", targetCustom) - require.Equal(t, targetCustom, latestTarget.Custom, "Target created does not contain the expected custom data") + latestTarget := addTargetWithCustom(t, repo, "latest", "../fixtures/intermediate-ca.crt", &targetCustom) + require.Equal(t, targetCustom, *latestTarget.Custom, "Target created does not contain the expected custom data") currentTarget := addTarget(t, repo, "current", "../fixtures/intermediate-ca.crt") diff --git a/cmd/notary/integration_test.go b/cmd/notary/integration_test.go index 3111e0d8d9..52ce7f93c2 100644 --- a/cmd/notary/integration_test.go +++ b/cmd/notary/integration_test.go @@ -301,7 +301,7 @@ func TestClientTUFInteraction(t *testing.T) { parsedTargets := data.SignedTargets{} err = json.Unmarshal(rawTargets, &parsedTargets) require.NoError(t, err) - require.Equal(t, *parsedTargets.Signed.Targets[target2].Custom, customData) + require.Equal(t, parsedTargets.Signed.Targets[target2].Custom, customData) // trigger a lookup error with < 2 args _, err = runCommand(t, tempDir, "-s", server.URL, "lookup", "gun") @@ -647,7 +647,7 @@ func TestClientTUFAddByHashInteraction(t *testing.T) { parsedTargets := data.SignedTargets{} err = json.Unmarshal(rawTargets, &parsedTargets) require.NoError(t, err) - require.Equal(t, *parsedTargets.Signed.Targets[target4].Custom, customData) + require.Equal(t, parsedTargets.Signed.Targets[target4].Custom, customData) // lookup target and repo - see target output, err = runCommand(t, tempDir, "-s", server.URL, "lookup", "gun", target4) diff --git a/cmd/notary/tuf.go b/cmd/notary/tuf.go index d6210bfff2..50eab68e7b 100644 --- a/cmd/notary/tuf.go +++ b/cmd/notary/tuf.go @@ -254,12 +254,13 @@ func getTargetHashes(t *tufCommander) (data.Hashes, error) { } // Open and read a file containing the targetCustom data -func getTargetCustom(targetCustomFilename string) (json.RawMessage, error) { - var nilCustom json.RawMessage - targetCustom, err := ioutil.ReadFile(targetCustomFilename) +func getTargetCustom(targetCustomFilename string) (*json.RawMessage, error) { + var targetCustom *json.RawMessage + rawTargetCustom, err := ioutil.ReadFile(targetCustomFilename) if err != nil { - return nilCustom, err + return targetCustom, err } + json.Unmarshal(rawTargetCustom, targetCustom) return targetCustom, nil } @@ -276,7 +277,7 @@ func (t *tufCommander) tufAddByHash(cmd *cobra.Command, args []string) error { gun := data.GUN(args[0]) targetName := args[1] targetSize := args[2] - var targetCustom []byte + var targetCustom *json.RawMessage if t.custom != "" { targetCustom, err = getTargetCustom(t.custom) if err != nil { @@ -342,7 +343,7 @@ func (t *tufCommander) tufAdd(cmd *cobra.Command, args []string) error { gun := data.GUN(args[0]) targetName := args[1] targetPath := args[2] - var targetCustom []byte + var targetCustom *json.RawMessage if t.custom != "" { targetCustom, err = getTargetCustom(t.custom) if err != nil { From b3df16a02b5f369a2ec5f06159faa7cdaea87567 Mon Sep 17 00:00:00 2001 From: Ashwini Oruganti Date: Fri, 5 May 2017 11:57:03 -0700 Subject: [PATCH 09/21] You can't unmarshal something into a nil Signed-off-by: Ashwini Oruganti --- cmd/notary/integration_test.go | 4 ++-- cmd/notary/tuf.go | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cmd/notary/integration_test.go b/cmd/notary/integration_test.go index 52ce7f93c2..3111e0d8d9 100644 --- a/cmd/notary/integration_test.go +++ b/cmd/notary/integration_test.go @@ -301,7 +301,7 @@ func TestClientTUFInteraction(t *testing.T) { parsedTargets := data.SignedTargets{} err = json.Unmarshal(rawTargets, &parsedTargets) require.NoError(t, err) - require.Equal(t, parsedTargets.Signed.Targets[target2].Custom, customData) + require.Equal(t, *parsedTargets.Signed.Targets[target2].Custom, customData) // trigger a lookup error with < 2 args _, err = runCommand(t, tempDir, "-s", server.URL, "lookup", "gun") @@ -647,7 +647,7 @@ func TestClientTUFAddByHashInteraction(t *testing.T) { parsedTargets := data.SignedTargets{} err = json.Unmarshal(rawTargets, &parsedTargets) require.NoError(t, err) - require.Equal(t, parsedTargets.Signed.Targets[target4].Custom, customData) + require.Equal(t, *parsedTargets.Signed.Targets[target4].Custom, customData) // lookup target and repo - see target output, err = runCommand(t, tempDir, "-s", server.URL, "lookup", "gun", target4) diff --git a/cmd/notary/tuf.go b/cmd/notary/tuf.go index 50eab68e7b..0390b0086e 100644 --- a/cmd/notary/tuf.go +++ b/cmd/notary/tuf.go @@ -255,13 +255,16 @@ func getTargetHashes(t *tufCommander) (data.Hashes, error) { // Open and read a file containing the targetCustom data func getTargetCustom(targetCustomFilename string) (*json.RawMessage, error) { - var targetCustom *json.RawMessage + var targetCustom json.RawMessage rawTargetCustom, err := ioutil.ReadFile(targetCustomFilename) if err != nil { - return targetCustom, err + return nil, err + } + err = json.Unmarshal(rawTargetCustom, &targetCustom) + if err != nil { + return nil, err } - json.Unmarshal(rawTargetCustom, targetCustom) - return targetCustom, nil + return &targetCustom, nil } func (t *tufCommander) tufAddByHash(cmd *cobra.Command, args []string) error { From 6bea8abbc066d0901e384c8b3e41441628c75e89 Mon Sep 17 00:00:00 2001 From: Ashwini Oruganti Date: Fri, 5 May 2017 12:20:41 -0700 Subject: [PATCH 10/21] Fix lint errors Signed-off-by: Ashwini Oruganti --- client/client.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/client.go b/client/client.go index a9b29213bd..16bf8b7ee4 100644 --- a/client/client.go +++ b/client/client.go @@ -129,9 +129,9 @@ func (r *NotaryRepository) GetGUN() data.GUN { // Target represents a simplified version of the data TUF operates on, so external // applications don't have to depend on TUF data types. type Target struct { - Name string // the name of the target - Hashes data.Hashes // the hash of the target - Length int64 // the size in bytes of the target + Name string // the name of the target + Hashes data.Hashes // the hash of the target + Length int64 // the size in bytes of the target Custom *canonicaljson.RawMessage // the custom data provided to describe the file at TARGETPATH } From c7a92ca15653464e2578ae073844d696588ea6c6 Mon Sep 17 00:00:00 2001 From: Ashwini Oruganti Date: Fri, 5 May 2017 14:31:49 -0700 Subject: [PATCH 11/21] Update the `custom` flag description Signed-off-by: Ashwini Oruganti --- cmd/notary/tuf.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/notary/tuf.go b/cmd/notary/tuf.go index 0390b0086e..e793fbb130 100644 --- a/cmd/notary/tuf.go +++ b/cmd/notary/tuf.go @@ -170,7 +170,7 @@ func (t *tufCommander) AddToCommand(cmd *cobra.Command) { cmdTUFAddHash.Flags().StringVar(&t.sha256, notary.SHA256, "", "hex encoded sha256 of the target to add") cmdTUFAddHash.Flags().StringVar(&t.sha512, notary.SHA512, "", "hex encoded sha512 of the target to add") cmdTUFAddHash.Flags().BoolVarP(&t.autoPublish, "publish", "p", false, htAutoPublish) - cmdTUFAddHash.Flags().StringVar(&t.custom, "custom", "", "Name of the file containing custom data for this target") + cmdTUFAddHash.Flags().StringVar(&t.custom, "custom", "", "Path to the file containing custom data for this target") cmd.AddCommand(cmdTUFAddHash) cmdTUFVerify := cmdTUFVerifyTemplate.ToCommand(t.tufVerify) From e4599222707cfa97fc0c1afeeb2e490e6e382449 Mon Sep 17 00:00:00 2001 From: Ashwini Oruganti Date: Fri, 5 May 2017 14:44:08 -0700 Subject: [PATCH 12/21] Update flag description for tufAdd as well. Signed-off-by: Ashwini Oruganti --- cmd/notary/tuf.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/notary/tuf.go b/cmd/notary/tuf.go index e793fbb130..cfd249ac49 100644 --- a/cmd/notary/tuf.go +++ b/cmd/notary/tuf.go @@ -157,7 +157,7 @@ func (t *tufCommander) AddToCommand(cmd *cobra.Command) { cmdTUFAdd := cmdTUFAddTemplate.ToCommand(t.tufAdd) cmdTUFAdd.Flags().StringSliceVarP(&t.roles, "roles", "r", nil, "Delegation roles to add this target to") cmdTUFAdd.Flags().BoolVarP(&t.autoPublish, "publish", "p", false, htAutoPublish) - cmdTUFAdd.Flags().StringVar(&t.custom, "custom", "", "Name of the file containing custom data for this target") + cmdTUFAdd.Flags().StringVar(&t.custom, "custom", "", "Path to the file containing custom data for this target") cmd.AddCommand(cmdTUFAdd) cmdTUFRemove := cmdTUFRemoveTemplate.ToCommand(t.tufRemove) From 369c42323fe8a44fc7ed7fce352dd2c55ec96155 Mon Sep 17 00:00:00 2001 From: Ashwini Oruganti Date: Mon, 8 May 2017 14:21:33 -0700 Subject: [PATCH 13/21] Use new(json.RawMessage) memory in getTargetCustom Signed-off-by: Ashwini Oruganti --- cmd/notary/tuf.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/notary/tuf.go b/cmd/notary/tuf.go index cfd249ac49..ddfe123db3 100644 --- a/cmd/notary/tuf.go +++ b/cmd/notary/tuf.go @@ -255,16 +255,16 @@ func getTargetHashes(t *tufCommander) (data.Hashes, error) { // Open and read a file containing the targetCustom data func getTargetCustom(targetCustomFilename string) (*json.RawMessage, error) { - var targetCustom json.RawMessage + targetCustom := new(json.RawMessage) rawTargetCustom, err := ioutil.ReadFile(targetCustomFilename) if err != nil { return nil, err } - err = json.Unmarshal(rawTargetCustom, &targetCustom) - if err != nil { + + if err := targetCustom.UnmarshalJSON(rawTargetCustom); err != nil { return nil, err } - return &targetCustom, nil + return targetCustom, nil } func (t *tufCommander) tufAddByHash(cmd *cobra.Command, args []string) error { From a6089e0d28e2d6544f1b0b01bb86d97d8ec44739 Mon Sep 17 00:00:00 2001 From: Ashwini Oruganti Date: Mon, 8 May 2017 14:27:43 -0700 Subject: [PATCH 14/21] Import canonical/json as canonicaljson To avoid confusing it with encoding/json. Signed-off-by: Ashwini Oruganti --- cmd/notary/tuf.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/notary/tuf.go b/cmd/notary/tuf.go index ddfe123db3..928f1f8958 100644 --- a/cmd/notary/tuf.go +++ b/cmd/notary/tuf.go @@ -21,7 +21,7 @@ import ( "github.com/docker/distribution/registry/client/auth/challenge" "github.com/docker/distribution/registry/client/transport" "github.com/docker/go-connections/tlsconfig" - "github.com/docker/go/canonical/json" + canonicaljson "github.com/docker/go/canonical/json" "github.com/docker/notary" notaryclient "github.com/docker/notary/client" "github.com/docker/notary/cryptoservice" @@ -254,8 +254,8 @@ func getTargetHashes(t *tufCommander) (data.Hashes, error) { } // Open and read a file containing the targetCustom data -func getTargetCustom(targetCustomFilename string) (*json.RawMessage, error) { - targetCustom := new(json.RawMessage) +func getTargetCustom(targetCustomFilename string) (*canonicaljson.RawMessage, error) { + targetCustom := new(canonicaljson.RawMessage) rawTargetCustom, err := ioutil.ReadFile(targetCustomFilename) if err != nil { return nil, err @@ -280,7 +280,7 @@ func (t *tufCommander) tufAddByHash(cmd *cobra.Command, args []string) error { gun := data.GUN(args[0]) targetName := args[1] targetSize := args[2] - var targetCustom *json.RawMessage + var targetCustom *canonicaljson.RawMessage if t.custom != "" { targetCustom, err = getTargetCustom(t.custom) if err != nil { @@ -346,7 +346,7 @@ func (t *tufCommander) tufAdd(cmd *cobra.Command, args []string) error { gun := data.GUN(args[0]) targetName := args[1] targetPath := args[2] - var targetCustom *json.RawMessage + var targetCustom *canonicaljson.RawMessage if t.custom != "" { targetCustom, err = getTargetCustom(t.custom) if err != nil { From 7ccda7e6d72ff2d8bebee20c444a091b1088efc9 Mon Sep 17 00:00:00 2001 From: Ashwini Oruganti Date: Mon, 8 May 2017 10:33:47 -0700 Subject: [PATCH 15/21] Auto-parse the DB URL and add parseTime=true if the backend is mysql Also updates the mysql driver to a newer release Signed-off-by: Ashwini Oruganti --- utils/configuration.go | 13 + vendor.conf | 5 +- .../github.com/go-sql-driver/mysql/README.md | 443 ++++++++++++++ .../github.com/go-sql-driver/mysql/buffer.go | 27 +- .../go-sql-driver/mysql/collations.go | 22 +- .../go-sql-driver/mysql/connection.go | 89 +-- .../github.com/go-sql-driver/mysql/const.go | 3 +- .../github.com/go-sql-driver/mysql/driver.go | 113 ++-- vendor/github.com/go-sql-driver/mysql/dsn.go | 548 ++++++++++++++++++ .../github.com/go-sql-driver/mysql/errors.go | 23 +- .../github.com/go-sql-driver/mysql/infile.go | 56 +- .../github.com/go-sql-driver/mysql/packets.go | 307 +++++++--- vendor/github.com/go-sql-driver/mysql/rows.go | 14 +- .../go-sql-driver/mysql/statement.go | 14 +- .../github.com/go-sql-driver/mysql/utils.go | 257 +------- 15 files changed, 1466 insertions(+), 468 deletions(-) create mode 100644 vendor/github.com/go-sql-driver/mysql/README.md create mode 100644 vendor/github.com/go-sql-driver/mysql/dsn.go diff --git a/utils/configuration.go b/utils/configuration.go index 057995ec54..53d0472542 100644 --- a/utils/configuration.go +++ b/utils/configuration.go @@ -14,6 +14,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/bugsnag/bugsnag-go" "github.com/docker/go-connections/tlsconfig" + "github.com/go-sql-driver/mysql" "github.com/spf13/viper" "github.com/docker/notary" @@ -109,6 +110,18 @@ func ParseSQLStorage(configuration *viper.Viper) (*Storage, error) { "must provide a non-empty database source for %s", store.Backend, ) + case store.Backend == notary.MySQLBackend: + // Parse the url into a config object, and grab the string from it. + + // cfg, err := sql.Open("mysql", store.Source) + url_config, err := mysql.ParseDSN(store.Source) + if err != nil { + return nil, err + } + if !url_config.ParseTime { + url_config.ParseTime = true + } + store.Source = url_config.FormatDSN() } return &store, nil } diff --git a/vendor.conf b/vendor.conf index ce4e1e6fc1..b50f2f671b 100644 --- a/vendor.conf +++ b/vendor.conf @@ -5,9 +5,9 @@ github.com/bugsnag/bugsnag-go 13fd6b8acda029830ef9904df6b63be0a83369d0 github.com/coreos/etcd 6acb3d67fbe131b3b2d5d010e00ec80182be4628 github.com/docker/distribution v2.6.0 github.com/docker/go-connections f549a9393d05688dff0992ef3efd8bbe6c628aeb -github.com/docker/go/canonical d30aec9fd63c35133f8f79c3412ad91a3b08be06 +github.com/docker/go d30aec9fd63c35133f8f79c3412ad91a3b08be06 github.com/dvsekhvalnov/jose2go v1.2 -github.com/go-sql-driver/mysql 0cc29e9fe8e25c2c58cf47bcab566e029bbaa88b +github.com/go-sql-driver/mysql v1.3 github.com/gorilla/mux e444e69cbd2e2e3e0749a2f3c717cec491552bbf github.com/jinzhu/gorm 5409931a1bb87e484d68d649af9367c207713ea2 github.com/jinzhu/inflection 1c35d901db3da928c72a72d8458480cc9ade058f @@ -24,6 +24,7 @@ golang.org/x/crypto 5bcd134fee4dd1475da17714aac19c0aa0142e2f golang.org/x/net 6a513affb38dc9788b449d59ffed099b8de18fa0 google.golang.org/grpc v1.0.5 + gopkg.in/dancannon/gorethink.v3 v3.0.0 # dependencies of gorethink.v3 gopkg.in/gorethink/gorethink.v2 v2.2.2 diff --git a/vendor/github.com/go-sql-driver/mysql/README.md b/vendor/github.com/go-sql-driver/mysql/README.md new file mode 100644 index 0000000000..a16012f819 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/README.md @@ -0,0 +1,443 @@ +# Go-MySQL-Driver + +A MySQL-Driver for Go's [database/sql](http://golang.org/pkg/database/sql) package + +![Go-MySQL-Driver logo](https://raw.github.com/wiki/go-sql-driver/mysql/gomysql_m.png "Golang Gopher holding the MySQL Dolphin") + +--------------------------------------- + * [Features](#features) + * [Requirements](#requirements) + * [Installation](#installation) + * [Usage](#usage) + * [DSN (Data Source Name)](#dsn-data-source-name) + * [Password](#password) + * [Protocol](#protocol) + * [Address](#address) + * [Parameters](#parameters) + * [Examples](#examples) + * [LOAD DATA LOCAL INFILE support](#load-data-local-infile-support) + * [time.Time support](#timetime-support) + * [Unicode support](#unicode-support) + * [Testing / Development](#testing--development) + * [License](#license) + +--------------------------------------- + +## Features + * Lightweight and [fast](https://github.com/go-sql-driver/sql-benchmark "golang MySQL-Driver performance") + * Native Go implementation. No C-bindings, just pure Go + * Connections over TCP/IPv4, TCP/IPv6, Unix domain sockets or [custom protocols](http://godoc.org/github.com/go-sql-driver/mysql#DialFunc) + * Automatic handling of broken connections + * Automatic Connection Pooling *(by database/sql package)* + * Supports queries larger than 16MB + * Full [`sql.RawBytes`](http://golang.org/pkg/database/sql/#RawBytes) support. + * Intelligent `LONG DATA` handling in prepared statements + * Secure `LOAD DATA LOCAL INFILE` support with file Whitelisting and `io.Reader` support + * Optional `time.Time` parsing + * Optional placeholder interpolation + +## Requirements + * Go 1.2 or higher + * MySQL (4.1+), MariaDB, Percona Server, Google CloudSQL or Sphinx (2.2.3+) + +--------------------------------------- + +## Installation +Simple install the package to your [$GOPATH](http://code.google.com/p/go-wiki/wiki/GOPATH "GOPATH") with the [go tool](http://golang.org/cmd/go/ "go command") from shell: +```bash +$ go get github.com/go-sql-driver/mysql +``` +Make sure [Git is installed](http://git-scm.com/downloads) on your machine and in your system's `PATH`. + +## Usage +_Go MySQL Driver_ is an implementation of Go's `database/sql/driver` interface. You only need to import the driver and can use the full [`database/sql`](http://golang.org/pkg/database/sql) API then. + +Use `mysql` as `driverName` and a valid [DSN](#dsn-data-source-name) as `dataSourceName`: +```go +import "database/sql" +import _ "github.com/go-sql-driver/mysql" + +db, err := sql.Open("mysql", "user:password@/dbname") +``` + +[Examples are available in our Wiki](https://github.com/go-sql-driver/mysql/wiki/Examples "Go-MySQL-Driver Examples"). + + +### DSN (Data Source Name) + +The Data Source Name has a common format, like e.g. [PEAR DB](http://pear.php.net/manual/en/package.database.db.intro-dsn.php) uses it, but without type-prefix (optional parts marked by squared brackets): +``` +[username[:password]@][protocol[(address)]]/dbname[?param1=value1&...¶mN=valueN] +``` + +A DSN in its fullest form: +``` +username:password@protocol(address)/dbname?param=value +``` + +Except for the databasename, all values are optional. So the minimal DSN is: +``` +/dbname +``` + +If you do not want to preselect a database, leave `dbname` empty: +``` +/ +``` +This has the same effect as an empty DSN string: +``` + +``` + +Alternatively, [Config.FormatDSN](https://godoc.org/github.com/go-sql-driver/mysql#Config.FormatDSN) can be used to create a DSN string by filling a struct. + +#### Password +Passwords can consist of any character. Escaping is **not** necessary. + +#### Protocol +See [net.Dial](http://golang.org/pkg/net/#Dial) for more information which networks are available. +In general you should use an Unix domain socket if available and TCP otherwise for best performance. + +#### Address +For TCP and UDP networks, addresses have the form `host:port`. +If `host` is a literal IPv6 address, it must be enclosed in square brackets. +The functions [net.JoinHostPort](http://golang.org/pkg/net/#JoinHostPort) and [net.SplitHostPort](http://golang.org/pkg/net/#SplitHostPort) manipulate addresses in this form. + +For Unix domain sockets the address is the absolute path to the MySQL-Server-socket, e.g. `/var/run/mysqld/mysqld.sock` or `/tmp/mysql.sock`. + +#### Parameters +*Parameters are case-sensitive!* + +Notice that any of `true`, `TRUE`, `True` or `1` is accepted to stand for a true boolean value. Not surprisingly, false can be specified as any of: `false`, `FALSE`, `False` or `0`. + +##### `allowAllFiles` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +`allowAllFiles=true` disables the file Whitelist for `LOAD DATA LOCAL INFILE` and allows *all* files. +[*Might be insecure!*](http://dev.mysql.com/doc/refman/5.7/en/load-data-local.html) + +##### `allowCleartextPasswords` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +`allowCleartextPasswords=true` allows using the [cleartext client side plugin](http://dev.mysql.com/doc/en/cleartext-authentication-plugin.html) if required by an account, such as one defined with the [PAM authentication plugin](http://dev.mysql.com/doc/en/pam-authentication-plugin.html). Sending passwords in clear text may be a security problem in some configurations. To avoid problems if there is any possibility that the password would be intercepted, clients should connect to MySQL Server using a method that protects the password. Possibilities include [TLS / SSL](#tls), IPsec, or a private network. + +##### `allowNativePasswords` + +``` +Type: bool +Valid Values: true, false +Default: false +``` +`allowNativePasswords=true` allows the usage of the mysql native password method. + +##### `allowOldPasswords` + +``` +Type: bool +Valid Values: true, false +Default: false +``` +`allowOldPasswords=true` allows the usage of the insecure old password method. This should be avoided, but is necessary in some cases. See also [the old_passwords wiki page](https://github.com/go-sql-driver/mysql/wiki/old_passwords). + +##### `charset` + +``` +Type: string +Valid Values: +Default: none +``` + +Sets the charset used for client-server interaction (`"SET NAMES "`). If multiple charsets are set (separated by a comma), the following charset is used if setting the charset failes. This enables for example support for `utf8mb4` ([introduced in MySQL 5.5.3](http://dev.mysql.com/doc/refman/5.5/en/charset-unicode-utf8mb4.html)) with fallback to `utf8` for older servers (`charset=utf8mb4,utf8`). + +Usage of the `charset` parameter is discouraged because it issues additional queries to the server. +Unless you need the fallback behavior, please use `collation` instead. + +##### `collation` + +``` +Type: string +Valid Values: +Default: utf8_general_ci +``` + +Sets the collation used for client-server interaction on connection. In contrast to `charset`, `collation` does not issue additional queries. If the specified collation is unavailable on the target server, the connection will fail. + +A list of valid charsets for a server is retrievable with `SHOW COLLATION`. + +##### `clientFoundRows` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +`clientFoundRows=true` causes an UPDATE to return the number of matching rows instead of the number of rows changed. + +##### `columnsWithAlias` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +When `columnsWithAlias` is true, calls to `sql.Rows.Columns()` will return the table alias and the column name separated by a dot. For example: + +``` +SELECT u.id FROM users as u +``` + +will return `u.id` instead of just `id` if `columnsWithAlias=true`. + +##### `interpolateParams` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +If `interpolateParams` is true, placeholders (`?`) in calls to `db.Query()` and `db.Exec()` are interpolated into a single query string with given parameters. This reduces the number of roundtrips, since the driver has to prepare a statement, execute it with given parameters and close the statement again with `interpolateParams=false`. + +*This can not be used together with the multibyte encodings BIG5, CP932, GB2312, GBK or SJIS. These are blacklisted as they may [introduce a SQL injection vulnerability](http://stackoverflow.com/a/12118602/3430118)!* + +##### `loc` + +``` +Type: string +Valid Values: +Default: UTC +``` + +Sets the location for time.Time values (when using `parseTime=true`). *"Local"* sets the system's location. See [time.LoadLocation](http://golang.org/pkg/time/#LoadLocation) for details. + +Note that this sets the location for time.Time values but does not change MySQL's [time_zone setting](https://dev.mysql.com/doc/refman/5.5/en/time-zone-support.html). For that see the [time_zone system variable](#system-variables), which can also be set as a DSN parameter. + +Please keep in mind, that param values must be [url.QueryEscape](http://golang.org/pkg/net/url/#QueryEscape)'ed. Alternatively you can manually replace the `/` with `%2F`. For example `US/Pacific` would be `loc=US%2FPacific`. + +##### `maxAllowedPacket` +``` +Type: decimal number +Default: 0 +``` + +Max packet size allowed in bytes. Use `maxAllowedPacket=0` to automatically fetch the `max_allowed_packet` variable from server. + +##### `multiStatements` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +Allow multiple statements in one query. While this allows batch queries, it also greatly increases the risk of SQL injections. Only the result of the first query is returned, all other results are silently discarded. + +When `multiStatements` is used, `?` parameters must only be used in the first statement. + +##### `parseTime` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +`parseTime=true` changes the output type of `DATE` and `DATETIME` values to `time.Time` instead of `[]byte` / `string` + + +##### `readTimeout` + +``` +Type: decimal number +Default: 0 +``` + +I/O read timeout. The value must be a decimal number with an unit suffix ( *"ms"*, *"s"*, *"m"*, *"h"* ), such as *"30s"*, *"0.5m"* or *"1m30s"*. + +##### `strict` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +`strict=true` enables a driver-side strict mode in which MySQL warnings are treated as errors. This mode should not be used in production as it may lead to data corruption in certain situations. + +A server-side strict mode, which is safe for production use, can be set via the [`sql_mode`](https://dev.mysql.com/doc/refman/5.7/en/sql-mode.html) system variable. + +By default MySQL also treats notes as warnings. Use [`sql_notes=false`](http://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_sql_notes) to ignore notes. + +##### `timeout` + +``` +Type: decimal number +Default: OS default +``` + +*Driver* side connection timeout. The value must be a decimal number with an unit suffix ( *"ms"*, *"s"*, *"m"*, *"h"* ), such as *"30s"*, *"0.5m"* or *"1m30s"*. To set a server side timeout, use the parameter [`wait_timeout`](http://dev.mysql.com/doc/refman/5.6/en/server-system-variables.html#sysvar_wait_timeout). + +##### `tls` + +``` +Type: bool / string +Valid Values: true, false, skip-verify, +Default: false +``` + +`tls=true` enables TLS / SSL encrypted connection to the server. Use `skip-verify` if you want to use a self-signed or invalid certificate (server side). Use a custom value registered with [`mysql.RegisterTLSConfig`](http://godoc.org/github.com/go-sql-driver/mysql#RegisterTLSConfig). + +##### `writeTimeout` + +``` +Type: decimal number +Default: 0 +``` + +I/O write timeout. The value must be a decimal number with an unit suffix ( *"ms"*, *"s"*, *"m"*, *"h"* ), such as *"30s"*, *"0.5m"* or *"1m30s"*. + + +##### System Variables + +Any other parameters are interpreted as system variables: + * `=`: `SET =` + * `=`: `SET =` + * `=%27%27`: `SET =''` + +Rules: +* The values for string variables must be quoted with ' +* The values must also be [url.QueryEscape](http://golang.org/pkg/net/url/#QueryEscape)'ed! + (which implies values of string variables must be wrapped with `%27`) + +Examples: + * `autocommit=1`: `SET autocommit=1` + * [`time_zone=%27Europe%2FParis%27`](https://dev.mysql.com/doc/refman/5.5/en/time-zone-support.html): `SET time_zone='Europe/Paris'` + * [`tx_isolation=%27REPEATABLE-READ%27`](https://dev.mysql.com/doc/refman/5.5/en/server-system-variables.html#sysvar_tx_isolation): `SET tx_isolation='REPEATABLE-READ'` + + +#### Examples +``` +user@unix(/path/to/socket)/dbname +``` + +``` +root:pw@unix(/tmp/mysql.sock)/myDatabase?loc=Local +``` + +``` +user:password@tcp(localhost:5555)/dbname?tls=skip-verify&autocommit=true +``` + +Treat warnings as errors by setting the system variable [`sql_mode`](https://dev.mysql.com/doc/refman/5.7/en/sql-mode.html): +``` +user:password@/dbname?sql_mode=TRADITIONAL +``` + +TCP via IPv6: +``` +user:password@tcp([de:ad:be:ef::ca:fe]:80)/dbname?timeout=90s&collation=utf8mb4_unicode_ci +``` + +TCP on a remote host, e.g. Amazon RDS: +``` +id:password@tcp(your-amazonaws-uri.com:3306)/dbname +``` + +Google Cloud SQL on App Engine (First Generation MySQL Server): +``` +user@cloudsql(project-id:instance-name)/dbname +``` + +Google Cloud SQL on App Engine (Second Generation MySQL Server): +``` +user@cloudsql(project-id:regionname:instance-name)/dbname +``` + +TCP using default port (3306) on localhost: +``` +user:password@tcp/dbname?charset=utf8mb4,utf8&sys_var=esc%40ped +``` + +Use the default protocol (tcp) and host (localhost:3306): +``` +user:password@/dbname +``` + +No Database preselected: +``` +user:password@/ +``` + +### `LOAD DATA LOCAL INFILE` support +For this feature you need direct access to the package. Therefore you must change the import path (no `_`): +```go +import "github.com/go-sql-driver/mysql" +``` + +Files must be whitelisted by registering them with `mysql.RegisterLocalFile(filepath)` (recommended) or the Whitelist check must be deactivated by using the DSN parameter `allowAllFiles=true` ([*Might be insecure!*](http://dev.mysql.com/doc/refman/5.7/en/load-data-local.html)). + +To use a `io.Reader` a handler function must be registered with `mysql.RegisterReaderHandler(name, handler)` which returns a `io.Reader` or `io.ReadCloser`. The Reader is available with the filepath `Reader::` then. Choose different names for different handlers and `DeregisterReaderHandler` when you don't need it anymore. + +See the [godoc of Go-MySQL-Driver](http://godoc.org/github.com/go-sql-driver/mysql "golang mysql driver documentation") for details. + + +### `time.Time` support +The default internal output type of MySQL `DATE` and `DATETIME` values is `[]byte` which allows you to scan the value into a `[]byte`, `string` or `sql.RawBytes` variable in your programm. + +However, many want to scan MySQL `DATE` and `DATETIME` values into `time.Time` variables, which is the logical opposite in Go to `DATE` and `DATETIME` in MySQL. You can do that by changing the internal output type from `[]byte` to `time.Time` with the DSN parameter `parseTime=true`. You can set the default [`time.Time` location](http://golang.org/pkg/time/#Location) with the `loc` DSN parameter. + +**Caution:** As of Go 1.1, this makes `time.Time` the only variable type you can scan `DATE` and `DATETIME` values into. This breaks for example [`sql.RawBytes` support](https://github.com/go-sql-driver/mysql/wiki/Examples#rawbytes). + +Alternatively you can use the [`NullTime`](http://godoc.org/github.com/go-sql-driver/mysql#NullTime) type as the scan destination, which works with both `time.Time` and `string` / `[]byte`. + + +### Unicode support +Since version 1.1 Go-MySQL-Driver automatically uses the collation `utf8_general_ci` by default. + +Other collations / charsets can be set using the [`collation`](#collation) DSN parameter. + +Version 1.0 of the driver recommended adding `&charset=utf8` (alias for `SET NAMES utf8`) to the DSN to enable proper UTF-8 support. This is not necessary anymore. The [`collation`](#collation) parameter should be preferred to set another collation / charset than the default. + +See http://dev.mysql.com/doc/refman/5.7/en/charset-unicode.html for more details on MySQL's Unicode support. + + +## Testing / Development +To run the driver tests you may need to adjust the configuration. See the [Testing Wiki-Page](https://github.com/go-sql-driver/mysql/wiki/Testing "Testing") for details. + +Go-MySQL-Driver is not feature-complete yet. Your help is very appreciated. +If you want to contribute, you can work on an [open issue](https://github.com/go-sql-driver/mysql/issues?state=open) or review a [pull request](https://github.com/go-sql-driver/mysql/pulls). + +See the [Contribution Guidelines](https://github.com/go-sql-driver/mysql/blob/master/CONTRIBUTING.md) for details. + +--------------------------------------- + +## License +Go-MySQL-Driver is licensed under the [Mozilla Public License Version 2.0](https://raw.github.com/go-sql-driver/mysql/master/LICENSE) + +Mozilla summarizes the license scope as follows: +> MPL: The copyleft applies to any files containing MPLed code. + + +That means: + * You can **use** the **unchanged** source code both in private and commercially + * When distributing, you **must publish** the source code of any **changed files** licensed under the MPL 2.0 under a) the MPL 2.0 itself or b) a compatible license (e.g. GPL 3.0 or Apache License 2.0) + * You **needn't publish** the source code of your library as long as the files licensed under the MPL 2.0 are **unchanged** + +Please read the [MPL 2.0 FAQ](http://www.mozilla.org/MPL/2.0/FAQ.html) if you have further questions regarding the license. + +You can read the full terms here: [LICENSE](https://raw.github.com/go-sql-driver/mysql/master/LICENSE) + +![Go Gopher and MySQL Dolphin](https://raw.github.com/wiki/go-sql-driver/mysql/go-mysql-driver_m.jpg "Golang Gopher transporting the MySQL Dolphin in a wheelbarrow") + diff --git a/vendor/github.com/go-sql-driver/mysql/buffer.go b/vendor/github.com/go-sql-driver/mysql/buffer.go index 509ce89e46..2001feacd3 100644 --- a/vendor/github.com/go-sql-driver/mysql/buffer.go +++ b/vendor/github.com/go-sql-driver/mysql/buffer.go @@ -8,7 +8,11 @@ package mysql -import "io" +import ( + "io" + "net" + "time" +) const defaultBufSize = 4096 @@ -18,17 +22,18 @@ const defaultBufSize = 4096 // The buffer is similar to bufio.Reader / Writer but zero-copy-ish // Also highly optimized for this particular use case. type buffer struct { - buf []byte - rd io.Reader - idx int - length int + buf []byte + nc net.Conn + idx int + length int + timeout time.Duration } -func newBuffer(rd io.Reader) buffer { +func newBuffer(nc net.Conn) buffer { var b [defaultBufSize]byte return buffer{ buf: b[:], - rd: rd, + nc: nc, } } @@ -54,7 +59,13 @@ func (b *buffer) fill(need int) error { b.idx = 0 for { - nn, err := b.rd.Read(b.buf[n:]) + if b.timeout > 0 { + if err := b.nc.SetReadDeadline(time.Now().Add(b.timeout)); err != nil { + return err + } + } + + nn, err := b.nc.Read(b.buf[n:]) n += nn switch err { diff --git a/vendor/github.com/go-sql-driver/mysql/collations.go b/vendor/github.com/go-sql-driver/mysql/collations.go index 6c1d613d5b..82079cfb93 100644 --- a/vendor/github.com/go-sql-driver/mysql/collations.go +++ b/vendor/github.com/go-sql-driver/mysql/collations.go @@ -8,7 +8,7 @@ package mysql -const defaultCollation byte = 33 // utf8_general_ci +const defaultCollation = "utf8_general_ci" // A list of available collations mapped to the internal ID. // To update this map use the following MySQL query: @@ -237,14 +237,14 @@ var collations = map[string]byte{ // A blacklist of collations which is unsafe to interpolate parameters. // These multibyte encodings may contains 0x5c (`\`) in their trailing bytes. -var unsafeCollations = map[byte]bool{ - 1: true, // big5_chinese_ci - 13: true, // sjis_japanese_ci - 28: true, // gbk_chinese_ci - 84: true, // big5_bin - 86: true, // gb2312_bin - 87: true, // gbk_bin - 88: true, // sjis_bin - 95: true, // cp932_japanese_ci - 96: true, // cp932_bin +var unsafeCollations = map[string]bool{ + "big5_chinese_ci": true, + "sjis_japanese_ci": true, + "gbk_chinese_ci": true, + "big5_bin": true, + "gb2312_bin": true, + "gbk_bin": true, + "sjis_bin": true, + "cp932_japanese_ci": true, + "cp932_bin": true, } diff --git a/vendor/github.com/go-sql-driver/mysql/connection.go b/vendor/github.com/go-sql-driver/mysql/connection.go index a6d39bec95..d82c728f3b 100644 --- a/vendor/github.com/go-sql-driver/mysql/connection.go +++ b/vendor/github.com/go-sql-driver/mysql/connection.go @@ -9,9 +9,7 @@ package mysql import ( - "crypto/tls" "database/sql/driver" - "errors" "net" "strconv" "strings" @@ -23,9 +21,10 @@ type mysqlConn struct { netConn net.Conn affectedRows uint64 insertId uint64 - cfg *config - maxPacketAllowed int + cfg *Config + maxAllowedPacket int maxWriteSize int + writeTimeout time.Duration flags clientFlag status statusFlag sequence uint8 @@ -33,27 +32,9 @@ type mysqlConn struct { strict bool } -type config struct { - user string - passwd string - net string - addr string - dbname string - params map[string]string - loc *time.Location - tls *tls.Config - timeout time.Duration - collation uint8 - allowAllFiles bool - allowOldPasswords bool - clientFoundRows bool - columnsWithAlias bool - interpolateParams bool -} - // Handles parameters set in DSN after the connection is established func (mc *mysqlConn) handleParams() (err error) { - for param, val := range mc.cfg.params { + for param, val := range mc.cfg.Params { switch param { // Charset case "charset": @@ -69,27 +50,6 @@ func (mc *mysqlConn) handleParams() (err error) { return } - // time.Time parsing - case "parseTime": - var isBool bool - mc.parseTime, isBool = readBool(val) - if !isBool { - return errors.New("Invalid Bool value: " + val) - } - - // Strict mode - case "strict": - var isBool bool - mc.strict, isBool = readBool(val) - if !isBool { - return errors.New("Invalid Bool value: " + val) - } - - // Compression - case "compress": - err = errors.New("Compression not implemented yet") - return - // System Vars default: err = mc.exec("SET " + param + "=" + val + "") @@ -119,20 +79,29 @@ func (mc *mysqlConn) Close() (err error) { // Makes Close idempotent if mc.netConn != nil { err = mc.writeCommandPacket(comQuit) - if err == nil { - err = mc.netConn.Close() - } else { - mc.netConn.Close() - } - mc.netConn = nil } - mc.cfg = nil - mc.buf.rd = nil + mc.cleanup() return } +// Closes the network connection and unsets internal variables. Do not call this +// function after successfully authentication, call Close instead. This function +// is called before auth or on auth failure because MySQL will have already +// closed the network connection. +func (mc *mysqlConn) cleanup() { + // Makes cleanup idempotent + if mc.netConn != nil { + if err := mc.netConn.Close(); err != nil { + errLog.Print(err) + } + mc.netConn = nil + } + mc.cfg = nil + mc.buf.nc = nil +} + func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) { if mc.netConn == nil { errLog.Print(ErrInvalidConn) @@ -166,6 +135,11 @@ func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) { } func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (string, error) { + // Number of ? should be same to len(args) + if strings.Count(query, "?") != len(args) { + return "", driver.ErrSkip + } + buf := mc.buf.takeCompleteBuffer() if buf == nil { // can not take the buffer. Something must be wrong with the connection @@ -207,7 +181,7 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin if v.IsZero() { buf = append(buf, "'0000-00-00'"...) } else { - v := v.In(mc.cfg.loc) + v := v.In(mc.cfg.Loc) v = v.Add(time.Nanosecond * 500) // To round under microsecond year := v.Year() year100 := year / 100 @@ -252,7 +226,7 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin if v == nil { buf = append(buf, "NULL"...) } else { - buf = append(buf, '\'') + buf = append(buf, "_binary'"...) if mc.status&statusNoBackslashEscapes == 0 { buf = escapeBytesBackslash(buf, v) } else { @@ -272,7 +246,7 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin return "", driver.ErrSkip } - if len(buf)+4 > mc.maxPacketAllowed { + if len(buf)+4 > mc.maxAllowedPacket { return "", driver.ErrSkip } } @@ -288,7 +262,7 @@ func (mc *mysqlConn) Exec(query string, args []driver.Value) (driver.Result, err return nil, driver.ErrBadConn } if len(args) != 0 { - if !mc.cfg.interpolateParams { + if !mc.cfg.InterpolateParams { return nil, driver.ErrSkip } // try to interpolate the parameters to save extra roundtrips for preparing and closing a statement @@ -339,7 +313,7 @@ func (mc *mysqlConn) Query(query string, args []driver.Value) (driver.Rows, erro return nil, driver.ErrBadConn } if len(args) != 0 { - if !mc.cfg.interpolateParams { + if !mc.cfg.InterpolateParams { return nil, driver.ErrSkip } // try client-side prepare to reduce roundtrip @@ -385,6 +359,7 @@ func (mc *mysqlConn) getSystemVar(name string) ([]byte, error) { if err == nil { rows := new(textRows) rows.mc = mc + rows.columns = []mysqlField{{fieldType: fieldTypeVarChar}} if resLen > 0 { // Columns diff --git a/vendor/github.com/go-sql-driver/mysql/const.go b/vendor/github.com/go-sql-driver/mysql/const.go index dddc12908f..88cfff3fd8 100644 --- a/vendor/github.com/go-sql-driver/mysql/const.go +++ b/vendor/github.com/go-sql-driver/mysql/const.go @@ -107,7 +107,8 @@ const ( fieldTypeBit ) const ( - fieldTypeNewDecimal byte = iota + 0xf6 + fieldTypeJSON byte = iota + 0xf5 + fieldTypeNewDecimal fieldTypeEnum fieldTypeSet fieldTypeTinyBLOB diff --git a/vendor/github.com/go-sql-driver/mysql/driver.go b/vendor/github.com/go-sql-driver/mysql/driver.go index 3cbbe6031c..0022d1f1e5 100644 --- a/vendor/github.com/go-sql-driver/mysql/driver.go +++ b/vendor/github.com/go-sql-driver/mysql/driver.go @@ -4,7 +4,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// Package mysql provides a MySQL driver for Go's database/sql package // // The driver should be used via the database/sql package: // @@ -22,7 +22,7 @@ import ( "net" ) -// This struct is exported to make the driver directly accessible. +// MySQLDriver is exported to make the driver directly accessible. // In general the driver is used via the database/sql package. type MySQLDriver struct{} @@ -50,20 +50,22 @@ func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { // New mysqlConn mc := &mysqlConn{ - maxPacketAllowed: maxPacketSize, + maxAllowedPacket: maxPacketSize, maxWriteSize: maxPacketSize - 1, } - mc.cfg, err = parseDSN(dsn) + mc.cfg, err = ParseDSN(dsn) if err != nil { return nil, err } + mc.parseTime = mc.cfg.ParseTime + mc.strict = mc.cfg.Strict // Connect to Server - if dial, ok := dials[mc.cfg.net]; ok { - mc.netConn, err = dial(mc.cfg.addr) + if dial, ok := dials[mc.cfg.Net]; ok { + mc.netConn, err = dial(mc.cfg.Addr) } else { - nd := net.Dialer{Timeout: mc.cfg.timeout} - mc.netConn, err = nd.Dial(mc.cfg.net, mc.cfg.addr) + nd := net.Dialer{Timeout: mc.cfg.Timeout} + mc.netConn, err = nd.Dial(mc.cfg.Net, mc.cfg.Addr) } if err != nil { return nil, err @@ -81,48 +83,45 @@ func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { mc.buf = newBuffer(mc.netConn) + // Set I/O timeouts + mc.buf.timeout = mc.cfg.ReadTimeout + mc.writeTimeout = mc.cfg.WriteTimeout + // Reading Handshake Initialization Packet cipher, err := mc.readInitPacket() if err != nil { - mc.Close() + mc.cleanup() return nil, err } // Send Client Authentication Packet if err = mc.writeAuthPacket(cipher); err != nil { - mc.Close() + mc.cleanup() return nil, err } - // Read Result Packet - err = mc.readResultOK() - if err != nil { - // Retry with old authentication method, if allowed - if mc.cfg != nil && mc.cfg.allowOldPasswords && err == ErrOldPassword { - if err = mc.writeOldAuthPacket(cipher); err != nil { - mc.Close() - return nil, err - } - if err = mc.readResultOK(); err != nil { - mc.Close() - return nil, err - } - } else { + // Handle response to auth packet, switch methods if possible + if err = handleAuthResult(mc, cipher); err != nil { + // Authentication failed and MySQL has already closed the connection + // (https://dev.mysql.com/doc/internals/en/authentication-fails.html). + // Do not send COM_QUIT, just cleanup and return the error. + mc.cleanup() + return nil, err + } + + if mc.cfg.MaxAllowedPacket > 0 { + mc.maxAllowedPacket = mc.cfg.MaxAllowedPacket + } else { + // Get max allowed packet size + maxap, err := mc.getSystemVar("max_allowed_packet") + if err != nil { mc.Close() return nil, err } - - } - - // Get max allowed packet size - maxap, err := mc.getSystemVar("max_allowed_packet") - if err != nil { - mc.Close() - return nil, err + mc.maxAllowedPacket = stringToInt(maxap) - 1 } - mc.maxPacketAllowed = stringToInt(maxap) - 1 - if mc.maxPacketAllowed < maxPacketSize { - mc.maxWriteSize = mc.maxPacketAllowed + if mc.maxAllowedPacket < maxPacketSize { + mc.maxWriteSize = mc.maxAllowedPacket } // Handle DSN Params @@ -135,6 +134,50 @@ func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { return mc, nil } +func handleAuthResult(mc *mysqlConn, oldCipher []byte) error { + // Read Result Packet + cipher, err := mc.readResultOK() + if err == nil { + return nil // auth successful + } + + if mc.cfg == nil { + return err // auth failed and retry not possible + } + + // Retry auth if configured to do so. + if mc.cfg.AllowOldPasswords && err == ErrOldPassword { + // Retry with old authentication method. Note: there are edge cases + // where this should work but doesn't; this is currently "wontfix": + // https://github.com/go-sql-driver/mysql/issues/184 + + // If CLIENT_PLUGIN_AUTH capability is not supported, no new cipher is + // sent and we have to keep using the cipher sent in the init packet. + if cipher == nil { + cipher = oldCipher + } + + if err = mc.writeOldAuthPacket(cipher); err != nil { + return err + } + _, err = mc.readResultOK() + } else if mc.cfg.AllowCleartextPasswords && err == ErrCleartextPassword { + // Retry with clear text password for + // http://dev.mysql.com/doc/refman/5.7/en/cleartext-authentication-plugin.html + // http://dev.mysql.com/doc/refman/5.7/en/pam-authentication-plugin.html + if err = mc.writeClearAuthPacket(); err != nil { + return err + } + _, err = mc.readResultOK() + } else if mc.cfg.AllowNativePasswords && err == ErrNativePassword { + if err = mc.writeNativeAuthPacket(cipher); err != nil { + return err + } + _, err = mc.readResultOK() + } + return err +} + func init() { sql.Register("mysql", &MySQLDriver{}) } diff --git a/vendor/github.com/go-sql-driver/mysql/dsn.go b/vendor/github.com/go-sql-driver/mysql/dsn.go new file mode 100644 index 0000000000..ac00dcedd5 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/dsn.go @@ -0,0 +1,548 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2016 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "crypto/tls" + "errors" + "fmt" + "net" + "net/url" + "strconv" + "strings" + "time" +) + +var ( + errInvalidDSNUnescaped = errors.New("invalid DSN: did you forget to escape a param value?") + errInvalidDSNAddr = errors.New("invalid DSN: network address not terminated (missing closing brace)") + errInvalidDSNNoSlash = errors.New("invalid DSN: missing the slash separating the database name") + errInvalidDSNUnsafeCollation = errors.New("invalid DSN: interpolateParams can not be used with unsafe collations") +) + +// Config is a configuration parsed from a DSN string +type Config struct { + User string // Username + Passwd string // Password (requires User) + Net string // Network type + Addr string // Network address (requires Net) + DBName string // Database name + Params map[string]string // Connection parameters + Collation string // Connection collation + Loc *time.Location // Location for time.Time values + MaxAllowedPacket int // Max packet size allowed + TLSConfig string // TLS configuration name + tls *tls.Config // TLS configuration + Timeout time.Duration // Dial timeout + ReadTimeout time.Duration // I/O read timeout + WriteTimeout time.Duration // I/O write timeout + + AllowAllFiles bool // Allow all files to be used with LOAD DATA LOCAL INFILE + AllowCleartextPasswords bool // Allows the cleartext client side plugin + AllowNativePasswords bool // Allows the native password authentication method + AllowOldPasswords bool // Allows the old insecure password method + ClientFoundRows bool // Return number of matching rows instead of rows changed + ColumnsWithAlias bool // Prepend table alias to column names + InterpolateParams bool // Interpolate placeholders into query string + MultiStatements bool // Allow multiple statements in one query + ParseTime bool // Parse time values to time.Time + Strict bool // Return warnings as errors +} + +// FormatDSN formats the given Config into a DSN string which can be passed to +// the driver. +func (cfg *Config) FormatDSN() string { + var buf bytes.Buffer + + // [username[:password]@] + if len(cfg.User) > 0 { + buf.WriteString(cfg.User) + if len(cfg.Passwd) > 0 { + buf.WriteByte(':') + buf.WriteString(cfg.Passwd) + } + buf.WriteByte('@') + } + + // [protocol[(address)]] + if len(cfg.Net) > 0 { + buf.WriteString(cfg.Net) + if len(cfg.Addr) > 0 { + buf.WriteByte('(') + buf.WriteString(cfg.Addr) + buf.WriteByte(')') + } + } + + // /dbname + buf.WriteByte('/') + buf.WriteString(cfg.DBName) + + // [?param1=value1&...¶mN=valueN] + hasParam := false + + if cfg.AllowAllFiles { + hasParam = true + buf.WriteString("?allowAllFiles=true") + } + + if cfg.AllowCleartextPasswords { + if hasParam { + buf.WriteString("&allowCleartextPasswords=true") + } else { + hasParam = true + buf.WriteString("?allowCleartextPasswords=true") + } + } + + if cfg.AllowNativePasswords { + if hasParam { + buf.WriteString("&allowNativePasswords=true") + } else { + hasParam = true + buf.WriteString("?allowNativePasswords=true") + } + } + + if cfg.AllowOldPasswords { + if hasParam { + buf.WriteString("&allowOldPasswords=true") + } else { + hasParam = true + buf.WriteString("?allowOldPasswords=true") + } + } + + if cfg.ClientFoundRows { + if hasParam { + buf.WriteString("&clientFoundRows=true") + } else { + hasParam = true + buf.WriteString("?clientFoundRows=true") + } + } + + if col := cfg.Collation; col != defaultCollation && len(col) > 0 { + if hasParam { + buf.WriteString("&collation=") + } else { + hasParam = true + buf.WriteString("?collation=") + } + buf.WriteString(col) + } + + if cfg.ColumnsWithAlias { + if hasParam { + buf.WriteString("&columnsWithAlias=true") + } else { + hasParam = true + buf.WriteString("?columnsWithAlias=true") + } + } + + if cfg.InterpolateParams { + if hasParam { + buf.WriteString("&interpolateParams=true") + } else { + hasParam = true + buf.WriteString("?interpolateParams=true") + } + } + + if cfg.Loc != time.UTC && cfg.Loc != nil { + if hasParam { + buf.WriteString("&loc=") + } else { + hasParam = true + buf.WriteString("?loc=") + } + buf.WriteString(url.QueryEscape(cfg.Loc.String())) + } + + if cfg.MultiStatements { + if hasParam { + buf.WriteString("&multiStatements=true") + } else { + hasParam = true + buf.WriteString("?multiStatements=true") + } + } + + if cfg.ParseTime { + if hasParam { + buf.WriteString("&parseTime=true") + } else { + hasParam = true + buf.WriteString("?parseTime=true") + } + } + + if cfg.ReadTimeout > 0 { + if hasParam { + buf.WriteString("&readTimeout=") + } else { + hasParam = true + buf.WriteString("?readTimeout=") + } + buf.WriteString(cfg.ReadTimeout.String()) + } + + if cfg.Strict { + if hasParam { + buf.WriteString("&strict=true") + } else { + hasParam = true + buf.WriteString("?strict=true") + } + } + + if cfg.Timeout > 0 { + if hasParam { + buf.WriteString("&timeout=") + } else { + hasParam = true + buf.WriteString("?timeout=") + } + buf.WriteString(cfg.Timeout.String()) + } + + if len(cfg.TLSConfig) > 0 { + if hasParam { + buf.WriteString("&tls=") + } else { + hasParam = true + buf.WriteString("?tls=") + } + buf.WriteString(url.QueryEscape(cfg.TLSConfig)) + } + + if cfg.WriteTimeout > 0 { + if hasParam { + buf.WriteString("&writeTimeout=") + } else { + hasParam = true + buf.WriteString("?writeTimeout=") + } + buf.WriteString(cfg.WriteTimeout.String()) + } + + if cfg.MaxAllowedPacket > 0 { + if hasParam { + buf.WriteString("&maxAllowedPacket=") + } else { + hasParam = true + buf.WriteString("?maxAllowedPacket=") + } + buf.WriteString(strconv.Itoa(cfg.MaxAllowedPacket)) + + } + + // other params + if cfg.Params != nil { + for param, value := range cfg.Params { + if hasParam { + buf.WriteByte('&') + } else { + hasParam = true + buf.WriteByte('?') + } + + buf.WriteString(param) + buf.WriteByte('=') + buf.WriteString(url.QueryEscape(value)) + } + } + + return buf.String() +} + +// ParseDSN parses the DSN string to a Config +func ParseDSN(dsn string) (cfg *Config, err error) { + // New config with some default values + cfg = &Config{ + Loc: time.UTC, + Collation: defaultCollation, + } + + // [user[:password]@][net[(addr)]]/dbname[?param1=value1¶mN=valueN] + // Find the last '/' (since the password or the net addr might contain a '/') + foundSlash := false + for i := len(dsn) - 1; i >= 0; i-- { + if dsn[i] == '/' { + foundSlash = true + var j, k int + + // left part is empty if i <= 0 + if i > 0 { + // [username[:password]@][protocol[(address)]] + // Find the last '@' in dsn[:i] + for j = i; j >= 0; j-- { + if dsn[j] == '@' { + // username[:password] + // Find the first ':' in dsn[:j] + for k = 0; k < j; k++ { + if dsn[k] == ':' { + cfg.Passwd = dsn[k+1 : j] + break + } + } + cfg.User = dsn[:k] + + break + } + } + + // [protocol[(address)]] + // Find the first '(' in dsn[j+1:i] + for k = j + 1; k < i; k++ { + if dsn[k] == '(' { + // dsn[i-1] must be == ')' if an address is specified + if dsn[i-1] != ')' { + if strings.ContainsRune(dsn[k+1:i], ')') { + return nil, errInvalidDSNUnescaped + } + return nil, errInvalidDSNAddr + } + cfg.Addr = dsn[k+1 : i-1] + break + } + } + cfg.Net = dsn[j+1 : k] + } + + // dbname[?param1=value1&...¶mN=valueN] + // Find the first '?' in dsn[i+1:] + for j = i + 1; j < len(dsn); j++ { + if dsn[j] == '?' { + if err = parseDSNParams(cfg, dsn[j+1:]); err != nil { + return + } + break + } + } + cfg.DBName = dsn[i+1 : j] + + break + } + } + + if !foundSlash && len(dsn) > 0 { + return nil, errInvalidDSNNoSlash + } + + if cfg.InterpolateParams && unsafeCollations[cfg.Collation] { + return nil, errInvalidDSNUnsafeCollation + } + + // Set default network if empty + if cfg.Net == "" { + cfg.Net = "tcp" + } + + // Set default address if empty + if cfg.Addr == "" { + switch cfg.Net { + case "tcp": + cfg.Addr = "127.0.0.1:3306" + case "unix": + cfg.Addr = "/tmp/mysql.sock" + default: + return nil, errors.New("default addr for network '" + cfg.Net + "' unknown") + } + + } + + return +} + +// parseDSNParams parses the DSN "query string" +// Values must be url.QueryEscape'ed +func parseDSNParams(cfg *Config, params string) (err error) { + for _, v := range strings.Split(params, "&") { + param := strings.SplitN(v, "=", 2) + if len(param) != 2 { + continue + } + + // cfg params + switch value := param[1]; param[0] { + + // Disable INFILE whitelist / enable all files + case "allowAllFiles": + var isBool bool + cfg.AllowAllFiles, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Use cleartext authentication mode (MySQL 5.5.10+) + case "allowCleartextPasswords": + var isBool bool + cfg.AllowCleartextPasswords, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Use native password authentication + case "allowNativePasswords": + var isBool bool + cfg.AllowNativePasswords, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Use old authentication mode (pre MySQL 4.1) + case "allowOldPasswords": + var isBool bool + cfg.AllowOldPasswords, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Switch "rowsAffected" mode + case "clientFoundRows": + var isBool bool + cfg.ClientFoundRows, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Collation + case "collation": + cfg.Collation = value + break + + case "columnsWithAlias": + var isBool bool + cfg.ColumnsWithAlias, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Compression + case "compress": + return errors.New("compression not implemented yet") + + // Enable client side placeholder substitution + case "interpolateParams": + var isBool bool + cfg.InterpolateParams, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Time Location + case "loc": + if value, err = url.QueryUnescape(value); err != nil { + return + } + cfg.Loc, err = time.LoadLocation(value) + if err != nil { + return + } + + // multiple statements in one query + case "multiStatements": + var isBool bool + cfg.MultiStatements, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // time.Time parsing + case "parseTime": + var isBool bool + cfg.ParseTime, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // I/O read Timeout + case "readTimeout": + cfg.ReadTimeout, err = time.ParseDuration(value) + if err != nil { + return + } + + // Strict mode + case "strict": + var isBool bool + cfg.Strict, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Dial Timeout + case "timeout": + cfg.Timeout, err = time.ParseDuration(value) + if err != nil { + return + } + + // TLS-Encryption + case "tls": + boolValue, isBool := readBool(value) + if isBool { + if boolValue { + cfg.TLSConfig = "true" + cfg.tls = &tls.Config{} + } else { + cfg.TLSConfig = "false" + } + } else if vl := strings.ToLower(value); vl == "skip-verify" { + cfg.TLSConfig = vl + cfg.tls = &tls.Config{InsecureSkipVerify: true} + } else { + name, err := url.QueryUnescape(value) + if err != nil { + return fmt.Errorf("invalid value for TLS config name: %v", err) + } + + if tlsConfig, ok := tlsConfigRegister[name]; ok { + if len(tlsConfig.ServerName) == 0 && !tlsConfig.InsecureSkipVerify { + host, _, err := net.SplitHostPort(cfg.Addr) + if err == nil { + tlsConfig.ServerName = host + } + } + + cfg.TLSConfig = name + cfg.tls = tlsConfig + } else { + return errors.New("invalid value / unknown config name: " + name) + } + } + + // I/O write Timeout + case "writeTimeout": + cfg.WriteTimeout, err = time.ParseDuration(value) + if err != nil { + return + } + case "maxAllowedPacket": + cfg.MaxAllowedPacket, err = strconv.Atoi(value) + if err != nil { + return + } + default: + // lazy init + if cfg.Params == nil { + cfg.Params = make(map[string]string) + } + + if cfg.Params[param[0]], err = url.QueryUnescape(value); err != nil { + return + } + } + } + + return +} diff --git a/vendor/github.com/go-sql-driver/mysql/errors.go b/vendor/github.com/go-sql-driver/mysql/errors.go index 97d7b39962..857854e147 100644 --- a/vendor/github.com/go-sql-driver/mysql/errors.go +++ b/vendor/github.com/go-sql-driver/mysql/errors.go @@ -19,18 +19,21 @@ import ( // Various errors the driver might return. Can change between driver versions. var ( - ErrInvalidConn = errors.New("Invalid Connection") - ErrMalformPkt = errors.New("Malformed Packet") - ErrNoTLS = errors.New("TLS encryption requested but server does not support TLS") - ErrOldPassword = errors.New("This server only supports the insecure old password authentication. If you still want to use it, please add 'allowOldPasswords=1' to your DSN. See also https://github.com/go-sql-driver/mysql/wiki/old_passwords") - ErrOldProtocol = errors.New("MySQL-Server does not support required Protocol 41+") - ErrPktSync = errors.New("Commands out of sync. You can't run this command now") - ErrPktSyncMul = errors.New("Commands out of sync. Did you run multiple statements at once?") - ErrPktTooLarge = errors.New("Packet for query is too large. You can change this value on the server by adjusting the 'max_allowed_packet' variable.") - ErrBusyBuffer = errors.New("Busy buffer") + ErrInvalidConn = errors.New("invalid connection") + ErrMalformPkt = errors.New("malformed packet") + ErrNoTLS = errors.New("TLS requested but server does not support TLS") + ErrCleartextPassword = errors.New("this user requires clear text authentication. If you still want to use it, please add 'allowCleartextPasswords=1' to your DSN") + ErrNativePassword = errors.New("this user requires mysql native password authentication.") + ErrOldPassword = errors.New("this user requires old password authentication. If you still want to use it, please add 'allowOldPasswords=1' to your DSN. See also https://github.com/go-sql-driver/mysql/wiki/old_passwords") + ErrUnknownPlugin = errors.New("this authentication plugin is not supported") + ErrOldProtocol = errors.New("MySQL server does not support required protocol 41+") + ErrPktSync = errors.New("commands out of sync. You can't run this command now") + ErrPktSyncMul = errors.New("commands out of sync. Did you run multiple statements at once?") + ErrPktTooLarge = errors.New("packet for query is too large. Try adjusting the 'max_allowed_packet' variable on the server") + ErrBusyBuffer = errors.New("busy buffer") ) -var errLog Logger = log.New(os.Stderr, "[MySQL] ", log.Ldate|log.Ltime|log.Lshortfile) +var errLog = Logger(log.New(os.Stderr, "[mysql] ", log.Ldate|log.Ltime|log.Lshortfile)) // Logger is used to log critical error messages. type Logger interface { diff --git a/vendor/github.com/go-sql-driver/mysql/infile.go b/vendor/github.com/go-sql-driver/mysql/infile.go index 121a04c712..547357cfa7 100644 --- a/vendor/github.com/go-sql-driver/mysql/infile.go +++ b/vendor/github.com/go-sql-driver/mysql/infile.go @@ -13,11 +13,14 @@ import ( "io" "os" "strings" + "sync" ) var ( - fileRegister map[string]bool - readerRegister map[string]func() io.Reader + fileRegister map[string]bool + fileRegisterLock sync.RWMutex + readerRegister map[string]func() io.Reader + readerRegisterLock sync.RWMutex ) // RegisterLocalFile adds the given file to the file whitelist, @@ -32,17 +35,21 @@ var ( // ... // func RegisterLocalFile(filePath string) { + fileRegisterLock.Lock() // lazy map init if fileRegister == nil { fileRegister = make(map[string]bool) } fileRegister[strings.Trim(filePath, `"`)] = true + fileRegisterLock.Unlock() } // DeregisterLocalFile removes the given filepath from the whitelist. func DeregisterLocalFile(filePath string) { + fileRegisterLock.Lock() delete(fileRegister, strings.Trim(filePath, `"`)) + fileRegisterLock.Unlock() } // RegisterReaderHandler registers a handler function which is used @@ -61,18 +68,22 @@ func DeregisterLocalFile(filePath string) { // ... // func RegisterReaderHandler(name string, handler func() io.Reader) { + readerRegisterLock.Lock() // lazy map init if readerRegister == nil { readerRegister = make(map[string]func() io.Reader) } readerRegister[name] = handler + readerRegisterLock.Unlock() } // DeregisterReaderHandler removes the ReaderHandler function with // the given name from the registry. func DeregisterReaderHandler(name string) { + readerRegisterLock.Lock() delete(readerRegister, name) + readerRegisterLock.Unlock() } func deferredClose(err *error, closer io.Closer) { @@ -85,14 +96,22 @@ func deferredClose(err *error, closer io.Closer) { func (mc *mysqlConn) handleInFileRequest(name string) (err error) { var rdr io.Reader var data []byte + packetSize := 16 * 1024 // 16KB is small enough for disk readahead and large enough for TCP + if mc.maxWriteSize < packetSize { + packetSize = mc.maxWriteSize + } + + if idx := strings.Index(name, "Reader::"); idx == 0 || (idx > 0 && name[idx-1] == '/') { // io.Reader + // The server might return an an absolute path. See issue #355. + name = name[idx+8:] + + readerRegisterLock.RLock() + handler, inMap := readerRegister[name] + readerRegisterLock.RUnlock() - if strings.HasPrefix(name, "Reader::") { // io.Reader - name = name[8:] - if handler, inMap := readerRegister[name]; inMap { + if inMap { rdr = handler() if rdr != nil { - data = make([]byte, 4+mc.maxWriteSize) - if cl, ok := rdr.(io.Closer); ok { defer deferredClose(&err, cl) } @@ -104,7 +123,10 @@ func (mc *mysqlConn) handleInFileRequest(name string) (err error) { } } else { // File name = strings.Trim(name, `"`) - if mc.cfg.allowAllFiles || fileRegister[name] { + fileRegisterLock.RLock() + fr := fileRegister[name] + fileRegisterLock.RUnlock() + if mc.cfg.AllowAllFiles || fr { var file *os.File var fi os.FileInfo @@ -114,22 +136,19 @@ func (mc *mysqlConn) handleInFileRequest(name string) (err error) { // get file size if fi, err = file.Stat(); err == nil { rdr = file - if fileSize := int(fi.Size()); fileSize <= mc.maxWriteSize { - data = make([]byte, 4+fileSize) - } else if fileSize <= mc.maxPacketAllowed { - data = make([]byte, 4+mc.maxWriteSize) - } else { - err = fmt.Errorf("Local File '%s' too large: Size: %d, Max: %d", name, fileSize, mc.maxPacketAllowed) + if fileSize := int(fi.Size()); fileSize < packetSize { + packetSize = fileSize } } } } else { - err = fmt.Errorf("Local File '%s' is not registered. Use the DSN parameter 'allowAllFiles=true' to allow all files", name) + err = fmt.Errorf("local file '%s' is not registered", name) } } // send content packets if err == nil { + data := make([]byte, 4+packetSize) var n int for err == nil { n, err = rdr.Read(data[4:]) @@ -154,9 +173,10 @@ func (mc *mysqlConn) handleInFileRequest(name string) (err error) { // read OK packet if err == nil { - return mc.readResultOK() - } else { - mc.readPacket() + _, err = mc.readResultOK() + return err } + + mc.readPacket() return err } diff --git a/vendor/github.com/go-sql-driver/mysql/packets.go b/vendor/github.com/go-sql-driver/mysql/packets.go index 290a3887a7..aafe9793ea 100644 --- a/vendor/github.com/go-sql-driver/mysql/packets.go +++ b/vendor/github.com/go-sql-driver/mysql/packets.go @@ -13,6 +13,7 @@ import ( "crypto/tls" "database/sql/driver" "encoding/binary" + "errors" "fmt" "io" "math" @@ -24,9 +25,9 @@ import ( // Read packet to buffer 'data' func (mc *mysqlConn) readPacket() ([]byte, error) { - var payload []byte + var prevData []byte for { - // Read packet header + // read packet header data, err := mc.buf.readNext(4) if err != nil { errLog.Print(err) @@ -34,26 +35,32 @@ func (mc *mysqlConn) readPacket() ([]byte, error) { return nil, driver.ErrBadConn } - // Packet Length [24 bit] + // packet length [24 bit] pktLen := int(uint32(data[0]) | uint32(data[1])<<8 | uint32(data[2])<<16) - if pktLen < 1 { - errLog.Print(ErrMalformPkt) - mc.Close() - return nil, driver.ErrBadConn - } - - // Check Packet Sync [8 bit] + // check packet sync [8 bit] if data[3] != mc.sequence { if data[3] > mc.sequence { return nil, ErrPktSyncMul - } else { - return nil, ErrPktSync } + return nil, ErrPktSync } mc.sequence++ - // Read packet body [pktLen bytes] + // packets with length 0 terminate a previous packet which is a + // multiple of (2^24)−1 bytes long + if pktLen == 0 { + // there was no previous packet + if prevData == nil { + errLog.Print(ErrMalformPkt) + mc.Close() + return nil, driver.ErrBadConn + } + + return prevData, nil + } + + // read packet body [pktLen bytes] data, err = mc.buf.readNext(pktLen) if err != nil { errLog.Print(err) @@ -61,18 +68,17 @@ func (mc *mysqlConn) readPacket() ([]byte, error) { return nil, driver.ErrBadConn } - isLastPacket := (pktLen < maxPacketSize) + // return data if this was the last packet + if pktLen < maxPacketSize { + // zero allocations for non-split packets + if prevData == nil { + return data, nil + } - // Zero allocations for non-splitting packets - if isLastPacket && payload == nil { - return data, nil + return append(prevData, data...), nil } - payload = append(payload, data...) - - if isLastPacket { - return payload, nil - } + prevData = append(prevData, data...) } } @@ -80,7 +86,7 @@ func (mc *mysqlConn) readPacket() ([]byte, error) { func (mc *mysqlConn) writePacket(data []byte) error { pktLen := len(data) - 4 - if pktLen > mc.maxPacketAllowed { + if pktLen > mc.maxAllowedPacket { return ErrPktTooLarge } @@ -100,6 +106,12 @@ func (mc *mysqlConn) writePacket(data []byte) error { data[3] = mc.sequence // Write packet + if mc.writeTimeout > 0 { + if err := mc.netConn.SetWriteDeadline(time.Now().Add(mc.writeTimeout)); err != nil { + return err + } + } + n, err := mc.netConn.Write(data[:4+size]) if err == nil && n == 4+size { mc.sequence++ @@ -140,7 +152,7 @@ func (mc *mysqlConn) readInitPacket() ([]byte, error) { // protocol version [1 byte] if data[0] < minProtocolVersion { return nil, fmt.Errorf( - "Unsupported MySQL Protocol Version %d. Protocol Version %d or higher is required", + "unsupported protocol version %d. Version %d or higher is required", data[0], minProtocolVersion, ) @@ -196,7 +208,11 @@ func (mc *mysqlConn) readInitPacket() ([]byte, error) { // return //} //return ErrMalformPkt - return cipher, nil + + // make a memory safe copy of the cipher slice + var b [20]byte + copy(b[:], cipher) + return b[:], nil } // make a memory safe copy of the cipher slice @@ -214,9 +230,11 @@ func (mc *mysqlConn) writeAuthPacket(cipher []byte) error { clientLongPassword | clientTransactions | clientLocalFiles | + clientPluginAuth | + clientMultiResults | mc.flags&clientLongFlag - if mc.cfg.clientFoundRows { + if mc.cfg.ClientFoundRows { clientFlags |= clientFoundRows } @@ -225,13 +243,17 @@ func (mc *mysqlConn) writeAuthPacket(cipher []byte) error { clientFlags |= clientSSL } + if mc.cfg.MultiStatements { + clientFlags |= clientMultiStatements + } + // User Password - scrambleBuff := scramblePassword(cipher, []byte(mc.cfg.passwd)) + scrambleBuff := scramblePassword(cipher, []byte(mc.cfg.Passwd)) - pktLen := 4 + 4 + 1 + 23 + len(mc.cfg.user) + 1 + 1 + len(scrambleBuff) + pktLen := 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + 1 + len(scrambleBuff) + 21 + 1 // To specify a db name - if n := len(mc.cfg.dbname); n > 0 { + if n := len(mc.cfg.DBName); n > 0 { clientFlags |= clientConnectWithDB pktLen += n + 1 } @@ -257,7 +279,14 @@ func (mc *mysqlConn) writeAuthPacket(cipher []byte) error { data[11] = 0x00 // Charset [1 byte] - data[12] = mc.cfg.collation + var found bool + data[12], found = collations[mc.cfg.Collation] + if !found { + // Note possibility for false negatives: + // could be triggered although the collation is valid if the + // collations map does not contain entries the server supports. + return errors.New("unknown collation") + } // SSL Connection Request Packet // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::SSLRequest @@ -273,15 +302,18 @@ func (mc *mysqlConn) writeAuthPacket(cipher []byte) error { return err } mc.netConn = tlsConn - mc.buf.rd = tlsConn + mc.buf.nc = tlsConn } // Filler [23 bytes] (all 0x00) - pos := 13 + 23 + pos := 13 + for ; pos < 13+23; pos++ { + data[pos] = 0 + } // User [null terminated string] - if len(mc.cfg.user) > 0 { - pos += copy(data[pos:], mc.cfg.user) + if len(mc.cfg.User) > 0 { + pos += copy(data[pos:], mc.cfg.User) } data[pos] = 0x00 pos++ @@ -291,11 +323,16 @@ func (mc *mysqlConn) writeAuthPacket(cipher []byte) error { pos += 1 + copy(data[pos+1:], scrambleBuff) // Databasename [null terminated string] - if len(mc.cfg.dbname) > 0 { - pos += copy(data[pos:], mc.cfg.dbname) + if len(mc.cfg.DBName) > 0 { + pos += copy(data[pos:], mc.cfg.DBName) data[pos] = 0x00 + pos++ } + // Assume native client during response + pos += copy(data[pos:], "mysql_native_password") + data[pos] = 0x00 + // Send Auth packet return mc.writePacket(data) } @@ -304,9 +341,9 @@ func (mc *mysqlConn) writeAuthPacket(cipher []byte) error { // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchResponse func (mc *mysqlConn) writeOldAuthPacket(cipher []byte) error { // User password - scrambleBuff := scrambleOldPassword(cipher, []byte(mc.cfg.passwd)) + scrambleBuff := scrambleOldPassword(cipher, []byte(mc.cfg.Passwd)) - // Calculate the packet lenght and add a tailing 0 + // Calculate the packet length and add a tailing 0 pktLen := len(scrambleBuff) + 1 data := mc.buf.takeSmallBuffer(4 + pktLen) if data == nil { @@ -322,6 +359,45 @@ func (mc *mysqlConn) writeOldAuthPacket(cipher []byte) error { return mc.writePacket(data) } +// Client clear text authentication packet +// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchResponse +func (mc *mysqlConn) writeClearAuthPacket() error { + // Calculate the packet length and add a tailing 0 + pktLen := len(mc.cfg.Passwd) + 1 + data := mc.buf.takeSmallBuffer(4 + pktLen) + if data == nil { + // can not take the buffer. Something must be wrong with the connection + errLog.Print(ErrBusyBuffer) + return driver.ErrBadConn + } + + // Add the clear password [null terminated string] + copy(data[4:], mc.cfg.Passwd) + data[4+pktLen-1] = 0x00 + + return mc.writePacket(data) +} + +// Native password authentication method +// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchResponse +func (mc *mysqlConn) writeNativeAuthPacket(cipher []byte) error { + scrambleBuff := scramblePassword(cipher, []byte(mc.cfg.Passwd)) + + // Calculate the packet length and add a tailing 0 + pktLen := len(scrambleBuff) + data := mc.buf.takeSmallBuffer(4 + pktLen) + if data == nil { + // can not take the buffer. Something must be wrong with the connection + errLog.Print(ErrBusyBuffer) + return driver.ErrBadConn + } + + // Add the scramble + copy(data[4:], scrambleBuff) + + return mc.writePacket(data) +} + /****************************************************************************** * Command Packets * ******************************************************************************/ @@ -395,24 +471,43 @@ func (mc *mysqlConn) writeCommandPacketUint32(command byte, arg uint32) error { ******************************************************************************/ // Returns error if Packet is not an 'Result OK'-Packet -func (mc *mysqlConn) readResultOK() error { +func (mc *mysqlConn) readResultOK() ([]byte, error) { data, err := mc.readPacket() if err == nil { // packet indicator switch data[0] { case iOK: - return mc.handleOkPacket(data) + return nil, mc.handleOkPacket(data) case iEOF: - // someone is using old_passwords - return ErrOldPassword + if len(data) > 1 { + pluginEndIndex := bytes.IndexByte(data, 0x00) + plugin := string(data[1:pluginEndIndex]) + cipher := data[pluginEndIndex+1 : len(data)-1] + + if plugin == "mysql_old_password" { + // using old_passwords + return cipher, ErrOldPassword + } else if plugin == "mysql_clear_password" { + // using clear text password + return cipher, ErrCleartextPassword + } else if plugin == "mysql_native_password" { + // using mysql default authentication method + return cipher, ErrNativePassword + } else { + return cipher, ErrUnknownPlugin + } + } else { + // https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::OldAuthSwitchRequest + return nil, ErrOldPassword + } default: // Error otherwise - return mc.handleErrorPacket(data) + return nil, mc.handleErrorPacket(data) } } - return err + return nil, err } // Result Set Header Packet @@ -470,6 +565,10 @@ func (mc *mysqlConn) handleErrorPacket(data []byte) error { } } +func readStatus(b []byte) statusFlag { + return statusFlag(b[0]) | statusFlag(b[1])<<8 +} + // Ok Packet // http://dev.mysql.com/doc/internals/en/generic-response-packets.html#packet-OK_Packet func (mc *mysqlConn) handleOkPacket(data []byte) error { @@ -484,18 +583,21 @@ func (mc *mysqlConn) handleOkPacket(data []byte) error { mc.insertId, _, m = readLengthEncodedInteger(data[1+n:]) // server_status [2 bytes] - mc.status = statusFlag(data[1+n+m]) | statusFlag(data[1+n+m+1])<<8 + mc.status = readStatus(data[1+n+m : 1+n+m+2]) + if err := mc.discardResults(); err != nil { + return err + } // warning count [2 bytes] if !mc.strict { return nil - } else { - pos := 1 + n + m + 2 - if binary.LittleEndian.Uint16(data[pos:pos+2]) > 0 { - return mc.getWarnings() - } - return nil } + + pos := 1 + n + m + 2 + if binary.LittleEndian.Uint16(data[pos:pos+2]) > 0 { + return mc.getWarnings() + } + return nil } // Read Packets as Field Packets until EOF-Packet or an Error appears @@ -514,7 +616,7 @@ func (mc *mysqlConn) readColumns(count int) ([]mysqlField, error) { if i == count { return columns, nil } - return nil, fmt.Errorf("ColumnsCount mismatch n:%d len:%d", count, len(columns)) + return nil, fmt.Errorf("column count mismatch n:%d len:%d", count, len(columns)) } // Catalog @@ -531,7 +633,7 @@ func (mc *mysqlConn) readColumns(count int) ([]mysqlField, error) { pos += n // Table [len coded string] - if mc.cfg.columnsWithAlias { + if mc.cfg.ColumnsWithAlias { tableName, _, n, err := readLengthEncodedString(data[pos:]) if err != nil { return nil, err @@ -603,8 +705,17 @@ func (rows *textRows) readRow(dest []driver.Value) error { // EOF Packet if data[0] == iEOF && len(data) == 5 { + // server_status [2 bytes] + rows.mc.status = readStatus(data[3:]) + err = rows.mc.discardResults() + if err == nil { + err = io.EOF + } else { + // connection unusable + rows.mc.Close() + } rows.mc = nil - return io.EOF + return err } if data[0] == iERR { rows.mc = nil @@ -630,7 +741,7 @@ func (rows *textRows) readRow(dest []driver.Value) error { fieldTypeDate, fieldTypeNewDate: dest[i], err = parseDateTime( string(dest[i].([]byte)), - mc.cfg.loc, + mc.cfg.Loc, ) if err == nil { continue @@ -655,12 +766,19 @@ func (rows *textRows) readRow(dest []driver.Value) error { func (mc *mysqlConn) readUntilEOF() error { for { data, err := mc.readPacket() + if err != nil { + return err + } - // No Err and no EOF Packet - if err == nil && data[0] != iEOF { - continue + switch data[0] { + case iERR: + return mc.handleErrorPacket(data) + case iEOF: + if len(data) == 5 { + mc.status = readStatus(data[3:]) + } + return nil } - return err // Err or EOF } } @@ -692,20 +810,20 @@ func (stmt *mysqlStmt) readPrepareResultPacket() (uint16, error) { // Warning count [16 bit uint] if !stmt.mc.strict { return columnCount, nil - } else { - // Check for warnings count > 0, only available in MySQL > 4.1 - if len(data) >= 12 && binary.LittleEndian.Uint16(data[10:12]) > 0 { - return columnCount, stmt.mc.getWarnings() - } - return columnCount, nil } + + // Check for warnings count > 0, only available in MySQL > 4.1 + if len(data) >= 12 && binary.LittleEndian.Uint16(data[10:12]) > 0 { + return columnCount, stmt.mc.getWarnings() + } + return columnCount, nil } return 0, err } // http://dev.mysql.com/doc/internals/en/com-stmt-send-long-data.html func (stmt *mysqlStmt) writeCommandLongData(paramID int, arg []byte) error { - maxLen := stmt.mc.maxPacketAllowed - 1 + maxLen := stmt.mc.maxAllowedPacket - 1 pktLen := maxLen // After the header (bytes 0-3) follows before the data: @@ -760,7 +878,7 @@ func (stmt *mysqlStmt) writeCommandLongData(paramID int, arg []byte) error { func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { if len(args) != stmt.paramCount { return fmt.Errorf( - "Arguments count mismatch (Got: %d Has: %d)", + "argument count mismatch (got: %d; has: %d)", len(args), stmt.paramCount, ) @@ -896,7 +1014,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { paramTypes[i+i] = fieldTypeString paramTypes[i+i+1] = 0x00 - if len(v) < mc.maxPacketAllowed-pos-len(paramValues)-(len(args)-(i+1))*64 { + if len(v) < mc.maxAllowedPacket-pos-len(paramValues)-(len(args)-(i+1))*64 { paramValues = appendLengthEncodedInteger(paramValues, uint64(len(v)), ) @@ -918,7 +1036,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { paramTypes[i+i] = fieldTypeString paramTypes[i+i+1] = 0x00 - if len(v) < mc.maxPacketAllowed-pos-len(paramValues)-(len(args)-(i+1))*64 { + if len(v) < mc.maxAllowedPacket-pos-len(paramValues)-(len(args)-(i+1))*64 { paramValues = appendLengthEncodedInteger(paramValues, uint64(len(v)), ) @@ -937,7 +1055,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { if v.IsZero() { val = []byte("0000-00-00") } else { - val = []byte(v.In(mc.cfg.loc).Format(timeFormat)) + val = []byte(v.In(mc.cfg.Loc).Format(timeFormat)) } paramValues = appendLengthEncodedInteger(paramValues, @@ -946,7 +1064,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { paramValues = append(paramValues, val...) default: - return fmt.Errorf("Can't convert type: %T", arg) + return fmt.Errorf("can not convert type: %T", arg) } } @@ -964,6 +1082,28 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { return mc.writePacket(data) } +func (mc *mysqlConn) discardResults() error { + for mc.status&statusMoreResultsExists != 0 { + resLen, err := mc.readResultSetHeaderPacket() + if err != nil { + return err + } + if resLen > 0 { + // columns + if err := mc.readUntilEOF(); err != nil { + return err + } + // rows + if err := mc.readUntilEOF(); err != nil { + return err + } + } else { + mc.status &^= statusMoreResultsExists + } + } + return nil +} + // http://dev.mysql.com/doc/internals/en/binary-protocol-resultset-row.html func (rows *binaryRows) readRow(dest []driver.Value) error { data, err := rows.mc.readPacket() @@ -973,11 +1113,20 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { // packet indicator [1 byte] if data[0] != iOK { - rows.mc = nil // EOF Packet if data[0] == iEOF && len(data) == 5 { - return io.EOF + rows.mc.status = readStatus(data[3:]) + err = rows.mc.discardResults() + if err == nil { + err = io.EOF + } else { + // connection unusable + rows.mc.Close() + } + rows.mc = nil + return err } + rows.mc = nil // Error otherwise return rows.mc.handleErrorPacket(data) @@ -1044,7 +1193,7 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { continue case fieldTypeFloat: - dest[i] = float64(math.Float32frombits(binary.LittleEndian.Uint32(data[pos : pos+4]))) + dest[i] = float32(math.Float32frombits(binary.LittleEndian.Uint32(data[pos : pos+4]))) pos += 4 continue @@ -1057,7 +1206,7 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { case fieldTypeDecimal, fieldTypeNewDecimal, fieldTypeVarChar, fieldTypeBit, fieldTypeEnum, fieldTypeSet, fieldTypeTinyBLOB, fieldTypeMediumBLOB, fieldTypeLongBLOB, fieldTypeBLOB, - fieldTypeVarString, fieldTypeString, fieldTypeGeometry: + fieldTypeVarString, fieldTypeString, fieldTypeGeometry, fieldTypeJSON: var isNull bool var n int dest[i], isNull, n, err = readLengthEncodedString(data[pos:]) @@ -1094,13 +1243,13 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { dstlen = 8 + 1 + decimals default: return fmt.Errorf( - "MySQL protocol error, illegal decimals value %d", + "protocol error, illegal decimals value %d", rows.columns[i].decimals, ) } dest[i], err = formatBinaryDateTime(data[pos:pos+int(num)], dstlen, true) case rows.mc.parseTime: - dest[i], err = parseBinaryDateTime(num, data[pos:], rows.mc.cfg.loc) + dest[i], err = parseBinaryDateTime(num, data[pos:], rows.mc.cfg.Loc) default: var dstlen uint8 if rows.columns[i].fieldType == fieldTypeDate { @@ -1113,7 +1262,7 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { dstlen = 19 + 1 + decimals default: return fmt.Errorf( - "MySQL protocol error, illegal decimals value %d", + "protocol error, illegal decimals value %d", rows.columns[i].decimals, ) } @@ -1130,7 +1279,7 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { // Please report if this happens! default: - return fmt.Errorf("Unknown FieldType %d", rows.columns[i].fieldType) + return fmt.Errorf("unknown field type %d", rows.columns[i].fieldType) } } diff --git a/vendor/github.com/go-sql-driver/mysql/rows.go b/vendor/github.com/go-sql-driver/mysql/rows.go index 9d97d6d4f1..c08255eeec 100644 --- a/vendor/github.com/go-sql-driver/mysql/rows.go +++ b/vendor/github.com/go-sql-driver/mysql/rows.go @@ -38,9 +38,13 @@ type emptyRows struct{} func (rows *mysqlRows) Columns() []string { columns := make([]string, len(rows.columns)) - if rows.mc.cfg.columnsWithAlias { + if rows.mc != nil && rows.mc.cfg.ColumnsWithAlias { for i := range columns { - columns[i] = rows.columns[i].tableName + "." + rows.columns[i].name + if tableName := rows.columns[i].tableName; len(tableName) > 0 { + columns[i] = tableName + "." + rows.columns[i].name + } else { + columns[i] = rows.columns[i].name + } } } else { for i := range columns { @@ -61,6 +65,12 @@ func (rows *mysqlRows) Close() error { // Remove unread packets from stream err := mc.readUntilEOF() + if err == nil { + if err = mc.discardResults(); err != nil { + return err + } + } + rows.mc = nil return err } diff --git a/vendor/github.com/go-sql-driver/mysql/statement.go b/vendor/github.com/go-sql-driver/mysql/statement.go index f9dae03fab..7f9b045857 100644 --- a/vendor/github.com/go-sql-driver/mysql/statement.go +++ b/vendor/github.com/go-sql-driver/mysql/statement.go @@ -12,6 +12,7 @@ import ( "database/sql/driver" "fmt" "reflect" + "strconv" ) type mysqlStmt struct { @@ -23,7 +24,10 @@ type mysqlStmt struct { func (stmt *mysqlStmt) Close() error { if stmt.mc == nil || stmt.mc.netConn == nil { - errLog.Print(ErrInvalidConn) + // driver.Stmt.Close can be called more than once, thus this function + // has to be idempotent. + // See also Issue #450 and golang/go#16019. + //errLog.Print(ErrInvalidConn) return driver.ErrBadConn } @@ -100,9 +104,9 @@ func (stmt *mysqlStmt) Query(args []driver.Value) (driver.Rows, error) { } rows := new(binaryRows) - rows.mc = mc if resLen > 0 { + rows.mc = mc // Columns // If not cached, read them and cache them if stmt.columns == nil { @@ -119,7 +123,7 @@ func (stmt *mysqlStmt) Query(args []driver.Value) (driver.Rows, error) { type converter struct{} -func (converter) ConvertValue(v interface{}) (driver.Value, error) { +func (c converter) ConvertValue(v interface{}) (driver.Value, error) { if driver.IsValue(v) { return v, nil } @@ -131,7 +135,7 @@ func (converter) ConvertValue(v interface{}) (driver.Value, error) { if rv.IsNil() { return nil, nil } - return driver.DefaultParameterConverter.ConvertValue(rv.Elem().Interface()) + return c.ConvertValue(rv.Elem().Interface()) case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: return rv.Int(), nil case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32: @@ -139,7 +143,7 @@ func (converter) ConvertValue(v interface{}) (driver.Value, error) { case reflect.Uint64: u64 := rv.Uint() if u64 >= 1<<63 { - return fmt.Sprintf("%d", u64), nil + return strconv.FormatUint(u64, 10), nil } return int64(u64), nil case reflect.Float32, reflect.Float64: diff --git a/vendor/github.com/go-sql-driver/mysql/utils.go b/vendor/github.com/go-sql-driver/mysql/utils.go index 6693d29709..d523b7ffdf 100644 --- a/vendor/github.com/go-sql-driver/mysql/utils.go +++ b/vendor/github.com/go-sql-driver/mysql/utils.go @@ -13,28 +13,16 @@ import ( "crypto/tls" "database/sql/driver" "encoding/binary" - "errors" "fmt" "io" - "net" - "net/url" "strings" "time" ) var ( tlsConfigRegister map[string]*tls.Config // Register for custom tls.Configs - - errInvalidDSNUnescaped = errors.New("Invalid DSN: Did you forget to escape a param value?") - errInvalidDSNAddr = errors.New("Invalid DSN: Network Address not terminated (missing closing brace)") - errInvalidDSNNoSlash = errors.New("Invalid DSN: Missing the slash separating the database name") - errInvalidDSNUnsafeCollation = errors.New("Invalid DSN: interpolateParams can be used with ascii, latin1, utf8 and utf8mb4 charset") ) -func init() { - tlsConfigRegister = make(map[string]*tls.Config) -} - // RegisterTLSConfig registers a custom tls.Config to be used with sql.Open. // Use the key as a value in the DSN where tls=value. // @@ -60,7 +48,11 @@ func init() { // func RegisterTLSConfig(key string, config *tls.Config) error { if _, isBool := readBool(key); isBool || strings.ToLower(key) == "skip-verify" { - return fmt.Errorf("Key '%s' is reserved", key) + return fmt.Errorf("key '%s' is reserved", key) + } + + if tlsConfigRegister == nil { + tlsConfigRegister = make(map[string]*tls.Config) } tlsConfigRegister[key] = config @@ -69,228 +61,9 @@ func RegisterTLSConfig(key string, config *tls.Config) error { // DeregisterTLSConfig removes the tls.Config associated with key. func DeregisterTLSConfig(key string) { - delete(tlsConfigRegister, key) -} - -// parseDSN parses the DSN string to a config -func parseDSN(dsn string) (cfg *config, err error) { - // New config with some default values - cfg = &config{ - loc: time.UTC, - collation: defaultCollation, - } - - // TODO: use strings.IndexByte when we can depend on Go 1.2 - - // [user[:password]@][net[(addr)]]/dbname[?param1=value1¶mN=valueN] - // Find the last '/' (since the password or the net addr might contain a '/') - foundSlash := false - for i := len(dsn) - 1; i >= 0; i-- { - if dsn[i] == '/' { - foundSlash = true - var j, k int - - // left part is empty if i <= 0 - if i > 0 { - // [username[:password]@][protocol[(address)]] - // Find the last '@' in dsn[:i] - for j = i; j >= 0; j-- { - if dsn[j] == '@' { - // username[:password] - // Find the first ':' in dsn[:j] - for k = 0; k < j; k++ { - if dsn[k] == ':' { - cfg.passwd = dsn[k+1 : j] - break - } - } - cfg.user = dsn[:k] - - break - } - } - - // [protocol[(address)]] - // Find the first '(' in dsn[j+1:i] - for k = j + 1; k < i; k++ { - if dsn[k] == '(' { - // dsn[i-1] must be == ')' if an address is specified - if dsn[i-1] != ')' { - if strings.ContainsRune(dsn[k+1:i], ')') { - return nil, errInvalidDSNUnescaped - } - return nil, errInvalidDSNAddr - } - cfg.addr = dsn[k+1 : i-1] - break - } - } - cfg.net = dsn[j+1 : k] - } - - // dbname[?param1=value1&...¶mN=valueN] - // Find the first '?' in dsn[i+1:] - for j = i + 1; j < len(dsn); j++ { - if dsn[j] == '?' { - if err = parseDSNParams(cfg, dsn[j+1:]); err != nil { - return - } - break - } - } - cfg.dbname = dsn[i+1 : j] - - break - } - } - - if !foundSlash && len(dsn) > 0 { - return nil, errInvalidDSNNoSlash - } - - if cfg.interpolateParams && unsafeCollations[cfg.collation] { - return nil, errInvalidDSNUnsafeCollation + if tlsConfigRegister != nil { + delete(tlsConfigRegister, key) } - - // Set default network if empty - if cfg.net == "" { - cfg.net = "tcp" - } - - // Set default address if empty - if cfg.addr == "" { - switch cfg.net { - case "tcp": - cfg.addr = "127.0.0.1:3306" - case "unix": - cfg.addr = "/tmp/mysql.sock" - default: - return nil, errors.New("Default addr for network '" + cfg.net + "' unknown") - } - - } - - return -} - -// parseDSNParams parses the DSN "query string" -// Values must be url.QueryEscape'ed -func parseDSNParams(cfg *config, params string) (err error) { - for _, v := range strings.Split(params, "&") { - param := strings.SplitN(v, "=", 2) - if len(param) != 2 { - continue - } - - // cfg params - switch value := param[1]; param[0] { - - // Enable client side placeholder substitution - case "interpolateParams": - var isBool bool - cfg.interpolateParams, isBool = readBool(value) - if !isBool { - return fmt.Errorf("Invalid Bool value: %s", value) - } - - // Disable INFILE whitelist / enable all files - case "allowAllFiles": - var isBool bool - cfg.allowAllFiles, isBool = readBool(value) - if !isBool { - return fmt.Errorf("Invalid Bool value: %s", value) - } - - // Use old authentication mode (pre MySQL 4.1) - case "allowOldPasswords": - var isBool bool - cfg.allowOldPasswords, isBool = readBool(value) - if !isBool { - return fmt.Errorf("Invalid Bool value: %s", value) - } - - // Switch "rowsAffected" mode - case "clientFoundRows": - var isBool bool - cfg.clientFoundRows, isBool = readBool(value) - if !isBool { - return fmt.Errorf("Invalid Bool value: %s", value) - } - - // Collation - case "collation": - collation, ok := collations[value] - if !ok { - // Note possibility for false negatives: - // could be triggered although the collation is valid if the - // collations map does not contain entries the server supports. - err = errors.New("unknown collation") - return - } - cfg.collation = collation - break - - case "columnsWithAlias": - var isBool bool - cfg.columnsWithAlias, isBool = readBool(value) - if !isBool { - return fmt.Errorf("Invalid Bool value: %s", value) - } - - // Time Location - case "loc": - if value, err = url.QueryUnescape(value); err != nil { - return - } - cfg.loc, err = time.LoadLocation(value) - if err != nil { - return - } - - // Dial Timeout - case "timeout": - cfg.timeout, err = time.ParseDuration(value) - if err != nil { - return - } - - // TLS-Encryption - case "tls": - boolValue, isBool := readBool(value) - if isBool { - if boolValue { - cfg.tls = &tls.Config{} - } - } else { - if strings.ToLower(value) == "skip-verify" { - cfg.tls = &tls.Config{InsecureSkipVerify: true} - } else if tlsConfig, ok := tlsConfigRegister[value]; ok { - if len(tlsConfig.ServerName) == 0 && !tlsConfig.InsecureSkipVerify { - host, _, err := net.SplitHostPort(cfg.addr) - if err == nil { - tlsConfig.ServerName = host - } - } - - cfg.tls = tlsConfig - } else { - return fmt.Errorf("Invalid value / unknown config name: %s", value) - } - } - - default: - // lazy init - if cfg.params == nil { - cfg.params = make(map[string]string) - } - - if cfg.params[param[0]], err = url.QueryUnescape(value); err != nil { - return - } - } - } - - return } // Returns the bool value of the input. @@ -487,7 +260,7 @@ func parseDateTime(str string, loc *time.Location) (t time.Time, err error) { } t, err = time.Parse(timeFormat[:len(str)], str) default: - err = fmt.Errorf("Invalid Time-String: %s", str) + err = fmt.Errorf("invalid time string: %s", str) return } @@ -536,7 +309,7 @@ func parseBinaryDateTime(num uint64, data []byte, loc *time.Location) (driver.Va loc, ), nil } - return nil, fmt.Errorf("Invalid DATETIME-packet length %d", num) + return nil, fmt.Errorf("invalid DATETIME packet length %d", num) } // zeroDateTime is used in formatBinaryDateTime to avoid an allocation @@ -571,7 +344,7 @@ func formatBinaryDateTime(src []byte, length uint8, justTime bool) (driver.Value switch len(src) { case 8, 12: default: - return nil, fmt.Errorf("Invalid TIME-packet length %d", len(src)) + return nil, fmt.Errorf("invalid TIME packet length %d", len(src)) } // +2 to enable negative time and 100+ hours dst = make([]byte, 0, length+2) @@ -605,7 +378,7 @@ func formatBinaryDateTime(src []byte, length uint8, justTime bool) (driver.Value if length > 10 { t += "TIME" } - return nil, fmt.Errorf("illegal %s-packet length %d", t, len(src)) + return nil, fmt.Errorf("illegal %s packet length %d", t, len(src)) } dst = make([]byte, 0, length) // start with the date @@ -771,6 +544,10 @@ func skipLengthEncodedString(b []byte) (int, error) { // returns the number read, whether the value is NULL and the number of bytes read func readLengthEncodedInteger(b []byte) (uint64, bool, int) { + // See issue #349 + if len(b) == 0 { + return 0, true, 1 + } switch b[0] { // 251: NULL @@ -867,7 +644,7 @@ func escapeBytesBackslash(buf, v []byte) []byte { pos += 2 default: buf[pos] = c - pos += 1 + pos++ } } @@ -912,7 +689,7 @@ func escapeStringBackslash(buf []byte, v string) []byte { pos += 2 default: buf[pos] = c - pos += 1 + pos++ } } From 5a50c0c2d79cbce52becd6822e664de97f132214 Mon Sep 17 00:00:00 2001 From: Ashwini Oruganti Date: Mon, 8 May 2017 10:46:35 -0700 Subject: [PATCH 16/21] Update tests to expect 'parseTime=true' in the url Signed-off-by: Ashwini Oruganti --- utils/configuration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/configuration_test.go b/utils/configuration_test.go index 6cd88409f3..081982a5c5 100644 --- a/utils/configuration_test.go +++ b/utils/configuration_test.go @@ -203,7 +203,7 @@ func TestParseSQLStorageDBStore(t *testing.T) { expected := Storage{ Backend: "mysql", - Source: "username:passord@tcp(hostname:1234)/dbname", + Source: "username:passord@tcp(hostname:1234)/dbname?parseTime=true", } store, err := ParseSQLStorage(config) @@ -333,7 +333,7 @@ func TestParseSQLStorageWithEnvironmentVariables(t *testing.T) { expected := Storage{ Backend: "mysql", - Source: "username:passord@tcp(hostname:1234)/dbname", + Source: "username:passord@tcp(hostname:1234)/dbname?parseTime=true", } store, err := ParseSQLStorage(config) From cc8a75e7976bcd18b6d0946431bf797ce5a1896e Mon Sep 17 00:00:00 2001 From: Ashwini Oruganti Date: Mon, 8 May 2017 10:48:53 -0700 Subject: [PATCH 17/21] Remove a conditional check for parseTime=false, since we can set it to truie either way without any issues Signed-off-by: Ashwini Oruganti --- utils/configuration.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/utils/configuration.go b/utils/configuration.go index 53d0472542..4ab17406bb 100644 --- a/utils/configuration.go +++ b/utils/configuration.go @@ -118,9 +118,8 @@ func ParseSQLStorage(configuration *viper.Viper) (*Storage, error) { if err != nil { return nil, err } - if !url_config.ParseTime { - url_config.ParseTime = true - } + + url_config.ParseTime = true store.Source = url_config.FormatDSN() } return &store, nil From ead82a27f31f347a1e35972eafec6c757e7cb71a Mon Sep 17 00:00:00 2001 From: Ashwini Oruganti Date: Mon, 8 May 2017 10:54:56 -0700 Subject: [PATCH 18/21] Camel case: url_config -> urlConfig Signed-off-by: Ashwini Oruganti --- utils/configuration.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/configuration.go b/utils/configuration.go index 4ab17406bb..f35d8782b7 100644 --- a/utils/configuration.go +++ b/utils/configuration.go @@ -114,12 +114,12 @@ func ParseSQLStorage(configuration *viper.Viper) (*Storage, error) { // Parse the url into a config object, and grab the string from it. // cfg, err := sql.Open("mysql", store.Source) - url_config, err := mysql.ParseDSN(store.Source) + urlConfig, err := mysql.ParseDSN(store.Source) if err != nil { return nil, err } - url_config.ParseTime = true + urlConfig.ParseTime = true store.Source = url_config.FormatDSN() } return &store, nil From c7d5981e01433a9226135cfa253a04f04053c2af Mon Sep 17 00:00:00 2001 From: Ashwini Oruganti Date: Mon, 8 May 2017 11:06:36 -0700 Subject: [PATCH 19/21] Clean up inline comments Signed-off-by: Ashwini Oruganti --- utils/configuration.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/utils/configuration.go b/utils/configuration.go index f35d8782b7..4d94bbd4e1 100644 --- a/utils/configuration.go +++ b/utils/configuration.go @@ -111,9 +111,6 @@ func ParseSQLStorage(configuration *viper.Viper) (*Storage, error) { store.Backend, ) case store.Backend == notary.MySQLBackend: - // Parse the url into a config object, and grab the string from it. - - // cfg, err := sql.Open("mysql", store.Source) urlConfig, err := mysql.ParseDSN(store.Source) if err != nil { return nil, err From 24ff1fb2801320c09874556e49660a766002e24a Mon Sep 17 00:00:00 2001 From: Ashwini Oruganti Date: Mon, 8 May 2017 11:12:10 -0700 Subject: [PATCH 20/21] Camel case more. Signed-off-by: Ashwini Oruganti --- utils/configuration.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/configuration.go b/utils/configuration.go index 4d94bbd4e1..5b8461b41f 100644 --- a/utils/configuration.go +++ b/utils/configuration.go @@ -117,7 +117,7 @@ func ParseSQLStorage(configuration *viper.Viper) (*Storage, error) { } urlConfig.ParseTime = true - store.Source = url_config.FormatDSN() + store.Source = urlConfig.FormatDSN() } return &store, nil } From 8cafc48b3a693b1a0779ffb051fea447b40c6318 Mon Sep 17 00:00:00 2001 From: Ashwini Oruganti Date: Mon, 8 May 2017 17:13:27 -0700 Subject: [PATCH 21/21] Test the error for an invalid URL Signed-off-by: Ashwini Oruganti --- utils/configuration.go | 4 +++- utils/configuration_test.go | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/utils/configuration.go b/utils/configuration.go index 5b8461b41f..0cd36e3760 100644 --- a/utils/configuration.go +++ b/utils/configuration.go @@ -113,7 +113,9 @@ func ParseSQLStorage(configuration *viper.Viper) (*Storage, error) { case store.Backend == notary.MySQLBackend: urlConfig, err := mysql.ParseDSN(store.Source) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to parse the database source for %s", + store.Backend, + ) } urlConfig.ParseTime = true diff --git a/utils/configuration_test.go b/utils/configuration_test.go index 081982a5c5..2a6401b86a 100644 --- a/utils/configuration_test.go +++ b/utils/configuration_test.go @@ -192,6 +192,20 @@ func TestParseInvalidSQLStorageNoDBSource(t *testing.T) { } } +// If an invalid DB source is provided, an error is returned. +func TestParseInvalidDBSourceInSQLStorage(t *testing.T) { + config := configure(`{ + "storage": { + "backend": "mysql", + "db_url": "foobar" + } + }`) + _, err := ParseSQLStorage(config) + require.Error(t, err) + require.Contains(t, err.Error(), + fmt.Sprintf("failed to parse the database source for mysql")) +} + // A supported backend with DB source will be successfully parsed. func TestParseSQLStorageDBStore(t *testing.T) { config := configure(`{