diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c650e30050..64d2326f9b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing to notary -## Before reporting an issue... +## Before reporting an issue... ### If your problem is with... @@ -26,7 +26,7 @@ By following these simple rules you will get better and faster feedback on your - search the bugtracker for an already reported issue -### If you found an issue that describes your problem: +### If you found an issue that describes your problem: - please read other user comments first, and confirm this is the same issue: a given error condition might be indicative of different problems - you may also find a workaround in the comments - please refrain from adding "same thing here" or "+1" comments diff --git a/client/client.go b/client/client.go index 55a5ba4b75..16bf8b7ee4 100644 --- a/client/client.go +++ b/client/client.go @@ -13,6 +13,7 @@ import ( "time" "github.com/Sirupsen/logrus" + canonicaljson "github.com/docker/go/canonical/json" "github.com/docker/notary" "github.com/docker/notary/client/changelist" "github.com/docker/notary/cryptoservice" @@ -128,9 +129,10 @@ func (r *NotaryRepository) GetGUN() data.GUN { // Target represents a simplified version of the data TUF operates on, so external // applications don't have to depend on TUF data types. type Target struct { - Name string // the name of the target - Hashes data.Hashes // the hash of the target - Length int64 // the size in bytes of the target + Name string // the name of the target + Hashes data.Hashes // the hash of the target + Length int64 // the size in bytes of the target + Custom *canonicaljson.RawMessage // the custom data provided to describe the file at TARGETPATH } // TargetWithRole represents a Target that exists in a particular role - this is @@ -141,7 +143,7 @@ type TargetWithRole struct { } // NewTarget is a helper method that returns a Target -func NewTarget(targetName string, targetPath string) (*Target, error) { +func NewTarget(targetName, targetPath string, targetCustom *canonicaljson.RawMessage) (*Target, error) { b, err := ioutil.ReadFile(targetPath) if err != nil { return nil, err @@ -152,7 +154,7 @@ func NewTarget(targetName string, targetPath string) (*Target, error) { return nil, err } - return &Target{Name: targetName, Hashes: meta.Hashes, Length: meta.Length}, nil + return &Target{Name: targetName, Hashes: meta.Hashes, Length: meta.Length, Custom: targetCustom}, nil } func rootCertKey(gun data.GUN, privKey data.PrivateKey) (data.PublicKey, error) { @@ -360,7 +362,7 @@ func (r *NotaryRepository) AddTarget(target *Target, roles ...data.RoleName) err } logrus.Debugf("Adding target \"%s\" with sha256 \"%x\" and size %d bytes.\n", target.Name, target.Hashes["sha256"], target.Length) - meta := data.FileMeta{Length: target.Length, Hashes: target.Hashes} + meta := data.FileMeta{Length: target.Length, Hashes: target.Hashes, Custom: target.Custom} metaJSON, err := json.Marshal(meta) if err != nil { return err @@ -417,6 +419,7 @@ func (r *NotaryRepository) ListTargets(roles ...data.RoleName) ([]*TargetWithRol Name: targetName, Hashes: targetMeta.Hashes, Length: targetMeta.Length, + Custom: targetMeta.Custom, }, Role: validRole.Name, } @@ -472,7 +475,7 @@ func (r *NotaryRepository) GetTargetByName(name string, roles ...data.RoleName) } // Check that we didn't error, and that we assigned to our target if err := r.tufRepo.WalkTargets(name, role, getTargetVisitorFunc, skipRoles...); err == nil && foundTarget { - return &TargetWithRole{Target: Target{Name: name, Hashes: resultMeta.Hashes, Length: resultMeta.Length}, Role: resultRoleName}, nil + return &TargetWithRole{Target: Target{Name: name, Hashes: resultMeta.Hashes, Length: resultMeta.Length, Custom: resultMeta.Custom}, Role: resultRoleName}, nil } } return nil, fmt.Errorf("No trust data for %s", name) @@ -516,7 +519,7 @@ func (r *NotaryRepository) GetAllTargetMetadataByName(name string) ([]TargetSign for targetName, resultMeta := range targetMetaToAdd { targetInfo := TargetSignedStruct{ Role: validRole, - Target: Target{Name: targetName, Hashes: resultMeta.Hashes, Length: resultMeta.Length}, + Target: Target{Name: targetName, Hashes: resultMeta.Hashes, Length: resultMeta.Length, Custom: resultMeta.Custom}, Signatures: tgt.Signatures, } targetInfoList = append(targetInfoList, targetInfo) diff --git a/client/client_test.go b/client/client_test.go index 67cbbbd399..54741612b1 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -631,7 +631,13 @@ func testInitRepoPasswordInvalid(t *testing.T, rootType string) { func addTarget(t *testing.T, repo *NotaryRepository, targetName, targetFile string, roles ...data.RoleName) *Target { - target, err := NewTarget(targetName, targetFile) + var targetCustom *json.RawMessage + return addTargetWithCustom(t, repo, targetName, targetFile, targetCustom, roles...) +} + +func addTargetWithCustom(t *testing.T, repo *NotaryRepository, targetName, + targetFile string, targetCustom *json.RawMessage, roles ...data.RoleName) *Target { + target, err := NewTarget(targetName, targetFile, targetCustom) require.NoError(t, err, "error creating target") err = repo.AddTarget(target, roles...) require.NoError(t, err, "error adding target") @@ -815,7 +821,8 @@ func testAddTargetToSpecifiedInvalidRoles(t *testing.T, clearCache bool) { } for _, invalidRole := range invalidRoles { - target, err := NewTarget("latest", "../fixtures/intermediate-ca.crt") + var targetCustom *json.RawMessage + target, err := NewTarget("latest", "../fixtures/intermediate-ca.crt", targetCustom) require.NoError(t, err, "error creating target") err = repo.AddTarget(target, data.CanonicalTargetsRole, invalidRole) @@ -877,7 +884,8 @@ func TestAddTargetWithInvalidTarget(t *testing.T) { repo, _ := initializeRepo(t, data.ECDSAKey, "docker.com/notary", ts.URL, false) defer os.RemoveAll(repo.baseDir) - target, err := NewTarget("latest", "../fixtures/intermediate-ca.crt") + var targetCustom *json.RawMessage + target, err := NewTarget("latest", "../fixtures/intermediate-ca.crt", targetCustom) require.NoError(t, err, "error creating target") // Clear the hashes @@ -889,7 +897,8 @@ func TestAddTargetWithInvalidTarget(t *testing.T) { // to be propagated. func TestAddTargetErrorWritingChanges(t *testing.T) { testErrorWritingChangefiles(t, func(repo *NotaryRepository) error { - target, err := NewTarget("latest", "../fixtures/intermediate-ca.crt") + var targetCustom *json.RawMessage + target, err := NewTarget("latest", "../fixtures/intermediate-ca.crt", targetCustom) require.NoError(t, err, "error creating target") return repo.AddTarget(target, data.CanonicalTargetsRole) }) @@ -1192,8 +1201,14 @@ func testListTarget(t *testing.T, rootType string) { // tests need to manually bootstrap timestamp as client doesn't generate it err := repo.tufRepo.InitTimestamp() require.NoError(t, err, "error creating repository: %s", err) + var targetCustom json.RawMessage + rawTargetCustom := []byte("\"Lorem ipsum dolor sit\"") + err = json.Unmarshal(rawTargetCustom, &targetCustom) + require.NoError(t, err) + + latestTarget := addTargetWithCustom(t, repo, "latest", "../fixtures/intermediate-ca.crt", &targetCustom) + require.Equal(t, targetCustom, *latestTarget.Custom, "Target created does not contain the expected custom data") - latestTarget := addTarget(t, repo, "latest", "../fixtures/intermediate-ca.crt") currentTarget := addTarget(t, repo, "current", "../fixtures/intermediate-ca.crt") // Apply the changelist. Normally, this would be done by Publish diff --git a/cmd/notary/integration_test.go b/cmd/notary/integration_test.go index 529ca4ba85..6320ceed78 100644 --- a/cmd/notary/integration_test.go +++ b/cmd/notary/integration_test.go @@ -23,8 +23,11 @@ import ( "testing" "time" + "encoding/json" + "github.com/Sirupsen/logrus" ctxu "github.com/docker/distribution/context" + canonicaljson "github.com/docker/go/canonical/json" "github.com/docker/notary" "github.com/docker/notary/client" "github.com/docker/notary/cryptoservice" @@ -198,8 +201,9 @@ func TestClientTUFInteraction(t *testing.T) { defer os.Remove(tempFile.Name()) var ( - output string - target = "sdgkadga" + output string + target = "sdgkadga" + target2 = "foobar" ) // -- tests -- @@ -251,6 +255,62 @@ func TestClientTUFInteraction(t *testing.T) { output, err = runCommand(t, tempDir, "-s", server.URL, "list", "gun") require.NoError(t, err) require.False(t, strings.Contains(string(output), target)) + + // Test a target with custom data. + tempFileForTargetCustom, err := ioutil.TempFile("", "targetCustom") + require.NoError(t, err) + var customData canonicaljson.RawMessage + err = canonicaljson.Unmarshal([]byte("\"Lorem ipsum dolor sit amet, consectetur adipiscing elit\""), &customData) + require.NoError(t, err) + _, err = tempFileForTargetCustom.Write(customData) + require.NoError(t, err) + tempFileForTargetCustom.Close() + defer os.Remove(tempFileForTargetCustom.Name()) + + // add a target + _, err = runCommand(t, tempDir, "add", "gun", target2, tempFile.Name(), "--custom", tempFileForTargetCustom.Name()) + require.NoError(t, err) + + // check status - see target + output, err = runCommand(t, tempDir, "status", "gun") + require.NoError(t, err) + require.Contains(t, output, target2) + + // publish repo + _, err = runCommand(t, tempDir, "-s", server.URL, "publish", "gun") + require.NoError(t, err) + + // check status - no targets + output, err = runCommand(t, tempDir, "status", "gun") + require.NoError(t, err) + require.False(t, strings.Contains(string(output), target2)) + + // list repo - see target + output, err = runCommand(t, tempDir, "-s", server.URL, "list", "gun") + require.NoError(t, err) + require.Contains(t, output, target2) + + // Check the file this was written to to inspect metadata + cache, err := nstorage.NewFileStore( + filepath.Join(tempDir, "tuf", filepath.FromSlash("gun"), "metadata"), + "json", + ) + require.NoError(t, err) + rawTargets, err := cache.Get("targets") + require.NoError(t, err) + parsedTargets := data.SignedTargets{} + err = json.Unmarshal(rawTargets, &parsedTargets) + require.NoError(t, err) + require.Equal(t, *parsedTargets.Signed.Targets[target2].Custom, customData) + + // trigger a lookup error with < 2 args + _, err = runCommand(t, tempDir, "-s", server.URL, "lookup", "gun") + require.Error(t, err) + + // lookup target and repo - see target + output, err = runCommand(t, tempDir, "-s", server.URL, "lookup", "gun", target2) + require.NoError(t, err) + require.Contains(t, output, target2) } func TestClientDeleteTUFInteraction(t *testing.T) { @@ -422,6 +482,7 @@ func TestClientTUFAddByHashInteraction(t *testing.T) { target1 = "sdgkadga" target2 = "asdfasdf" target3 = "qwerty" + target4 = "foobar" ) // -- tests -- @@ -541,6 +602,57 @@ func TestClientTUFAddByHashInteraction(t *testing.T) { // publish repo _, err = runCommand(t, tempDir, "-s", server.URL, "publish", "gun") require.NoError(t, err) + + tempFile, err := ioutil.TempFile("", "targetCustom") + require.NoError(t, err) + var customData canonicaljson.RawMessage + err = canonicaljson.Unmarshal([]byte("\"Lorem ipsum dolor sit amet, consectetur adipiscing elit\""), &customData) + require.NoError(t, err) + _, err = tempFile.Write(customData) + require.NoError(t, err) + tempFile.Close() + defer os.Remove(tempFile.Name()) + + // add a target by sha512 and custom data + _, err = runCommand(t, tempDir, "addhash", "gun", target4, "3", "--sha512", targetSha512Hex, "--custom", tempFile.Name()) + require.NoError(t, err) + + // check status - see target + output, err = runCommand(t, tempDir, "status", "gun") + require.NoError(t, err) + require.Contains(t, output, target4) + + // publish repo + _, err = runCommand(t, tempDir, "-s", server.URL, "publish", "gun") + require.NoError(t, err) + + // check status - no targets + output, err = runCommand(t, tempDir, "status", "gun") + require.NoError(t, err) + require.False(t, strings.Contains(string(output), target4)) + + // list repo - see target + output, err = runCommand(t, tempDir, "-s", server.URL, "list", "gun") + require.NoError(t, err) + require.Contains(t, output, target4) + + // Check the file this was written to to inspect metadata + cache, err := nstorage.NewFileStore( + filepath.Join(tempDir, "tuf", filepath.FromSlash("gun"), "metadata"), + "json", + ) + require.NoError(t, err) + rawTargets, err := cache.Get("targets") + require.NoError(t, err) + parsedTargets := data.SignedTargets{} + err = json.Unmarshal(rawTargets, &parsedTargets) + require.NoError(t, err) + require.Equal(t, *parsedTargets.Signed.Targets[target4].Custom, customData) + + // lookup target and repo - see target + output, err = runCommand(t, tempDir, "-s", server.URL, "lookup", "gun", target4) + require.NoError(t, err) + require.Contains(t, output, target4) } // Initialize repo and test delegations commands by adding, listing, and removing delegations diff --git a/cmd/notary/tuf.go b/cmd/notary/tuf.go index c6dcf88b6e..928f1f8958 100644 --- a/cmd/notary/tuf.go +++ b/cmd/notary/tuf.go @@ -21,6 +21,7 @@ import ( "github.com/docker/distribution/registry/client/auth/challenge" "github.com/docker/distribution/registry/client/transport" "github.com/docker/go-connections/tlsconfig" + canonicaljson "github.com/docker/go/canonical/json" "github.com/docker/notary" notaryclient "github.com/docker/notary/client" "github.com/docker/notary/cryptoservice" @@ -116,6 +117,7 @@ type tufCommander struct { sha256 string sha512 string rootKey string + custom string input string output string @@ -155,6 +157,7 @@ func (t *tufCommander) AddToCommand(cmd *cobra.Command) { cmdTUFAdd := cmdTUFAddTemplate.ToCommand(t.tufAdd) cmdTUFAdd.Flags().StringSliceVarP(&t.roles, "roles", "r", nil, "Delegation roles to add this target to") cmdTUFAdd.Flags().BoolVarP(&t.autoPublish, "publish", "p", false, htAutoPublish) + cmdTUFAdd.Flags().StringVar(&t.custom, "custom", "", "Path to the file containing custom data for this target") cmd.AddCommand(cmdTUFAdd) cmdTUFRemove := cmdTUFRemoveTemplate.ToCommand(t.tufRemove) @@ -167,6 +170,7 @@ func (t *tufCommander) AddToCommand(cmd *cobra.Command) { cmdTUFAddHash.Flags().StringVar(&t.sha256, notary.SHA256, "", "hex encoded sha256 of the target to add") cmdTUFAddHash.Flags().StringVar(&t.sha512, notary.SHA512, "", "hex encoded sha512 of the target to add") cmdTUFAddHash.Flags().BoolVarP(&t.autoPublish, "publish", "p", false, htAutoPublish) + cmdTUFAddHash.Flags().StringVar(&t.custom, "custom", "", "Path to the file containing custom data for this target") cmd.AddCommand(cmdTUFAddHash) cmdTUFVerify := cmdTUFVerifyTemplate.ToCommand(t.tufVerify) @@ -249,6 +253,20 @@ func getTargetHashes(t *tufCommander) (data.Hashes, error) { return targetHash, nil } +// Open and read a file containing the targetCustom data +func getTargetCustom(targetCustomFilename string) (*canonicaljson.RawMessage, error) { + targetCustom := new(canonicaljson.RawMessage) + rawTargetCustom, err := ioutil.ReadFile(targetCustomFilename) + if err != nil { + return nil, err + } + + if err := targetCustom.UnmarshalJSON(rawTargetCustom); err != nil { + return nil, err + } + return targetCustom, nil +} + func (t *tufCommander) tufAddByHash(cmd *cobra.Command, args []string) error { if len(args) < 3 || t.sha256 == "" && t.sha512 == "" { cmd.Usage() @@ -262,6 +280,13 @@ func (t *tufCommander) tufAddByHash(cmd *cobra.Command, args []string) error { gun := data.GUN(args[0]) targetName := args[1] targetSize := args[2] + var targetCustom *canonicaljson.RawMessage + if t.custom != "" { + targetCustom, err = getTargetCustom(t.custom) + if err != nil { + return err + } + } targetInt64Len, err := strconv.ParseInt(targetSize, 0, 64) if err != nil { @@ -287,7 +312,7 @@ func (t *tufCommander) tufAddByHash(cmd *cobra.Command, args []string) error { } // Manually construct the target with the given byte size and hashes - target := ¬aryclient.Target{Name: targetName, Hashes: targetHashes, Length: targetInt64Len} + target := ¬aryclient.Target{Name: targetName, Hashes: targetHashes, Length: targetInt64Len, Custom: targetCustom} roleNames := data.NewRoleList(t.roles) @@ -321,6 +346,13 @@ func (t *tufCommander) tufAdd(cmd *cobra.Command, args []string) error { gun := data.GUN(args[0]) targetName := args[1] targetPath := args[2] + var targetCustom *canonicaljson.RawMessage + if t.custom != "" { + targetCustom, err = getTargetCustom(t.custom) + if err != nil { + return err + } + } trustPin, err := getTrustPinning(config) if err != nil { @@ -335,7 +367,7 @@ func (t *tufCommander) tufAdd(cmd *cobra.Command, args []string) error { return err } - target, err := notaryclient.NewTarget(targetName, targetPath) + target, err := notaryclient.NewTarget(targetName, targetPath, targetCustom) if err != nil { return err } diff --git a/migrations/migrate.sh b/migrations/migrate.sh index 928b3b93e4..9fd988dcba 100755 --- a/migrations/migrate.sh +++ b/migrations/migrate.sh @@ -10,7 +10,7 @@ case $SERVICE_NAME in MIGRATIONS_PATH=${MIGRATIONS_PATH:-migrations/server/mysql} DB_URL=${DB_URL:-mysql://server@tcp(mysql:3306)/notaryserver} # have to poll for DB to come up - until migrate -path=$MIGRATIONS_PATH -url=$DB_URL version > /dev/null + until migrate -path=$MIGRATIONS_PATH -database="${DB_URL}" up do iter=$(( iter+1 )) if [[ $iter -gt 30 ]]; then @@ -20,24 +20,13 @@ case $SERVICE_NAME in echo "waiting for $DB_URL to come up." sleep 1 done - pre=$(migrate -path=$MIGRATIONS_PATH -url="${DB_URL}" version) - if migrate -path=$MIGRATIONS_PATH -url="${DB_URL}" up ; then - post=$(migrate -path=$MIGRATIONS_PATH -url="${DB_URL}" version) - if [ "$pre" != "$post" ]; then - echo "notaryserver database migrated to latest version" - else - echo "notaryserver database already at latest version" - fi - else - echo "notaryserver database migration failed" - exit 1 - fi + echo "notaryserver database migrated to latest version" ;; notary_signer) MIGRATIONS_PATH=${MIGRATIONS_PATH:-migrations/signer/mysql} DB_URL=${DB_URL:-mysql://signer@tcp(mysql:3306)/notarysigner} # have to poll for DB to come up - until migrate -path=$MIGRATIONS_PATH -url=$DB_URL up version > /dev/null + until migrate -path=$MIGRATIONS_PATH -database="${DB_URL}" up do iter=$(( iter+1 )) if [[ $iter -gt 30 ]]; then @@ -47,17 +36,6 @@ case $SERVICE_NAME in echo "waiting for $DB_URL to come up." sleep 1 done - pre=$(migrate -path=$MIGRATIONS_PATH -url="${DB_URL}" version) - if migrate -path=$MIGRATIONS_PATH -url="${DB_URL}" up ; then - post=$(migrate -path=$MIGRATIONS_PATH -url="${DB_URL}" version) - if [ "$pre" != "$post" ]; then - echo "notarysigner database migrated to latest version" - else - echo "notarysigner database already at latest version" - fi - else - echo "notarysigner database migration failed" - exit 1 - fi + echo "notarysigner database migrated to latest version" ;; esac diff --git a/server.Dockerfile b/server.Dockerfile index 64606231c1..92135bebe8 100644 --- a/server.Dockerfile +++ b/server.Dockerfile @@ -3,10 +3,8 @@ MAINTAINER David Lawrence "david.lawrence@docker.com" RUN apk add --update git gcc libc-dev && rm -rf /var/cache/apk/* -# Pin to the specific v1 version -RUN git clone -b v1 https://github.com/mattes/migrate.git /go/src/github.com/mattes/migrate/ && \ - go get github.com/mattes/migrate && \ - go build -tags 'mysql' -o /usr/local/bin/migrate github.com/mattes/migrate +# Pin to the specific v3.0.0 version +RUN go get -tags 'mysql postgres file' github.com/mattes/migrate/cli && mv /go/bin/cli /go/bin/migrate ENV NOTARYPKG github.com/docker/notary diff --git a/signer.Dockerfile b/signer.Dockerfile index 77e20342bb..da4963b4b5 100644 --- a/signer.Dockerfile +++ b/signer.Dockerfile @@ -3,10 +3,8 @@ MAINTAINER David Lawrence "david.lawrence@docker.com" RUN apk add --update git gcc libc-dev && rm -rf /var/cache/apk/* -# Pin to the specific v1 version -RUN git clone -b v1 https://github.com/mattes/migrate.git /go/src/github.com/mattes/migrate/ && \ - go get github.com/mattes/migrate && \ - go build -tags 'mysql' -o /usr/local/bin/migrate github.com/mattes/migrate +# Pin to the specific v3.0.0 version +RUN go get -tags 'mysql postgres file' github.com/mattes/migrate/cli && mv /go/bin/cli /go/bin/migrate ENV NOTARYPKG github.com/docker/notary diff --git a/utils/configuration.go b/utils/configuration.go index 057995ec54..0cd36e3760 100644 --- a/utils/configuration.go +++ b/utils/configuration.go @@ -14,6 +14,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/bugsnag/bugsnag-go" "github.com/docker/go-connections/tlsconfig" + "github.com/go-sql-driver/mysql" "github.com/spf13/viper" "github.com/docker/notary" @@ -109,6 +110,16 @@ func ParseSQLStorage(configuration *viper.Viper) (*Storage, error) { "must provide a non-empty database source for %s", store.Backend, ) + case store.Backend == notary.MySQLBackend: + urlConfig, err := mysql.ParseDSN(store.Source) + if err != nil { + return nil, fmt.Errorf("failed to parse the database source for %s", + store.Backend, + ) + } + + urlConfig.ParseTime = true + store.Source = urlConfig.FormatDSN() } return &store, nil } diff --git a/utils/configuration_test.go b/utils/configuration_test.go index 6cd88409f3..2a6401b86a 100644 --- a/utils/configuration_test.go +++ b/utils/configuration_test.go @@ -192,6 +192,20 @@ func TestParseInvalidSQLStorageNoDBSource(t *testing.T) { } } +// If an invalid DB source is provided, an error is returned. +func TestParseInvalidDBSourceInSQLStorage(t *testing.T) { + config := configure(`{ + "storage": { + "backend": "mysql", + "db_url": "foobar" + } + }`) + _, err := ParseSQLStorage(config) + require.Error(t, err) + require.Contains(t, err.Error(), + fmt.Sprintf("failed to parse the database source for mysql")) +} + // A supported backend with DB source will be successfully parsed. func TestParseSQLStorageDBStore(t *testing.T) { config := configure(`{ @@ -203,7 +217,7 @@ func TestParseSQLStorageDBStore(t *testing.T) { expected := Storage{ Backend: "mysql", - Source: "username:passord@tcp(hostname:1234)/dbname", + Source: "username:passord@tcp(hostname:1234)/dbname?parseTime=true", } store, err := ParseSQLStorage(config) @@ -333,7 +347,7 @@ func TestParseSQLStorageWithEnvironmentVariables(t *testing.T) { expected := Storage{ Backend: "mysql", - Source: "username:passord@tcp(hostname:1234)/dbname", + Source: "username:passord@tcp(hostname:1234)/dbname?parseTime=true", } store, err := ParseSQLStorage(config) diff --git a/vendor.conf b/vendor.conf index ce4e1e6fc1..b50f2f671b 100644 --- a/vendor.conf +++ b/vendor.conf @@ -5,9 +5,9 @@ github.com/bugsnag/bugsnag-go 13fd6b8acda029830ef9904df6b63be0a83369d0 github.com/coreos/etcd 6acb3d67fbe131b3b2d5d010e00ec80182be4628 github.com/docker/distribution v2.6.0 github.com/docker/go-connections f549a9393d05688dff0992ef3efd8bbe6c628aeb -github.com/docker/go/canonical d30aec9fd63c35133f8f79c3412ad91a3b08be06 +github.com/docker/go d30aec9fd63c35133f8f79c3412ad91a3b08be06 github.com/dvsekhvalnov/jose2go v1.2 -github.com/go-sql-driver/mysql 0cc29e9fe8e25c2c58cf47bcab566e029bbaa88b +github.com/go-sql-driver/mysql v1.3 github.com/gorilla/mux e444e69cbd2e2e3e0749a2f3c717cec491552bbf github.com/jinzhu/gorm 5409931a1bb87e484d68d649af9367c207713ea2 github.com/jinzhu/inflection 1c35d901db3da928c72a72d8458480cc9ade058f @@ -24,6 +24,7 @@ golang.org/x/crypto 5bcd134fee4dd1475da17714aac19c0aa0142e2f golang.org/x/net 6a513affb38dc9788b449d59ffed099b8de18fa0 google.golang.org/grpc v1.0.5 + gopkg.in/dancannon/gorethink.v3 v3.0.0 # dependencies of gorethink.v3 gopkg.in/gorethink/gorethink.v2 v2.2.2 diff --git a/vendor/github.com/go-sql-driver/mysql/README.md b/vendor/github.com/go-sql-driver/mysql/README.md new file mode 100644 index 0000000000..a16012f819 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/README.md @@ -0,0 +1,443 @@ +# Go-MySQL-Driver + +A MySQL-Driver for Go's [database/sql](http://golang.org/pkg/database/sql) package + +![Go-MySQL-Driver logo](https://raw.github.com/wiki/go-sql-driver/mysql/gomysql_m.png "Golang Gopher holding the MySQL Dolphin") + +--------------------------------------- + * [Features](#features) + * [Requirements](#requirements) + * [Installation](#installation) + * [Usage](#usage) + * [DSN (Data Source Name)](#dsn-data-source-name) + * [Password](#password) + * [Protocol](#protocol) + * [Address](#address) + * [Parameters](#parameters) + * [Examples](#examples) + * [LOAD DATA LOCAL INFILE support](#load-data-local-infile-support) + * [time.Time support](#timetime-support) + * [Unicode support](#unicode-support) + * [Testing / Development](#testing--development) + * [License](#license) + +--------------------------------------- + +## Features + * Lightweight and [fast](https://github.com/go-sql-driver/sql-benchmark "golang MySQL-Driver performance") + * Native Go implementation. No C-bindings, just pure Go + * Connections over TCP/IPv4, TCP/IPv6, Unix domain sockets or [custom protocols](http://godoc.org/github.com/go-sql-driver/mysql#DialFunc) + * Automatic handling of broken connections + * Automatic Connection Pooling *(by database/sql package)* + * Supports queries larger than 16MB + * Full [`sql.RawBytes`](http://golang.org/pkg/database/sql/#RawBytes) support. + * Intelligent `LONG DATA` handling in prepared statements + * Secure `LOAD DATA LOCAL INFILE` support with file Whitelisting and `io.Reader` support + * Optional `time.Time` parsing + * Optional placeholder interpolation + +## Requirements + * Go 1.2 or higher + * MySQL (4.1+), MariaDB, Percona Server, Google CloudSQL or Sphinx (2.2.3+) + +--------------------------------------- + +## Installation +Simple install the package to your [$GOPATH](http://code.google.com/p/go-wiki/wiki/GOPATH "GOPATH") with the [go tool](http://golang.org/cmd/go/ "go command") from shell: +```bash +$ go get github.com/go-sql-driver/mysql +``` +Make sure [Git is installed](http://git-scm.com/downloads) on your machine and in your system's `PATH`. + +## Usage +_Go MySQL Driver_ is an implementation of Go's `database/sql/driver` interface. You only need to import the driver and can use the full [`database/sql`](http://golang.org/pkg/database/sql) API then. + +Use `mysql` as `driverName` and a valid [DSN](#dsn-data-source-name) as `dataSourceName`: +```go +import "database/sql" +import _ "github.com/go-sql-driver/mysql" + +db, err := sql.Open("mysql", "user:password@/dbname") +``` + +[Examples are available in our Wiki](https://github.com/go-sql-driver/mysql/wiki/Examples "Go-MySQL-Driver Examples"). + + +### DSN (Data Source Name) + +The Data Source Name has a common format, like e.g. [PEAR DB](http://pear.php.net/manual/en/package.database.db.intro-dsn.php) uses it, but without type-prefix (optional parts marked by squared brackets): +``` +[username[:password]@][protocol[(address)]]/dbname[?param1=value1&...¶mN=valueN] +``` + +A DSN in its fullest form: +``` +username:password@protocol(address)/dbname?param=value +``` + +Except for the databasename, all values are optional. So the minimal DSN is: +``` +/dbname +``` + +If you do not want to preselect a database, leave `dbname` empty: +``` +/ +``` +This has the same effect as an empty DSN string: +``` + +``` + +Alternatively, [Config.FormatDSN](https://godoc.org/github.com/go-sql-driver/mysql#Config.FormatDSN) can be used to create a DSN string by filling a struct. + +#### Password +Passwords can consist of any character. Escaping is **not** necessary. + +#### Protocol +See [net.Dial](http://golang.org/pkg/net/#Dial) for more information which networks are available. +In general you should use an Unix domain socket if available and TCP otherwise for best performance. + +#### Address +For TCP and UDP networks, addresses have the form `host:port`. +If `host` is a literal IPv6 address, it must be enclosed in square brackets. +The functions [net.JoinHostPort](http://golang.org/pkg/net/#JoinHostPort) and [net.SplitHostPort](http://golang.org/pkg/net/#SplitHostPort) manipulate addresses in this form. + +For Unix domain sockets the address is the absolute path to the MySQL-Server-socket, e.g. `/var/run/mysqld/mysqld.sock` or `/tmp/mysql.sock`. + +#### Parameters +*Parameters are case-sensitive!* + +Notice that any of `true`, `TRUE`, `True` or `1` is accepted to stand for a true boolean value. Not surprisingly, false can be specified as any of: `false`, `FALSE`, `False` or `0`. + +##### `allowAllFiles` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +`allowAllFiles=true` disables the file Whitelist for `LOAD DATA LOCAL INFILE` and allows *all* files. +[*Might be insecure!*](http://dev.mysql.com/doc/refman/5.7/en/load-data-local.html) + +##### `allowCleartextPasswords` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +`allowCleartextPasswords=true` allows using the [cleartext client side plugin](http://dev.mysql.com/doc/en/cleartext-authentication-plugin.html) if required by an account, such as one defined with the [PAM authentication plugin](http://dev.mysql.com/doc/en/pam-authentication-plugin.html). Sending passwords in clear text may be a security problem in some configurations. To avoid problems if there is any possibility that the password would be intercepted, clients should connect to MySQL Server using a method that protects the password. Possibilities include [TLS / SSL](#tls), IPsec, or a private network. + +##### `allowNativePasswords` + +``` +Type: bool +Valid Values: true, false +Default: false +``` +`allowNativePasswords=true` allows the usage of the mysql native password method. + +##### `allowOldPasswords` + +``` +Type: bool +Valid Values: true, false +Default: false +``` +`allowOldPasswords=true` allows the usage of the insecure old password method. This should be avoided, but is necessary in some cases. See also [the old_passwords wiki page](https://github.com/go-sql-driver/mysql/wiki/old_passwords). + +##### `charset` + +``` +Type: string +Valid Values: +Default: none +``` + +Sets the charset used for client-server interaction (`"SET NAMES "`). If multiple charsets are set (separated by a comma), the following charset is used if setting the charset failes. This enables for example support for `utf8mb4` ([introduced in MySQL 5.5.3](http://dev.mysql.com/doc/refman/5.5/en/charset-unicode-utf8mb4.html)) with fallback to `utf8` for older servers (`charset=utf8mb4,utf8`). + +Usage of the `charset` parameter is discouraged because it issues additional queries to the server. +Unless you need the fallback behavior, please use `collation` instead. + +##### `collation` + +``` +Type: string +Valid Values: +Default: utf8_general_ci +``` + +Sets the collation used for client-server interaction on connection. In contrast to `charset`, `collation` does not issue additional queries. If the specified collation is unavailable on the target server, the connection will fail. + +A list of valid charsets for a server is retrievable with `SHOW COLLATION`. + +##### `clientFoundRows` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +`clientFoundRows=true` causes an UPDATE to return the number of matching rows instead of the number of rows changed. + +##### `columnsWithAlias` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +When `columnsWithAlias` is true, calls to `sql.Rows.Columns()` will return the table alias and the column name separated by a dot. For example: + +``` +SELECT u.id FROM users as u +``` + +will return `u.id` instead of just `id` if `columnsWithAlias=true`. + +##### `interpolateParams` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +If `interpolateParams` is true, placeholders (`?`) in calls to `db.Query()` and `db.Exec()` are interpolated into a single query string with given parameters. This reduces the number of roundtrips, since the driver has to prepare a statement, execute it with given parameters and close the statement again with `interpolateParams=false`. + +*This can not be used together with the multibyte encodings BIG5, CP932, GB2312, GBK or SJIS. These are blacklisted as they may [introduce a SQL injection vulnerability](http://stackoverflow.com/a/12118602/3430118)!* + +##### `loc` + +``` +Type: string +Valid Values: +Default: UTC +``` + +Sets the location for time.Time values (when using `parseTime=true`). *"Local"* sets the system's location. See [time.LoadLocation](http://golang.org/pkg/time/#LoadLocation) for details. + +Note that this sets the location for time.Time values but does not change MySQL's [time_zone setting](https://dev.mysql.com/doc/refman/5.5/en/time-zone-support.html). For that see the [time_zone system variable](#system-variables), which can also be set as a DSN parameter. + +Please keep in mind, that param values must be [url.QueryEscape](http://golang.org/pkg/net/url/#QueryEscape)'ed. Alternatively you can manually replace the `/` with `%2F`. For example `US/Pacific` would be `loc=US%2FPacific`. + +##### `maxAllowedPacket` +``` +Type: decimal number +Default: 0 +``` + +Max packet size allowed in bytes. Use `maxAllowedPacket=0` to automatically fetch the `max_allowed_packet` variable from server. + +##### `multiStatements` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +Allow multiple statements in one query. While this allows batch queries, it also greatly increases the risk of SQL injections. Only the result of the first query is returned, all other results are silently discarded. + +When `multiStatements` is used, `?` parameters must only be used in the first statement. + +##### `parseTime` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +`parseTime=true` changes the output type of `DATE` and `DATETIME` values to `time.Time` instead of `[]byte` / `string` + + +##### `readTimeout` + +``` +Type: decimal number +Default: 0 +``` + +I/O read timeout. The value must be a decimal number with an unit suffix ( *"ms"*, *"s"*, *"m"*, *"h"* ), such as *"30s"*, *"0.5m"* or *"1m30s"*. + +##### `strict` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +`strict=true` enables a driver-side strict mode in which MySQL warnings are treated as errors. This mode should not be used in production as it may lead to data corruption in certain situations. + +A server-side strict mode, which is safe for production use, can be set via the [`sql_mode`](https://dev.mysql.com/doc/refman/5.7/en/sql-mode.html) system variable. + +By default MySQL also treats notes as warnings. Use [`sql_notes=false`](http://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_sql_notes) to ignore notes. + +##### `timeout` + +``` +Type: decimal number +Default: OS default +``` + +*Driver* side connection timeout. The value must be a decimal number with an unit suffix ( *"ms"*, *"s"*, *"m"*, *"h"* ), such as *"30s"*, *"0.5m"* or *"1m30s"*. To set a server side timeout, use the parameter [`wait_timeout`](http://dev.mysql.com/doc/refman/5.6/en/server-system-variables.html#sysvar_wait_timeout). + +##### `tls` + +``` +Type: bool / string +Valid Values: true, false, skip-verify, +Default: false +``` + +`tls=true` enables TLS / SSL encrypted connection to the server. Use `skip-verify` if you want to use a self-signed or invalid certificate (server side). Use a custom value registered with [`mysql.RegisterTLSConfig`](http://godoc.org/github.com/go-sql-driver/mysql#RegisterTLSConfig). + +##### `writeTimeout` + +``` +Type: decimal number +Default: 0 +``` + +I/O write timeout. The value must be a decimal number with an unit suffix ( *"ms"*, *"s"*, *"m"*, *"h"* ), such as *"30s"*, *"0.5m"* or *"1m30s"*. + + +##### System Variables + +Any other parameters are interpreted as system variables: + * `=`: `SET =` + * `=`: `SET =` + * `=%27%27`: `SET =''` + +Rules: +* The values for string variables must be quoted with ' +* The values must also be [url.QueryEscape](http://golang.org/pkg/net/url/#QueryEscape)'ed! + (which implies values of string variables must be wrapped with `%27`) + +Examples: + * `autocommit=1`: `SET autocommit=1` + * [`time_zone=%27Europe%2FParis%27`](https://dev.mysql.com/doc/refman/5.5/en/time-zone-support.html): `SET time_zone='Europe/Paris'` + * [`tx_isolation=%27REPEATABLE-READ%27`](https://dev.mysql.com/doc/refman/5.5/en/server-system-variables.html#sysvar_tx_isolation): `SET tx_isolation='REPEATABLE-READ'` + + +#### Examples +``` +user@unix(/path/to/socket)/dbname +``` + +``` +root:pw@unix(/tmp/mysql.sock)/myDatabase?loc=Local +``` + +``` +user:password@tcp(localhost:5555)/dbname?tls=skip-verify&autocommit=true +``` + +Treat warnings as errors by setting the system variable [`sql_mode`](https://dev.mysql.com/doc/refman/5.7/en/sql-mode.html): +``` +user:password@/dbname?sql_mode=TRADITIONAL +``` + +TCP via IPv6: +``` +user:password@tcp([de:ad:be:ef::ca:fe]:80)/dbname?timeout=90s&collation=utf8mb4_unicode_ci +``` + +TCP on a remote host, e.g. Amazon RDS: +``` +id:password@tcp(your-amazonaws-uri.com:3306)/dbname +``` + +Google Cloud SQL on App Engine (First Generation MySQL Server): +``` +user@cloudsql(project-id:instance-name)/dbname +``` + +Google Cloud SQL on App Engine (Second Generation MySQL Server): +``` +user@cloudsql(project-id:regionname:instance-name)/dbname +``` + +TCP using default port (3306) on localhost: +``` +user:password@tcp/dbname?charset=utf8mb4,utf8&sys_var=esc%40ped +``` + +Use the default protocol (tcp) and host (localhost:3306): +``` +user:password@/dbname +``` + +No Database preselected: +``` +user:password@/ +``` + +### `LOAD DATA LOCAL INFILE` support +For this feature you need direct access to the package. Therefore you must change the import path (no `_`): +```go +import "github.com/go-sql-driver/mysql" +``` + +Files must be whitelisted by registering them with `mysql.RegisterLocalFile(filepath)` (recommended) or the Whitelist check must be deactivated by using the DSN parameter `allowAllFiles=true` ([*Might be insecure!*](http://dev.mysql.com/doc/refman/5.7/en/load-data-local.html)). + +To use a `io.Reader` a handler function must be registered with `mysql.RegisterReaderHandler(name, handler)` which returns a `io.Reader` or `io.ReadCloser`. The Reader is available with the filepath `Reader::` then. Choose different names for different handlers and `DeregisterReaderHandler` when you don't need it anymore. + +See the [godoc of Go-MySQL-Driver](http://godoc.org/github.com/go-sql-driver/mysql "golang mysql driver documentation") for details. + + +### `time.Time` support +The default internal output type of MySQL `DATE` and `DATETIME` values is `[]byte` which allows you to scan the value into a `[]byte`, `string` or `sql.RawBytes` variable in your programm. + +However, many want to scan MySQL `DATE` and `DATETIME` values into `time.Time` variables, which is the logical opposite in Go to `DATE` and `DATETIME` in MySQL. You can do that by changing the internal output type from `[]byte` to `time.Time` with the DSN parameter `parseTime=true`. You can set the default [`time.Time` location](http://golang.org/pkg/time/#Location) with the `loc` DSN parameter. + +**Caution:** As of Go 1.1, this makes `time.Time` the only variable type you can scan `DATE` and `DATETIME` values into. This breaks for example [`sql.RawBytes` support](https://github.com/go-sql-driver/mysql/wiki/Examples#rawbytes). + +Alternatively you can use the [`NullTime`](http://godoc.org/github.com/go-sql-driver/mysql#NullTime) type as the scan destination, which works with both `time.Time` and `string` / `[]byte`. + + +### Unicode support +Since version 1.1 Go-MySQL-Driver automatically uses the collation `utf8_general_ci` by default. + +Other collations / charsets can be set using the [`collation`](#collation) DSN parameter. + +Version 1.0 of the driver recommended adding `&charset=utf8` (alias for `SET NAMES utf8`) to the DSN to enable proper UTF-8 support. This is not necessary anymore. The [`collation`](#collation) parameter should be preferred to set another collation / charset than the default. + +See http://dev.mysql.com/doc/refman/5.7/en/charset-unicode.html for more details on MySQL's Unicode support. + + +## Testing / Development +To run the driver tests you may need to adjust the configuration. See the [Testing Wiki-Page](https://github.com/go-sql-driver/mysql/wiki/Testing "Testing") for details. + +Go-MySQL-Driver is not feature-complete yet. Your help is very appreciated. +If you want to contribute, you can work on an [open issue](https://github.com/go-sql-driver/mysql/issues?state=open) or review a [pull request](https://github.com/go-sql-driver/mysql/pulls). + +See the [Contribution Guidelines](https://github.com/go-sql-driver/mysql/blob/master/CONTRIBUTING.md) for details. + +--------------------------------------- + +## License +Go-MySQL-Driver is licensed under the [Mozilla Public License Version 2.0](https://raw.github.com/go-sql-driver/mysql/master/LICENSE) + +Mozilla summarizes the license scope as follows: +> MPL: The copyleft applies to any files containing MPLed code. + + +That means: + * You can **use** the **unchanged** source code both in private and commercially + * When distributing, you **must publish** the source code of any **changed files** licensed under the MPL 2.0 under a) the MPL 2.0 itself or b) a compatible license (e.g. GPL 3.0 or Apache License 2.0) + * You **needn't publish** the source code of your library as long as the files licensed under the MPL 2.0 are **unchanged** + +Please read the [MPL 2.0 FAQ](http://www.mozilla.org/MPL/2.0/FAQ.html) if you have further questions regarding the license. + +You can read the full terms here: [LICENSE](https://raw.github.com/go-sql-driver/mysql/master/LICENSE) + +![Go Gopher and MySQL Dolphin](https://raw.github.com/wiki/go-sql-driver/mysql/go-mysql-driver_m.jpg "Golang Gopher transporting the MySQL Dolphin in a wheelbarrow") + diff --git a/vendor/github.com/go-sql-driver/mysql/buffer.go b/vendor/github.com/go-sql-driver/mysql/buffer.go index 509ce89e46..2001feacd3 100644 --- a/vendor/github.com/go-sql-driver/mysql/buffer.go +++ b/vendor/github.com/go-sql-driver/mysql/buffer.go @@ -8,7 +8,11 @@ package mysql -import "io" +import ( + "io" + "net" + "time" +) const defaultBufSize = 4096 @@ -18,17 +22,18 @@ const defaultBufSize = 4096 // The buffer is similar to bufio.Reader / Writer but zero-copy-ish // Also highly optimized for this particular use case. type buffer struct { - buf []byte - rd io.Reader - idx int - length int + buf []byte + nc net.Conn + idx int + length int + timeout time.Duration } -func newBuffer(rd io.Reader) buffer { +func newBuffer(nc net.Conn) buffer { var b [defaultBufSize]byte return buffer{ buf: b[:], - rd: rd, + nc: nc, } } @@ -54,7 +59,13 @@ func (b *buffer) fill(need int) error { b.idx = 0 for { - nn, err := b.rd.Read(b.buf[n:]) + if b.timeout > 0 { + if err := b.nc.SetReadDeadline(time.Now().Add(b.timeout)); err != nil { + return err + } + } + + nn, err := b.nc.Read(b.buf[n:]) n += nn switch err { diff --git a/vendor/github.com/go-sql-driver/mysql/collations.go b/vendor/github.com/go-sql-driver/mysql/collations.go index 6c1d613d5b..82079cfb93 100644 --- a/vendor/github.com/go-sql-driver/mysql/collations.go +++ b/vendor/github.com/go-sql-driver/mysql/collations.go @@ -8,7 +8,7 @@ package mysql -const defaultCollation byte = 33 // utf8_general_ci +const defaultCollation = "utf8_general_ci" // A list of available collations mapped to the internal ID. // To update this map use the following MySQL query: @@ -237,14 +237,14 @@ var collations = map[string]byte{ // A blacklist of collations which is unsafe to interpolate parameters. // These multibyte encodings may contains 0x5c (`\`) in their trailing bytes. -var unsafeCollations = map[byte]bool{ - 1: true, // big5_chinese_ci - 13: true, // sjis_japanese_ci - 28: true, // gbk_chinese_ci - 84: true, // big5_bin - 86: true, // gb2312_bin - 87: true, // gbk_bin - 88: true, // sjis_bin - 95: true, // cp932_japanese_ci - 96: true, // cp932_bin +var unsafeCollations = map[string]bool{ + "big5_chinese_ci": true, + "sjis_japanese_ci": true, + "gbk_chinese_ci": true, + "big5_bin": true, + "gb2312_bin": true, + "gbk_bin": true, + "sjis_bin": true, + "cp932_japanese_ci": true, + "cp932_bin": true, } diff --git a/vendor/github.com/go-sql-driver/mysql/connection.go b/vendor/github.com/go-sql-driver/mysql/connection.go index a6d39bec95..d82c728f3b 100644 --- a/vendor/github.com/go-sql-driver/mysql/connection.go +++ b/vendor/github.com/go-sql-driver/mysql/connection.go @@ -9,9 +9,7 @@ package mysql import ( - "crypto/tls" "database/sql/driver" - "errors" "net" "strconv" "strings" @@ -23,9 +21,10 @@ type mysqlConn struct { netConn net.Conn affectedRows uint64 insertId uint64 - cfg *config - maxPacketAllowed int + cfg *Config + maxAllowedPacket int maxWriteSize int + writeTimeout time.Duration flags clientFlag status statusFlag sequence uint8 @@ -33,27 +32,9 @@ type mysqlConn struct { strict bool } -type config struct { - user string - passwd string - net string - addr string - dbname string - params map[string]string - loc *time.Location - tls *tls.Config - timeout time.Duration - collation uint8 - allowAllFiles bool - allowOldPasswords bool - clientFoundRows bool - columnsWithAlias bool - interpolateParams bool -} - // Handles parameters set in DSN after the connection is established func (mc *mysqlConn) handleParams() (err error) { - for param, val := range mc.cfg.params { + for param, val := range mc.cfg.Params { switch param { // Charset case "charset": @@ -69,27 +50,6 @@ func (mc *mysqlConn) handleParams() (err error) { return } - // time.Time parsing - case "parseTime": - var isBool bool - mc.parseTime, isBool = readBool(val) - if !isBool { - return errors.New("Invalid Bool value: " + val) - } - - // Strict mode - case "strict": - var isBool bool - mc.strict, isBool = readBool(val) - if !isBool { - return errors.New("Invalid Bool value: " + val) - } - - // Compression - case "compress": - err = errors.New("Compression not implemented yet") - return - // System Vars default: err = mc.exec("SET " + param + "=" + val + "") @@ -119,20 +79,29 @@ func (mc *mysqlConn) Close() (err error) { // Makes Close idempotent if mc.netConn != nil { err = mc.writeCommandPacket(comQuit) - if err == nil { - err = mc.netConn.Close() - } else { - mc.netConn.Close() - } - mc.netConn = nil } - mc.cfg = nil - mc.buf.rd = nil + mc.cleanup() return } +// Closes the network connection and unsets internal variables. Do not call this +// function after successfully authentication, call Close instead. This function +// is called before auth or on auth failure because MySQL will have already +// closed the network connection. +func (mc *mysqlConn) cleanup() { + // Makes cleanup idempotent + if mc.netConn != nil { + if err := mc.netConn.Close(); err != nil { + errLog.Print(err) + } + mc.netConn = nil + } + mc.cfg = nil + mc.buf.nc = nil +} + func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) { if mc.netConn == nil { errLog.Print(ErrInvalidConn) @@ -166,6 +135,11 @@ func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) { } func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (string, error) { + // Number of ? should be same to len(args) + if strings.Count(query, "?") != len(args) { + return "", driver.ErrSkip + } + buf := mc.buf.takeCompleteBuffer() if buf == nil { // can not take the buffer. Something must be wrong with the connection @@ -207,7 +181,7 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin if v.IsZero() { buf = append(buf, "'0000-00-00'"...) } else { - v := v.In(mc.cfg.loc) + v := v.In(mc.cfg.Loc) v = v.Add(time.Nanosecond * 500) // To round under microsecond year := v.Year() year100 := year / 100 @@ -252,7 +226,7 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin if v == nil { buf = append(buf, "NULL"...) } else { - buf = append(buf, '\'') + buf = append(buf, "_binary'"...) if mc.status&statusNoBackslashEscapes == 0 { buf = escapeBytesBackslash(buf, v) } else { @@ -272,7 +246,7 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin return "", driver.ErrSkip } - if len(buf)+4 > mc.maxPacketAllowed { + if len(buf)+4 > mc.maxAllowedPacket { return "", driver.ErrSkip } } @@ -288,7 +262,7 @@ func (mc *mysqlConn) Exec(query string, args []driver.Value) (driver.Result, err return nil, driver.ErrBadConn } if len(args) != 0 { - if !mc.cfg.interpolateParams { + if !mc.cfg.InterpolateParams { return nil, driver.ErrSkip } // try to interpolate the parameters to save extra roundtrips for preparing and closing a statement @@ -339,7 +313,7 @@ func (mc *mysqlConn) Query(query string, args []driver.Value) (driver.Rows, erro return nil, driver.ErrBadConn } if len(args) != 0 { - if !mc.cfg.interpolateParams { + if !mc.cfg.InterpolateParams { return nil, driver.ErrSkip } // try client-side prepare to reduce roundtrip @@ -385,6 +359,7 @@ func (mc *mysqlConn) getSystemVar(name string) ([]byte, error) { if err == nil { rows := new(textRows) rows.mc = mc + rows.columns = []mysqlField{{fieldType: fieldTypeVarChar}} if resLen > 0 { // Columns diff --git a/vendor/github.com/go-sql-driver/mysql/const.go b/vendor/github.com/go-sql-driver/mysql/const.go index dddc12908f..88cfff3fd8 100644 --- a/vendor/github.com/go-sql-driver/mysql/const.go +++ b/vendor/github.com/go-sql-driver/mysql/const.go @@ -107,7 +107,8 @@ const ( fieldTypeBit ) const ( - fieldTypeNewDecimal byte = iota + 0xf6 + fieldTypeJSON byte = iota + 0xf5 + fieldTypeNewDecimal fieldTypeEnum fieldTypeSet fieldTypeTinyBLOB diff --git a/vendor/github.com/go-sql-driver/mysql/driver.go b/vendor/github.com/go-sql-driver/mysql/driver.go index 3cbbe6031c..0022d1f1e5 100644 --- a/vendor/github.com/go-sql-driver/mysql/driver.go +++ b/vendor/github.com/go-sql-driver/mysql/driver.go @@ -4,7 +4,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// Package mysql provides a MySQL driver for Go's database/sql package // // The driver should be used via the database/sql package: // @@ -22,7 +22,7 @@ import ( "net" ) -// This struct is exported to make the driver directly accessible. +// MySQLDriver is exported to make the driver directly accessible. // In general the driver is used via the database/sql package. type MySQLDriver struct{} @@ -50,20 +50,22 @@ func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { // New mysqlConn mc := &mysqlConn{ - maxPacketAllowed: maxPacketSize, + maxAllowedPacket: maxPacketSize, maxWriteSize: maxPacketSize - 1, } - mc.cfg, err = parseDSN(dsn) + mc.cfg, err = ParseDSN(dsn) if err != nil { return nil, err } + mc.parseTime = mc.cfg.ParseTime + mc.strict = mc.cfg.Strict // Connect to Server - if dial, ok := dials[mc.cfg.net]; ok { - mc.netConn, err = dial(mc.cfg.addr) + if dial, ok := dials[mc.cfg.Net]; ok { + mc.netConn, err = dial(mc.cfg.Addr) } else { - nd := net.Dialer{Timeout: mc.cfg.timeout} - mc.netConn, err = nd.Dial(mc.cfg.net, mc.cfg.addr) + nd := net.Dialer{Timeout: mc.cfg.Timeout} + mc.netConn, err = nd.Dial(mc.cfg.Net, mc.cfg.Addr) } if err != nil { return nil, err @@ -81,48 +83,45 @@ func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { mc.buf = newBuffer(mc.netConn) + // Set I/O timeouts + mc.buf.timeout = mc.cfg.ReadTimeout + mc.writeTimeout = mc.cfg.WriteTimeout + // Reading Handshake Initialization Packet cipher, err := mc.readInitPacket() if err != nil { - mc.Close() + mc.cleanup() return nil, err } // Send Client Authentication Packet if err = mc.writeAuthPacket(cipher); err != nil { - mc.Close() + mc.cleanup() return nil, err } - // Read Result Packet - err = mc.readResultOK() - if err != nil { - // Retry with old authentication method, if allowed - if mc.cfg != nil && mc.cfg.allowOldPasswords && err == ErrOldPassword { - if err = mc.writeOldAuthPacket(cipher); err != nil { - mc.Close() - return nil, err - } - if err = mc.readResultOK(); err != nil { - mc.Close() - return nil, err - } - } else { + // Handle response to auth packet, switch methods if possible + if err = handleAuthResult(mc, cipher); err != nil { + // Authentication failed and MySQL has already closed the connection + // (https://dev.mysql.com/doc/internals/en/authentication-fails.html). + // Do not send COM_QUIT, just cleanup and return the error. + mc.cleanup() + return nil, err + } + + if mc.cfg.MaxAllowedPacket > 0 { + mc.maxAllowedPacket = mc.cfg.MaxAllowedPacket + } else { + // Get max allowed packet size + maxap, err := mc.getSystemVar("max_allowed_packet") + if err != nil { mc.Close() return nil, err } - - } - - // Get max allowed packet size - maxap, err := mc.getSystemVar("max_allowed_packet") - if err != nil { - mc.Close() - return nil, err + mc.maxAllowedPacket = stringToInt(maxap) - 1 } - mc.maxPacketAllowed = stringToInt(maxap) - 1 - if mc.maxPacketAllowed < maxPacketSize { - mc.maxWriteSize = mc.maxPacketAllowed + if mc.maxAllowedPacket < maxPacketSize { + mc.maxWriteSize = mc.maxAllowedPacket } // Handle DSN Params @@ -135,6 +134,50 @@ func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { return mc, nil } +func handleAuthResult(mc *mysqlConn, oldCipher []byte) error { + // Read Result Packet + cipher, err := mc.readResultOK() + if err == nil { + return nil // auth successful + } + + if mc.cfg == nil { + return err // auth failed and retry not possible + } + + // Retry auth if configured to do so. + if mc.cfg.AllowOldPasswords && err == ErrOldPassword { + // Retry with old authentication method. Note: there are edge cases + // where this should work but doesn't; this is currently "wontfix": + // https://github.com/go-sql-driver/mysql/issues/184 + + // If CLIENT_PLUGIN_AUTH capability is not supported, no new cipher is + // sent and we have to keep using the cipher sent in the init packet. + if cipher == nil { + cipher = oldCipher + } + + if err = mc.writeOldAuthPacket(cipher); err != nil { + return err + } + _, err = mc.readResultOK() + } else if mc.cfg.AllowCleartextPasswords && err == ErrCleartextPassword { + // Retry with clear text password for + // http://dev.mysql.com/doc/refman/5.7/en/cleartext-authentication-plugin.html + // http://dev.mysql.com/doc/refman/5.7/en/pam-authentication-plugin.html + if err = mc.writeClearAuthPacket(); err != nil { + return err + } + _, err = mc.readResultOK() + } else if mc.cfg.AllowNativePasswords && err == ErrNativePassword { + if err = mc.writeNativeAuthPacket(cipher); err != nil { + return err + } + _, err = mc.readResultOK() + } + return err +} + func init() { sql.Register("mysql", &MySQLDriver{}) } diff --git a/vendor/github.com/go-sql-driver/mysql/dsn.go b/vendor/github.com/go-sql-driver/mysql/dsn.go new file mode 100644 index 0000000000..ac00dcedd5 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/dsn.go @@ -0,0 +1,548 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2016 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "crypto/tls" + "errors" + "fmt" + "net" + "net/url" + "strconv" + "strings" + "time" +) + +var ( + errInvalidDSNUnescaped = errors.New("invalid DSN: did you forget to escape a param value?") + errInvalidDSNAddr = errors.New("invalid DSN: network address not terminated (missing closing brace)") + errInvalidDSNNoSlash = errors.New("invalid DSN: missing the slash separating the database name") + errInvalidDSNUnsafeCollation = errors.New("invalid DSN: interpolateParams can not be used with unsafe collations") +) + +// Config is a configuration parsed from a DSN string +type Config struct { + User string // Username + Passwd string // Password (requires User) + Net string // Network type + Addr string // Network address (requires Net) + DBName string // Database name + Params map[string]string // Connection parameters + Collation string // Connection collation + Loc *time.Location // Location for time.Time values + MaxAllowedPacket int // Max packet size allowed + TLSConfig string // TLS configuration name + tls *tls.Config // TLS configuration + Timeout time.Duration // Dial timeout + ReadTimeout time.Duration // I/O read timeout + WriteTimeout time.Duration // I/O write timeout + + AllowAllFiles bool // Allow all files to be used with LOAD DATA LOCAL INFILE + AllowCleartextPasswords bool // Allows the cleartext client side plugin + AllowNativePasswords bool // Allows the native password authentication method + AllowOldPasswords bool // Allows the old insecure password method + ClientFoundRows bool // Return number of matching rows instead of rows changed + ColumnsWithAlias bool // Prepend table alias to column names + InterpolateParams bool // Interpolate placeholders into query string + MultiStatements bool // Allow multiple statements in one query + ParseTime bool // Parse time values to time.Time + Strict bool // Return warnings as errors +} + +// FormatDSN formats the given Config into a DSN string which can be passed to +// the driver. +func (cfg *Config) FormatDSN() string { + var buf bytes.Buffer + + // [username[:password]@] + if len(cfg.User) > 0 { + buf.WriteString(cfg.User) + if len(cfg.Passwd) > 0 { + buf.WriteByte(':') + buf.WriteString(cfg.Passwd) + } + buf.WriteByte('@') + } + + // [protocol[(address)]] + if len(cfg.Net) > 0 { + buf.WriteString(cfg.Net) + if len(cfg.Addr) > 0 { + buf.WriteByte('(') + buf.WriteString(cfg.Addr) + buf.WriteByte(')') + } + } + + // /dbname + buf.WriteByte('/') + buf.WriteString(cfg.DBName) + + // [?param1=value1&...¶mN=valueN] + hasParam := false + + if cfg.AllowAllFiles { + hasParam = true + buf.WriteString("?allowAllFiles=true") + } + + if cfg.AllowCleartextPasswords { + if hasParam { + buf.WriteString("&allowCleartextPasswords=true") + } else { + hasParam = true + buf.WriteString("?allowCleartextPasswords=true") + } + } + + if cfg.AllowNativePasswords { + if hasParam { + buf.WriteString("&allowNativePasswords=true") + } else { + hasParam = true + buf.WriteString("?allowNativePasswords=true") + } + } + + if cfg.AllowOldPasswords { + if hasParam { + buf.WriteString("&allowOldPasswords=true") + } else { + hasParam = true + buf.WriteString("?allowOldPasswords=true") + } + } + + if cfg.ClientFoundRows { + if hasParam { + buf.WriteString("&clientFoundRows=true") + } else { + hasParam = true + buf.WriteString("?clientFoundRows=true") + } + } + + if col := cfg.Collation; col != defaultCollation && len(col) > 0 { + if hasParam { + buf.WriteString("&collation=") + } else { + hasParam = true + buf.WriteString("?collation=") + } + buf.WriteString(col) + } + + if cfg.ColumnsWithAlias { + if hasParam { + buf.WriteString("&columnsWithAlias=true") + } else { + hasParam = true + buf.WriteString("?columnsWithAlias=true") + } + } + + if cfg.InterpolateParams { + if hasParam { + buf.WriteString("&interpolateParams=true") + } else { + hasParam = true + buf.WriteString("?interpolateParams=true") + } + } + + if cfg.Loc != time.UTC && cfg.Loc != nil { + if hasParam { + buf.WriteString("&loc=") + } else { + hasParam = true + buf.WriteString("?loc=") + } + buf.WriteString(url.QueryEscape(cfg.Loc.String())) + } + + if cfg.MultiStatements { + if hasParam { + buf.WriteString("&multiStatements=true") + } else { + hasParam = true + buf.WriteString("?multiStatements=true") + } + } + + if cfg.ParseTime { + if hasParam { + buf.WriteString("&parseTime=true") + } else { + hasParam = true + buf.WriteString("?parseTime=true") + } + } + + if cfg.ReadTimeout > 0 { + if hasParam { + buf.WriteString("&readTimeout=") + } else { + hasParam = true + buf.WriteString("?readTimeout=") + } + buf.WriteString(cfg.ReadTimeout.String()) + } + + if cfg.Strict { + if hasParam { + buf.WriteString("&strict=true") + } else { + hasParam = true + buf.WriteString("?strict=true") + } + } + + if cfg.Timeout > 0 { + if hasParam { + buf.WriteString("&timeout=") + } else { + hasParam = true + buf.WriteString("?timeout=") + } + buf.WriteString(cfg.Timeout.String()) + } + + if len(cfg.TLSConfig) > 0 { + if hasParam { + buf.WriteString("&tls=") + } else { + hasParam = true + buf.WriteString("?tls=") + } + buf.WriteString(url.QueryEscape(cfg.TLSConfig)) + } + + if cfg.WriteTimeout > 0 { + if hasParam { + buf.WriteString("&writeTimeout=") + } else { + hasParam = true + buf.WriteString("?writeTimeout=") + } + buf.WriteString(cfg.WriteTimeout.String()) + } + + if cfg.MaxAllowedPacket > 0 { + if hasParam { + buf.WriteString("&maxAllowedPacket=") + } else { + hasParam = true + buf.WriteString("?maxAllowedPacket=") + } + buf.WriteString(strconv.Itoa(cfg.MaxAllowedPacket)) + + } + + // other params + if cfg.Params != nil { + for param, value := range cfg.Params { + if hasParam { + buf.WriteByte('&') + } else { + hasParam = true + buf.WriteByte('?') + } + + buf.WriteString(param) + buf.WriteByte('=') + buf.WriteString(url.QueryEscape(value)) + } + } + + return buf.String() +} + +// ParseDSN parses the DSN string to a Config +func ParseDSN(dsn string) (cfg *Config, err error) { + // New config with some default values + cfg = &Config{ + Loc: time.UTC, + Collation: defaultCollation, + } + + // [user[:password]@][net[(addr)]]/dbname[?param1=value1¶mN=valueN] + // Find the last '/' (since the password or the net addr might contain a '/') + foundSlash := false + for i := len(dsn) - 1; i >= 0; i-- { + if dsn[i] == '/' { + foundSlash = true + var j, k int + + // left part is empty if i <= 0 + if i > 0 { + // [username[:password]@][protocol[(address)]] + // Find the last '@' in dsn[:i] + for j = i; j >= 0; j-- { + if dsn[j] == '@' { + // username[:password] + // Find the first ':' in dsn[:j] + for k = 0; k < j; k++ { + if dsn[k] == ':' { + cfg.Passwd = dsn[k+1 : j] + break + } + } + cfg.User = dsn[:k] + + break + } + } + + // [protocol[(address)]] + // Find the first '(' in dsn[j+1:i] + for k = j + 1; k < i; k++ { + if dsn[k] == '(' { + // dsn[i-1] must be == ')' if an address is specified + if dsn[i-1] != ')' { + if strings.ContainsRune(dsn[k+1:i], ')') { + return nil, errInvalidDSNUnescaped + } + return nil, errInvalidDSNAddr + } + cfg.Addr = dsn[k+1 : i-1] + break + } + } + cfg.Net = dsn[j+1 : k] + } + + // dbname[?param1=value1&...¶mN=valueN] + // Find the first '?' in dsn[i+1:] + for j = i + 1; j < len(dsn); j++ { + if dsn[j] == '?' { + if err = parseDSNParams(cfg, dsn[j+1:]); err != nil { + return + } + break + } + } + cfg.DBName = dsn[i+1 : j] + + break + } + } + + if !foundSlash && len(dsn) > 0 { + return nil, errInvalidDSNNoSlash + } + + if cfg.InterpolateParams && unsafeCollations[cfg.Collation] { + return nil, errInvalidDSNUnsafeCollation + } + + // Set default network if empty + if cfg.Net == "" { + cfg.Net = "tcp" + } + + // Set default address if empty + if cfg.Addr == "" { + switch cfg.Net { + case "tcp": + cfg.Addr = "127.0.0.1:3306" + case "unix": + cfg.Addr = "/tmp/mysql.sock" + default: + return nil, errors.New("default addr for network '" + cfg.Net + "' unknown") + } + + } + + return +} + +// parseDSNParams parses the DSN "query string" +// Values must be url.QueryEscape'ed +func parseDSNParams(cfg *Config, params string) (err error) { + for _, v := range strings.Split(params, "&") { + param := strings.SplitN(v, "=", 2) + if len(param) != 2 { + continue + } + + // cfg params + switch value := param[1]; param[0] { + + // Disable INFILE whitelist / enable all files + case "allowAllFiles": + var isBool bool + cfg.AllowAllFiles, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Use cleartext authentication mode (MySQL 5.5.10+) + case "allowCleartextPasswords": + var isBool bool + cfg.AllowCleartextPasswords, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Use native password authentication + case "allowNativePasswords": + var isBool bool + cfg.AllowNativePasswords, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Use old authentication mode (pre MySQL 4.1) + case "allowOldPasswords": + var isBool bool + cfg.AllowOldPasswords, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Switch "rowsAffected" mode + case "clientFoundRows": + var isBool bool + cfg.ClientFoundRows, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Collation + case "collation": + cfg.Collation = value + break + + case "columnsWithAlias": + var isBool bool + cfg.ColumnsWithAlias, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Compression + case "compress": + return errors.New("compression not implemented yet") + + // Enable client side placeholder substitution + case "interpolateParams": + var isBool bool + cfg.InterpolateParams, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Time Location + case "loc": + if value, err = url.QueryUnescape(value); err != nil { + return + } + cfg.Loc, err = time.LoadLocation(value) + if err != nil { + return + } + + // multiple statements in one query + case "multiStatements": + var isBool bool + cfg.MultiStatements, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // time.Time parsing + case "parseTime": + var isBool bool + cfg.ParseTime, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // I/O read Timeout + case "readTimeout": + cfg.ReadTimeout, err = time.ParseDuration(value) + if err != nil { + return + } + + // Strict mode + case "strict": + var isBool bool + cfg.Strict, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + + // Dial Timeout + case "timeout": + cfg.Timeout, err = time.ParseDuration(value) + if err != nil { + return + } + + // TLS-Encryption + case "tls": + boolValue, isBool := readBool(value) + if isBool { + if boolValue { + cfg.TLSConfig = "true" + cfg.tls = &tls.Config{} + } else { + cfg.TLSConfig = "false" + } + } else if vl := strings.ToLower(value); vl == "skip-verify" { + cfg.TLSConfig = vl + cfg.tls = &tls.Config{InsecureSkipVerify: true} + } else { + name, err := url.QueryUnescape(value) + if err != nil { + return fmt.Errorf("invalid value for TLS config name: %v", err) + } + + if tlsConfig, ok := tlsConfigRegister[name]; ok { + if len(tlsConfig.ServerName) == 0 && !tlsConfig.InsecureSkipVerify { + host, _, err := net.SplitHostPort(cfg.Addr) + if err == nil { + tlsConfig.ServerName = host + } + } + + cfg.TLSConfig = name + cfg.tls = tlsConfig + } else { + return errors.New("invalid value / unknown config name: " + name) + } + } + + // I/O write Timeout + case "writeTimeout": + cfg.WriteTimeout, err = time.ParseDuration(value) + if err != nil { + return + } + case "maxAllowedPacket": + cfg.MaxAllowedPacket, err = strconv.Atoi(value) + if err != nil { + return + } + default: + // lazy init + if cfg.Params == nil { + cfg.Params = make(map[string]string) + } + + if cfg.Params[param[0]], err = url.QueryUnescape(value); err != nil { + return + } + } + } + + return +} diff --git a/vendor/github.com/go-sql-driver/mysql/errors.go b/vendor/github.com/go-sql-driver/mysql/errors.go index 97d7b39962..857854e147 100644 --- a/vendor/github.com/go-sql-driver/mysql/errors.go +++ b/vendor/github.com/go-sql-driver/mysql/errors.go @@ -19,18 +19,21 @@ import ( // Various errors the driver might return. Can change between driver versions. var ( - ErrInvalidConn = errors.New("Invalid Connection") - ErrMalformPkt = errors.New("Malformed Packet") - ErrNoTLS = errors.New("TLS encryption requested but server does not support TLS") - ErrOldPassword = errors.New("This server only supports the insecure old password authentication. If you still want to use it, please add 'allowOldPasswords=1' to your DSN. See also https://github.com/go-sql-driver/mysql/wiki/old_passwords") - ErrOldProtocol = errors.New("MySQL-Server does not support required Protocol 41+") - ErrPktSync = errors.New("Commands out of sync. You can't run this command now") - ErrPktSyncMul = errors.New("Commands out of sync. Did you run multiple statements at once?") - ErrPktTooLarge = errors.New("Packet for query is too large. You can change this value on the server by adjusting the 'max_allowed_packet' variable.") - ErrBusyBuffer = errors.New("Busy buffer") + ErrInvalidConn = errors.New("invalid connection") + ErrMalformPkt = errors.New("malformed packet") + ErrNoTLS = errors.New("TLS requested but server does not support TLS") + ErrCleartextPassword = errors.New("this user requires clear text authentication. If you still want to use it, please add 'allowCleartextPasswords=1' to your DSN") + ErrNativePassword = errors.New("this user requires mysql native password authentication.") + ErrOldPassword = errors.New("this user requires old password authentication. If you still want to use it, please add 'allowOldPasswords=1' to your DSN. See also https://github.com/go-sql-driver/mysql/wiki/old_passwords") + ErrUnknownPlugin = errors.New("this authentication plugin is not supported") + ErrOldProtocol = errors.New("MySQL server does not support required protocol 41+") + ErrPktSync = errors.New("commands out of sync. You can't run this command now") + ErrPktSyncMul = errors.New("commands out of sync. Did you run multiple statements at once?") + ErrPktTooLarge = errors.New("packet for query is too large. Try adjusting the 'max_allowed_packet' variable on the server") + ErrBusyBuffer = errors.New("busy buffer") ) -var errLog Logger = log.New(os.Stderr, "[MySQL] ", log.Ldate|log.Ltime|log.Lshortfile) +var errLog = Logger(log.New(os.Stderr, "[mysql] ", log.Ldate|log.Ltime|log.Lshortfile)) // Logger is used to log critical error messages. type Logger interface { diff --git a/vendor/github.com/go-sql-driver/mysql/infile.go b/vendor/github.com/go-sql-driver/mysql/infile.go index 121a04c712..547357cfa7 100644 --- a/vendor/github.com/go-sql-driver/mysql/infile.go +++ b/vendor/github.com/go-sql-driver/mysql/infile.go @@ -13,11 +13,14 @@ import ( "io" "os" "strings" + "sync" ) var ( - fileRegister map[string]bool - readerRegister map[string]func() io.Reader + fileRegister map[string]bool + fileRegisterLock sync.RWMutex + readerRegister map[string]func() io.Reader + readerRegisterLock sync.RWMutex ) // RegisterLocalFile adds the given file to the file whitelist, @@ -32,17 +35,21 @@ var ( // ... // func RegisterLocalFile(filePath string) { + fileRegisterLock.Lock() // lazy map init if fileRegister == nil { fileRegister = make(map[string]bool) } fileRegister[strings.Trim(filePath, `"`)] = true + fileRegisterLock.Unlock() } // DeregisterLocalFile removes the given filepath from the whitelist. func DeregisterLocalFile(filePath string) { + fileRegisterLock.Lock() delete(fileRegister, strings.Trim(filePath, `"`)) + fileRegisterLock.Unlock() } // RegisterReaderHandler registers a handler function which is used @@ -61,18 +68,22 @@ func DeregisterLocalFile(filePath string) { // ... // func RegisterReaderHandler(name string, handler func() io.Reader) { + readerRegisterLock.Lock() // lazy map init if readerRegister == nil { readerRegister = make(map[string]func() io.Reader) } readerRegister[name] = handler + readerRegisterLock.Unlock() } // DeregisterReaderHandler removes the ReaderHandler function with // the given name from the registry. func DeregisterReaderHandler(name string) { + readerRegisterLock.Lock() delete(readerRegister, name) + readerRegisterLock.Unlock() } func deferredClose(err *error, closer io.Closer) { @@ -85,14 +96,22 @@ func deferredClose(err *error, closer io.Closer) { func (mc *mysqlConn) handleInFileRequest(name string) (err error) { var rdr io.Reader var data []byte + packetSize := 16 * 1024 // 16KB is small enough for disk readahead and large enough for TCP + if mc.maxWriteSize < packetSize { + packetSize = mc.maxWriteSize + } + + if idx := strings.Index(name, "Reader::"); idx == 0 || (idx > 0 && name[idx-1] == '/') { // io.Reader + // The server might return an an absolute path. See issue #355. + name = name[idx+8:] + + readerRegisterLock.RLock() + handler, inMap := readerRegister[name] + readerRegisterLock.RUnlock() - if strings.HasPrefix(name, "Reader::") { // io.Reader - name = name[8:] - if handler, inMap := readerRegister[name]; inMap { + if inMap { rdr = handler() if rdr != nil { - data = make([]byte, 4+mc.maxWriteSize) - if cl, ok := rdr.(io.Closer); ok { defer deferredClose(&err, cl) } @@ -104,7 +123,10 @@ func (mc *mysqlConn) handleInFileRequest(name string) (err error) { } } else { // File name = strings.Trim(name, `"`) - if mc.cfg.allowAllFiles || fileRegister[name] { + fileRegisterLock.RLock() + fr := fileRegister[name] + fileRegisterLock.RUnlock() + if mc.cfg.AllowAllFiles || fr { var file *os.File var fi os.FileInfo @@ -114,22 +136,19 @@ func (mc *mysqlConn) handleInFileRequest(name string) (err error) { // get file size if fi, err = file.Stat(); err == nil { rdr = file - if fileSize := int(fi.Size()); fileSize <= mc.maxWriteSize { - data = make([]byte, 4+fileSize) - } else if fileSize <= mc.maxPacketAllowed { - data = make([]byte, 4+mc.maxWriteSize) - } else { - err = fmt.Errorf("Local File '%s' too large: Size: %d, Max: %d", name, fileSize, mc.maxPacketAllowed) + if fileSize := int(fi.Size()); fileSize < packetSize { + packetSize = fileSize } } } } else { - err = fmt.Errorf("Local File '%s' is not registered. Use the DSN parameter 'allowAllFiles=true' to allow all files", name) + err = fmt.Errorf("local file '%s' is not registered", name) } } // send content packets if err == nil { + data := make([]byte, 4+packetSize) var n int for err == nil { n, err = rdr.Read(data[4:]) @@ -154,9 +173,10 @@ func (mc *mysqlConn) handleInFileRequest(name string) (err error) { // read OK packet if err == nil { - return mc.readResultOK() - } else { - mc.readPacket() + _, err = mc.readResultOK() + return err } + + mc.readPacket() return err } diff --git a/vendor/github.com/go-sql-driver/mysql/packets.go b/vendor/github.com/go-sql-driver/mysql/packets.go index 290a3887a7..aafe9793ea 100644 --- a/vendor/github.com/go-sql-driver/mysql/packets.go +++ b/vendor/github.com/go-sql-driver/mysql/packets.go @@ -13,6 +13,7 @@ import ( "crypto/tls" "database/sql/driver" "encoding/binary" + "errors" "fmt" "io" "math" @@ -24,9 +25,9 @@ import ( // Read packet to buffer 'data' func (mc *mysqlConn) readPacket() ([]byte, error) { - var payload []byte + var prevData []byte for { - // Read packet header + // read packet header data, err := mc.buf.readNext(4) if err != nil { errLog.Print(err) @@ -34,26 +35,32 @@ func (mc *mysqlConn) readPacket() ([]byte, error) { return nil, driver.ErrBadConn } - // Packet Length [24 bit] + // packet length [24 bit] pktLen := int(uint32(data[0]) | uint32(data[1])<<8 | uint32(data[2])<<16) - if pktLen < 1 { - errLog.Print(ErrMalformPkt) - mc.Close() - return nil, driver.ErrBadConn - } - - // Check Packet Sync [8 bit] + // check packet sync [8 bit] if data[3] != mc.sequence { if data[3] > mc.sequence { return nil, ErrPktSyncMul - } else { - return nil, ErrPktSync } + return nil, ErrPktSync } mc.sequence++ - // Read packet body [pktLen bytes] + // packets with length 0 terminate a previous packet which is a + // multiple of (2^24)−1 bytes long + if pktLen == 0 { + // there was no previous packet + if prevData == nil { + errLog.Print(ErrMalformPkt) + mc.Close() + return nil, driver.ErrBadConn + } + + return prevData, nil + } + + // read packet body [pktLen bytes] data, err = mc.buf.readNext(pktLen) if err != nil { errLog.Print(err) @@ -61,18 +68,17 @@ func (mc *mysqlConn) readPacket() ([]byte, error) { return nil, driver.ErrBadConn } - isLastPacket := (pktLen < maxPacketSize) + // return data if this was the last packet + if pktLen < maxPacketSize { + // zero allocations for non-split packets + if prevData == nil { + return data, nil + } - // Zero allocations for non-splitting packets - if isLastPacket && payload == nil { - return data, nil + return append(prevData, data...), nil } - payload = append(payload, data...) - - if isLastPacket { - return payload, nil - } + prevData = append(prevData, data...) } } @@ -80,7 +86,7 @@ func (mc *mysqlConn) readPacket() ([]byte, error) { func (mc *mysqlConn) writePacket(data []byte) error { pktLen := len(data) - 4 - if pktLen > mc.maxPacketAllowed { + if pktLen > mc.maxAllowedPacket { return ErrPktTooLarge } @@ -100,6 +106,12 @@ func (mc *mysqlConn) writePacket(data []byte) error { data[3] = mc.sequence // Write packet + if mc.writeTimeout > 0 { + if err := mc.netConn.SetWriteDeadline(time.Now().Add(mc.writeTimeout)); err != nil { + return err + } + } + n, err := mc.netConn.Write(data[:4+size]) if err == nil && n == 4+size { mc.sequence++ @@ -140,7 +152,7 @@ func (mc *mysqlConn) readInitPacket() ([]byte, error) { // protocol version [1 byte] if data[0] < minProtocolVersion { return nil, fmt.Errorf( - "Unsupported MySQL Protocol Version %d. Protocol Version %d or higher is required", + "unsupported protocol version %d. Version %d or higher is required", data[0], minProtocolVersion, ) @@ -196,7 +208,11 @@ func (mc *mysqlConn) readInitPacket() ([]byte, error) { // return //} //return ErrMalformPkt - return cipher, nil + + // make a memory safe copy of the cipher slice + var b [20]byte + copy(b[:], cipher) + return b[:], nil } // make a memory safe copy of the cipher slice @@ -214,9 +230,11 @@ func (mc *mysqlConn) writeAuthPacket(cipher []byte) error { clientLongPassword | clientTransactions | clientLocalFiles | + clientPluginAuth | + clientMultiResults | mc.flags&clientLongFlag - if mc.cfg.clientFoundRows { + if mc.cfg.ClientFoundRows { clientFlags |= clientFoundRows } @@ -225,13 +243,17 @@ func (mc *mysqlConn) writeAuthPacket(cipher []byte) error { clientFlags |= clientSSL } + if mc.cfg.MultiStatements { + clientFlags |= clientMultiStatements + } + // User Password - scrambleBuff := scramblePassword(cipher, []byte(mc.cfg.passwd)) + scrambleBuff := scramblePassword(cipher, []byte(mc.cfg.Passwd)) - pktLen := 4 + 4 + 1 + 23 + len(mc.cfg.user) + 1 + 1 + len(scrambleBuff) + pktLen := 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + 1 + len(scrambleBuff) + 21 + 1 // To specify a db name - if n := len(mc.cfg.dbname); n > 0 { + if n := len(mc.cfg.DBName); n > 0 { clientFlags |= clientConnectWithDB pktLen += n + 1 } @@ -257,7 +279,14 @@ func (mc *mysqlConn) writeAuthPacket(cipher []byte) error { data[11] = 0x00 // Charset [1 byte] - data[12] = mc.cfg.collation + var found bool + data[12], found = collations[mc.cfg.Collation] + if !found { + // Note possibility for false negatives: + // could be triggered although the collation is valid if the + // collations map does not contain entries the server supports. + return errors.New("unknown collation") + } // SSL Connection Request Packet // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::SSLRequest @@ -273,15 +302,18 @@ func (mc *mysqlConn) writeAuthPacket(cipher []byte) error { return err } mc.netConn = tlsConn - mc.buf.rd = tlsConn + mc.buf.nc = tlsConn } // Filler [23 bytes] (all 0x00) - pos := 13 + 23 + pos := 13 + for ; pos < 13+23; pos++ { + data[pos] = 0 + } // User [null terminated string] - if len(mc.cfg.user) > 0 { - pos += copy(data[pos:], mc.cfg.user) + if len(mc.cfg.User) > 0 { + pos += copy(data[pos:], mc.cfg.User) } data[pos] = 0x00 pos++ @@ -291,11 +323,16 @@ func (mc *mysqlConn) writeAuthPacket(cipher []byte) error { pos += 1 + copy(data[pos+1:], scrambleBuff) // Databasename [null terminated string] - if len(mc.cfg.dbname) > 0 { - pos += copy(data[pos:], mc.cfg.dbname) + if len(mc.cfg.DBName) > 0 { + pos += copy(data[pos:], mc.cfg.DBName) data[pos] = 0x00 + pos++ } + // Assume native client during response + pos += copy(data[pos:], "mysql_native_password") + data[pos] = 0x00 + // Send Auth packet return mc.writePacket(data) } @@ -304,9 +341,9 @@ func (mc *mysqlConn) writeAuthPacket(cipher []byte) error { // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchResponse func (mc *mysqlConn) writeOldAuthPacket(cipher []byte) error { // User password - scrambleBuff := scrambleOldPassword(cipher, []byte(mc.cfg.passwd)) + scrambleBuff := scrambleOldPassword(cipher, []byte(mc.cfg.Passwd)) - // Calculate the packet lenght and add a tailing 0 + // Calculate the packet length and add a tailing 0 pktLen := len(scrambleBuff) + 1 data := mc.buf.takeSmallBuffer(4 + pktLen) if data == nil { @@ -322,6 +359,45 @@ func (mc *mysqlConn) writeOldAuthPacket(cipher []byte) error { return mc.writePacket(data) } +// Client clear text authentication packet +// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchResponse +func (mc *mysqlConn) writeClearAuthPacket() error { + // Calculate the packet length and add a tailing 0 + pktLen := len(mc.cfg.Passwd) + 1 + data := mc.buf.takeSmallBuffer(4 + pktLen) + if data == nil { + // can not take the buffer. Something must be wrong with the connection + errLog.Print(ErrBusyBuffer) + return driver.ErrBadConn + } + + // Add the clear password [null terminated string] + copy(data[4:], mc.cfg.Passwd) + data[4+pktLen-1] = 0x00 + + return mc.writePacket(data) +} + +// Native password authentication method +// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchResponse +func (mc *mysqlConn) writeNativeAuthPacket(cipher []byte) error { + scrambleBuff := scramblePassword(cipher, []byte(mc.cfg.Passwd)) + + // Calculate the packet length and add a tailing 0 + pktLen := len(scrambleBuff) + data := mc.buf.takeSmallBuffer(4 + pktLen) + if data == nil { + // can not take the buffer. Something must be wrong with the connection + errLog.Print(ErrBusyBuffer) + return driver.ErrBadConn + } + + // Add the scramble + copy(data[4:], scrambleBuff) + + return mc.writePacket(data) +} + /****************************************************************************** * Command Packets * ******************************************************************************/ @@ -395,24 +471,43 @@ func (mc *mysqlConn) writeCommandPacketUint32(command byte, arg uint32) error { ******************************************************************************/ // Returns error if Packet is not an 'Result OK'-Packet -func (mc *mysqlConn) readResultOK() error { +func (mc *mysqlConn) readResultOK() ([]byte, error) { data, err := mc.readPacket() if err == nil { // packet indicator switch data[0] { case iOK: - return mc.handleOkPacket(data) + return nil, mc.handleOkPacket(data) case iEOF: - // someone is using old_passwords - return ErrOldPassword + if len(data) > 1 { + pluginEndIndex := bytes.IndexByte(data, 0x00) + plugin := string(data[1:pluginEndIndex]) + cipher := data[pluginEndIndex+1 : len(data)-1] + + if plugin == "mysql_old_password" { + // using old_passwords + return cipher, ErrOldPassword + } else if plugin == "mysql_clear_password" { + // using clear text password + return cipher, ErrCleartextPassword + } else if plugin == "mysql_native_password" { + // using mysql default authentication method + return cipher, ErrNativePassword + } else { + return cipher, ErrUnknownPlugin + } + } else { + // https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::OldAuthSwitchRequest + return nil, ErrOldPassword + } default: // Error otherwise - return mc.handleErrorPacket(data) + return nil, mc.handleErrorPacket(data) } } - return err + return nil, err } // Result Set Header Packet @@ -470,6 +565,10 @@ func (mc *mysqlConn) handleErrorPacket(data []byte) error { } } +func readStatus(b []byte) statusFlag { + return statusFlag(b[0]) | statusFlag(b[1])<<8 +} + // Ok Packet // http://dev.mysql.com/doc/internals/en/generic-response-packets.html#packet-OK_Packet func (mc *mysqlConn) handleOkPacket(data []byte) error { @@ -484,18 +583,21 @@ func (mc *mysqlConn) handleOkPacket(data []byte) error { mc.insertId, _, m = readLengthEncodedInteger(data[1+n:]) // server_status [2 bytes] - mc.status = statusFlag(data[1+n+m]) | statusFlag(data[1+n+m+1])<<8 + mc.status = readStatus(data[1+n+m : 1+n+m+2]) + if err := mc.discardResults(); err != nil { + return err + } // warning count [2 bytes] if !mc.strict { return nil - } else { - pos := 1 + n + m + 2 - if binary.LittleEndian.Uint16(data[pos:pos+2]) > 0 { - return mc.getWarnings() - } - return nil } + + pos := 1 + n + m + 2 + if binary.LittleEndian.Uint16(data[pos:pos+2]) > 0 { + return mc.getWarnings() + } + return nil } // Read Packets as Field Packets until EOF-Packet or an Error appears @@ -514,7 +616,7 @@ func (mc *mysqlConn) readColumns(count int) ([]mysqlField, error) { if i == count { return columns, nil } - return nil, fmt.Errorf("ColumnsCount mismatch n:%d len:%d", count, len(columns)) + return nil, fmt.Errorf("column count mismatch n:%d len:%d", count, len(columns)) } // Catalog @@ -531,7 +633,7 @@ func (mc *mysqlConn) readColumns(count int) ([]mysqlField, error) { pos += n // Table [len coded string] - if mc.cfg.columnsWithAlias { + if mc.cfg.ColumnsWithAlias { tableName, _, n, err := readLengthEncodedString(data[pos:]) if err != nil { return nil, err @@ -603,8 +705,17 @@ func (rows *textRows) readRow(dest []driver.Value) error { // EOF Packet if data[0] == iEOF && len(data) == 5 { + // server_status [2 bytes] + rows.mc.status = readStatus(data[3:]) + err = rows.mc.discardResults() + if err == nil { + err = io.EOF + } else { + // connection unusable + rows.mc.Close() + } rows.mc = nil - return io.EOF + return err } if data[0] == iERR { rows.mc = nil @@ -630,7 +741,7 @@ func (rows *textRows) readRow(dest []driver.Value) error { fieldTypeDate, fieldTypeNewDate: dest[i], err = parseDateTime( string(dest[i].([]byte)), - mc.cfg.loc, + mc.cfg.Loc, ) if err == nil { continue @@ -655,12 +766,19 @@ func (rows *textRows) readRow(dest []driver.Value) error { func (mc *mysqlConn) readUntilEOF() error { for { data, err := mc.readPacket() + if err != nil { + return err + } - // No Err and no EOF Packet - if err == nil && data[0] != iEOF { - continue + switch data[0] { + case iERR: + return mc.handleErrorPacket(data) + case iEOF: + if len(data) == 5 { + mc.status = readStatus(data[3:]) + } + return nil } - return err // Err or EOF } } @@ -692,20 +810,20 @@ func (stmt *mysqlStmt) readPrepareResultPacket() (uint16, error) { // Warning count [16 bit uint] if !stmt.mc.strict { return columnCount, nil - } else { - // Check for warnings count > 0, only available in MySQL > 4.1 - if len(data) >= 12 && binary.LittleEndian.Uint16(data[10:12]) > 0 { - return columnCount, stmt.mc.getWarnings() - } - return columnCount, nil } + + // Check for warnings count > 0, only available in MySQL > 4.1 + if len(data) >= 12 && binary.LittleEndian.Uint16(data[10:12]) > 0 { + return columnCount, stmt.mc.getWarnings() + } + return columnCount, nil } return 0, err } // http://dev.mysql.com/doc/internals/en/com-stmt-send-long-data.html func (stmt *mysqlStmt) writeCommandLongData(paramID int, arg []byte) error { - maxLen := stmt.mc.maxPacketAllowed - 1 + maxLen := stmt.mc.maxAllowedPacket - 1 pktLen := maxLen // After the header (bytes 0-3) follows before the data: @@ -760,7 +878,7 @@ func (stmt *mysqlStmt) writeCommandLongData(paramID int, arg []byte) error { func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { if len(args) != stmt.paramCount { return fmt.Errorf( - "Arguments count mismatch (Got: %d Has: %d)", + "argument count mismatch (got: %d; has: %d)", len(args), stmt.paramCount, ) @@ -896,7 +1014,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { paramTypes[i+i] = fieldTypeString paramTypes[i+i+1] = 0x00 - if len(v) < mc.maxPacketAllowed-pos-len(paramValues)-(len(args)-(i+1))*64 { + if len(v) < mc.maxAllowedPacket-pos-len(paramValues)-(len(args)-(i+1))*64 { paramValues = appendLengthEncodedInteger(paramValues, uint64(len(v)), ) @@ -918,7 +1036,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { paramTypes[i+i] = fieldTypeString paramTypes[i+i+1] = 0x00 - if len(v) < mc.maxPacketAllowed-pos-len(paramValues)-(len(args)-(i+1))*64 { + if len(v) < mc.maxAllowedPacket-pos-len(paramValues)-(len(args)-(i+1))*64 { paramValues = appendLengthEncodedInteger(paramValues, uint64(len(v)), ) @@ -937,7 +1055,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { if v.IsZero() { val = []byte("0000-00-00") } else { - val = []byte(v.In(mc.cfg.loc).Format(timeFormat)) + val = []byte(v.In(mc.cfg.Loc).Format(timeFormat)) } paramValues = appendLengthEncodedInteger(paramValues, @@ -946,7 +1064,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { paramValues = append(paramValues, val...) default: - return fmt.Errorf("Can't convert type: %T", arg) + return fmt.Errorf("can not convert type: %T", arg) } } @@ -964,6 +1082,28 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { return mc.writePacket(data) } +func (mc *mysqlConn) discardResults() error { + for mc.status&statusMoreResultsExists != 0 { + resLen, err := mc.readResultSetHeaderPacket() + if err != nil { + return err + } + if resLen > 0 { + // columns + if err := mc.readUntilEOF(); err != nil { + return err + } + // rows + if err := mc.readUntilEOF(); err != nil { + return err + } + } else { + mc.status &^= statusMoreResultsExists + } + } + return nil +} + // http://dev.mysql.com/doc/internals/en/binary-protocol-resultset-row.html func (rows *binaryRows) readRow(dest []driver.Value) error { data, err := rows.mc.readPacket() @@ -973,11 +1113,20 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { // packet indicator [1 byte] if data[0] != iOK { - rows.mc = nil // EOF Packet if data[0] == iEOF && len(data) == 5 { - return io.EOF + rows.mc.status = readStatus(data[3:]) + err = rows.mc.discardResults() + if err == nil { + err = io.EOF + } else { + // connection unusable + rows.mc.Close() + } + rows.mc = nil + return err } + rows.mc = nil // Error otherwise return rows.mc.handleErrorPacket(data) @@ -1044,7 +1193,7 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { continue case fieldTypeFloat: - dest[i] = float64(math.Float32frombits(binary.LittleEndian.Uint32(data[pos : pos+4]))) + dest[i] = float32(math.Float32frombits(binary.LittleEndian.Uint32(data[pos : pos+4]))) pos += 4 continue @@ -1057,7 +1206,7 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { case fieldTypeDecimal, fieldTypeNewDecimal, fieldTypeVarChar, fieldTypeBit, fieldTypeEnum, fieldTypeSet, fieldTypeTinyBLOB, fieldTypeMediumBLOB, fieldTypeLongBLOB, fieldTypeBLOB, - fieldTypeVarString, fieldTypeString, fieldTypeGeometry: + fieldTypeVarString, fieldTypeString, fieldTypeGeometry, fieldTypeJSON: var isNull bool var n int dest[i], isNull, n, err = readLengthEncodedString(data[pos:]) @@ -1094,13 +1243,13 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { dstlen = 8 + 1 + decimals default: return fmt.Errorf( - "MySQL protocol error, illegal decimals value %d", + "protocol error, illegal decimals value %d", rows.columns[i].decimals, ) } dest[i], err = formatBinaryDateTime(data[pos:pos+int(num)], dstlen, true) case rows.mc.parseTime: - dest[i], err = parseBinaryDateTime(num, data[pos:], rows.mc.cfg.loc) + dest[i], err = parseBinaryDateTime(num, data[pos:], rows.mc.cfg.Loc) default: var dstlen uint8 if rows.columns[i].fieldType == fieldTypeDate { @@ -1113,7 +1262,7 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { dstlen = 19 + 1 + decimals default: return fmt.Errorf( - "MySQL protocol error, illegal decimals value %d", + "protocol error, illegal decimals value %d", rows.columns[i].decimals, ) } @@ -1130,7 +1279,7 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { // Please report if this happens! default: - return fmt.Errorf("Unknown FieldType %d", rows.columns[i].fieldType) + return fmt.Errorf("unknown field type %d", rows.columns[i].fieldType) } } diff --git a/vendor/github.com/go-sql-driver/mysql/rows.go b/vendor/github.com/go-sql-driver/mysql/rows.go index 9d97d6d4f1..c08255eeec 100644 --- a/vendor/github.com/go-sql-driver/mysql/rows.go +++ b/vendor/github.com/go-sql-driver/mysql/rows.go @@ -38,9 +38,13 @@ type emptyRows struct{} func (rows *mysqlRows) Columns() []string { columns := make([]string, len(rows.columns)) - if rows.mc.cfg.columnsWithAlias { + if rows.mc != nil && rows.mc.cfg.ColumnsWithAlias { for i := range columns { - columns[i] = rows.columns[i].tableName + "." + rows.columns[i].name + if tableName := rows.columns[i].tableName; len(tableName) > 0 { + columns[i] = tableName + "." + rows.columns[i].name + } else { + columns[i] = rows.columns[i].name + } } } else { for i := range columns { @@ -61,6 +65,12 @@ func (rows *mysqlRows) Close() error { // Remove unread packets from stream err := mc.readUntilEOF() + if err == nil { + if err = mc.discardResults(); err != nil { + return err + } + } + rows.mc = nil return err } diff --git a/vendor/github.com/go-sql-driver/mysql/statement.go b/vendor/github.com/go-sql-driver/mysql/statement.go index f9dae03fab..7f9b045857 100644 --- a/vendor/github.com/go-sql-driver/mysql/statement.go +++ b/vendor/github.com/go-sql-driver/mysql/statement.go @@ -12,6 +12,7 @@ import ( "database/sql/driver" "fmt" "reflect" + "strconv" ) type mysqlStmt struct { @@ -23,7 +24,10 @@ type mysqlStmt struct { func (stmt *mysqlStmt) Close() error { if stmt.mc == nil || stmt.mc.netConn == nil { - errLog.Print(ErrInvalidConn) + // driver.Stmt.Close can be called more than once, thus this function + // has to be idempotent. + // See also Issue #450 and golang/go#16019. + //errLog.Print(ErrInvalidConn) return driver.ErrBadConn } @@ -100,9 +104,9 @@ func (stmt *mysqlStmt) Query(args []driver.Value) (driver.Rows, error) { } rows := new(binaryRows) - rows.mc = mc if resLen > 0 { + rows.mc = mc // Columns // If not cached, read them and cache them if stmt.columns == nil { @@ -119,7 +123,7 @@ func (stmt *mysqlStmt) Query(args []driver.Value) (driver.Rows, error) { type converter struct{} -func (converter) ConvertValue(v interface{}) (driver.Value, error) { +func (c converter) ConvertValue(v interface{}) (driver.Value, error) { if driver.IsValue(v) { return v, nil } @@ -131,7 +135,7 @@ func (converter) ConvertValue(v interface{}) (driver.Value, error) { if rv.IsNil() { return nil, nil } - return driver.DefaultParameterConverter.ConvertValue(rv.Elem().Interface()) + return c.ConvertValue(rv.Elem().Interface()) case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: return rv.Int(), nil case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32: @@ -139,7 +143,7 @@ func (converter) ConvertValue(v interface{}) (driver.Value, error) { case reflect.Uint64: u64 := rv.Uint() if u64 >= 1<<63 { - return fmt.Sprintf("%d", u64), nil + return strconv.FormatUint(u64, 10), nil } return int64(u64), nil case reflect.Float32, reflect.Float64: diff --git a/vendor/github.com/go-sql-driver/mysql/utils.go b/vendor/github.com/go-sql-driver/mysql/utils.go index 6693d29709..d523b7ffdf 100644 --- a/vendor/github.com/go-sql-driver/mysql/utils.go +++ b/vendor/github.com/go-sql-driver/mysql/utils.go @@ -13,28 +13,16 @@ import ( "crypto/tls" "database/sql/driver" "encoding/binary" - "errors" "fmt" "io" - "net" - "net/url" "strings" "time" ) var ( tlsConfigRegister map[string]*tls.Config // Register for custom tls.Configs - - errInvalidDSNUnescaped = errors.New("Invalid DSN: Did you forget to escape a param value?") - errInvalidDSNAddr = errors.New("Invalid DSN: Network Address not terminated (missing closing brace)") - errInvalidDSNNoSlash = errors.New("Invalid DSN: Missing the slash separating the database name") - errInvalidDSNUnsafeCollation = errors.New("Invalid DSN: interpolateParams can be used with ascii, latin1, utf8 and utf8mb4 charset") ) -func init() { - tlsConfigRegister = make(map[string]*tls.Config) -} - // RegisterTLSConfig registers a custom tls.Config to be used with sql.Open. // Use the key as a value in the DSN where tls=value. // @@ -60,7 +48,11 @@ func init() { // func RegisterTLSConfig(key string, config *tls.Config) error { if _, isBool := readBool(key); isBool || strings.ToLower(key) == "skip-verify" { - return fmt.Errorf("Key '%s' is reserved", key) + return fmt.Errorf("key '%s' is reserved", key) + } + + if tlsConfigRegister == nil { + tlsConfigRegister = make(map[string]*tls.Config) } tlsConfigRegister[key] = config @@ -69,228 +61,9 @@ func RegisterTLSConfig(key string, config *tls.Config) error { // DeregisterTLSConfig removes the tls.Config associated with key. func DeregisterTLSConfig(key string) { - delete(tlsConfigRegister, key) -} - -// parseDSN parses the DSN string to a config -func parseDSN(dsn string) (cfg *config, err error) { - // New config with some default values - cfg = &config{ - loc: time.UTC, - collation: defaultCollation, - } - - // TODO: use strings.IndexByte when we can depend on Go 1.2 - - // [user[:password]@][net[(addr)]]/dbname[?param1=value1¶mN=valueN] - // Find the last '/' (since the password or the net addr might contain a '/') - foundSlash := false - for i := len(dsn) - 1; i >= 0; i-- { - if dsn[i] == '/' { - foundSlash = true - var j, k int - - // left part is empty if i <= 0 - if i > 0 { - // [username[:password]@][protocol[(address)]] - // Find the last '@' in dsn[:i] - for j = i; j >= 0; j-- { - if dsn[j] == '@' { - // username[:password] - // Find the first ':' in dsn[:j] - for k = 0; k < j; k++ { - if dsn[k] == ':' { - cfg.passwd = dsn[k+1 : j] - break - } - } - cfg.user = dsn[:k] - - break - } - } - - // [protocol[(address)]] - // Find the first '(' in dsn[j+1:i] - for k = j + 1; k < i; k++ { - if dsn[k] == '(' { - // dsn[i-1] must be == ')' if an address is specified - if dsn[i-1] != ')' { - if strings.ContainsRune(dsn[k+1:i], ')') { - return nil, errInvalidDSNUnescaped - } - return nil, errInvalidDSNAddr - } - cfg.addr = dsn[k+1 : i-1] - break - } - } - cfg.net = dsn[j+1 : k] - } - - // dbname[?param1=value1&...¶mN=valueN] - // Find the first '?' in dsn[i+1:] - for j = i + 1; j < len(dsn); j++ { - if dsn[j] == '?' { - if err = parseDSNParams(cfg, dsn[j+1:]); err != nil { - return - } - break - } - } - cfg.dbname = dsn[i+1 : j] - - break - } - } - - if !foundSlash && len(dsn) > 0 { - return nil, errInvalidDSNNoSlash - } - - if cfg.interpolateParams && unsafeCollations[cfg.collation] { - return nil, errInvalidDSNUnsafeCollation + if tlsConfigRegister != nil { + delete(tlsConfigRegister, key) } - - // Set default network if empty - if cfg.net == "" { - cfg.net = "tcp" - } - - // Set default address if empty - if cfg.addr == "" { - switch cfg.net { - case "tcp": - cfg.addr = "127.0.0.1:3306" - case "unix": - cfg.addr = "/tmp/mysql.sock" - default: - return nil, errors.New("Default addr for network '" + cfg.net + "' unknown") - } - - } - - return -} - -// parseDSNParams parses the DSN "query string" -// Values must be url.QueryEscape'ed -func parseDSNParams(cfg *config, params string) (err error) { - for _, v := range strings.Split(params, "&") { - param := strings.SplitN(v, "=", 2) - if len(param) != 2 { - continue - } - - // cfg params - switch value := param[1]; param[0] { - - // Enable client side placeholder substitution - case "interpolateParams": - var isBool bool - cfg.interpolateParams, isBool = readBool(value) - if !isBool { - return fmt.Errorf("Invalid Bool value: %s", value) - } - - // Disable INFILE whitelist / enable all files - case "allowAllFiles": - var isBool bool - cfg.allowAllFiles, isBool = readBool(value) - if !isBool { - return fmt.Errorf("Invalid Bool value: %s", value) - } - - // Use old authentication mode (pre MySQL 4.1) - case "allowOldPasswords": - var isBool bool - cfg.allowOldPasswords, isBool = readBool(value) - if !isBool { - return fmt.Errorf("Invalid Bool value: %s", value) - } - - // Switch "rowsAffected" mode - case "clientFoundRows": - var isBool bool - cfg.clientFoundRows, isBool = readBool(value) - if !isBool { - return fmt.Errorf("Invalid Bool value: %s", value) - } - - // Collation - case "collation": - collation, ok := collations[value] - if !ok { - // Note possibility for false negatives: - // could be triggered although the collation is valid if the - // collations map does not contain entries the server supports. - err = errors.New("unknown collation") - return - } - cfg.collation = collation - break - - case "columnsWithAlias": - var isBool bool - cfg.columnsWithAlias, isBool = readBool(value) - if !isBool { - return fmt.Errorf("Invalid Bool value: %s", value) - } - - // Time Location - case "loc": - if value, err = url.QueryUnescape(value); err != nil { - return - } - cfg.loc, err = time.LoadLocation(value) - if err != nil { - return - } - - // Dial Timeout - case "timeout": - cfg.timeout, err = time.ParseDuration(value) - if err != nil { - return - } - - // TLS-Encryption - case "tls": - boolValue, isBool := readBool(value) - if isBool { - if boolValue { - cfg.tls = &tls.Config{} - } - } else { - if strings.ToLower(value) == "skip-verify" { - cfg.tls = &tls.Config{InsecureSkipVerify: true} - } else if tlsConfig, ok := tlsConfigRegister[value]; ok { - if len(tlsConfig.ServerName) == 0 && !tlsConfig.InsecureSkipVerify { - host, _, err := net.SplitHostPort(cfg.addr) - if err == nil { - tlsConfig.ServerName = host - } - } - - cfg.tls = tlsConfig - } else { - return fmt.Errorf("Invalid value / unknown config name: %s", value) - } - } - - default: - // lazy init - if cfg.params == nil { - cfg.params = make(map[string]string) - } - - if cfg.params[param[0]], err = url.QueryUnescape(value); err != nil { - return - } - } - } - - return } // Returns the bool value of the input. @@ -487,7 +260,7 @@ func parseDateTime(str string, loc *time.Location) (t time.Time, err error) { } t, err = time.Parse(timeFormat[:len(str)], str) default: - err = fmt.Errorf("Invalid Time-String: %s", str) + err = fmt.Errorf("invalid time string: %s", str) return } @@ -536,7 +309,7 @@ func parseBinaryDateTime(num uint64, data []byte, loc *time.Location) (driver.Va loc, ), nil } - return nil, fmt.Errorf("Invalid DATETIME-packet length %d", num) + return nil, fmt.Errorf("invalid DATETIME packet length %d", num) } // zeroDateTime is used in formatBinaryDateTime to avoid an allocation @@ -571,7 +344,7 @@ func formatBinaryDateTime(src []byte, length uint8, justTime bool) (driver.Value switch len(src) { case 8, 12: default: - return nil, fmt.Errorf("Invalid TIME-packet length %d", len(src)) + return nil, fmt.Errorf("invalid TIME packet length %d", len(src)) } // +2 to enable negative time and 100+ hours dst = make([]byte, 0, length+2) @@ -605,7 +378,7 @@ func formatBinaryDateTime(src []byte, length uint8, justTime bool) (driver.Value if length > 10 { t += "TIME" } - return nil, fmt.Errorf("illegal %s-packet length %d", t, len(src)) + return nil, fmt.Errorf("illegal %s packet length %d", t, len(src)) } dst = make([]byte, 0, length) // start with the date @@ -771,6 +544,10 @@ func skipLengthEncodedString(b []byte) (int, error) { // returns the number read, whether the value is NULL and the number of bytes read func readLengthEncodedInteger(b []byte) (uint64, bool, int) { + // See issue #349 + if len(b) == 0 { + return 0, true, 1 + } switch b[0] { // 251: NULL @@ -867,7 +644,7 @@ func escapeBytesBackslash(buf, v []byte) []byte { pos += 2 default: buf[pos] = c - pos += 1 + pos++ } } @@ -912,7 +689,7 @@ func escapeStringBackslash(buf []byte, v string) []byte { pos += 2 default: buf[pos] = c - pos += 1 + pos++ } }