setup mysql tables as utf8mb4 and convert them

models/branches.go

@@ -25,7 +25,7 @@ const (
 type ProtectedBranch struct {
 	ID                    int64  `xorm:"pk autoincr"`
 	RepoID                int64  `xorm:"UNIQUE(s)"`
-	BranchName            string `xorm:"UNIQUE(s)"`
+	BranchName            string `xorm:"VARCHAR(191) UNIQUE(s)"`
 	CanPush               bool   `xorm:"NOT NULL DEFAULT false"`
 	EnableWhitelist       bool
 	WhitelistUserIDs      []int64        `xorm:"JSON TEXT"`
@@ -287,8 +287,8 @@ func (repo *Repository) DeleteProtectedBranch(id int64) (err error) {
 type DeletedBranch struct {
 	ID          int64          `xorm:"pk autoincr"`
 	RepoID      int64          `xorm:"UNIQUE(s) INDEX NOT NULL"`
-	Name        string         `xorm:"UNIQUE(s) NOT NULL"`
-	Commit      string         `xorm:"UNIQUE(s) NOT NULL"`
+	Name        string         `xorm:"VARCHAR(191) UNIQUE(s) NOT NULL"`
+	Commit      string         `xorm:"VARCHAR(191) UNIQUE(s) NOT NULL"`
 	DeletedByID int64          `xorm:"INDEX"`
 	DeletedBy   *User          `xorm:"-"`
 	DeletedUnix util.TimeStamp `xorm:"INDEX created"`

models/external_login_user.go

@@ -8,7 +8,7 @@ import "github.com/markbates/goth"
 
 // ExternalLoginUser makes the connecting between some existing user and additional external login sources
 type ExternalLoginUser struct {
-	ExternalID    string `xorm:"pk NOT NULL"`
+	ExternalID    string `xorm:"VARCHAR(191) pk NOT NULL"`
 	UserID        int64  `xorm:"INDEX NOT NULL"`
 	LoginSourceID int64  `xorm:"pk NOT NULL"`
 }

models/issue_reaction.go

@@ -18,7 +18,7 @@ import (
 // Reaction represents a reactions on issues and comments.
 type Reaction struct {
 	ID          int64          `xorm:"pk autoincr"`
-	Type        string         `xorm:"INDEX UNIQUE(s) NOT NULL"`
+	Type        string         `xorm:"VARCHAR(191) INDEX UNIQUE(s) NOT NULL"`
 	IssueID     int64          `xorm:"INDEX UNIQUE(s) NOT NULL"`
 	CommentID   int64          `xorm:"INDEX UNIQUE(s)"`
 	UserID      int64          `xorm:"INDEX UNIQUE(s) NOT NULL"`

models/lfs.go

@@ -9,7 +9,7 @@ import (
 // LFSMetaObject stores metadata for LFS tracked files.
 type LFSMetaObject struct {
 	ID           int64          `xorm:"pk autoincr"`
-	Oid          string         `xorm:"UNIQUE(s) INDEX NOT NULL"`
+	Oid          string         `xorm:"VARCHAR(191) UNIQUE(s) INDEX NOT NULL"`
 	Size         int64          `xorm:"NOT NULL"`
 	RepositoryID int64          `xorm:"UNIQUE(s) INDEX NOT NULL"`
 	Existing     bool           `xorm:"-"`

models/login_source.go

@@ -142,7 +142,7 @@ func (cfg *OAuth2Config) ToDB() ([]byte, error) {
 type LoginSource struct {
 	ID            int64 `xorm:"pk autoincr"`
 	Type          LoginType
-	Name          string          `xorm:"UNIQUE"`
+	Name          string          `xorm:"VARCHAR(191) UNIQUE"`
 	IsActived     bool            `xorm:"INDEX NOT NULL DEFAULT false"`
 	IsSyncEnabled bool            `xorm:"INDEX NOT NULL DEFAULT false"`
 	Cfg           core.Conversion `xorm:"TEXT"`

models/migrations/migrations.go

@@ -182,6 +182,8 @@ var migrations = []Migration{
 	NewMigration("add language column for user setting", addLanguageSetting),
 	// v64 -> v65
 	NewMigration("add multiple assignees", addMultipleAssignees),
+	// v65 -> v66
+	NewMigration("change columns to utf8mb4 on mysql databases", mysqlColumnsToUTF8MB4),
 }
 
 // Migrate database to current version

models/migrations/v65.go

@@ -0,0 +1,67 @@
+// Copyright 2018 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"fmt"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/go-sql-driver/mysql"
+	"github.com/go-xorm/xorm"
+)
+
+func mysqlColumnsToUTF8MB4(x *xorm.Engine) (err error) {
+	if !setting.UseMySQL {
+		log.Info("Nothing to do")
+		return nil
+	}
+
+	const maxvc = 191
+	migrationSuccess := true
+
+	tables, err := x.DBMetas()
+	if err != nil {
+		return fmt.Errorf("cannot get tables: %v", err)
+	}
+	for _, table := range tables {
+		readyForConversion := true
+		for _, col := range table.Columns() {
+			if !(len(col.Indexes) > 0 || col.IsPrimaryKey) {
+				continue
+			}
+			if !(col.SQLType.Name == "VARCHAR" && col.Length > maxvc) {
+				continue
+			}
+			log.Info("reducing column %s.%s from %d to %d bytes", table.Name, col.Name, col.Length, maxvc)
+			sqlstmt := fmt.Sprintf("alter table `%s` change column `%s` `%s` varchar(%d)", table.Name, col.Name, col.Name, maxvc)
+			if _, err := x.Exec(sqlstmt); err != nil {
+				if e, ok := err.(*mysql.MySQLError); ok {
+					if e.Number == 1265 || e.Number == 1406 {
+						log.Warn("failed. Please cut all data of this column down to a maximum of %d bytes", maxvc)
+					} else {
+						log.Warn("failed with %v", err)
+					}
+					readyForConversion = false
+					migrationSuccess = false
+					continue
+				}
+				return fmt.Errorf("something went horribly wrong: %v", err)
+			}
+		}
+		if readyForConversion {
+			log.Info("%s: converting table to utf8mb4", table.Name)
+			if _, err := x.Exec("alter table `" + table.Name + "` convert to character set utf8mb4"); err != nil {
+				log.Warn("conversion of %s failed: %v", table.Name, err)
+				migrationSuccess = false
+			}
+		}
+	}
+	if !migrationSuccess {
+		return fmt.Errorf("conversion of some of the tables failed. Please check the logs and re-run gitea")
+	}
+	return nil
+}

models/models.go

@@ -206,10 +206,10 @@ func getEngine() (*xorm.Engine, error) {
 	switch DbCfg.Type {
 	case "mysql":
 		if DbCfg.Host[0] == '/' { // looks like a unix socket
-			connStr = fmt.Sprintf("%s:%s@unix(%s)/%s%scharset=utf8&parseTime=true",
+			connStr = fmt.Sprintf("%s:%s@unix(%s)/%s%scharset=utf8mb4&parseTime=true",
 				DbCfg.User, DbCfg.Passwd, DbCfg.Host, DbCfg.Name, Param)
 		} else {
-			connStr = fmt.Sprintf("%s:%s@tcp(%s)/%s%scharset=utf8&parseTime=true",
+			connStr = fmt.Sprintf("%s:%s@tcp(%s)/%s%scharset=utf8mb4&parseTime=true",
 				DbCfg.User, DbCfg.Passwd, DbCfg.Host, DbCfg.Name, Param)
 		}
 	case "postgres":

models/notification.go

@@ -45,7 +45,7 @@ type Notification struct {
 	Source NotificationSource `xorm:"SMALLINT INDEX NOT NULL"`
 
 	IssueID  int64  `xorm:"INDEX NOT NULL"`
-	CommitID string `xorm:"INDEX"`
+	CommitID string `xorm:"VARCHAR(191) INDEX"`
 
 	UpdatedBy int64 `xorm:"INDEX NOT NULL"`
 

models/release.go

@@ -25,7 +25,7 @@ type Release struct {
 	Repo             *Repository `xorm:"-"`
 	PublisherID      int64       `xorm:"INDEX"`
 	Publisher        *User       `xorm:"-"`
-	TagName          string      `xorm:"INDEX UNIQUE(n)"`
+	TagName          string      `xorm:"VARCHAR(191) INDEX UNIQUE(n)"`
 	LowerTagName     string
 	Target           string
 	Title            string

models/repo.go

@@ -165,8 +165,8 @@ type Repository struct {
 	OwnerID       int64  `xorm:"UNIQUE(s)"`
 	OwnerName     string `xorm:"-"`
 	Owner         *User  `xorm:"-"`
-	LowerName     string `xorm:"UNIQUE(s) INDEX NOT NULL"`
-	Name          string `xorm:"INDEX NOT NULL"`
+	LowerName     string `xorm:"VARCHAR(191) UNIQUE(s) INDEX NOT NULL"`
+	Name          string `xorm:"VARCHAR(191) INDEX NOT NULL"`
 	Description   string
 	Website       string
 	DefaultBranch string

models/repo_redirect.go

@@ -10,7 +10,7 @@ import "strings"
 type RepoRedirect struct {
 	ID             int64  `xorm:"pk autoincr"`
 	OwnerID        int64  `xorm:"UNIQUE(s)"`
-	LowerName      string `xorm:"UNIQUE(s) INDEX NOT NULL"`
+	LowerName      string `xorm:"VARCHAR(191) UNIQUE(s) INDEX NOT NULL"`
 	RedirectRepoID int64  // repoID to redirect to
 }
 

models/user.go

@@ -76,8 +76,8 @@ var (
 // User represents the object of individual and member of organization.
 type User struct {
 	ID        int64  `xorm:"pk autoincr"`
-	LowerName string `xorm:"UNIQUE NOT NULL"`
-	Name      string `xorm:"UNIQUE NOT NULL"`
+	LowerName string `xorm:"VARCHAR(191) UNIQUE NOT NULL"`
+	Name      string `xorm:"VARCHAR(191) UNIQUE NOT NULL"`
 	FullName  string
 	// Email is the primary email address (to be used for communication)
 	Email            string `xorm:"NOT NULL"`

models/user_mail.go

@@ -20,7 +20,7 @@ var (
 type EmailAddress struct {
 	ID          int64  `xorm:"pk autoincr"`
 	UID         int64  `xorm:"INDEX NOT NULL"`
-	Email       string `xorm:"UNIQUE NOT NULL"`
+	Email       string `xorm:"VARCHAR(191) UNIQUE NOT NULL"`
 	IsActivated bool
 	IsPrimary   bool `xorm:"-"`
 }

models/user_openid.go

@@ -20,7 +20,7 @@ var (
 type UserOpenID struct {
 	ID   int64  `xorm:"pk autoincr"`
 	UID  int64  `xorm:"INDEX NOT NULL"`
-	URI  string `xorm:"UNIQUE NOT NULL"`
+	URI  string `xorm:"VARCHAR(191) UNIQUE NOT NULL"`
 	Show bool   `xorm:"DEFAULT false"`
 }