Fix postgres ID sequences broken by recreate-table (#15015) (#15029)

Backport #15015

Unfortunately there is a subtle problem with recreatetable on postgres which
leads to the sequences not being renamed and not being left at 0.

Fix #14725

Signed-off-by: Andrew Thornton <art27@cantab.net>

cmd/doctor.go

@@ -670,6 +670,23 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) {
 		}
 	}
 
+	if setting.Database.UsePostgreSQL {
+		count, err = models.CountBadSequences()
+		if err != nil {
+			return nil, err
+		}
+		if count > 0 {
+			if ctx.Bool("fix") {
+				err := models.FixBadSequences()
+				if err != nil {
+					return nil, err
+				}
+				results = append(results, fmt.Sprintf("%d sequences updated", count))
+			} else {
+				results = append(results, fmt.Sprintf("%d sequences with incorrect values", count))
+			}
+		}
+	}
 	//ToDo: function to recalc all counters
 
 	return results, nil

models/consistency.go

@@ -5,10 +5,13 @@
 package models
 
 import (
+	"fmt"
 	"reflect"
+	"regexp"
 	"strings"
 	"testing"
 
+	"code.gitea.io/gitea/modules/setting"
 	"github.com/stretchr/testify/assert"
 	"xorm.io/builder"
 )
@@ -295,3 +298,61 @@ func FixNullArchivedRepository() (int64, error) {
 		IsArchived: false,
 	})
 }
+
+// CountBadSequences looks for broken sequences from recreate-table mistakes
+func CountBadSequences() (int64, error) {
+	if !setting.Database.UsePostgreSQL {
+		return 0, nil
+	}
+
+	sess := x.NewSession()
+	defer sess.Close()
+
+	var sequences []string
+	schema := sess.Engine().Dialect().URI().Schema
+
+	sess.Engine().SetSchema("")
+	if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__%_id_seq%' AND sequence_catalog = ?", setting.Database.Name).Find(&sequences); err != nil {
+		return 0, err
+	}
+	sess.Engine().SetSchema(schema)
+
+	return int64(len(sequences)), nil
+}
+
+// FixBadSequences fixes for broken sequences from recreate-table mistakes
+func FixBadSequences() error {
+	if !setting.Database.UsePostgreSQL {
+		return nil
+	}
+
+	sess := x.NewSession()
+	defer sess.Close()
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+
+	var sequences []string
+	schema := sess.Engine().Dialect().URI().Schema
+
+	sess.Engine().SetSchema("")
+	if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__%_id_seq%' AND sequence_catalog = ?", setting.Database.Name).Find(&sequences); err != nil {
+		return err
+	}
+	sess.Engine().SetSchema(schema)
+
+	sequenceRegexp := regexp.MustCompile(`tmp_recreate__(\w+)_id_seq.*`)
+
+	for _, sequence := range sequences {
+		tableName := sequenceRegexp.FindStringSubmatch(sequence)[1]
+		newSequenceName := tableName + "_id_seq"
+		if _, err := sess.Exec(fmt.Sprintf("ALTER SEQUENCE `%s` RENAME TO `%s`", sequence, newSequenceName)); err != nil {
+			return err
+		}
+		if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM `%s`), 1), false)", newSequenceName, tableName)); err != nil {
+			return err
+		}
+	}
+
+	return sess.Commit()
+}

models/migrations/migrations.go

@@ -516,6 +516,31 @@ func recreateTable(sess *xorm.Session, bean interface{}) error {
 			return err
 		}
 	case setting.Database.UsePostgreSQL:
+		var originalSequences []string
+		type sequenceData struct {
+			LastValue int  `xorm:"'last_value'"`
+			IsCalled  bool `xorm:"'is_called'"`
+		}
+		sequenceMap := map[string]sequenceData{}
+
+		schema := sess.Engine().Dialect().URI().Schema
+		sess.Engine().SetSchema("")
+		if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE ? || '_%' AND sequence_catalog = ?", tableName, setting.Database.Name).Find(&originalSequences); err != nil {
+			log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
+			return err
+		}
+		sess.Engine().SetSchema(schema)
+
+		for _, sequence := range originalSequences {
+			sequenceData := sequenceData{}
+			if _, err := sess.Table(sequence).Cols("last_value", "is_called").Get(&sequenceData); err != nil {
+				log.Error("Unable to get last_value and is_called from %s. Error: %v", sequence, err)
+				return err
+			}
+			sequenceMap[sequence] = sequenceData
+
+		}
+
 		// CASCADE causes postgres to drop all the constraints on the old table
 		if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s` CASCADE", tableName)); err != nil {
 			log.Error("Unable to drop old table %s. Error: %v", tableName, err)
@@ -529,7 +554,6 @@ func recreateTable(sess *xorm.Session, bean interface{}) error {
 		}
 
 		var indices []string
-		schema := sess.Engine().Dialect().URI().Schema
 		sess.Engine().SetSchema("")
 		if err := sess.Table("pg_indexes").Cols("indexname").Where("tablename = ? ", tableName).Find(&indices); err != nil {
 			log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
@@ -545,6 +569,43 @@ func recreateTable(sess *xorm.Session, bean interface{}) error {
 			}
 		}
 
+		var sequences []string
+		sess.Engine().SetSchema("")
+		if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__' || ? || '_%' AND sequence_catalog = ?", tableName, setting.Database.Name).Find(&sequences); err != nil {
+			log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
+			return err
+		}
+		sess.Engine().SetSchema(schema)
+
+		for _, sequence := range sequences {
+			newSequenceName := strings.Replace(sequence, "tmp_recreate__", "", 1)
+			if _, err := sess.Exec(fmt.Sprintf("ALTER SEQUENCE `%s` RENAME TO `%s`", sequence, newSequenceName)); err != nil {
+				log.Error("Unable to rename %s sequence to %s. Error: %v", sequence, newSequenceName, err)
+				return err
+			}
+			val, ok := sequenceMap[newSequenceName]
+			if newSequenceName == tableName+"_id_seq" {
+				if ok && val.LastValue != 0 {
+					if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', %d, %t)", newSequenceName, val.LastValue, val.IsCalled)); err != nil {
+						log.Error("Unable to reset %s to %d. Error: %v", newSequenceName, val, err)
+						return err
+					}
+				} else {
+					// We're going to try to guess this
+					if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM `%s`), 1), false)", newSequenceName, tableName)); err != nil {
+						log.Error("Unable to reset %s. Error: %v", newSequenceName, err)
+						return err
+					}
+				}
+			} else if ok {
+				if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', %d, %t)", newSequenceName, val.LastValue, val.IsCalled)); err != nil {
+					log.Error("Unable to reset %s to %d. Error: %v", newSequenceName, val, err)
+					return err
+				}
+			}
+
+		}
+
 	case setting.Database.UseMSSQL:
 		// MSSQL will drop all the constraints on the old table
 		if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s`", tableName)); err != nil {