Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Tweak the testcontainers setup #1460

Merged
merged 13 commits into from
Nov 18, 2024
5 changes: 5 additions & 0 deletions go/base/context.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,11 @@ type MigrationContext struct {
AzureMySQL bool
AttemptInstantDDL bool

// SkipPortValidation allows skipping the port validation in `ValidateConnection`
// This is useful when connecting to a MySQL instance where the external port
// may not match the internal port.
SkipPortValidation bool

config ContextConfig
configMutex *sync.Mutex
ConfigFile string
Expand Down
11 changes: 10 additions & 1 deletion go/base/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,18 +63,27 @@ func StringContainsAll(s string, substrings ...string) bool {

func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig, migrationContext *MigrationContext, name string) (string, error) {
versionQuery := `select @@global.version`
var port, extraPort int

var version string
if err := db.QueryRow(versionQuery).Scan(&version); err != nil {
return "", err
}

if migrationContext.SkipPortValidation {
return version, nil
}

var extraPort int

extraPortQuery := `select @@global.extra_port`
if err := db.QueryRow(extraPortQuery).Scan(&extraPort); err != nil { //nolint:staticcheck
// swallow this error. not all servers support extra_port
}

// AliyunRDS set users port to "NULL", replace it by gh-ost param
// GCP set users port to "NULL", replace it by gh-ost param
// Azure MySQL set users port to a different value by design, replace it by gh-ost para
var port int
if migrationContext.AliyunRDS || migrationContext.GoogleCloudPlatform || migrationContext.AzureMySQL {
port = connectionConfig.Key.Port
} else {
Expand Down
210 changes: 170 additions & 40 deletions go/logic/applier_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import (

"github.com/github/gh-ost/go/base"
"github.com/github/gh-ost/go/binlog"
"github.com/github/gh-ost/go/mysql"
"github.com/github/gh-ost/go/sql"
)

Expand Down Expand Up @@ -183,6 +182,7 @@ func TestApplierBuildDMLEventQuery(t *testing.T) {
func TestApplierInstantDDL(t *testing.T) {
migrationContext := base.NewMigrationContext()
migrationContext.DatabaseName = "test"
migrationContext.SkipPortValidation = true
migrationContext.OriginalTableName = "mytable"
migrationContext.AlterStatementOptions = "ADD INDEX (foo)"
applier := NewApplier(migrationContext)
Expand All @@ -197,14 +197,16 @@ type ApplierTestSuite struct {
suite.Suite

mysqlContainer testcontainers.Container
db *gosql.DB
}

func (suite *ApplierTestSuite) SetupSuite() {
ctx := context.Background()
req := testcontainers.ContainerRequest{
Image: "mysql:8.0",
Env: map[string]string{"MYSQL_ROOT_PASSWORD": "root-password"},
WaitingFor: wait.ForLog("port: 3306 MySQL Community Server - GPL"),
Image: "mysql:8.0.40",
Env: map[string]string{"MYSQL_ROOT_PASSWORD": "root-password"},
ExposedPorts: []string{"3306/tcp"},
WaitingFor: wait.ForListeningPort("3306/tcp"),
}

mysqlContainer, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
Expand All @@ -214,47 +216,52 @@ func (suite *ApplierTestSuite) SetupSuite() {
suite.Require().NoError(err)

suite.mysqlContainer = mysqlContainer

dsn, err := GetDSN(ctx, mysqlContainer)
suite.Require().NoError(err)

db, err := gosql.Open("mysql", dsn)
suite.Require().NoError(err)

suite.db = db
}

func (suite *ApplierTestSuite) TeardownSuite() {
ctx := context.Background()

suite.Require().NoError(suite.mysqlContainer.Terminate(ctx))
suite.Assert().NoError(suite.db.Close())
suite.Assert().NoError(suite.mysqlContainer.Terminate(ctx))
}

func (suite *ApplierTestSuite) SetupTest() {
ctx := context.Background()

rc, _, err := suite.mysqlContainer.Exec(ctx, []string{"mysql", "-uroot", "-proot-password", "-e", "CREATE DATABASE test;"})
suite.Require().NoError(err)
suite.Require().Equalf(0, rc, "failed to created database: expected exit code 0, got %d", rc)

rc, _, err = suite.mysqlContainer.Exec(ctx, []string{"mysql", "-uroot", "-proot-password", "-e", "CREATE TABLE test.testing (id INT, item_id INT);"})
_, err := suite.db.ExecContext(ctx, "CREATE DATABASE test")
suite.Require().NoError(err)
suite.Require().Equalf(0, rc, "failed to created table: expected exit code 0, got %d", rc)
}

func (suite *ApplierTestSuite) TearDownTest() {
ctx := context.Background()

rc, _, err := suite.mysqlContainer.Exec(ctx, []string{"mysql", "-uroot", "-proot-password", "-e", "DROP DATABASE test;"})
_, err := suite.db.ExecContext(ctx, "DROP DATABASE test")
suite.Require().NoError(err)
suite.Require().Equalf(0, rc, "failed to created database: expected exit code 0, got %d", rc)
}

func (suite *ApplierTestSuite) TestInitDBConnections() {
ctx := context.Background()

host, err := suite.mysqlContainer.ContainerIP(ctx)
var err error

_, err = suite.db.ExecContext(ctx, "CREATE TABLE test.testing (id INT, item_id INT);")
suite.Require().NoError(err)

connectionConfig, err := GetConnectionConfig(ctx, suite.mysqlContainer)
suite.Require().NoError(err)

migrationContext := base.NewMigrationContext()
migrationContext.ApplierConnectionConfig = mysql.NewConnectionConfig()
migrationContext.ApplierConnectionConfig.Key.Hostname = host
migrationContext.ApplierConnectionConfig.Key.Port = 3306
migrationContext.ApplierConnectionConfig.User = "root"
migrationContext.ApplierConnectionConfig.Password = "root-password"
migrationContext.ApplierConnectionConfig = connectionConfig
migrationContext.DatabaseName = "test"
migrationContext.SkipPortValidation = true
migrationContext.OriginalTableName = "testing"
migrationContext.SetConnectionConfig("innodb")

Expand All @@ -274,16 +281,21 @@ func (suite *ApplierTestSuite) TestInitDBConnections() {
func (suite *ApplierTestSuite) TestApplyDMLEventQueries() {
ctx := context.Background()

host, err := suite.mysqlContainer.ContainerIP(ctx)
var err error

_, err = suite.db.ExecContext(ctx, "CREATE TABLE test.testing (id INT, item_id INT);")
suite.Require().NoError(err)

_, err = suite.db.ExecContext(ctx, "CREATE TABLE test._testing_gho (id INT, item_id INT);")
suite.Require().NoError(err)

connectionConfig, err := GetConnectionConfig(ctx, suite.mysqlContainer)
suite.Require().NoError(err)

migrationContext := base.NewMigrationContext()
migrationContext.ApplierConnectionConfig = mysql.NewConnectionConfig()
migrationContext.ApplierConnectionConfig.Key.Hostname = host
migrationContext.ApplierConnectionConfig.Key.Port = 3306
migrationContext.ApplierConnectionConfig.User = "root"
migrationContext.ApplierConnectionConfig.Password = "root-password"
migrationContext.ApplierConnectionConfig = connectionConfig
migrationContext.DatabaseName = "test"
migrationContext.SkipPortValidation = true
migrationContext.OriginalTableName = "testing"
migrationContext.SetConnectionConfig("innodb")

Expand All @@ -297,10 +309,6 @@ func (suite *ApplierTestSuite) TestApplyDMLEventQueries() {
err = applier.InitDBConnections()
suite.Require().NoError(err)

rc, _, err := suite.mysqlContainer.Exec(ctx, []string{"mysql", "-uroot", "-proot-password", "-e", "CREATE TABLE test._testing_gho (id INT, item_id INT);"})
suite.Require().NoError(err)
suite.Require().Equalf(0, rc, "failed to created table: expected exit code 0, got %d", rc)

dmlEvents := []*binlog.BinlogDMLEvent{
{
DatabaseName: "test",
Expand All @@ -313,11 +321,7 @@ func (suite *ApplierTestSuite) TestApplyDMLEventQueries() {
suite.Require().NoError(err)

// Check that the row was inserted
db, err := gosql.Open("mysql", "root:root-password@tcp("+host+":3306)/test")
suite.Require().NoError(err)
defer db.Close()

rows, err := db.Query("SELECT * FROM test._testing_gho")
rows, err := suite.db.Query("SELECT * FROM test._testing_gho")
suite.Require().NoError(err)
defer rows.Close()

Expand All @@ -340,16 +344,18 @@ func (suite *ApplierTestSuite) TestApplyDMLEventQueries() {
func (suite *ApplierTestSuite) TestValidateOrDropExistingTables() {
ctx := context.Background()

host, err := suite.mysqlContainer.ContainerIP(ctx)
var err error

_, err = suite.db.ExecContext(ctx, "CREATE TABLE test.testing (id INT, item_id INT);")
suite.Require().NoError(err)

connectionConfig, err := GetConnectionConfig(ctx, suite.mysqlContainer)
suite.Require().NoError(err)

migrationContext := base.NewMigrationContext()
migrationContext.ApplierConnectionConfig = mysql.NewConnectionConfig()
migrationContext.ApplierConnectionConfig.Key.Hostname = host
migrationContext.ApplierConnectionConfig.Key.Port = 3306
migrationContext.ApplierConnectionConfig.User = "root"
migrationContext.ApplierConnectionConfig.Password = "root-password"
migrationContext.ApplierConnectionConfig = connectionConfig
migrationContext.DatabaseName = "test"
migrationContext.SkipPortValidation = true
migrationContext.OriginalTableName = "testing"
migrationContext.SetConnectionConfig("innodb")

Expand All @@ -367,6 +373,130 @@ func (suite *ApplierTestSuite) TestValidateOrDropExistingTables() {
suite.Require().NoError(err)
}

func (suite *ApplierTestSuite) TestValidateOrDropExistingTablesWithGhostTableExisting() {
ctx := context.Background()

var err error

_, err = suite.db.ExecContext(ctx, "CREATE TABLE test.testing (id INT, item_id INT);")
suite.Require().NoError(err)

_, err = suite.db.ExecContext(ctx, "CREATE TABLE test._testing_gho (id INT, item_id INT);")
suite.Require().NoError(err)

connectionConfig, err := GetConnectionConfig(ctx, suite.mysqlContainer)
suite.Require().NoError(err)

migrationContext := base.NewMigrationContext()
migrationContext.ApplierConnectionConfig = connectionConfig
migrationContext.DatabaseName = "test"
migrationContext.SkipPortValidation = true
migrationContext.OriginalTableName = "testing"
migrationContext.SetConnectionConfig("innodb")

migrationContext.OriginalTableColumns = sql.NewColumnList([]string{"id", "item_id"})
migrationContext.SharedColumns = sql.NewColumnList([]string{"id", "item_id"})
migrationContext.MappedSharedColumns = sql.NewColumnList([]string{"id", "item_id"})

applier := NewApplier(migrationContext)
defer applier.Teardown()

err = applier.InitDBConnections()
suite.Require().NoError(err)

err = applier.ValidateOrDropExistingTables()
suite.Require().Error(err)
suite.Require().EqualError(err, "Table `_testing_gho` already exists. Panicking. Use --initially-drop-ghost-table to force dropping it, though I really prefer that you drop it or rename it away")
}

func (suite *ApplierTestSuite) TestValidateOrDropExistingTablesWithGhostTableExistingAndInitiallyDropGhostTableSet() {
ctx := context.Background()

var err error

_, err = suite.db.ExecContext(ctx, "CREATE TABLE test.testing (id INT, item_id INT);")
suite.Require().NoError(err)

_, err = suite.db.ExecContext(ctx, "CREATE TABLE test._testing_gho (id INT, item_id INT);")
suite.Require().NoError(err)

connectionConfig, err := GetConnectionConfig(ctx, suite.mysqlContainer)
suite.Require().NoError(err)

migrationContext := base.NewMigrationContext()
migrationContext.ApplierConnectionConfig = connectionConfig
migrationContext.DatabaseName = "test"
migrationContext.SkipPortValidation = true
migrationContext.OriginalTableName = "testing"
migrationContext.SetConnectionConfig("innodb")

migrationContext.InitiallyDropGhostTable = true

applier := NewApplier(migrationContext)
defer applier.Teardown()

err = applier.InitDBConnections()
suite.Require().NoError(err)

err = applier.ValidateOrDropExistingTables()
suite.Require().NoError(err)

// Check that the ghost table was dropped
var tableName string
//nolint:execinquery
err = suite.db.QueryRow("SHOW TABLES IN test LIKE '_testing_gho'").Scan(&tableName)
suite.Require().Error(err)
suite.Require().Equal(gosql.ErrNoRows, err)
}

func (suite *ApplierTestSuite) TestCreateGhostTable() {
ctx := context.Background()

var err error

_, err = suite.db.ExecContext(ctx, "CREATE TABLE test.testing (id INT, item_id INT);")
suite.Require().NoError(err)

connectionConfig, err := GetConnectionConfig(ctx, suite.mysqlContainer)
suite.Require().NoError(err)

migrationContext := base.NewMigrationContext()
migrationContext.ApplierConnectionConfig = connectionConfig
migrationContext.DatabaseName = "test"
migrationContext.SkipPortValidation = true
migrationContext.OriginalTableName = "testing"
migrationContext.SetConnectionConfig("innodb")

migrationContext.OriginalTableColumns = sql.NewColumnList([]string{"id", "item_id"})
migrationContext.SharedColumns = sql.NewColumnList([]string{"id", "item_id"})
migrationContext.MappedSharedColumns = sql.NewColumnList([]string{"id", "item_id"})

migrationContext.InitiallyDropGhostTable = true

applier := NewApplier(migrationContext)
defer applier.Teardown()

err = applier.InitDBConnections()
suite.Require().NoError(err)

err = applier.CreateGhostTable()
suite.Require().NoError(err)

// Check that the ghost table was created
var tableName string
//nolint:execinquery
err = suite.db.QueryRow("SHOW TABLES IN test LIKE '_testing_gho'").Scan(&tableName)
suite.Require().NoError(err)
suite.Require().Equal("_testing_gho", tableName)

// Check that the ghost table has the same columns as the original table
var createDDL string
//nolint:execinquery
err = suite.db.QueryRow("SHOW CREATE TABLE test._testing_gho").Scan(&tableName, &createDDL)
suite.Require().NoError(err)
suite.Require().Equal("CREATE TABLE `_testing_gho` (\n `id` int DEFAULT NULL,\n `item_id` int DEFAULT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci", createDDL)
}

func TestApplier(t *testing.T) {
suite.Run(t, new(ApplierTestSuite))
}
Loading