Skip to content
This repository has been archived by the owner on Jul 24, 2024. It is now read-only.

glue: create schema/table directly with info #216

Merged
merged 7 commits into from
Apr 2, 2020
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ require (
github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011
github.com/pingcap/kvproto v0.0.0-20200317112120-78042b285b75
github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd
github.com/pingcap/parser v0.0.0-20200317021010-cd90cc2a7d87
github.com/pingcap/parser v0.0.0-20200326020624-68d423641be5
github.com/pingcap/pd/v4 v4.0.0-beta.1.0.20200305072537-61d9f9cc35d3
github.com/pingcap/tidb v1.1.0-beta.0.20200325094938-30e1edae0897
github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200306084441-875bd09aa3d5+incompatible
Expand All @@ -37,8 +37,10 @@ require (
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738
go.opencensus.io v0.22.2 // indirect
go.uber.org/zap v1.14.0
go.uber.org/zap v1.14.1
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
google.golang.org/api v0.14.0
google.golang.org/grpc v1.25.1
)

replace github.com/pingcap/tidb => github.com/kennytm/tidb v0.0.0-20200328193758-f9d04bda076b
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should update this to new tidb

16 changes: 8 additions & 8 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
github.com/juju/ratelimit v1.0.1 h1:+7AIFJVQ0EQgq/K9+0Krm7m530Du7tIz0METWzN0RgY=
github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kennytm/tidb v0.0.0-20200328193758-f9d04bda076b h1:VSeoD3A0M2/ZJ1I5Ew1/AlHcV+tDd0xkpy5nNekfBVI=
github.com/kennytm/tidb v0.0.0-20200328193758-f9d04bda076b/go.mod h1:WEpSvruX5Me+t+r4RaMSx079ClHKEkh/lCIaYa1b5cU=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
Expand Down Expand Up @@ -370,17 +372,15 @@ github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9 h1:AJD9pZYm72vMgPcQDww
github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd h1:CV3VsP3Z02MVtdpTMfEgRJ4T9NGgGTxdHpJerent7rM=
github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
github.com/pingcap/parser v0.0.0-20200317021010-cd90cc2a7d87 h1:533jEUp3mtfWjk0el+awLbyGVxiHcUIGWcR1Y7gB+fg=
github.com/pingcap/parser v0.0.0-20200317021010-cd90cc2a7d87/go.mod h1:9v0Edh8IbgjGYW2ArJr19E+bvL8zKahsFp+ixWeId+4=
github.com/pingcap/parser v0.0.0-20200326020624-68d423641be5 h1:fXVqoeYfV+xI8K2he5NNv00c6YksrjeM6+vkNo1ZK2Q=
github.com/pingcap/parser v0.0.0-20200326020624-68d423641be5/go.mod h1:9v0Edh8IbgjGYW2ArJr19E+bvL8zKahsFp+ixWeId+4=
github.com/pingcap/pd/v4 v4.0.0-beta.1.0.20200305072537-61d9f9cc35d3 h1:Yrp99FnjHAEuDrSBql2l0IqCtJX7KwJbTsD5hIArkvk=
github.com/pingcap/pd/v4 v4.0.0-beta.1.0.20200305072537-61d9f9cc35d3/go.mod h1:25GfNw6+Jcr9kca5rtmTb4gKCJ4jOpow2zV2S9Dgafs=
github.com/pingcap/sysutil v0.0.0-20200206130906-2bfa6dc40bcd/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=
github.com/pingcap/sysutil v0.0.0-20200302022240-21c8c70d0ab1 h1:YUnUZ914SHFMsOSe/xgH5DKK/thtRma8X8hcszRo3CA=
github.com/pingcap/sysutil v0.0.0-20200302022240-21c8c70d0ab1/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=
github.com/pingcap/sysutil v0.0.0-20200309085538-962fd285f3bb h1:bDbgLaNTRNK6Qw7KjvEqqfCQstY8WMEcXyXTU7yzYKg=
github.com/pingcap/sysutil v0.0.0-20200309085538-962fd285f3bb/go.mod h1:EB/852NMQ+aRKioCpToQ94Wl7fktV+FNnxf3CX/TTXI=
github.com/pingcap/tidb v1.1.0-beta.0.20200325094938-30e1edae0897 h1:wTNFJMM6GNmG09YoN3/3K8BqiK74zKkurjm4iY+m2mI=
github.com/pingcap/tidb v1.1.0-beta.0.20200325094938-30e1edae0897/go.mod h1:4CGOiKZSaOU/Da3QYMtp0c3uBE2SxpcLOpESXmeQhcs=
github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200306084441-875bd09aa3d5+incompatible h1:84F7MFMfdAYObrznvRslmVu43aoihrlL+7mMyMlOi0o=
github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200306084441-875bd09aa3d5+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI=
Expand Down Expand Up @@ -535,8 +535,8 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.12.0 h1:dySoUQPFBGj6xwjmBzageVL8jGi8uxc6bEmJQjA06bw=
go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
go.uber.org/zap v1.14.0 h1:/pduUoebOeeJzTDFuoMgC6nRkiasr1sBCIEorly7m4o=
go.uber.org/zap v1.14.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
go.uber.org/zap v1.14.1 h1:nYDKopTbvAPq/NrUVZwT15y2lpROBiLLyoRTbXOYWOo=
go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
Expand Down Expand Up @@ -676,8 +676,8 @@ golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200301222351-066e0c02454c h1:FD7jysxM+EJqg5UYYy3XYDsAiUickFsn4UiaanJkf8c=
golang.org/x/tools v0.0.0-20200301222351-066e0c02454c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200313205530-4303120df7d8 h1:gkI/wGGwpcG5W4hLCzZNGxA4wzWBGGDStRI1MrjDl2Q=
golang.org/x/tools v0.0.0-20200313205530-4303120df7d8/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
golang.org/x/tools v0.0.0-20200325203130-f53864d0dba1 h1:odiryKYJy7CjdrZxhrcE1Z8L9+kGyGZOnfpuauvdCeU=
golang.org/x/tools v0.0.0-20200325203130-f53864d0dba1/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand Down
5 changes: 2 additions & 3 deletions pkg/glue/glue.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
pd "github.com/pingcap/pd/v4/client"
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/meta/autoid"
)

// Glue is an abstraction of TiDB function calls used in BR.
Expand All @@ -28,8 +27,8 @@ type Glue interface {
// Session is an abstraction of the session.Session interface.
type Session interface {
Execute(ctx context.Context, sql string) error
ShowCreateDatabase(schema *model.DBInfo) (string, error)
ShowCreateTable(table *model.TableInfo, allocator autoid.Allocator) (string, error)
CreateDatabase(ctx context.Context, schema *model.DBInfo) error
CreateTable(ctx context.Context, dbName model.CIStr, table *model.TableInfo) error
Close()
}

Expand Down
26 changes: 9 additions & 17 deletions pkg/gluetidb/glue.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,13 @@
package gluetidb

import (
"bytes"
"context"

"github.com/pingcap/parser/model"
pd "github.com/pingcap/pd/v4/client"
"github.com/pingcap/tidb/ddl"
"github.com/pingcap/tidb/domain"
"github.com/pingcap/tidb/executor"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/meta/autoid"
"github.com/pingcap/tidb/session"

"github.com/pingcap/br/pkg/glue"
Expand Down Expand Up @@ -62,22 +60,16 @@ func (gs *tidbSession) Execute(ctx context.Context, sql string) error {
return err
}

// ShowCreateDatabase implements glue.Session
func (gs *tidbSession) ShowCreateDatabase(schema *model.DBInfo) (string, error) {
var buf bytes.Buffer
if err := executor.ConstructResultOfShowCreateDatabase(gs.se, schema, true, &buf); err != nil {
return "", err
}
return buf.String(), nil
// CreateDatabase implements glue.Session
func (gs *tidbSession) CreateDatabase(ctx context.Context, schema *model.DBInfo) error {
d := domain.GetDomain(gs.se).DDL()
return d.CreateSchemaWithInfo(gs.se, schema.Clone(), ddl.OnExistIgnore, true)
}

// ShowCreateTable implements glue.Session
func (gs *tidbSession) ShowCreateTable(table *model.TableInfo, allocator autoid.Allocator) (string, error) {
var buf bytes.Buffer
if err := executor.ConstructResultOfShowCreateTable(gs.se, table, allocator, &buf); err != nil {
return "", err
}
return buf.String(), nil
// CreateTable implements glue.Session
func (gs *tidbSession) CreateTable(ctx context.Context, dbName model.CIStr, table *model.TableInfo) error {
d := domain.GetDomain(gs.se).DDL()
return d.CreateTableWithInfo(gs.se, dbName, table.Clone(), ddl.OnExistIgnore, true)
}

// Close implements glue.Session
Expand Down
44 changes: 7 additions & 37 deletions pkg/restore/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"context"
"fmt"
"sort"
"strings"

"github.com/pingcap/errors"
"github.com/pingcap/log"
Expand Down Expand Up @@ -70,57 +69,28 @@ func (db *DB) ExecDDL(ctx context.Context, ddlJob *model.Job) error {

// CreateDatabase executes a CREATE DATABASE SQL.
func (db *DB) CreateDatabase(ctx context.Context, schema *model.DBInfo) error {
createSQL, err := db.se.ShowCreateDatabase(schema)
err := db.se.CreateDatabase(ctx, schema)
if err != nil {
log.Error("build create database SQL failed", zap.Stringer("db", schema.Name), zap.Error(err))
return errors.Trace(err)
}
err = db.se.Execute(ctx, createSQL)
if err != nil {
log.Error("create database failed", zap.String("query", createSQL), zap.Error(err))
log.Error("create database failed", zap.Stringer("db", schema.Name), zap.Error(err))
}
return errors.Trace(err)
}

// CreateTable executes a CREATE TABLE SQL.
func (db *DB) CreateTable(ctx context.Context, table *utils.Table) error {
tableInfo := table.Info
createSQL, err := db.se.ShowCreateTable(tableInfo, newIDAllocator(tableInfo.AutoIncID))
if err != nil {
log.Error(
"build create table SQL failed",
zap.Stringer("db", table.Db.Name),
zap.Stringer("table", tableInfo.Name),
zap.Error(err))
return errors.Trace(err)
}
switchDbSQL := fmt.Sprintf("use %s;", utils.EncloseName(table.Db.Name.O))
err = db.se.Execute(ctx, switchDbSQL)
if err != nil {
log.Error("switch db failed",
zap.String("SQL", switchDbSQL),
zap.Stringer("db", table.Db.Name),
zap.Error(err))
return errors.Trace(err)
}
// Insert `IF NOT EXISTS` statement to skip the created tables
words := strings.SplitN(createSQL, " ", 3)
if len(words) > 2 && strings.ToUpper(words[0]) == "CREATE" && strings.ToUpper(words[1]) == "TABLE" {
createSQL = "CREATE TABLE IF NOT EXISTS " + words[2]
}
err = db.se.Execute(ctx, createSQL)
err := db.se.CreateTable(ctx, table.Db.Name, table.Info)
if err != nil {
log.Error("create table failed",
zap.String("SQL", createSQL),
zap.Stringer("db", table.Db.Name),
zap.Stringer("table", table.Info.Name),
zap.Error(err))
return errors.Trace(err)
}
alterAutoIncIDSQL := fmt.Sprintf(
"alter table %s auto_increment = %d",
utils.EncloseName(tableInfo.Name.O),
tableInfo.AutoIncID)
"alter table %s.%s auto_increment = %d",
utils.EncloseName(table.Db.Name.O),
utils.EncloseName(table.Info.Name.O),
table.Info.AutoIncID)
err = db.se.Execute(ctx, alterAutoIncIDSQL)
if err != nil {
log.Error("alter AutoIncID failed",
Expand Down
39 changes: 0 additions & 39 deletions pkg/restore/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ import (
"github.com/pingcap/kvproto/pkg/metapb"
"github.com/pingcap/log"
"github.com/pingcap/parser/model"
"github.com/pingcap/tidb/meta/autoid"
"github.com/pingcap/tidb/tablecodec"
"github.com/pingcap/tidb/util/codec"
"go.uber.org/zap"
Expand All @@ -29,44 +28,6 @@ import (

var recordPrefixSep = []byte("_r")

// idAllocator always returns a specified ID
type idAllocator struct {
id int64
}

func newIDAllocator(id int64) *idAllocator {
return &idAllocator{id: id}
}

func (alloc *idAllocator) Alloc(tableID int64, n uint64, increment, offset int64) (min int64, max int64, err error) {
return alloc.id, alloc.id, nil
}

func (alloc *idAllocator) AllocSeqCache(sequenceID int64) (min int64, max int64, round int64, err error) {
// TODO fix this function after support backup sequence
return 0, 0, 0, nil
}

func (alloc *idAllocator) Rebase(tableID, newBase int64, allocIDs bool) error {
return nil
}

func (alloc *idAllocator) Base() int64 {
return alloc.id
}

func (alloc *idAllocator) End() int64 {
return alloc.id
}

func (alloc *idAllocator) NextGlobalAutoID(tableID int64) (int64, error) {
return alloc.id, nil
}

func (alloc *idAllocator) GetType() autoid.AllocatorType {
return autoid.RowIDAllocType
}

// GetRewriteRules returns the rewrite rule of the new table and the old table.
func GetRewriteRules(
newTable *model.TableInfo,
Expand Down
7 changes: 5 additions & 2 deletions tests/_utils/run_services
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ stop_services() {
start_services() {
stop_services

TIDB_CONFIG="${1-tests}/config/tidb.toml"
TIKV_CONFIG="${1-tests}/config/tikv.toml"

echo "Starting PD..."
mkdir -p "$TEST_DIR/pd"
bin/pd-server \
Expand All @@ -63,7 +66,7 @@ start_services() {
-A "$TIKV_ADDR$i" \
--status-addr "$TIKV_STATUS_ADDR$i" \
--log-file "$TEST_DIR/tikv${i}.log" \
-C "tests/config/tikv.toml" \
-C "$TIKV_CONFIG" \
-s "$TEST_DIR/tikv${i}" &
done

Expand All @@ -83,7 +86,7 @@ start_services() {
--status 10080 \
--store tikv \
--path "$PD_ADDR" \
--config "tests/config/tidb.toml" \
--config "$TIDB_CONFIG" \
--log-file "$TEST_DIR/tidb.log" &

echo "Verifying TiDB is started..."
Expand Down
8 changes: 8 additions & 0 deletions tests/br_alter_pk_server/config/tidb.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# config of tidb

# Schema lease duration
# There are lot of ddl in the tests, setting this
# to 360s to test whther BR is gracefully shutdown.
lease = "360s"

alter-primary-key = true
14 changes: 14 additions & 0 deletions tests/br_alter_pk_server/config/tikv.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# config of tikv

[coprocessor]
region-max-keys = 20
region-split-keys = 12

[rocksdb]
max-open-files = 4096
[raftdb]
max-open-files = 4096
[raftstore]
# true (default value) for high reliability, this can prevent data loss when power failure.
sync-log = false
capacity = "10GB"
42 changes: 42 additions & 0 deletions tests/br_alter_pk_server/run.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
#!/bin/bash
#
# Copyright 2020 PingCAP, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# See the License for the specific language governing permissions and
# limitations under the License.

set -eu

cur=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
source $cur/../_utils/run_services

DB="$TEST_NAME"

# prepare database
echo "Restart cluster with alter-primary-key = true"
start_services "$cur"

run_sql "drop schema if exists $DB;"
run_sql "create schema $DB;"

run_sql "create table $DB.a (a int primary key, b int unique);"
run_sql "insert into $DB.a values (42, 42);"

# backup
run_br --pd $PD_ADDR backup db --db "$DB" -s "local://$TEST_DIR/$DB"

# restore
run_sql "drop schema $DB;"
run_br --pd $PD_ADDR restore db --db "$DB" -s "local://$TEST_DIR/$DB"

run_sql "drop schema $DB;"
echo "Restart service with alter-primary-key = false"
start_services