From 5589aa35d497199cf843e2ee7306a7d9a0d0eca4 Mon Sep 17 00:00:00 2001 From: WangXiangUSTC Date: Tue, 25 Jun 2019 16:10:18 +0800 Subject: [PATCH 1/5] generate schema create file if not exists --- loader/loader.go | 7 ++++++- loader/util.go | 14 ++++++++++++++ loader/util_test.go | 19 +++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/loader/loader.go b/loader/loader.go index e9b0547a0e..9c2426cfb1 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -713,7 +713,12 @@ func (l *Loader) prepareTableFiles(files map[string]struct{}) error { } tables, ok := l.db2Tables[db] if !ok { - return errors.Errorf("invalid table schema file, cannot find db - %s", file) + log.Warnf("cann't find schema create file for db %s, will generate one", db) + if err := generateSchemaCreateFile(l.cfg.Dir, db); err != nil { + return errors.Trace(err) + } + l.db2Tables[db] = make(Tables2DataFiles) + l.totalFileCount.Add(1) } if _, ok := tables[table]; ok { diff --git a/loader/util.go b/loader/util.go index d50b0bf26a..e7f37bf662 100644 --- a/loader/util.go +++ b/loader/util.go @@ -18,8 +18,11 @@ import ( "crypto/sha1" "fmt" "os" + "path" "path/filepath" "strings" + + "github.com/pingcap/errors" ) // CollectDirFiles gets files in path @@ -73,3 +76,14 @@ func shortSha1(s string) string { func percent(a int64, b int64) string { return fmt.Sprintf("%.2f %%", float64(a)/float64(b)*100) } + +func generateSchemaCreateFile(dir string, schema string) error { + file, err := os.Create(path.Join(dir, fmt.Sprintf("%s-schema-create.sql", schema))) + if err != nil { + return errors.Trace(err) + } + defer file.Close() + + file.WriteString(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`", schema)) + return nil +} diff --git a/loader/util_test.go b/loader/util_test.go index a9c13c0d47..bddd895337 100644 --- a/loader/util_test.go +++ b/loader/util_test.go @@ -14,6 +14,10 @@ package loader import ( + "fmt" + "io/ioutil" + "os" + "path" "testing" . "github.com/pingcap/check" @@ -61,3 +65,18 @@ func (t *testUtilSuite) TestSQLReplace(c *C) { func (t *testUtilSuite) TestShortSha1(c *C) { c.Assert(shortSha1("/tmp/test_sha1_short_6"), Equals, "97b645") } + +func (t *testUtilSuite) TestGenerateSchemaCreateFile(c *C) { + dir := c.MkDir() + schema := "loader_test" + + err := generateSchemaCreateFile(dir, schema) + c.Assert(err, IsNil) + + file, err := os.Open(path.Join(dir, fmt.Sprintf("%s-schema-create.sql", schema))) + c.Assert(err, IsNil) + + data, err := ioutil.ReadAll(file) + c.Assert(err, IsNil) + c.Assert(string(data), Equals, "CREATE DATABASE IF NOT EXISTS `loader_test`") +} From 90e2da43e0cd4c775cbab296f392d01167ddfad2 Mon Sep 17 00:00:00 2001 From: WangXiangUSTC Date: Tue, 25 Jun 2019 19:07:12 +0800 Subject: [PATCH 2/5] minor update --- loader/loader.go | 5 +++-- loader/util.go | 2 +- loader/util_test.go | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/loader/loader.go b/loader/loader.go index 9c2426cfb1..6101361a4f 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -683,10 +683,10 @@ func (l *Loader) prepareDbFiles(files map[string]struct{}) error { } if schemaFileCount == 0 { - return errors.New("invalid mydumper files for there are no `-schema-create.sql` files found") + log.Warn("invalid mydumper files for there are no `-schema-create.sql` files found, and will generate later") } if len(l.db2Tables) == 0 { - return errors.New("no available `-schema-create.sql` files, check mydumper parameter matches black-white-list in task config") + log.Warn("no available `-schema-create.sql` files, check mydumper parameter matches black-white-list in task config, will generate later") } return nil @@ -718,6 +718,7 @@ func (l *Loader) prepareTableFiles(files map[string]struct{}) error { return errors.Trace(err) } l.db2Tables[db] = make(Tables2DataFiles) + tables = l.db2Tables[db] l.totalFileCount.Add(1) } diff --git a/loader/util.go b/loader/util.go index e7f37bf662..5949863636 100644 --- a/loader/util.go +++ b/loader/util.go @@ -84,6 +84,6 @@ func generateSchemaCreateFile(dir string, schema string) error { } defer file.Close() - file.WriteString(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`", schema)) + file.WriteString(fmt.Sprintf("CREATE DATABASE `%s`;\n", schema)) return nil } diff --git a/loader/util_test.go b/loader/util_test.go index bddd895337..679d602a0e 100644 --- a/loader/util_test.go +++ b/loader/util_test.go @@ -78,5 +78,5 @@ func (t *testUtilSuite) TestGenerateSchemaCreateFile(c *C) { data, err := ioutil.ReadAll(file) c.Assert(err, IsNil) - c.Assert(string(data), Equals, "CREATE DATABASE IF NOT EXISTS `loader_test`") + c.Assert(string(data), Equals, "CREATE DATABASE `loader_test`;\n") } From 674cc69f61b017c663d69769e3801c24715d27d9 Mon Sep 17 00:00:00 2001 From: WangXiangUSTC Date: Wed, 26 Jun 2019 12:42:21 +0800 Subject: [PATCH 3/5] Update loader/loader.go Co-Authored-By: kennytm --- loader/loader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loader/loader.go b/loader/loader.go index 6101361a4f..8a03f22db5 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -713,7 +713,7 @@ func (l *Loader) prepareTableFiles(files map[string]struct{}) error { } tables, ok := l.db2Tables[db] if !ok { - log.Warnf("cann't find schema create file for db %s, will generate one", db) + log.Warnf("can't find schema create file for db %s, will generate one", db) if err := generateSchemaCreateFile(l.cfg.Dir, db); err != nil { return errors.Trace(err) } From 25659eee5dfb4a70f92960b721fe9d7fac819f15 Mon Sep 17 00:00:00 2001 From: WangXiangUSTC Date: Wed, 26 Jun 2019 12:46:41 +0800 Subject: [PATCH 4/5] address comment --- loader/util.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/loader/util.go b/loader/util.go index 5949863636..88ca597e32 100644 --- a/loader/util.go +++ b/loader/util.go @@ -84,6 +84,6 @@ func generateSchemaCreateFile(dir string, schema string) error { } defer file.Close() - file.WriteString(fmt.Sprintf("CREATE DATABASE `%s`;\n", schema)) - return nil + _, err = fmt.Fprintf(file, "CREATE DATABASE `%s`;\n", schema) + return errors.Trace(err) } From d87c9591b3e8676567cb2997f23cde56b9918707 Mon Sep 17 00:00:00 2001 From: WangXiangUSTC Date: Wed, 26 Jun 2019 13:40:37 +0800 Subject: [PATCH 5/5] address comment --- loader/util.go | 6 +++++- loader/util_test.go | 30 +++++++++++++++++++++--------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/loader/util.go b/loader/util.go index 88ca597e32..a2c2e420ba 100644 --- a/loader/util.go +++ b/loader/util.go @@ -84,6 +84,10 @@ func generateSchemaCreateFile(dir string, schema string) error { } defer file.Close() - _, err = fmt.Fprintf(file, "CREATE DATABASE `%s`;\n", schema) + _, err = fmt.Fprintf(file, "CREATE DATABASE `%s`;\n", escapeName(schema)) return errors.Trace(err) } + +func escapeName(name string) string { + return strings.Replace(name, "`", "``", -1) +} diff --git a/loader/util_test.go b/loader/util_test.go index 679d602a0e..f35f103c8a 100644 --- a/loader/util_test.go +++ b/loader/util_test.go @@ -68,15 +68,27 @@ func (t *testUtilSuite) TestShortSha1(c *C) { func (t *testUtilSuite) TestGenerateSchemaCreateFile(c *C) { dir := c.MkDir() - schema := "loader_test" - - err := generateSchemaCreateFile(dir, schema) - c.Assert(err, IsNil) + testCases := []struct { + schema string + createSQL string + }{ + { + "loader_test", + "CREATE DATABASE `loader_test`;\n", + }, { + "loader`test", + "CREATE DATABASE `loader``test`;\n", + }, + } + for _, testCase := range testCases { + err := generateSchemaCreateFile(dir, testCase.schema) + c.Assert(err, IsNil) - file, err := os.Open(path.Join(dir, fmt.Sprintf("%s-schema-create.sql", schema))) - c.Assert(err, IsNil) + file, err := os.Open(path.Join(dir, fmt.Sprintf("%s-schema-create.sql", testCase.schema))) + c.Assert(err, IsNil) - data, err := ioutil.ReadAll(file) - c.Assert(err, IsNil) - c.Assert(string(data), Equals, "CREATE DATABASE `loader_test`;\n") + data, err := ioutil.ReadAll(file) + c.Assert(err, IsNil) + c.Assert(string(data), Equals, testCase.createSQL) + } }