Fix/use parseddls (#65)

* fix failed test

* use memefish ParseDDLs instead of strings.Split for fix parse with
comment

internal/loader_test.go

@@ -3,6 +3,8 @@ package internal
 import (
 	"fmt"
 	"testing"
+
+	"go.mercari.io/yo/models"
 )
 
 func Test_setIndexesToTables(t *testing.T) {
@@ -18,8 +20,20 @@ func Test_setIndexesToTables(t *testing.T) {
 				},
 			},
 			ix: map[string]*Index{
-				"TableA_Index1": &Index{},
-				"TableA_Index2": &Index{},
+				"TableA_Index1": &Index{
+					Type: &Type{
+						Table: &models.Table{
+							TableName: "TableA",
+						},
+					},
+				},
+				"TableA_Index2": &Index{
+					Type: &Type{
+						Table: &models.Table{
+							TableName: "TableA",
+						},
+					},
+				},
 			},
 			result: map[string]int{
 				"TableA": 2,
@@ -35,8 +49,20 @@ func Test_setIndexesToTables(t *testing.T) {
 				},
 			},
 			ix: map[string]*Index{
-				"TableA_Index1": &Index{},
-				"TableA_Index2": &Index{},
+				"TableA_Index1": &Index{
+					Type: &Type{
+						Table: &models.Table{
+							TableName: "TableA",
+						},
+					},
+				},
+				"TableA_Index2": &Index{
+					Type: &Type{
+						Table: &models.Table{
+							TableName: "TableA",
+						},
+					},
+				},
 			},
 			result: map[string]int{
 				"TableA": 2,
@@ -53,10 +79,34 @@ func Test_setIndexesToTables(t *testing.T) {
 				},
 			},
 			ix: map[string]*Index{
-				"TableA_Index1":               &Index{},
-				"TableA_Index2":               &Index{},
-				"TableB_Index1":               &Index{},
-				"TableB_Index2forTableA_Hoge": &Index{},
+				"TableA_Index1": &Index{
+					Type: &Type{
+						Table: &models.Table{
+							TableName: "TableA",
+						},
+					},
+				},
+				"TableA_Index2": &Index{
+					Type: &Type{
+						Table: &models.Table{
+							TableName: "TableA",
+						},
+					},
+				},
+				"TableB_Index1": &Index{
+					Type: &Type{
+						Table: &models.Table{
+							TableName: "TableB",
+						},
+					},
+				},
+				"TableB_Index2forTableA_Hoge": &Index{
+					Type: &Type{
+						Table: &models.Table{
+							TableName: "TableB",
+						},
+					},
+				},
 			},
 			result: map[string]int{
 				"TableA": 2,

loaders/parser.go

@@ -22,7 +22,6 @@ package loaders
 import (
 	"fmt"
 	"io/ioutil"
-	"strings"
 
 	"github.com/MakeNowJust/memefish/pkg/ast"
 	"github.com/MakeNowJust/memefish/pkg/parser"
@@ -37,21 +36,15 @@ func NewSpannerLoaderFromDDL(fpath string) (*SpannerLoaderFromDDL, error) {
 	}
 
 	tables := make(map[string]table)
-	stmts := strings.Split(string(b), ";")
-	for _, stmt := range stmts {
-		stmt := strings.TrimSpace(stmt)
-		if stmt == "" {
-			continue
-		}
-		ddl, err := (&parser.Parser{
-			Lexer: &parser.Lexer{
-				File: &token.File{FilePath: fpath, Buffer: stmt},
-			},
-		}).ParseDDL()
-		if err != nil {
-			return nil, err
-		}
-
+	ddls, err := (&parser.Parser{
+		Lexer: &parser.Lexer{
+			File: &token.File{FilePath: fpath, Buffer: string(b)},
+		},
+	}).ParseDDLs()
+	if err != nil {
+		return nil, err
+	}
+	for _, ddl := range ddls {
 		switch val := ddl.(type) {
 		case *ast.CreateTable:
 			v := tables[val.Name.Name]

test/testdata/schema.sql

@@ -31,6 +31,7 @@ CREATE INDEX CompositePrimaryKeysByXY     ON CompositePrimaryKeys(X, Y);
 CREATE INDEX CompositePrimaryKeysByError  ON CompositePrimaryKeys(Error);
 CREATE INDEX CompositePrimaryKeysByError2 ON CompositePrimaryKeys(Error) STORING(Z);
 CREATE INDEX CompositePrimaryKeysByError3 ON CompositePrimaryKeys(Error) STORING(Z, Y);
+-- CREATE INDEX ForTestCommentIndex     ON CompositePrimaryKeys(X, Y, Z);
 
 CREATE TABLE OutOfOrderPrimaryKeys (
   PKey1 STRING(32) NOT NULL,

test/testutil/data.go

@@ -25,12 +25,13 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"strings"
 	"time"
 
 	"cloud.google.com/go/spanner"
 	dbadmin "cloud.google.com/go/spanner/admin/database/apiv1"
 	insadmin "cloud.google.com/go/spanner/admin/instance/apiv1"
+	"github.com/MakeNowJust/memefish/pkg/parser"
+	mtoken "github.com/MakeNowJust/memefish/pkg/token"
 	dbadminpb "google.golang.org/genproto/googleapis/spanner/admin/database/v1"
 	instancepb "google.golang.org/genproto/googleapis/spanner/admin/instance/v1"
 	"google.golang.org/grpc/codes"
@@ -97,7 +98,7 @@ func SetupDatabase(ctx context.Context, projectName, instanceName, dbName string
 
 	dbAdminCli, err := dbadmin.NewDatabaseAdminClient(ctx)
 	if err != nil {
-		return fmt.Errorf("failed to create database admin client: %v")
+		return fmt.Errorf("failed to create database admin client: %v", err)
 	}
 	defer dbAdminCli.Close()
 
@@ -220,7 +221,8 @@ func ApplyTestSchema(ctx context.Context, adminClient *dbadmin.DatabaseAdminClie
 	dir := findProjectRootDir()
 
 	// Open test schema
-	file, err := os.Open(filepath.Join(dir, "./test/testdata/schema.sql"))
+	fpath := filepath.Join(dir, "./test/testdata/schema.sql")
+	file, err := os.Open(fpath)
 	if err != nil {
 		return fmt.Errorf("scheme file cannot open: %v", err)
 	}
@@ -233,19 +235,19 @@ func ApplyTestSchema(ctx context.Context, adminClient *dbadmin.DatabaseAdminClie
 
 	// find the first DDL to skip comments.
 	contents := string(b)
-	if pos := strings.Index(contents, "CREATE "); pos >= 0 {
-		contents = contents[pos:]
+	ddls, err := (&parser.Parser{
+		Lexer: &parser.Lexer{
+			File: &mtoken.File{FilePath: fpath, Buffer: contents},
+		},
+	}).ParseDDLs()
+	if err != nil {
+		return fmt.Errorf("parse error: %v", err)
 	}
-
 	// Split scheme definition to DDL statements.
 	// This assuemes there is no comments and each statement is separated by semi-colon
 	var statements []string
-	for _, s := range strings.Split(contents, ";") {
-		s = strings.TrimSpace(s)
-		if len(s) == 0 {
-			continue
-		}
-		statements = append(statements, s)
+	for _, ddl := range ddls {
+		statements = append(statements, ddl.SQL())
 	}
 
 	// Apply DDL statements to create tables