diff --git a/go/test/endtoend/vtgate/aggr_test.go b/go/test/endtoend/vtgate/aggr_test.go index f68a5808c34..adc8c93a84c 100644 --- a/go/test/endtoend/vtgate/aggr_test.go +++ b/go/test/endtoend/vtgate/aggr_test.go @@ -40,3 +40,22 @@ func TestAggregateTypes(t *testing.T) { assertMatches(t, conn, "select val1, count(distinct val2) k, count(*) from aggr_test group by val1 order by k desc, val1 limit 4", `[[VARCHAR("c") INT64(2) INT64(2)] [VARCHAR("a") INT64(1) INT64(2)] [VARCHAR("b") INT64(1) INT64(1)] [VARCHAR("e") INT64(1) INT64(2)]]`) exec(t, conn, "delete from aggr_test") } + +func TestGroupBy(t *testing.T) { + defer cluster.PanicHandler(t) + ctx := context.Background() + conn, err := mysql.Connect(ctx, &vtParams) + require.Nil(t, err) + defer conn.Close() + exec(t, conn, "insert into t3(id5, id6, id7) values(1,1,2), (2,2,4), (3,2,4), (4,1,2), (5,1,2), (6,3,6)") + // test ordering and group by int column + assertMatches(t, conn, "select id6, id7, count(*) k from t3 group by id6, id7 order by k", `[[INT64(3) INT64(6) INT64(1)] [INT64(2) INT64(4) INT64(2)] [INT64(1) INT64(2) INT64(3)]]`) + + defer func() { + exec(t, conn, "set workload = oltp") + exec(t, conn, "delete from t3") + }() + // Test the same queries in streaming mode + exec(t, conn, "set workload = olap") + assertMatches(t, conn, "select id6, id7, count(*) k from t3 group by id6, id7 order by k", `[[INT64(3) INT64(6) INT64(1)] [INT64(2) INT64(4) INT64(2)] [INT64(1) INT64(2) INT64(3)]]`) +} diff --git a/go/test/endtoend/vtgate/misc_test.go b/go/test/endtoend/vtgate/misc_test.go index 38570ef03ed..3dc1f55aa5a 100644 --- a/go/test/endtoend/vtgate/misc_test.go +++ b/go/test/endtoend/vtgate/misc_test.go @@ -558,6 +558,29 @@ func TestShowVariables(t *testing.T) { require.True(t, found, "Expected a row for version in show query") } +func TestOrderBy(t *testing.T) { + defer cluster.PanicHandler(t) + ctx := context.Background() + conn, err := mysql.Connect(ctx, &vtParams) + require.Nil(t, err) + defer conn.Close() + exec(t, conn, "insert into t4(id1, id2) values(1,'a'), (2,'Abc'), (3,'b'), (4,'c'), (5,'test')") + exec(t, conn, "insert into t4(id1, id2) values(6,'d'), (7,'e'), (8,'F')") + // test ordering of varchar column + assertMatches(t, conn, "select id1, id2 from t4 order by id2 desc", `[[INT64(5) VARCHAR("test")] [INT64(8) VARCHAR("F")] [INT64(7) VARCHAR("e")] [INT64(6) VARCHAR("d")] [INT64(4) VARCHAR("c")] [INT64(3) VARCHAR("b")] [INT64(2) VARCHAR("Abc")] [INT64(1) VARCHAR("a")]]`) + // test ordering of int column + assertMatches(t, conn, "select id1, id2 from t4 order by id1 desc", `[[INT64(8) VARCHAR("F")] [INT64(7) VARCHAR("e")] [INT64(6) VARCHAR("d")] [INT64(5) VARCHAR("test")] [INT64(4) VARCHAR("c")] [INT64(3) VARCHAR("b")] [INT64(2) VARCHAR("Abc")] [INT64(1) VARCHAR("a")]]`) + + defer func() { + exec(t, conn, "set workload = oltp") + exec(t, conn, "delete from t4") + }() + // Test the same queries in streaming mode + exec(t, conn, "set workload = olap") + assertMatches(t, conn, "select id1, id2 from t4 order by id2 desc", `[[INT64(5) VARCHAR("test")] [INT64(8) VARCHAR("F")] [INT64(7) VARCHAR("e")] [INT64(6) VARCHAR("d")] [INT64(4) VARCHAR("c")] [INT64(3) VARCHAR("b")] [INT64(2) VARCHAR("Abc")] [INT64(1) VARCHAR("a")]]`) + assertMatches(t, conn, "select id1, id2 from t4 order by id1 desc", `[[INT64(8) VARCHAR("F")] [INT64(7) VARCHAR("e")] [INT64(6) VARCHAR("d")] [INT64(5) VARCHAR("test")] [INT64(4) VARCHAR("c")] [INT64(3) VARCHAR("b")] [INT64(2) VARCHAR("Abc")] [INT64(1) VARCHAR("a")]]`) +} + func assertMatches(t *testing.T, conn *mysql.Conn, query, expected string) { t.Helper() qr := exec(t, conn, query) diff --git a/go/vt/vtgate/engine/cached_size.go b/go/vt/vtgate/engine/cached_size.go index 18e3a530d19..1d0b4304d0e 100644 --- a/go/vt/vtgate/engine/cached_size.go +++ b/go/vt/vtgate/engine/cached_size.go @@ -307,7 +307,7 @@ func (cached *MemorySort) CachedSize(alloc bool) int64 { size += cached.UpperLimit.CachedSize(false) // field OrderBy []vitess.io/vitess/go/vt/vtgate/engine.OrderbyParams { - size += int64(cap(cached.OrderBy)) * int64(9) + size += int64(cap(cached.OrderBy)) * int64(17) } // field Input vitess.io/vitess/go/vt/vtgate/engine.Primitive if cc, ok := cached.Input.(cachedObject); ok { @@ -334,7 +334,7 @@ func (cached *MergeSort) CachedSize(alloc bool) int64 { } // field OrderBy []vitess.io/vitess/go/vt/vtgate/engine.OrderbyParams { - size += int64(cap(cached.OrderBy)) * int64(9) + size += int64(cap(cached.OrderBy)) * int64(17) } return size } @@ -527,7 +527,7 @@ func (cached *Route) CachedSize(alloc bool) int64 { } // field OrderBy []vitess.io/vitess/go/vt/vtgate/engine.OrderbyParams { - size += int64(cap(cached.OrderBy)) * int64(9) + size += int64(cap(cached.OrderBy)) * int64(17) } // field SysTableTableSchema vitess.io/vitess/go/vt/vtgate/evalengine.Expr if cc, ok := cached.SysTableTableSchema.(cachedObject); ok { diff --git a/go/vt/vtgate/engine/comparer.go b/go/vt/vtgate/engine/comparer.go new file mode 100644 index 00000000000..a685c4816a2 --- /dev/null +++ b/go/vt/vtgate/engine/comparer.go @@ -0,0 +1,67 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package engine + +import ( + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/vtgate/evalengine" +) + +// comparer is the struct that has the logic for comparing two rows in the result set +type comparer struct { + orderBy, weightString int + desc bool +} + +// compare compares two rows given the comparer and returns which one should be earlier in the result set +// -1 if the first row should be earlier +// 1 is the second row should be earlier +// 0 if both the rows have equal ordering +func (c *comparer) compare(r1, r2 []sqltypes.Value) (int, error) { + cmp, err := evalengine.NullsafeCompare(r1[c.orderBy], r2[c.orderBy]) + if err != nil { + _, isComparisonErr := err.(evalengine.UnsupportedComparisonError) + if !(isComparisonErr && c.weightString != -1) { + return 0, err + } + // in case of a comparison error switch to using the weight string column for ordering + c.orderBy = c.weightString + c.weightString = -1 + cmp, err = evalengine.NullsafeCompare(r1[c.orderBy], r2[c.orderBy]) + if err != nil { + return 0, err + } + } + // change the result if descending ordering is required + if c.desc { + cmp = -cmp + } + return cmp, nil +} + +// extractSlices extracts the three fields of OrderbyParams into a slice of comparers +func extractSlices(input []OrderbyParams) []*comparer { + var result []*comparer + for _, order := range input { + result = append(result, &comparer{ + orderBy: order.Col, + weightString: order.WeightStringCol, + desc: order.Desc, + }) + } + return result +} diff --git a/go/vt/vtgate/engine/comparer_test.go b/go/vt/vtgate/engine/comparer_test.go new file mode 100644 index 00000000000..c1be2c25e82 --- /dev/null +++ b/go/vt/vtgate/engine/comparer_test.go @@ -0,0 +1,114 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package engine + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/sqltypes" +) + +func TestComparer(t *testing.T) { + tests := []struct { + comparer comparer + row1 []sqltypes.Value + row2 []sqltypes.Value + output int + }{ + { + comparer: comparer{ + orderBy: 0, + weightString: -1, + desc: true, + }, + row1: []sqltypes.Value{ + sqltypes.NewInt64(23), + }, + row2: []sqltypes.Value{ + sqltypes.NewInt64(34), + }, + output: 1, + }, { + comparer: comparer{ + orderBy: 0, + weightString: -1, + desc: false, + }, + row1: []sqltypes.Value{ + sqltypes.NewInt64(23), + }, + row2: []sqltypes.Value{ + sqltypes.NewInt64(23), + }, + output: 0, + }, { + comparer: comparer{ + orderBy: 0, + weightString: -1, + desc: false, + }, + row1: []sqltypes.Value{ + sqltypes.NewInt64(23), + }, + row2: []sqltypes.Value{ + sqltypes.NewInt64(12), + }, + output: 1, + }, { + comparer: comparer{ + orderBy: 1, + weightString: 0, + desc: false, + }, + row1: []sqltypes.Value{ + sqltypes.NewInt64(23), + sqltypes.NewVarChar("b"), + }, + row2: []sqltypes.Value{ + sqltypes.NewInt64(34), + sqltypes.NewVarChar("a"), + }, + output: -1, + }, { + comparer: comparer{ + orderBy: 1, + weightString: 0, + desc: true, + }, + row1: []sqltypes.Value{ + sqltypes.NewInt64(23), + sqltypes.NewVarChar("A"), + }, + row2: []sqltypes.Value{ + sqltypes.NewInt64(23), + sqltypes.NewVarChar("a"), + }, + output: 0, + }, + } + + for i, test := range tests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + got, err := test.comparer.compare(test.row1, test.row2) + require.NoError(t, err) + require.Equal(t, test.output, got) + }) + } +} diff --git a/go/vt/vtgate/engine/memory_sort.go b/go/vt/vtgate/engine/memory_sort.go index 9c4a2df4c02..560b029356e 100644 --- a/go/vt/vtgate/engine/memory_sort.go +++ b/go/vt/vtgate/engine/memory_sort.go @@ -76,8 +76,8 @@ func (ms *MemorySort) Execute(vcursor VCursor, bindVars map[string]*querypb.Bind return nil, err } sh := &sortHeap{ - rows: result.Rows, - orderBy: ms.OrderBy, + rows: result.Rows, + comparers: extractSlices(ms.OrderBy), } sort.Sort(sh) if sh.err != nil { @@ -104,8 +104,8 @@ func (ms *MemorySort) StreamExecute(vcursor VCursor, bindVars map[string]*queryp // You have to reverse the ordering because the highest values // must be dropped once the upper limit is reached. sh := &sortHeap{ - orderBy: ms.OrderBy, - reverse: true, + comparers: extractSlices(ms.OrderBy), + reverse: true, } err = ms.Input.StreamExecute(vcursor, bindVars, wantfields, func(qr *sqltypes.Result) error { if len(qr.Fields) != 0 { @@ -115,9 +115,11 @@ func (ms *MemorySort) StreamExecute(vcursor VCursor, bindVars map[string]*queryp } for _, row := range qr.Rows { heap.Push(sh, row) - } - for len(sh.rows) > count { - _ = heap.Pop(sh) + // Remove the highest element from the heap if the size is more than the count + // This optimization means that the maximum size of the heap is going to be (count + 1) + for len(sh.rows) > count { + _ = heap.Pop(sh) + } } if vcursor.ExceedsMaxMemoryRows(len(sh.rows)) { return fmt.Errorf("in-memory row count exceeded allowed limit of %d", vcursor.MaxMemoryRows()) @@ -215,10 +217,10 @@ func GenericJoin(input interface{}, f func(interface{}) string) string { // sortHeap is sorted based on the orderBy params. // Implementation is similar to scatterHeap type sortHeap struct { - rows [][]sqltypes.Value - orderBy []OrderbyParams - reverse bool - err error + rows [][]sqltypes.Value + comparers []*comparer + reverse bool + err error } // Len satisfies sort.Interface and heap.Interface. @@ -228,11 +230,11 @@ func (sh *sortHeap) Len() int { // Less satisfies sort.Interface and heap.Interface. func (sh *sortHeap) Less(i, j int) bool { - for _, order := range sh.orderBy { + for _, c := range sh.comparers { if sh.err != nil { return true } - cmp, err := evalengine.NullsafeCompare(sh.rows[i][order.Col], sh.rows[j][order.Col]) + cmp, err := c.compare(sh.rows[i], sh.rows[j]) if err != nil { sh.err = err return true @@ -240,17 +242,7 @@ func (sh *sortHeap) Less(i, j int) bool { if cmp == 0 { continue } - // This is equivalent to: - //if !sh.reverse { - // if order.Desc { - // cmp = -cmp - // } - //} else { - // if !order.Desc { - // cmp = -cmp - // } - //} - if sh.reverse != order.Desc { + if sh.reverse { cmp = -cmp } return cmp < 0 diff --git a/go/vt/vtgate/engine/memory_sort_test.go b/go/vt/vtgate/engine/memory_sort_test.go index dd7fd924a2f..1a39208f83c 100644 --- a/go/vt/vtgate/engine/memory_sort_test.go +++ b/go/vt/vtgate/engine/memory_sort_test.go @@ -17,10 +17,10 @@ limitations under the License. package engine import ( - "encoding/json" - "reflect" "testing" + "vitess.io/vitess/go/test/utils" + "github.com/stretchr/testify/require" "vitess.io/vitess/go/sqltypes" @@ -37,7 +37,7 @@ func TestMemorySortExecute(t *testing.T) { results: []*sqltypes.Result{sqltypes.MakeTestResult( fields, "a|1", - "b|2", + "g|2", "a|1", "c|4", "c|3", @@ -46,50 +46,164 @@ func TestMemorySortExecute(t *testing.T) { ms := &MemorySort{ OrderBy: []OrderbyParams{{ - Col: 1, + WeightStringCol: -1, + Col: 1, }}, Input: fp, } result, err := ms.Execute(nil, nil, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) wantResult := sqltypes.MakeTestResult( fields, "a|1", "a|1", - "b|2", + "g|2", "c|3", "c|4", ) - if !reflect.DeepEqual(result, wantResult) { - t.Errorf("oa.Execute:\n%v, want\n%v", result, wantResult) - } + utils.MustMatch(t, wantResult, result) fp.rewind() upperlimit, err := sqlparser.NewPlanValue(sqlparser.NewArgument(":__upper_limit")) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) ms.UpperLimit = upperlimit bv := map[string]*querypb.BindVariable{"__upper_limit": sqltypes.Int64BindVariable(3)} result, err = ms.Execute(nil, bv, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) wantResult = sqltypes.MakeTestResult( fields, "a|1", "a|1", - "b|2", + "g|2", + ) + utils.MustMatch(t, wantResult, result) +} + +func TestMemorySortStreamExecuteWeightString(t *testing.T) { + fields := sqltypes.MakeTestFields( + "weightString|normal", + "varbinary|varchar", ) - if !reflect.DeepEqual(result, wantResult) { - t.Errorf("oa.Execute:\n%v, want\n%v", result, wantResult) + fp := &fakePrimitive{ + results: []*sqltypes.Result{sqltypes.MakeTestResult( + fields, + "null|x", + "g|d", + "a|a", + "c|t", + "f|p", + )}, + } + + ms := &MemorySort{ + OrderBy: []OrderbyParams{{ + WeightStringCol: 0, + Col: 1, + }}, + Input: fp, + } + var results []*sqltypes.Result + + t.Run("order by weight string", func(t *testing.T) { + + err := ms.StreamExecute(&noopVCursor{}, nil, false, func(qr *sqltypes.Result) error { + results = append(results, qr) + return nil + }) + require.NoError(t, err) + + wantResults := sqltypes.MakeTestStreamingResults( + fields, + "null|x", + "a|a", + "c|t", + "f|p", + "g|d", + ) + utils.MustMatch(t, wantResults, results) + }) + + t.Run("Limit test", func(t *testing.T) { + fp.rewind() + upperlimit, err := sqlparser.NewPlanValue(sqlparser.NewArgument(":__upper_limit")) + require.NoError(t, err) + ms.UpperLimit = upperlimit + bv := map[string]*querypb.BindVariable{"__upper_limit": sqltypes.Int64BindVariable(3)} + + results = nil + err = ms.StreamExecute(&noopVCursor{}, bv, false, func(qr *sqltypes.Result) error { + results = append(results, qr) + return nil + }) + require.NoError(t, err) + + wantResults := sqltypes.MakeTestStreamingResults( + fields, + "null|x", + "a|a", + "c|t", + ) + utils.MustMatch(t, wantResults, results) + }) +} + +func TestMemorySortExecuteWeightString(t *testing.T) { + fields := sqltypes.MakeTestFields( + "c1|c2", + "varchar|varbinary", + ) + fp := &fakePrimitive{ + results: []*sqltypes.Result{sqltypes.MakeTestResult( + fields, + "a|1", + "g|2", + "a|1", + "c|4", + "c|3", + )}, + } + + ms := &MemorySort{ + OrderBy: []OrderbyParams{{ + WeightStringCol: 1, + Col: 0, + }}, + Input: fp, } + + result, err := ms.Execute(nil, nil, false) + require.NoError(t, err) + + wantResult := sqltypes.MakeTestResult( + fields, + "a|1", + "a|1", + "g|2", + "c|3", + "c|4", + ) + utils.MustMatch(t, wantResult, result) + + fp.rewind() + upperlimit, err := sqlparser.NewPlanValue(sqlparser.NewArgument(":__upper_limit")) + require.NoError(t, err) + ms.UpperLimit = upperlimit + bv := map[string]*querypb.BindVariable{"__upper_limit": sqltypes.Int64BindVariable(3)} + + result, err = ms.Execute(nil, bv, false) + require.NoError(t, err) + + wantResult = sqltypes.MakeTestResult( + fields, + "a|1", + "a|1", + "g|2", + ) + utils.MustMatch(t, wantResult, result) } func TestMemorySortStreamExecute(t *testing.T) { @@ -101,7 +215,7 @@ func TestMemorySortStreamExecute(t *testing.T) { results: []*sqltypes.Result{sqltypes.MakeTestResult( fields, "a|1", - "b|2", + "g|2", "a|1", "c|4", "c|3", @@ -110,7 +224,8 @@ func TestMemorySortStreamExecute(t *testing.T) { ms := &MemorySort{ OrderBy: []OrderbyParams{{ - Col: 1, + WeightStringCol: -1, + Col: 1, }}, Input: fp, } @@ -120,27 +235,21 @@ func TestMemorySortStreamExecute(t *testing.T) { results = append(results, qr) return nil }) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) wantResults := sqltypes.MakeTestStreamingResults( fields, "a|1", "a|1", - "b|2", + "g|2", "c|3", "c|4", ) - if !reflect.DeepEqual(results, wantResults) { - t.Errorf("oa.Execute:\n%v, want\n%v", results, wantResults) - } + utils.MustMatch(t, wantResults, results) fp.rewind() upperlimit, err := sqlparser.NewPlanValue(sqlparser.NewArgument(":__upper_limit")) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) ms.UpperLimit = upperlimit bv := map[string]*querypb.BindVariable{"__upper_limit": sqltypes.Int64BindVariable(3)} @@ -149,21 +258,15 @@ func TestMemorySortStreamExecute(t *testing.T) { results = append(results, qr) return nil }) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) wantResults = sqltypes.MakeTestStreamingResults( fields, "a|1", "a|1", - "b|2", + "g|2", ) - if !reflect.DeepEqual(results, wantResults) { - r, _ := json.Marshal(results) - w, _ := json.Marshal(wantResults) - t.Errorf("oa.Execute:\n%s, want\n%s", r, w) - } + utils.MustMatch(t, wantResults, results) } func TestMemorySortGetFields(t *testing.T) { @@ -178,12 +281,8 @@ func TestMemorySortGetFields(t *testing.T) { ms := &MemorySort{Input: fp} got, err := ms.GetFields(nil, nil) - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(got, result) { - t.Errorf("l.GetFields:\n%v, want\n%v", got, result) - } + require.NoError(t, err) + utils.MustMatch(t, result, got) } func TestMemorySortExecuteTruncate(t *testing.T) { @@ -195,7 +294,7 @@ func TestMemorySortExecuteTruncate(t *testing.T) { results: []*sqltypes.Result{sqltypes.MakeTestResult( fields, "a|1|1", - "b|2|1", + "g|2|1", "a|1|1", "c|4|1", "c|3|1", @@ -204,28 +303,25 @@ func TestMemorySortExecuteTruncate(t *testing.T) { ms := &MemorySort{ OrderBy: []OrderbyParams{{ - Col: 1, + WeightStringCol: -1, + Col: 1, }}, Input: fp, TruncateColumnCount: 2, } result, err := ms.Execute(nil, nil, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) wantResult := sqltypes.MakeTestResult( fields[:2], "a|1", "a|1", - "b|2", + "g|2", "c|3", "c|4", ) - if !reflect.DeepEqual(result, wantResult) { - t.Errorf("oa.Execute:\n%v, want\n%v", result, wantResult) - } + utils.MustMatch(t, wantResult, result) } func TestMemorySortStreamExecuteTruncate(t *testing.T) { @@ -237,7 +333,7 @@ func TestMemorySortStreamExecuteTruncate(t *testing.T) { results: []*sqltypes.Result{sqltypes.MakeTestResult( fields, "a|1|1", - "b|2|1", + "g|2|1", "a|1|1", "c|4|1", "c|3|1", @@ -246,7 +342,8 @@ func TestMemorySortStreamExecuteTruncate(t *testing.T) { ms := &MemorySort{ OrderBy: []OrderbyParams{{ - Col: 1, + WeightStringCol: -1, + Col: 1, }}, Input: fp, TruncateColumnCount: 2, @@ -257,21 +354,17 @@ func TestMemorySortStreamExecuteTruncate(t *testing.T) { results = append(results, qr) return nil }) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) wantResults := sqltypes.MakeTestStreamingResults( fields[:2], "a|1", "a|1", - "b|2", + "g|2", "c|3", "c|4", ) - if !reflect.DeepEqual(results, wantResults) { - t.Errorf("oa.Execute:\n%v, want\n%v", results, wantResults) - } + utils.MustMatch(t, wantResults, results) } func TestMemorySortMultiColumn(t *testing.T) { @@ -292,18 +385,18 @@ func TestMemorySortMultiColumn(t *testing.T) { ms := &MemorySort{ OrderBy: []OrderbyParams{{ - Col: 1, + Col: 1, + WeightStringCol: -1, }, { - Col: 0, - Desc: true, + Col: 0, + WeightStringCol: -1, + Desc: true, }}, Input: fp, } result, err := ms.Execute(nil, nil, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) wantResult := sqltypes.MakeTestResult( fields, @@ -313,22 +406,16 @@ func TestMemorySortMultiColumn(t *testing.T) { "c|3", "c|4", ) - if !reflect.DeepEqual(result, wantResult) { - t.Errorf("oa.Execute:\n%v, want\n%v", result, wantResult) - } + utils.MustMatch(t, wantResult, result) fp.rewind() upperlimit, err := sqlparser.NewPlanValue(sqlparser.NewArgument(":__upper_limit")) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) ms.UpperLimit = upperlimit bv := map[string]*querypb.BindVariable{"__upper_limit": sqltypes.Int64BindVariable(3)} result, err = ms.Execute(nil, bv, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) wantResult = sqltypes.MakeTestResult( fields, @@ -336,9 +423,7 @@ func TestMemorySortMultiColumn(t *testing.T) { "a|1", "b|2", ) - if !reflect.DeepEqual(result, wantResult) { - t.Errorf("oa.Execute:\n%v, want\n%v", result, wantResult) - } + utils.MustMatch(t, wantResult, result) } func TestMemorySortMaxMemoryRows(t *testing.T) { @@ -375,7 +460,8 @@ func TestMemorySortMaxMemoryRows(t *testing.T) { ms := &MemorySort{ OrderBy: []OrderbyParams{{ - Col: 1, + WeightStringCol: -1, + Col: 1, }}, Input: fp, } @@ -410,7 +496,8 @@ func TestMemorySortExecuteNoVarChar(t *testing.T) { ms := &MemorySort{ OrderBy: []OrderbyParams{{ - Col: 0, + WeightStringCol: -1, + Col: 0, }}, Input: fp, } diff --git a/go/vt/vtgate/engine/merge_sort.go b/go/vt/vtgate/engine/merge_sort.go index 90a33313ea0..b87aa981765 100644 --- a/go/vt/vtgate/engine/merge_sort.go +++ b/go/vt/vtgate/engine/merge_sort.go @@ -20,8 +20,6 @@ import ( "container/heap" "io" - "vitess.io/vitess/go/vt/vtgate/evalengine" - "context" "vitess.io/vitess/go/sqltypes" @@ -96,9 +94,10 @@ func (ms *MergeSort) StreamExecute(vcursor VCursor, bindVars map[string]*querypb return err } + comparers := extractSlices(ms.OrderBy) sh := &scatterHeap{ - rows: make([]streamRow, 0, len(handles)), - orderBy: ms.OrderBy, + rows: make([]streamRow, 0, len(handles)), + comparers: comparers, } // Prime the heap. One element must be pulled from @@ -236,9 +235,9 @@ type streamRow struct { // yielded an error, err is set. This must be checked // after every heap operation. type scatterHeap struct { - rows []streamRow - orderBy []OrderbyParams - err error + rows []streamRow + err error + comparers []*comparer } // Len satisfies sort.Interface and heap.Interface. @@ -248,11 +247,12 @@ func (sh *scatterHeap) Len() int { // Less satisfies sort.Interface and heap.Interface. func (sh *scatterHeap) Less(i, j int) bool { - for _, order := range sh.orderBy { + for _, c := range sh.comparers { if sh.err != nil { return true } - cmp, err := evalengine.NullsafeCompare(sh.rows[i].row[order.Col], sh.rows[j].row[order.Col]) + // First try to compare the columns that we want to order + cmp, err := c.compare(sh.rows[i].row, sh.rows[j].row) if err != nil { sh.err = err return true @@ -260,9 +260,6 @@ func (sh *scatterHeap) Less(i, j int) bool { if cmp == 0 { continue } - if order.Desc { - cmp = -cmp - } return cmp < 0 } return true diff --git a/go/vt/vtgate/engine/merge_sort_test.go b/go/vt/vtgate/engine/merge_sort_test.go index 1b37af94fa0..ede93d784b2 100644 --- a/go/vt/vtgate/engine/merge_sort_test.go +++ b/go/vt/vtgate/engine/merge_sort_test.go @@ -18,9 +18,10 @@ package engine import ( "errors" - "reflect" "testing" + "vitess.io/vitess/go/test/utils" + "github.com/stretchr/testify/require" "vitess.io/vitess/go/sqltypes" @@ -56,7 +57,8 @@ func TestMergeSortNormal(t *testing.T) { ), }} orderBy := []OrderbyParams{{ - Col: 0, + WeightStringCol: -1, + Col: 0, }} var results []*sqltypes.Result @@ -84,9 +86,65 @@ func TestMergeSortNormal(t *testing.T) { "---", "8|h", ) - if !reflect.DeepEqual(results, wantResults) { - t.Errorf("MergeSort:\n%s, want\n%s", sqltypes.PrintResults(results), sqltypes.PrintResults(wantResults)) - } + utils.MustMatch(t, wantResults, results) +} + +func TestMergeSortWeightString(t *testing.T) { + idColFields := sqltypes.MakeTestFields("id|col", "varbinary|varchar") + shardResults := []*shardResult{{ + results: sqltypes.MakeTestStreamingResults(idColFields, + "1|a", + "7|g", + ), + }, { + results: sqltypes.MakeTestStreamingResults(idColFields, + "2|b", + "---", + "3|c", + ), + }, { + results: sqltypes.MakeTestStreamingResults(idColFields, + "4|d", + "6|f", + ), + }, { + results: sqltypes.MakeTestStreamingResults(idColFields, + "4|d", + "---", + "8|h", + ), + }} + orderBy := []OrderbyParams{{ + WeightStringCol: 0, + Col: 1, + }} + + var results []*sqltypes.Result + err := testMergeSort(shardResults, orderBy, func(qr *sqltypes.Result) error { + results = append(results, qr) + return nil + }) + require.NoError(t, err) + + // Results are returned one row at a time. + wantResults := sqltypes.MakeTestStreamingResults(idColFields, + "1|a", + "---", + "2|b", + "---", + "3|c", + "---", + "4|d", + "---", + "4|d", + "---", + "6|f", + "---", + "7|g", + "---", + "8|h", + ) + utils.MustMatch(t, wantResults, results) } // TestMergeSortDescending tests the normal flow of a merge @@ -117,8 +175,9 @@ func TestMergeSortDescending(t *testing.T) { ), }} orderBy := []OrderbyParams{{ - Col: 0, - Desc: true, + WeightStringCol: -1, + Col: 0, + Desc: true, }} var results []*sqltypes.Result @@ -146,9 +205,7 @@ func TestMergeSortDescending(t *testing.T) { "---", "1|a", ) - if !reflect.DeepEqual(results, wantResults) { - t.Errorf("MergeSort:\n%s, want\n%s", sqltypes.PrintResults(results), sqltypes.PrintResults(wantResults)) - } + utils.MustMatch(t, wantResults, results) } func TestMergeSortEmptyResults(t *testing.T) { @@ -169,7 +226,8 @@ func TestMergeSortEmptyResults(t *testing.T) { results: sqltypes.MakeTestStreamingResults(idColFields), }} orderBy := []OrderbyParams{{ - Col: 0, + WeightStringCol: -1, + Col: 0, }} var results []*sqltypes.Result @@ -189,16 +247,15 @@ func TestMergeSortEmptyResults(t *testing.T) { "---", "7|g", ) - if !reflect.DeepEqual(results, wantResults) { - t.Errorf("MergeSort:\n%s, want\n%s", sqltypes.PrintResults(results), sqltypes.PrintResults(wantResults)) - } + utils.MustMatch(t, wantResults, results) } // TestMergeSortResultFailures tests failures at various // stages of result return. func TestMergeSortResultFailures(t *testing.T) { orderBy := []OrderbyParams{{ - Col: 0, + WeightStringCol: -1, + Col: 0, }} // Test early error. @@ -207,9 +264,7 @@ func TestMergeSortResultFailures(t *testing.T) { }} err := testMergeSort(shardResults, orderBy, func(qr *sqltypes.Result) error { return nil }) want := "early error" - if err == nil || err.Error() != want { - t.Errorf("MergeSort(): %v, want %v", err, want) - } + require.EqualError(t, err, want) // Test fail after fields. idFields := sqltypes.MakeTestFields("id", "int32") @@ -219,9 +274,7 @@ func TestMergeSortResultFailures(t *testing.T) { }} err = testMergeSort(shardResults, orderBy, func(qr *sqltypes.Result) error { return nil }) want = "fail after fields" - if err == nil || err.Error() != want { - t.Errorf("MergeSort(): %v, want %v", err, want) - } + require.EqualError(t, err, want) // Test fail after first row. shardResults = []*shardResult{{ @@ -230,9 +283,7 @@ func TestMergeSortResultFailures(t *testing.T) { }} err = testMergeSort(shardResults, orderBy, func(qr *sqltypes.Result) error { return nil }) want = "fail after first row" - if err == nil || err.Error() != want { - t.Errorf("MergeSort(): %v, want %v", err, want) - } + require.EqualError(t, err, want) } func TestMergeSortDataFailures(t *testing.T) { @@ -249,14 +300,13 @@ func TestMergeSortDataFailures(t *testing.T) { ), }} orderBy := []OrderbyParams{{ - Col: 0, + WeightStringCol: -1, + Col: 0, }} err := testMergeSort(shardResults, orderBy, func(qr *sqltypes.Result) error { return nil }) want := `strconv.ParseInt: parsing "2.1": invalid syntax` - if err == nil || err.Error() != want { - t.Errorf("MergeSort(): %v, want %v", err, want) - } + require.EqualError(t, err, want) // Create a new VCursor because the previous MergeSort will still // have lingering goroutines that can cause data race. @@ -272,9 +322,7 @@ func TestMergeSortDataFailures(t *testing.T) { }} err = testMergeSort(shardResults, orderBy, func(qr *sqltypes.Result) error { return nil }) want = `strconv.ParseInt: parsing "1.1": invalid syntax` - if err == nil || err.Error() != want { - t.Errorf("MergeSort(): %v, want %v", err, want) - } + require.EqualError(t, err, want) } func testMergeSort(shardResults []*shardResult, orderBy []OrderbyParams, callback func(qr *sqltypes.Result) error) error { diff --git a/go/vt/vtgate/engine/route.go b/go/vt/vtgate/engine/route.go index cb11e980e58..b77d8c115b3 100644 --- a/go/vt/vtgate/engine/route.go +++ b/go/vt/vtgate/engine/route.go @@ -125,8 +125,11 @@ func NewRoute(opcode RouteOpcode, keyspace *vindexes.Keyspace, query, fieldQuery // OrderbyParams specifies the parameters for ordering. // This is used for merge-sorting scatter queries. type OrderbyParams struct { - Col int - Desc bool + Col int + // WeightStringCol is the weight_string column that will be used for sorting. + // It is set to -1 if such a column is not added to the query + WeightStringCol int + Desc bool } func (obp OrderbyParams) String() string { @@ -584,27 +587,26 @@ func (route *Route) sort(in *sqltypes.Result) (*sqltypes.Result, error) { InsertID: in.InsertID, } + comparers := extractSlices(route.OrderBy) + sort.Slice(out.Rows, func(i, j int) bool { + var cmp int + if err != nil { + return true + } // If there are any errors below, the function sets // the external err and returns true. Once err is set, // all subsequent calls return true. This will make // Slice think that all elements are in the correct // order and return more quickly. - for _, order := range route.OrderBy { - if err != nil { - return true - } - var cmp int - cmp, err = evalengine.NullsafeCompare(out.Rows[i][order.Col], out.Rows[j][order.Col]) + for _, c := range comparers { + cmp, err = c.compare(out.Rows[i], out.Rows[j]) if err != nil { return true } if cmp == 0 { continue } - if order.Desc { - cmp = -cmp - } return cmp < 0 } return true diff --git a/go/vt/vtgate/engine/route_test.go b/go/vt/vtgate/engine/route_test.go index b0c4788bd9f..67afadce4c4 100644 --- a/go/vt/vtgate/engine/route_test.go +++ b/go/vt/vtgate/engine/route_test.go @@ -59,9 +59,7 @@ func TestSelectUnsharded(t *testing.T) { results: []*sqltypes.Result{defaultSelectResult}, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAnyShard()`, `ExecuteMultiShard ks.0: dummy_select {} false false`, @@ -70,9 +68,7 @@ func TestSelectUnsharded(t *testing.T) { vc.Rewind() result, err = wrapStreamExecute(sel, vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAnyShard()`, `StreamExecuteMulti dummy_select ks.0: {} `, @@ -188,9 +184,7 @@ func TestSelectScatter(t *testing.T) { results: []*sqltypes.Result{defaultSelectResult}, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAllShards()`, `ExecuteMultiShard ks.-20: dummy_select {} ks.20-: dummy_select {} false false`, @@ -199,9 +193,7 @@ func TestSelectScatter(t *testing.T) { vc.Rewind() result, err = wrapStreamExecute(sel, vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAllShards()`, `StreamExecuteMulti dummy_select ks.-20: {} ks.20-: {} `, @@ -228,9 +220,7 @@ func TestSelectEqualUnique(t *testing.T) { results: []*sqltypes.Result{defaultSelectResult}, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [type:INT64 value:"1" ] Destinations:DestinationKeyspaceID(166b40b44aba4bd6)`, `ExecuteMultiShard ks.-20: dummy_select {} false false`, @@ -239,9 +229,7 @@ func TestSelectEqualUnique(t *testing.T) { vc.Rewind() result, err = wrapStreamExecute(sel, vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [type:INT64 value:"1" ] Destinations:DestinationKeyspaceID(166b40b44aba4bd6)`, `StreamExecuteMulti dummy_select ks.-20: {} `, @@ -303,9 +291,7 @@ func TestSelectEqualUniqueScatter(t *testing.T) { results: []*sqltypes.Result{defaultSelectResult}, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [type:INT64 value:"1" ] Destinations:DestinationKeyRange(-)`, `ExecuteMultiShard ks.-20: dummy_select {} ks.20-: dummy_select {} false false`, @@ -314,9 +300,7 @@ func TestSelectEqualUniqueScatter(t *testing.T) { vc.Rewind() result, err = wrapStreamExecute(sel, vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [type:INT64 value:"1" ] Destinations:DestinationKeyRange(-)`, `StreamExecuteMulti dummy_select ks.-20: {} ks.20-: {} `, @@ -357,9 +341,7 @@ func TestSelectEqual(t *testing.T) { }, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `Execute select from, toc from lkp where from in ::from from: type:TUPLE values: false`, `ResolveDestinations ks [type:INT64 value:"1" ] Destinations:DestinationKeyspaceIDs(00,80)`, @@ -369,9 +351,7 @@ func TestSelectEqual(t *testing.T) { vc.Rewind() result, err = wrapStreamExecute(sel, vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `Execute select from, toc from lkp where from in ::from from: type:TUPLE values: false`, `ResolveDestinations ks [type:INT64 value:"1" ] Destinations:DestinationKeyspaceIDs(00,80)`, @@ -400,9 +380,7 @@ func TestSelectEqualNoRoute(t *testing.T) { vc := &loggingVCursor{shards: []string{"-20", "20-"}} result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `Execute select from, toc from lkp where from in ::from from: type:TUPLE values: false`, `ResolveDestinations ks [type:INT64 value:"1" ] Destinations:DestinationNone()`, @@ -411,9 +389,7 @@ func TestSelectEqualNoRoute(t *testing.T) { vc.Rewind() result, err = wrapStreamExecute(sel, vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `Execute select from, toc from lkp where from in ::from from: type:TUPLE values: false`, `ResolveDestinations ks [type:INT64 value:"1" ] Destinations:DestinationNone()`, @@ -449,9 +425,7 @@ func TestSelectINUnique(t *testing.T) { results: []*sqltypes.Result{defaultSelectResult}, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [type:INT64 value:"1" type:INT64 value:"2" type:INT64 value:"4" ] Destinations:DestinationKeyspaceID(166b40b44aba4bd6),DestinationKeyspaceID(06e7ea22ce92708f),DestinationKeyspaceID(d2fd8867d50d2dfe)`, `ExecuteMultiShard ` + @@ -463,9 +437,7 @@ func TestSelectINUnique(t *testing.T) { vc.Rewind() result, err = wrapStreamExecute(sel, vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [type:INT64 value:"1" type:INT64 value:"2" type:INT64 value:"4" ] Destinations:DestinationKeyspaceID(166b40b44aba4bd6),DestinationKeyspaceID(06e7ea22ce92708f),DestinationKeyspaceID(d2fd8867d50d2dfe)`, `StreamExecuteMulti dummy_select ks.-20: {__vals: type:TUPLE values: values: } ks.20-: {__vals: type:TUPLE values: } `, @@ -520,9 +492,7 @@ func TestSelectINNonUnique(t *testing.T) { }, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `Execute select from, toc from lkp where from in ::from from: type:TUPLE values: values: values: false`, `ResolveDestinations ks [type:INT64 value:"1" type:INT64 value:"2" type:INT64 value:"4" ] Destinations:DestinationKeyspaceIDs(00,80),DestinationKeyspaceIDs(00),DestinationKeyspaceIDs(80)`, @@ -535,9 +505,7 @@ func TestSelectINNonUnique(t *testing.T) { vc.Rewind() result, err = wrapStreamExecute(sel, vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `Execute select from, toc from lkp where from in ::from from: type:TUPLE values: values: values: false`, `ResolveDestinations ks [type:INT64 value:"1" type:INT64 value:"2" type:INT64 value:"4" ] Destinations:DestinationKeyspaceIDs(00,80),DestinationKeyspaceIDs(00),DestinationKeyspaceIDs(80)`, @@ -574,9 +542,7 @@ func TestSelectMultiEqual(t *testing.T) { results: []*sqltypes.Result{defaultSelectResult}, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [type:INT64 value:"1" type:INT64 value:"2" type:INT64 value:"4" ] Destinations:DestinationKeyspaceID(166b40b44aba4bd6),DestinationKeyspaceID(06e7ea22ce92708f),DestinationKeyspaceID(d2fd8867d50d2dfe)`, `ExecuteMultiShard ks.-20: dummy_select {} ks.20-: dummy_select {} false false`, @@ -585,9 +551,7 @@ func TestSelectMultiEqual(t *testing.T) { vc.Rewind() result, err = wrapStreamExecute(sel, vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [type:INT64 value:"1" type:INT64 value:"2" type:INT64 value:"4" ] Destinations:DestinationKeyspaceID(166b40b44aba4bd6),DestinationKeyspaceID(06e7ea22ce92708f),DestinationKeyspaceID(d2fd8867d50d2dfe)`, `StreamExecuteMulti dummy_select ks.-20: {} ks.20-: {} `, @@ -611,9 +575,7 @@ func TestSelectNext(t *testing.T) { results: []*sqltypes.Result{defaultSelectResult}, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAnyShard()`, `ExecuteMultiShard ks.-20: dummy_select {} false false`, @@ -645,9 +607,7 @@ func TestSelectDBA(t *testing.T) { results: []*sqltypes.Result{defaultSelectResult}, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAnyShard()`, `ExecuteMultiShard ks.-20: dummy_select {} false false`, @@ -679,9 +639,7 @@ func TestSelectReference(t *testing.T) { results: []*sqltypes.Result{defaultSelectResult}, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAnyShard()`, `ExecuteMultiShard ks.-20: dummy_select {} false false`, @@ -717,9 +675,7 @@ func TestRouteGetFields(t *testing.T) { vc := &loggingVCursor{shards: []string{"-20", "20-"}} result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, true) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `Execute select from, toc from lkp where from in ::from from: type:TUPLE values: false`, `ResolveDestinations ks [type:INT64 value:"1" ] Destinations:DestinationNone()`, @@ -730,9 +686,7 @@ func TestRouteGetFields(t *testing.T) { vc.Rewind() result, err = wrapStreamExecute(sel, vc, map[string]*querypb.BindVariable{}, true) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `Execute select from, toc from lkp where from in ::from from: type:TUPLE values: false`, `ResolveDestinations ks [type:INT64 value:"1" ] Destinations:DestinationNone()`, @@ -753,7 +707,8 @@ func TestRouteSort(t *testing.T) { "dummy_select_field", ) sel.OrderBy = []OrderbyParams{{ - Col: 0, + Col: 0, + WeightStringCol: -1, }} vc := &loggingVCursor{ @@ -772,9 +727,7 @@ func TestRouteSort(t *testing.T) { }, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAnyShard()`, `ExecuteMultiShard ks.0: dummy_select {} false false`, @@ -794,9 +747,7 @@ func TestRouteSort(t *testing.T) { sel.OrderBy[0].Desc = true vc.Rewind() result, err = sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) wantResult = sqltypes.MakeTestResult( sqltypes.MakeTestFields( "id", @@ -827,6 +778,108 @@ func TestRouteSort(t *testing.T) { require.EqualError(t, err, `types are not comparable: VARCHAR vs VARCHAR`) } +func TestRouteSortWeightStrings(t *testing.T) { + sel := NewRoute( + SelectUnsharded, + &vindexes.Keyspace{ + Name: "ks", + Sharded: false, + }, + "dummy_select", + "dummy_select_field", + ) + sel.OrderBy = []OrderbyParams{{ + Col: 1, + WeightStringCol: 0, + }} + + vc := &loggingVCursor{ + shards: []string{"0"}, + results: []*sqltypes.Result{ + sqltypes.MakeTestResult( + sqltypes.MakeTestFields( + "weightString|normal", + "varbinary|varchar", + ), + "v|x", + "g|d", + "a|a", + "c|t", + "f|p", + ), + }, + } + + var result *sqltypes.Result + var wantResult *sqltypes.Result + var err error + t.Run("Sort using Weight Strings", func(t *testing.T) { + result, err = sel.Execute(vc, map[string]*querypb.BindVariable{}, false) + require.NoError(t, err) + vc.ExpectLog(t, []string{ + `ResolveDestinations ks [] Destinations:DestinationAnyShard()`, + `ExecuteMultiShard ks.0: dummy_select {} false false`, + }) + wantResult = sqltypes.MakeTestResult( + sqltypes.MakeTestFields( + "weightString|normal", + "varbinary|varchar", + ), + "a|a", + "c|t", + "f|p", + "g|d", + "v|x", + ) + expectResult(t, "sel.Execute", result, wantResult) + }) + + t.Run("Descending ordering using weighted strings", func(t *testing.T) { + sel.OrderBy[0].Desc = true + vc.Rewind() + result, err = sel.Execute(vc, map[string]*querypb.BindVariable{}, false) + require.NoError(t, err) + wantResult = sqltypes.MakeTestResult( + sqltypes.MakeTestFields( + "weightString|normal", + "varbinary|varchar", + ), + "v|x", + "g|d", + "f|p", + "c|t", + "a|a", + ) + expectResult(t, "sel.Execute", result, wantResult) + }) + + t.Run("Error when no weight string set", func(t *testing.T) { + sel.OrderBy = []OrderbyParams{{ + Col: 1, + WeightStringCol: -1, + }} + + vc = &loggingVCursor{ + shards: []string{"0"}, + results: []*sqltypes.Result{ + sqltypes.MakeTestResult( + sqltypes.MakeTestFields( + "weightString|normal", + "varbinary|varchar", + ), + "v|x", + "g|d", + "a|a", + "c|t", + "f|p", + ), + }, + } + _, err = sel.Execute(vc, map[string]*querypb.BindVariable{}, false) + require.EqualError(t, err, `types are not comparable: VARCHAR vs VARCHAR`) + }) +} + func TestRouteSortTruncate(t *testing.T) { sel := NewRoute( SelectUnsharded, @@ -858,9 +911,7 @@ func TestRouteSortTruncate(t *testing.T) { }, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAnyShard()`, `ExecuteMultiShard ks.0: dummy_select {} false false`, @@ -904,9 +955,7 @@ func TestRouteStreamTruncate(t *testing.T) { }, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAnyShard()`, `ExecuteMultiShard ks.0: dummy_select {} false false`, @@ -951,9 +1000,7 @@ func TestRouteStreamSortTruncate(t *testing.T) { }, } result, err := wrapStreamExecute(sel, vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAnyShard()`, `StreamExecuteMulti dummy_select ks.0: {} `, @@ -1062,9 +1109,7 @@ func TestExecFail(t *testing.T) { }, } _, err = sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Errorf("unexpected ScatterErrorsAsWarnings error %v", err) - } + require.NoError(t, err, "unexpected ScatterErrorsAsWarnings error %v", err) // Ensure that the error code is preserved from SQLErrors and that it // turns into ERUnknownError for all others @@ -1100,9 +1145,7 @@ func TestExecFail(t *testing.T) { }, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Errorf("unexpected ScatterErrorsAsWarnings error %v", err) - } + require.NoError(t, err, "unexpected ScatterErrorsAsWarnings error %v", err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAllShards()`, `ExecuteMultiShard ks.-20: dummy_select {} ks.20-: dummy_select {} false false`, diff --git a/go/vt/vtgate/evalengine/arithmetic.go b/go/vt/vtgate/evalengine/arithmetic.go index 7ad75e6159b..3a887d77763 100644 --- a/go/vt/vtgate/evalengine/arithmetic.go +++ b/go/vt/vtgate/evalengine/arithmetic.go @@ -34,6 +34,17 @@ import ( // a Value, used for arithmetic operations. var zeroBytes = []byte("0") +// UnsupportedComparisonError represents the error where the comparison between the two types is unsupported on vitess +type UnsupportedComparisonError struct { + Type1 querypb.Type + Type2 querypb.Type +} + +// Error function implements the error interface +func (err UnsupportedComparisonError) Error() string { + return fmt.Sprintf("types are not comparable: %v vs %v", err.Type1, err.Type2) +} + // Add adds two values together // if v1 or v2 is null, then it returns null func Add(v1, v2 sqltypes.Value) (sqltypes.Value, error) { @@ -201,7 +212,10 @@ func NullsafeCompare(v1, v2 sqltypes.Value) (int, error) { if isByteComparable(v1) && isByteComparable(v2) { return bytes.Compare(v1.ToBytes(), v2.ToBytes()), nil } - return 0, fmt.Errorf("types are not comparable: %v vs %v", v1.Type(), v2.Type()) + return 0, UnsupportedComparisonError{ + Type1: v1.Type(), + Type2: v2.Type(), + } } // NullsafeHashcode returns an int64 hashcode that is guaranteed to be the same diff --git a/go/vt/vtgate/executor_select_test.go b/go/vt/vtgate/executor_select_test.go index 09bf6ee8ff2..f57b37b6ae8 100644 --- a/go/vt/vtgate/executor_select_test.go +++ b/go/vt/vtgate/executor_select_test.go @@ -221,13 +221,16 @@ func TestStreamLimitOffset(t *testing.T) { Fields: []*querypb.Field{ {Name: "id", Type: sqltypes.Int32}, {Name: "textcol", Type: sqltypes.VarChar}, + {Name: "weight_string(id)", Type: sqltypes.VarBinary}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewInt32(1), sqltypes.NewVarChar("1234"), + sqltypes.NULL, }, { sqltypes.NewInt32(4), sqltypes.NewVarChar("4567"), + sqltypes.NULL, }}, }}) @@ -235,10 +238,12 @@ func TestStreamLimitOffset(t *testing.T) { Fields: []*querypb.Field{ {Name: "id", Type: sqltypes.Int32}, {Name: "textcol", Type: sqltypes.VarChar}, + {Name: "weight_string(id)", Type: sqltypes.VarBinary}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewInt32(2), sqltypes.NewVarChar("2345"), + sqltypes.NULL, }}, }}) @@ -1159,6 +1164,7 @@ func TestSelectScatterOrderBy(t *testing.T) { Fields: []*querypb.Field{ {Name: "col1", Type: sqltypes.Int32}, {Name: "col2", Type: sqltypes.Int32}, + {Name: "weight_string(col2)", Type: sqltypes.VarBinary}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ @@ -1167,6 +1173,7 @@ func TestSelectScatterOrderBy(t *testing.T) { // This will allow us to test that cross-shard ordering // still works correctly. sqltypes.NewInt32(int32(i % 4)), + sqltypes.NULL, }}, }}) conns = append(conns, sbc) @@ -1178,7 +1185,7 @@ func TestSelectScatterOrderBy(t *testing.T) { require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "select col1, col2 from `user` order by col2 desc", + Sql: "select col1, col2, weight_string(col2) from `user` order by col2 desc", BindVariables: map[string]*querypb.BindVariable{}, }} for _, conn := range conns { @@ -1287,11 +1294,13 @@ func TestStreamSelectScatterOrderBy(t *testing.T) { Fields: []*querypb.Field{ {Name: "id", Type: sqltypes.Int32}, {Name: "col", Type: sqltypes.Int32}, + {Name: "weight_string(col)", Type: sqltypes.VarBinary}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ sqltypes.NewInt32(1), sqltypes.NewInt32(int32(i % 4)), + sqltypes.NULL, }}, }}) conns = append(conns, sbc) @@ -1303,7 +1312,7 @@ func TestStreamSelectScatterOrderBy(t *testing.T) { require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "select id, col from `user` order by col desc", + Sql: "select id, col, weight_string(col) from `user` order by col desc", BindVariables: map[string]*querypb.BindVariable{}, }} for _, conn := range conns { @@ -1401,11 +1410,13 @@ func TestSelectScatterAggregate(t *testing.T) { Fields: []*querypb.Field{ {Name: "col", Type: sqltypes.Int32}, {Name: "sum(foo)", Type: sqltypes.Int32}, + {Name: "weight_string(col)", Type: sqltypes.VarBinary}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ sqltypes.NewInt32(int32(i % 4)), sqltypes.NewInt32(int32(i)), + sqltypes.NULL, }}, }}) conns = append(conns, sbc) @@ -1417,7 +1428,7 @@ func TestSelectScatterAggregate(t *testing.T) { require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "select col, sum(foo) from `user` group by col order by col asc", + Sql: "select col, sum(foo), weight_string(col) from `user` group by col order by col asc", BindVariables: map[string]*querypb.BindVariable{}, }} for _, conn := range conns { @@ -1458,11 +1469,13 @@ func TestStreamSelectScatterAggregate(t *testing.T) { Fields: []*querypb.Field{ {Name: "col", Type: sqltypes.Int32}, {Name: "sum(foo)", Type: sqltypes.Int32}, + {Name: "weight_string(col)", Type: sqltypes.VarBinary}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ sqltypes.NewInt32(int32(i % 4)), sqltypes.NewInt32(int32(i)), + sqltypes.NULL, }}, }}) conns = append(conns, sbc) @@ -1474,7 +1487,7 @@ func TestStreamSelectScatterAggregate(t *testing.T) { require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "select col, sum(foo) from `user` group by col order by col asc", + Sql: "select col, sum(foo), weight_string(col) from `user` group by col order by col asc", BindVariables: map[string]*querypb.BindVariable{}, }} for _, conn := range conns { @@ -1516,11 +1529,13 @@ func TestSelectScatterLimit(t *testing.T) { Fields: []*querypb.Field{ {Name: "col1", Type: sqltypes.Int32}, {Name: "col2", Type: sqltypes.Int32}, + {Name: "weight_string(col2)", Type: sqltypes.VarBinary}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ sqltypes.NewInt32(1), sqltypes.NewInt32(int32(i % 4)), + sqltypes.NULL, }}, }}) conns = append(conns, sbc) @@ -1532,7 +1547,7 @@ func TestSelectScatterLimit(t *testing.T) { require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "select col1, col2 from `user` order by col2 desc limit :__upper_limit", + Sql: "select col1, col2, weight_string(col2) from `user` order by col2 desc limit :__upper_limit", BindVariables: map[string]*querypb.BindVariable{"__upper_limit": sqltypes.Int64BindVariable(3)}, }} for _, conn := range conns { @@ -1582,11 +1597,13 @@ func TestStreamSelectScatterLimit(t *testing.T) { Fields: []*querypb.Field{ {Name: "col1", Type: sqltypes.Int32}, {Name: "col2", Type: sqltypes.Int32}, + {Name: "weight_string(col2)", Type: sqltypes.VarBinary}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ sqltypes.NewInt32(1), sqltypes.NewInt32(int32(i % 4)), + sqltypes.NULL, }}, }}) conns = append(conns, sbc) @@ -1598,7 +1615,7 @@ func TestStreamSelectScatterLimit(t *testing.T) { require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "select col1, col2 from `user` order by col2 desc limit :__upper_limit", + Sql: "select col1, col2, weight_string(col2) from `user` order by col2 desc limit :__upper_limit", BindVariables: map[string]*querypb.BindVariable{"__upper_limit": sqltypes.Int64BindVariable(3)}, }} for _, conn := range conns { diff --git a/go/vt/vtgate/planbuilder/logical_plan.go b/go/vt/vtgate/planbuilder/logical_plan.go index bf17fca74c9..81e4fae0aa2 100644 --- a/go/vt/vtgate/planbuilder/logical_plan.go +++ b/go/vt/vtgate/planbuilder/logical_plan.go @@ -69,7 +69,7 @@ type logicalPlan interface { SupplyCol(col *sqlparser.ColName) (rc *resultColumn, colNumber int) // SupplyWeightString must supply a weight_string expression of the - // specified column. + // specified column. It returns an error if we cannot supply a weight column for it. SupplyWeightString(colNumber int) (weightcolNumber int, err error) // Primitive returns the underlying primitive. diff --git a/go/vt/vtgate/planbuilder/memory_sort.go b/go/vt/vtgate/planbuilder/memory_sort.go index 053c5ad5e87..1f96ed47d5e 100644 --- a/go/vt/vtgate/planbuilder/memory_sort.go +++ b/go/vt/vtgate/planbuilder/memory_sort.go @@ -83,8 +83,9 @@ func newMemorySort(plan logicalPlan, orderBy sqlparser.OrderBy) (*memorySort, er return nil, fmt.Errorf("unsupported: memory sort: order by must reference a column in the select list: %s", sqlparser.String(order)) } ob := engine.OrderbyParams{ - Col: colNumber, - Desc: order.Direction == sqlparser.DescOrder, + Col: colNumber, + WeightStringCol: -1, + Desc: order.Direction == sqlparser.DescOrder, } ms.eMemorySort.OrderBy = append(ms.eMemorySort.OrderBy, ob) } @@ -110,18 +111,23 @@ func (ms *memorySort) SetLimit(limit *sqlparser.Limit) error { func (ms *memorySort) Wireup(plan logicalPlan, jt *jointab) error { for i, orderby := range ms.eMemorySort.OrderBy { rc := ms.resultColumns[orderby.Col] - if sqltypes.IsText(rc.column.typ) { + // Add a weight_string column if we know that the column is a textual column or if its type is unknown + if sqltypes.IsText(rc.column.typ) || rc.column.typ == sqltypes.Null { // If a weight string was previously requested, reuse it. if weightcolNumber, ok := ms.weightStrings[rc]; ok { - ms.eMemorySort.OrderBy[i].Col = weightcolNumber + ms.eMemorySort.OrderBy[i].WeightStringCol = weightcolNumber continue } weightcolNumber, err := ms.input.SupplyWeightString(orderby.Col) if err != nil { + _, isUnsupportedErr := err.(UnsupportedSupplyWeightString) + if isUnsupportedErr { + continue + } return err } ms.weightStrings[rc] = weightcolNumber - ms.eMemorySort.OrderBy[i].Col = weightcolNumber + ms.eMemorySort.OrderBy[i].WeightStringCol = weightcolNumber ms.eMemorySort.TruncateColumnCount = len(ms.resultColumns) } } diff --git a/go/vt/vtgate/planbuilder/merge_sort.go b/go/vt/vtgate/planbuilder/merge_sort.go index ee28f7fec71..1fea7894cc4 100644 --- a/go/vt/vtgate/planbuilder/merge_sort.go +++ b/go/vt/vtgate/planbuilder/merge_sort.go @@ -66,15 +66,20 @@ func (ms *mergeSort) Wireup(plan logicalPlan, jt *jointab) error { rb := ms.input.(*route) for i, orderby := range rb.eroute.OrderBy { rc := ms.resultColumns[orderby.Col] - if sqltypes.IsText(rc.column.typ) { + // Add a weight_string column if we know that the column is a textual column or if its type is unknown + if sqltypes.IsText(rc.column.typ) || rc.column.typ == sqltypes.Null { // If a weight string was previously requested, reuse it. if colNumber, ok := ms.weightStrings[rc]; ok { - rb.eroute.OrderBy[i].Col = colNumber + rb.eroute.OrderBy[i].WeightStringCol = colNumber continue } var err error - rb.eroute.OrderBy[i].Col, err = rb.SupplyWeightString(orderby.Col) + rb.eroute.OrderBy[i].WeightStringCol, err = rb.SupplyWeightString(orderby.Col) if err != nil { + _, isUnsupportedErr := err.(UnsupportedSupplyWeightString) + if isUnsupportedErr { + continue + } return err } ms.truncateColumnCount = len(ms.resultColumns) diff --git a/go/vt/vtgate/planbuilder/ordered_aggregate.go b/go/vt/vtgate/planbuilder/ordered_aggregate.go index 7e2065b7f64..82d06035b2b 100644 --- a/go/vt/vtgate/planbuilder/ordered_aggregate.go +++ b/go/vt/vtgate/planbuilder/ordered_aggregate.go @@ -340,6 +340,10 @@ func (oa *orderedAggregate) Wireup(plan logicalPlan, jt *jointab) error { } weightcolNumber, err := oa.input.SupplyWeightString(colNumber) if err != nil { + _, isUnsupportedErr := err.(UnsupportedSupplyWeightString) + if isUnsupportedErr { + continue + } return err } oa.weightStrings[rc] = weightcolNumber diff --git a/go/vt/vtgate/planbuilder/ordering.go b/go/vt/vtgate/planbuilder/ordering.go index 1e4c87bba46..c8c52739090 100644 --- a/go/vt/vtgate/planbuilder/ordering.go +++ b/go/vt/vtgate/planbuilder/ordering.go @@ -290,8 +290,9 @@ func planRouteOrdering(orderBy sqlparser.OrderBy, node *route) (logicalPlan, err return nil, fmt.Errorf("unsupported: in scatter query: order by must reference a column in the select list: %s", sqlparser.String(order)) } ob := engine.OrderbyParams{ - Col: colNumber, - Desc: order.Direction == sqlparser.DescOrder, + Col: colNumber, + WeightStringCol: -1, + Desc: order.Direction == sqlparser.DescOrder, } node.eroute.OrderBy = append(node.eroute.OrderBy, ob) diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go index 0250f4215f5..56441311b31 100644 --- a/go/vt/vtgate/planbuilder/plan_test.go +++ b/go/vt/vtgate/planbuilder/plan_test.go @@ -401,6 +401,7 @@ func testFile(t *testing.T, filename, tempDir string, vschema *vschemaWrapper, c t.Run(filename, func(t *testing.T) { expected := &strings.Builder{} fail := checkAllTests + var outFirstPlanner string for tcase := range iterateExecFile(filename) { t.Run(fmt.Sprintf("%d V3: %s", tcase.lineno, tcase.comments), func(t *testing.T) { vschema.version = V3 @@ -414,6 +415,7 @@ func testFile(t *testing.T, filename, tempDir string, vschema *vschemaWrapper, c if err != nil { out = `"` + out + `"` } + outFirstPlanner = out expected.WriteString(fmt.Sprintf("%s\"%s\"\n%s\n", tcase.comments, escapeNewLines(tcase.input), out)) }) @@ -445,7 +447,7 @@ func testFile(t *testing.T, filename, tempDir string, vschema *vschemaWrapper, c out = `"` + out + `"` } - if tcase.output == out { + if outFirstPlanner == out { expected.WriteString(samePlanMarker) } else { expected.WriteString(fmt.Sprintf("%s\n", out)) diff --git a/go/vt/vtgate/planbuilder/sql_calc_found_rows.go b/go/vt/vtgate/planbuilder/sql_calc_found_rows.go index 2729dbfbc3a..21a7bae3d29 100644 --- a/go/vt/vtgate/planbuilder/sql_calc_found_rows.go +++ b/go/vt/vtgate/planbuilder/sql_calc_found_rows.go @@ -91,7 +91,7 @@ func (s *sqlCalcFoundRows) SupplyCol(col *sqlparser.ColName) (*resultColumn, int //SupplyWeightString implements the logicalPlan interface func (s *sqlCalcFoundRows) SupplyWeightString(int) (weightcolNumber int, err error) { - return 0, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "[BUG] unreachable: sqlCalcFoundRows.SupplyWeightString") + return 0, UnsupportedSupplyWeightString{Type: "sqlCalcFoundRows"} } // Rewrite implements the logicalPlan interface diff --git a/go/vt/vtgate/planbuilder/symtab.go b/go/vt/vtgate/planbuilder/symtab.go index 61cc1d331cf..afaffea1f97 100644 --- a/go/vt/vtgate/planbuilder/symtab.go +++ b/go/vt/vtgate/planbuilder/symtab.go @@ -22,6 +22,9 @@ import ( "strconv" "strings" + vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/vterrors" + "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vtgate/vindexes" @@ -564,9 +567,37 @@ func newResultColumn(expr *sqlparser.AliasedExpr, origin logicalPlan) *resultCol } else { // We don't generate an alias if the expression is non-trivial. // Just to be safe, generate an anonymous column for the expression. + typ, err := GetReturnType(expr.Expr) rc.column = &column{ origin: origin, } + if err == nil { + rc.column.typ = typ + } } return rc } + +// GetReturnType returns the type of the select expression that MySQL will return +func GetReturnType(input sqlparser.Expr) (querypb.Type, error) { + switch node := input.(type) { + case *sqlparser.FuncExpr: + functionName := strings.ToUpper(node.Name.String()) + switch functionName { + case "ABS": + // Returned value depends on the return type of the input + if len(node.Exprs) == 1 { + expr, isAliasedExpr := node.Exprs[0].(*sqlparser.AliasedExpr) + if isAliasedExpr { + return GetReturnType(expr.Expr) + } + } + case "COUNT": + return querypb.Type_INT64, nil + } + case *sqlparser.ColName: + col := node.Metadata.(*column) + return col.typ, nil + } + return 0, vterrors.Errorf(vtrpcpb.Code_UNIMPLEMENTED, "cannot evaluate return type for %T", input) +} diff --git a/go/vt/vtgate/planbuilder/symtab_test.go b/go/vt/vtgate/planbuilder/symtab_test.go index bd9ac8a6a90..c327ad9c385 100644 --- a/go/vt/vtgate/planbuilder/symtab_test.go +++ b/go/vt/vtgate/planbuilder/symtab_test.go @@ -16,6 +16,16 @@ limitations under the License. package planbuilder +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + querypb "vitess.io/vitess/go/vt/proto/query" + "vitess.io/vitess/go/vt/sqlparser" +) + /* func TestSymtabAddVSchemaTable(t *testing.T) { tname := sqlparser.TableName{Name: sqlparser.NewTableIdent("t")} @@ -178,3 +188,53 @@ func TestSymtabAddVSchemaTable(t *testing.T) { } } */ + +func TestGetReturnType(t *testing.T) { + tests := []struct { + input sqlparser.Expr + output querypb.Type + expectedErr error + }{{ + input: &sqlparser.FuncExpr{Name: sqlparser.NewColIdent("Abs"), Exprs: sqlparser.SelectExprs{ + &sqlparser.AliasedExpr{ + Expr: &sqlparser.ColName{ + Name: sqlparser.NewColIdent("A"), + Metadata: &column{ + typ: querypb.Type_DECIMAL, + }, + }, + }, + }}, + output: querypb.Type_DECIMAL, + expectedErr: nil, + }, { + input: &sqlparser.FuncExpr{Name: sqlparser.NewColIdent("Count"), Exprs: sqlparser.SelectExprs{ + &sqlparser.StarExpr{}, + }}, + output: querypb.Type_INT64, + expectedErr: nil, + }, { + input: &sqlparser.FuncExpr{Name: sqlparser.NewColIdent("cOunt"), Exprs: sqlparser.SelectExprs{ + &sqlparser.StarExpr{}, + }}, + output: querypb.Type_INT64, + expectedErr: nil, + }, { + input: &sqlparser.FuncExpr{Name: sqlparser.NewColIdent("Abs"), Exprs: sqlparser.SelectExprs{ + &sqlparser.StarExpr{}, + }}, + expectedErr: fmt.Errorf("cannot evaluate return type for *sqlparser.FuncExpr"), + }} + + for _, test := range tests { + t.Run(sqlparser.String(test.input), func(t *testing.T) { + got, err := GetReturnType(test.input) + if test.expectedErr != nil { + require.EqualError(t, err, test.expectedErr.Error()) + } else { + require.NoError(t, err) + require.Equal(t, test.output, got) + } + }) + } +} diff --git a/go/vt/vtgate/planbuilder/testdata/aggr_cases.txt b/go/vt/vtgate/planbuilder/testdata/aggr_cases.txt index 747d50f67ac..7c76066eec1 100644 --- a/go/vt/vtgate/planbuilder/testdata/aggr_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/aggr_cases.txt @@ -116,9 +116,37 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select count(*), a, textcol1, b, weight_string(textcol1) from `user` where 1 != 1 group by a, textcol1, b", - "OrderBy": "1 ASC, 4 ASC, 3 ASC", - "Query": "select count(*), a, textcol1, b, weight_string(textcol1) from `user` group by a, textcol1, b order by a asc, textcol1 asc, b asc", + "FieldQuery": "select count(*), a, textcol1, b, weight_string(textcol1), weight_string(a), weight_string(b) from `user` where 1 != 1 group by a, textcol1, b", + "OrderBy": "1 ASC, 2 ASC, 3 ASC", + "Query": "select count(*), a, textcol1, b, weight_string(textcol1), weight_string(a), weight_string(b) from `user` group by a, textcol1, b order by a asc, textcol1 asc, b asc", + "Table": "`user`" + } + ] + } +} + +# scatter group by a integer column. Do not add weight strings for this. +"select count(*), intcol from user group by intcol" +{ + "QueryType": "SELECT", + "Original": "select count(*), intcol from user group by intcol", + "Instructions": { + "OperatorType": "Aggregate", + "Variant": "Ordered", + "Aggregates": "count(0)", + "Distinct": "false", + "GroupBy": "1", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "SelectScatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select count(*), intcol from `user` where 1 != 1 group by intcol", + "OrderBy": "1 ASC", + "Query": "select count(*), intcol from `user` group by intcol order by intcol asc", "Table": "`user`" } ] @@ -133,7 +161,7 @@ Gen4 plan same as above "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "0 ASC, 4 ASC", + "OrderBy": "0 ASC, 2 ASC", "Inputs": [ { "OperatorType": "Aggregate", @@ -149,9 +177,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select count(*) as k, a, textcol1, b, weight_string(textcol1) from `user` where 1 != 1 group by a, textcol1, b", - "OrderBy": "4 ASC, 1 ASC, 3 ASC", - "Query": "select count(*) as k, a, textcol1, b, weight_string(textcol1) from `user` group by a, textcol1, b order by textcol1 asc, a asc, b asc", + "FieldQuery": "select count(*) as k, a, textcol1, b, weight_string(textcol1), weight_string(a), weight_string(b) from `user` where 1 != 1 group by a, textcol1, b", + "OrderBy": "2 ASC, 1 ASC, 3 ASC", + "Query": "select count(*) as k, a, textcol1, b, weight_string(textcol1), weight_string(a), weight_string(b) from `user` group by a, textcol1, b order by textcol1 asc, a asc, b asc", "Table": "`user`" } ] @@ -282,9 +310,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col1, col2 from `user` where 1 != 1 group by col1", + "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1", "OrderBy": "0 ASC, 1 ASC, 0 ASC", - "Query": "select distinct col1, col2 from `user` group by col1 order by col1 asc, col2 asc, col1 asc", + "Query": "select distinct col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1 order by col1 asc, col2 asc, col1 asc", "Table": "`user`" } ] @@ -383,9 +411,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col, count(*) from `user` where 1 != 1 group by col", + "FieldQuery": "select col, count(*), weight_string(col) from `user` where 1 != 1 group by col", "OrderBy": "0 ASC", - "Query": "select col, count(*) from `user` group by col order by col asc", + "Query": "select col, count(*), weight_string(col) from `user` group by col order by col asc", "Table": "`user`" } ] @@ -415,9 +443,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select `name`, count(*) from `user` where 1 != 1 group by `name`", + "FieldQuery": "select `name`, count(*), weight_string(`name`) from `user` where 1 != 1 group by `name`", "OrderBy": "0 ASC", - "Query": "select `name`, count(*) from `user` group by `name` order by `name` asc", + "Query": "select `name`, count(*), weight_string(`name`) from `user` group by `name` order by `name` asc", "Table": "`user`" } ] @@ -596,9 +624,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col from `user` where 1 != 1", + "FieldQuery": "select col, weight_string(col) from `user` where 1 != 1", "OrderBy": "0 ASC", - "Query": "select distinct col from `user` order by col asc", + "Query": "select distinct col, weight_string(col) from `user` order by col asc", "Table": "`user`" } ] @@ -623,9 +651,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col from `user` where 1 != 1 group by col", + "FieldQuery": "select col, weight_string(col) from `user` where 1 != 1 group by col", "OrderBy": "0 ASC", - "Query": "select col from `user` group by col order by col asc", + "Query": "select col, weight_string(col) from `user` group by col order by col asc", "Table": "`user`" } ] @@ -669,9 +697,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col, count(distinct id) from `user` where 1 != 1 group by col", + "FieldQuery": "select col, count(distinct id), weight_string(col) from `user` where 1 != 1 group by col", "OrderBy": "0 ASC", - "Query": "select col, count(distinct id) from `user` group by col order by col asc", + "Query": "select col, count(distinct id), weight_string(col) from `user` group by col order by col asc", "Table": "`user`" } ] @@ -697,9 +725,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col1, col2 from `user` where 1 != 1 group by col1, col2", + "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, col2", "OrderBy": "0 ASC, 1 ASC", - "Query": "select col1, col2 from `user` group by col1, col2 order by col1 asc, col2 asc", + "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, col2 order by col1 asc, col2 asc", "Table": "`user`" } ] @@ -724,9 +752,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col2 from `user` where 1 != 1 group by col2", + "FieldQuery": "select col2, weight_string(col2) from `user` where 1 != 1 group by col2", "OrderBy": "0 ASC", - "Query": "select col2 from `user` group by col2 order by col2 asc", + "Query": "select col2, weight_string(col2) from `user` group by col2 order by col2 asc", "Table": "`user`" } ] @@ -752,9 +780,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col1, col2 from `user` where 1 != 1 group by col1, col2", + "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, col2", "OrderBy": "0 ASC, 1 ASC", - "Query": "select col1, col2 from `user` group by col1, col2 order by col1 asc, col2 asc", + "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, col2 order by col1 asc, col2 asc", "Table": "`user`" } ] @@ -780,9 +808,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col1, col2 from `user` where 1 != 1 group by col1, col2", + "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, col2", "OrderBy": "0 ASC, 1 ASC", - "Query": "select col1, col2 from `user` group by col1, col2 order by col1 asc, col2 asc", + "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, col2 order by col1 asc, col2 asc", "Table": "`user`" } ] @@ -808,9 +836,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col1, min(distinct col2) from `user` where 1 != 1 group by col1", + "FieldQuery": "select col1, min(distinct col2), weight_string(col1) from `user` where 1 != 1 group by col1", "OrderBy": "0 ASC", - "Query": "select col1, min(distinct col2) from `user` group by col1 order by col1 asc", + "Query": "select col1, min(distinct col2), weight_string(col1) from `user` group by col1 order by col1 asc", "Table": "`user`" } ] @@ -841,9 +869,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col1, col2 from `user` where 1 != 1 group by col1, col2", + "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, col2", "OrderBy": "0 ASC, 1 ASC", - "Query": "select col1, col2 from `user` group by col1, col2 order by col1 asc, col2 asc", + "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, col2 order by col1 asc, col2 asc", "Table": "`user`" } ] @@ -875,9 +903,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, count(*) from `user` where 1 != 1 group by b, a", + "FieldQuery": "select a, b, count(*), weight_string(b), weight_string(a) from `user` where 1 != 1 group by b, a", "OrderBy": "1 ASC, 0 ASC", - "Query": "select a, b, count(*) from `user` group by b, a order by b asc, a asc", + "Query": "select a, b, count(*), weight_string(b), weight_string(a) from `user` group by b, a order by b asc, a asc", "Table": "`user`" } ] @@ -903,9 +931,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, count(*) from `user` where 1 != 1 group by 2, 1", + "FieldQuery": "select a, b, count(*), weight_string(b), weight_string(a) from `user` where 1 != 1 group by 2, 1", "OrderBy": "1 ASC, 0 ASC", - "Query": "select a, b, count(*) from `user` group by 2, 1 order by b asc, a asc", + "Query": "select a, b, count(*), weight_string(b), weight_string(a) from `user` group by 2, 1 order by b asc, a asc", "Table": "`user`" } ] @@ -931,9 +959,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, count(*) from `user` where 1 != 1 group by b, a", + "FieldQuery": "select a, b, count(*), weight_string(b), weight_string(a) from `user` where 1 != 1 group by b, a", "OrderBy": "1 ASC, 0 ASC", - "Query": "select a, b, count(*) from `user` group by b, a order by b asc, a asc", + "Query": "select a, b, count(*), weight_string(b), weight_string(a) from `user` group by b, a order by b asc, a asc", "Table": "`user`" } ] @@ -958,9 +986,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col from `user` where 1 != 1 group by 1", + "FieldQuery": "select col, weight_string(col) from `user` where 1 != 1 group by 1", "OrderBy": "0 ASC", - "Query": "select col from `user` group by 1 order by col asc", + "Query": "select col, weight_string(col) from `user` group by 1 order by col asc", "Table": "`user`" } ] @@ -1020,9 +1048,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, c, d, count(*) from `user` where 1 != 1 group by 1, 2, 3", + "FieldQuery": "select a, b, c, d, count(*), weight_string(a), weight_string(b), weight_string(c) from `user` where 1 != 1 group by 1, 2, 3", "OrderBy": "0 ASC, 1 ASC, 2 ASC", - "Query": "select a, b, c, d, count(*) from `user` group by 1, 2, 3 order by 1 asc, 2 asc, 3 asc", + "Query": "select a, b, c, d, count(*), weight_string(a), weight_string(b), weight_string(c) from `user` group by 1, 2, 3 order by 1 asc, 2 asc, 3 asc", "Table": "`user`" } ] @@ -1048,9 +1076,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, c, d, count(*) from `user` where 1 != 1 group by 1, 2, 3", + "FieldQuery": "select a, b, c, d, count(*), weight_string(a), weight_string(b), weight_string(c) from `user` where 1 != 1 group by 1, 2, 3", "OrderBy": "0 ASC, 1 ASC, 2 ASC", - "Query": "select a, b, c, d, count(*) from `user` group by 1, 2, 3 order by a asc, b asc, c asc", + "Query": "select a, b, c, d, count(*), weight_string(a), weight_string(b), weight_string(c) from `user` group by 1, 2, 3 order by a asc, b asc, c asc", "Table": "`user`" } ] @@ -1076,9 +1104,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, c, d, count(*) from `user` where 1 != 1 group by 1, 2, 3, 4", + "FieldQuery": "select a, b, c, d, count(*), weight_string(d), weight_string(b), weight_string(a), weight_string(c) from `user` where 1 != 1 group by 1, 2, 3, 4", "OrderBy": "3 ASC, 1 ASC, 0 ASC, 2 ASC", - "Query": "select a, b, c, d, count(*) from `user` group by 1, 2, 3, 4 order by d asc, b asc, a asc, c asc", + "Query": "select a, b, c, d, count(*), weight_string(d), weight_string(b), weight_string(a), weight_string(c) from `user` group by 1, 2, 3, 4 order by d asc, b asc, a asc, c asc", "Table": "`user`" } ] @@ -1104,9 +1132,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, c, d, count(*) from `user` where 1 != 1 group by 3, 2, 1, 4", + "FieldQuery": "select a, b, c, d, count(*), weight_string(d), weight_string(b), weight_string(a), weight_string(c) from `user` where 1 != 1 group by 3, 2, 1, 4", "OrderBy": "3 ASC, 1 ASC, 0 ASC, 2 ASC", - "Query": "select a, b, c, d, count(*) from `user` group by 3, 2, 1, 4 order by d asc, b asc, a asc, c asc", + "Query": "select a, b, c, d, count(*), weight_string(d), weight_string(b), weight_string(a), weight_string(c) from `user` group by 3, 2, 1, 4 order by d asc, b asc, a asc, c asc", "Table": "`user`" } ] @@ -1132,9 +1160,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, c, count(*) from `user` where 1 != 1 group by 3, 2, 1", + "FieldQuery": "select a, b, c, count(*), weight_string(a), weight_string(c), weight_string(b) from `user` where 1 != 1 group by 3, 2, 1", "OrderBy": "0 DESC, 2 DESC, 1 ASC", - "Query": "select a, b, c, count(*) from `user` group by 3, 2, 1 order by 1 desc, 3 desc, b asc", + "Query": "select a, b, c, count(*), weight_string(a), weight_string(c), weight_string(b) from `user` group by 3, 2, 1 order by 1 desc, 3 desc, b asc", "Table": "`user`" } ] @@ -1168,9 +1196,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col, count(*) from `user` where 1 != 1 group by col", + "FieldQuery": "select col, count(*), weight_string(col) from `user` where 1 != 1 group by col", "OrderBy": "0 ASC", - "Query": "select col, count(*) from `user` group by col order by col asc limit :__upper_limit", + "Query": "select col, count(*), weight_string(col) from `user` group by col order by col asc limit :__upper_limit", "Table": "`user`" } ] @@ -1261,9 +1289,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select a, count(*) from `user` where 1 != 1", + "FieldQuery": "select a, count(*), weight_string(a) from `user` where 1 != 1", "OrderBy": "0 ASC", - "Query": "select a, count(*) from `user` order by a asc", + "Query": "select a, count(*), weight_string(a) from `user` order by a asc", "Table": "`user`" } ] @@ -1294,9 +1322,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select a, count(*) from `user` where 1 != 1 group by a", + "FieldQuery": "select a, count(*), weight_string(a) from `user` where 1 != 1 group by a", "OrderBy": "0 ASC, 0 ASC", - "Query": "select a, count(*) from `user` group by a order by a asc, a asc", + "Query": "select a, count(*), weight_string(a) from `user` group by a order by a asc, a asc", "Table": "`user`" } ] diff --git a/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.txt b/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.txt index cb743923963..53b22207510 100644 --- a/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.txt @@ -23,9 +23,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, count(*) from `user` where 1 != 1 group by a", + "FieldQuery": "select a, b, count(*), weight_string(b), weight_string(a) from `user` where 1 != 1 group by a", "OrderBy": "0 ASC", - "Query": "select a, b, count(*) from `user` group by a order by a asc", + "Query": "select a, b, count(*), weight_string(b), weight_string(a) from `user` group by a order by a asc", "Table": "`user`" } ] @@ -58,9 +58,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, count(*) as k from `user` where 1 != 1 group by a", + "FieldQuery": "select a, b, count(*) as k, weight_string(a) from `user` where 1 != 1 group by a", "OrderBy": "0 ASC", - "Query": "select a, b, count(*) as k from `user` group by a order by a asc", + "Query": "select a, b, count(*) as k, weight_string(a) from `user` group by a order by a asc", "Table": "`user`" } ] @@ -93,9 +93,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, count(*) as k from `user` where 1 != 1 group by a", + "FieldQuery": "select a, b, count(*) as k, weight_string(b), weight_string(a) from `user` where 1 != 1 group by a", "OrderBy": "0 ASC", - "Query": "select a, b, count(*) as k from `user` group by a order by a asc", + "Query": "select a, b, count(*) as k, weight_string(b), weight_string(a) from `user` group by a order by a asc", "Table": "`user`" } ] @@ -132,9 +132,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, count(*) as k from `user` where 1 != 1 group by a", + "FieldQuery": "select a, b, count(*) as k, weight_string(a) from `user` where 1 != 1 group by a", "OrderBy": "0 ASC", - "Query": "select a, b, count(*) as k from `user` group by a order by a asc", + "Query": "select a, b, count(*) as k, weight_string(a) from `user` group by a order by a asc", "Table": "`user`" } ] @@ -169,9 +169,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, count(*) as k from `user` where 1 != 1 group by a", + "FieldQuery": "select a, b, count(*) as k, weight_string(a) from `user` where 1 != 1 group by a", "OrderBy": "0 ASC", - "Query": "select a, b, count(*) as k from `user` group by a order by 1 asc", + "Query": "select a, b, count(*) as k, weight_string(a) from `user` group by a order by 1 asc", "Table": "`user`" } ] @@ -189,7 +189,7 @@ "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "2 ASC, 1 ASC, 2 ASC", + "OrderBy": "0 ASC, 1 ASC, 0 ASC", "Inputs": [ { "OperatorType": "Aggregate", @@ -206,7 +206,7 @@ "Sharded": true }, "FieldQuery": "select textcol1 as t, count(*) as k, weight_string(textcol1) from `user` where 1 != 1 group by textcol1", - "OrderBy": "2 ASC, 2 ASC", + "OrderBy": "0 ASC, 0 ASC", "Query": "select textcol1 as t, count(*) as k, weight_string(textcol1) from `user` group by textcol1 order by textcol1 asc, textcol1 asc", "Table": "`user`" } @@ -235,7 +235,7 @@ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "-1,-2", + "JoinColumnIndexes": "-1,-2,-3", "TableName": "`user`_user_extra", "Inputs": [ { @@ -245,8 +245,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select `user`.id, `user`.col from `user` where 1 != 1", - "Query": "select `user`.id, `user`.col from `user`", + "FieldQuery": "select `user`.id, `user`.col, weight_string(`user`.id) from `user` where 1 != 1", + "Query": "select `user`.id, `user`.col, weight_string(`user`.id) from `user`", "Table": "`user`" }, { @@ -281,7 +281,7 @@ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "-1,-2,1", + "JoinColumnIndexes": "-1,-2,1,2", "TableName": "`user`_music", "Inputs": [ { @@ -306,8 +306,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select music.col3 as c from music where 1 != 1", - "Query": "select music.col3 as c from music where music.id = :user_id", + "FieldQuery": "select music.col3 as c, weight_string(music.col3) from music where 1 != 1", + "Query": "select music.col3 as c, weight_string(music.col3) from music where music.id = :user_id", "Table": "music", "Values": [ ":user_id" @@ -333,7 +333,7 @@ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "-1,-2,1", + "JoinColumnIndexes": "-1,-2,1,-3,2,-4", "TableName": "`user`_music", "Inputs": [ { @@ -343,8 +343,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select `user`.col1 as a, `user`.col2, `user`.id from `user` where 1 != 1", - "Query": "select `user`.col1 as a, `user`.col2, `user`.id from `user` where `user`.id = 1", + "FieldQuery": "select `user`.col1 as a, `user`.col2, weight_string(`user`.col1), weight_string(`user`.col2), `user`.id from `user` where 1 != 1", + "Query": "select `user`.col1 as a, `user`.col2, weight_string(`user`.col1), weight_string(`user`.col2), `user`.id from `user` where `user`.id = 1", "Table": "`user`", "Values": [ 1 @@ -358,8 +358,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select music.col3 from music where 1 != 1", - "Query": "select music.col3 from music where music.id = :user_id", + "FieldQuery": "select music.col3, weight_string(music.col3) from music where 1 != 1", + "Query": "select music.col3, weight_string(music.col3) from music where music.id = :user_id", "Table": "music", "Values": [ ":user_id" @@ -380,12 +380,12 @@ "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "3 ASC, 2 ASC", + "OrderBy": "1 ASC, 2 ASC", "Inputs": [ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "-1,-2,1,-3", + "JoinColumnIndexes": "-1,-2,1,-3,2", "TableName": "`user`_unsharded", "Inputs": [ { @@ -406,8 +406,8 @@ "Name": "main", "Sharded": false }, - "FieldQuery": "select un.col2 from unsharded as un where 1 != 1", - "Query": "select un.col2 from unsharded as un", + "FieldQuery": "select un.col2, weight_string(un.col2) from unsharded as un where 1 != 1", + "Query": "select un.col2, weight_string(un.col2) from unsharded as un", "Table": "unsharded" } ] @@ -424,12 +424,12 @@ "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "3 ASC, 2 ASC", + "OrderBy": "1 ASC, 2 ASC", "Inputs": [ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "1,2,-1,3", + "JoinColumnIndexes": "1,2,-1,3,-2", "TableName": "unsharded_`user`", "Inputs": [ { @@ -439,8 +439,8 @@ "Name": "main", "Sharded": false }, - "FieldQuery": "select un.col2 from unsharded as un where 1 != 1", - "Query": "select un.col2 from unsharded as un", + "FieldQuery": "select un.col2, weight_string(un.col2) from unsharded as un where 1 != 1", + "Query": "select un.col2, weight_string(un.col2) from unsharded as un", "Table": "unsharded" }, { @@ -504,9 +504,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select a from `user` where 1 != 1", + "FieldQuery": "select a, weight_string(a) from `user` where 1 != 1", "OrderBy": "0 DESC", - "Query": "select a from `user` order by binary a desc", + "Query": "select a, weight_string(a) from `user` order by binary a desc", "Table": "`user`" } } @@ -529,9 +529,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select u.a from `user` as u where 1 != 1", + "FieldQuery": "select u.a, weight_string(u.a) from `user` as u where 1 != 1", "OrderBy": "0 DESC", - "Query": "select u.a from `user` as u order by binary a desc", + "Query": "select u.a, weight_string(u.a) from `user` as u order by binary a desc", "Table": "`user`" }, { diff --git a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.txt b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.txt index 28263baae24..a8e9497f8e4 100644 --- a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.txt @@ -191,9 +191,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col from `user` where 1 != 1", + "FieldQuery": "select col, weight_string(col) from `user` where 1 != 1", "OrderBy": "0 ASC", - "Query": "select col from `user` order by col asc", + "Query": "select col, weight_string(col) from `user` order by col asc", "Table": "`user`" } } @@ -210,9 +210,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select user_id, col1, col2 from authoritative where 1 != 1", + "FieldQuery": "select user_id, col1, col2, weight_string(user_id) from authoritative where 1 != 1", "OrderBy": "0 ASC", - "Query": "select user_id, col1, col2 from authoritative order by user_id asc", + "Query": "select user_id, col1, col2, weight_string(user_id) from authoritative order by user_id asc", "Table": "authoritative" } } @@ -230,7 +230,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select user_id, col1, col2, weight_string(col1) from authoritative where 1 != 1", - "OrderBy": "3 ASC", + "OrderBy": "1 ASC", "Query": "select user_id, col1, col2, weight_string(col1) from authoritative order by col1 asc", "Table": "authoritative" } @@ -248,9 +248,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select a, textcol1, b, weight_string(textcol1) from `user` where 1 != 1", - "OrderBy": "0 ASC, 3 ASC, 2 ASC", - "Query": "select a, textcol1, b, weight_string(textcol1) from `user` order by a asc, textcol1 asc, b asc", + "FieldQuery": "select a, textcol1, b, weight_string(a), weight_string(textcol1), weight_string(b) from `user` where 1 != 1", + "OrderBy": "0 ASC, 1 ASC, 2 ASC", + "Query": "select a, textcol1, b, weight_string(a), weight_string(textcol1), weight_string(b) from `user` order by a asc, textcol1 asc, b asc", "Table": "`user`" } } @@ -267,9 +267,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select a, `user`.textcol1, b, weight_string(`user`.textcol1) from `user` where 1 != 1", - "OrderBy": "0 ASC, 3 ASC, 2 ASC", - "Query": "select a, `user`.textcol1, b, weight_string(`user`.textcol1) from `user` order by a asc, textcol1 asc, b asc", + "FieldQuery": "select a, `user`.textcol1, b, weight_string(a), weight_string(`user`.textcol1), weight_string(b) from `user` where 1 != 1", + "OrderBy": "0 ASC, 1 ASC, 2 ASC", + "Query": "select a, `user`.textcol1, b, weight_string(a), weight_string(`user`.textcol1), weight_string(b) from `user` order by a asc, textcol1 asc, b asc", "Table": "`user`" } } @@ -286,9 +286,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select a, textcol1, b, textcol2, weight_string(textcol1), weight_string(textcol2) from `user` where 1 != 1", - "OrderBy": "0 ASC, 4 ASC, 2 ASC, 5 ASC", - "Query": "select a, textcol1, b, textcol2, weight_string(textcol1), weight_string(textcol2) from `user` order by a asc, textcol1 asc, b asc, textcol2 asc", + "FieldQuery": "select a, textcol1, b, textcol2, weight_string(a), weight_string(textcol1), weight_string(b), weight_string(textcol2) from `user` where 1 != 1", + "OrderBy": "0 ASC, 1 ASC, 2 ASC, 3 ASC", + "Query": "select a, textcol1, b, textcol2, weight_string(a), weight_string(textcol1), weight_string(b), weight_string(textcol2) from `user` order by a asc, textcol1 asc, b asc, textcol2 asc", "Table": "`user`" } } @@ -342,9 +342,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col from `user` where 1 != 1", + "FieldQuery": "select col, weight_string(col) from `user` where 1 != 1", "OrderBy": "0 ASC", - "Query": "select col from `user` where :__sq_has_values1 = 1 and col in ::__sq1 order by col asc", + "Query": "select col, weight_string(col) from `user` where :__sq_has_values1 = 1 and col in ::__sq1 order by col asc", "Table": "`user`" } ] diff --git a/go/vt/vtgate/planbuilder/testdata/schema_test.json b/go/vt/vtgate/planbuilder/testdata/schema_test.json index 17f6576dbe4..fd56617707b 100644 --- a/go/vt/vtgate/planbuilder/testdata/schema_test.json +++ b/go/vt/vtgate/planbuilder/testdata/schema_test.json @@ -111,6 +111,10 @@ "name": "textcol1", "type": "VARCHAR" }, + { + "name": "intcol", + "type": "INT16" + }, { "name": "textcol2", "type": "VARCHAR" diff --git a/go/vt/vtgate/planbuilder/testdata/select_cases.txt b/go/vt/vtgate/planbuilder/testdata/select_cases.txt index 1fcee99d2ec..d0e5f0629a8 100644 --- a/go/vt/vtgate/planbuilder/testdata/select_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/select_cases.txt @@ -849,9 +849,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select user_id from music where 1 != 1", + "FieldQuery": "select user_id, weight_string(user_id) from music where 1 != 1", "OrderBy": "0 ASC", - "Query": "select user_id from music order by user_id asc limit :__upper_limit", + "Query": "select user_id, weight_string(user_id) from music order by user_id asc limit :__upper_limit", "Table": "music" } ] @@ -1544,9 +1544,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select user_id, count(id) from music where 1 != 1 group by user_id", + "FieldQuery": "select user_id, count(id), weight_string(user_id) from music where 1 != 1 group by user_id", "OrderBy": "0 ASC", - "Query": "select user_id, count(id) from music group by user_id having count(user_id) = 1 order by user_id asc limit :__upper_limit", + "Query": "select user_id, count(id), weight_string(user_id) from music group by user_id having count(user_id) = 1 order by user_id asc limit :__upper_limit", "Table": "music" } ] diff --git a/go/vt/vtgate/planbuilder/testdata/union_cases.txt b/go/vt/vtgate/planbuilder/testdata/union_cases.txt index bc82ca929f4..c7572346b1d 100644 --- a/go/vt/vtgate/planbuilder/testdata/union_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/union_cases.txt @@ -15,21 +15,7 @@ "Table": "`user`" } } -{ - "QueryType": "SELECT", - "Original": "select id from user union all select id from music", - "Instructions": { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id from `user` where 1 != 1 union all select id from music where 1 != 1", - "Query": "select id from `user` union all select id from music", - "Table": "`user`" - } -} +Gen4 plan same as above # union distinct between two scatter selects "select id from user union select id from music" @@ -69,42 +55,7 @@ ] } } -{ - "QueryType": "SELECT", - "Original": "select id from user union select id from music", - "Instructions": { - "OperatorType": "Distinct", - "Inputs": [ - { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id from `user` where 1 != 1", - "Query": "select id from `user`", - "Table": "`user`" - }, - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id from music where 1 != 1", - "Query": "select id from music", - "Table": "music" - } - ] - } - ] - } -} +Gen4 plan same as above # union all between two SelectEqualUnique "select id from user where id = 1 union all select id from user where id = 5" @@ -147,45 +98,7 @@ ] } } -{ - "QueryType": "SELECT", - "Original": "select id from user where id = 1 union all select id from user where id = 5", - "Instructions": { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectEqualUnique", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id from `user` where 1 != 1", - "Query": "select id from `user` where id = 1", - "Table": "`user`", - "Values": [ - 1 - ], - "Vindex": "user_index" - }, - { - "OperatorType": "Route", - "Variant": "SelectEqualUnique", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id from `user` where 1 != 1", - "Query": "select id from `user` where id = 5", - "Table": "`user`", - "Values": [ - 5 - ], - "Vindex": "user_index" - } - ] - } -} +Gen4 plan same as above #almost dereks query - two queries with order by and limit being scattered to two different sets of tablets "(SELECT id FROM user ORDER BY id DESC LIMIT 1) UNION ALL (SELECT id FROM music ORDER BY id DESC LIMIT 1)" @@ -206,54 +119,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select id from `user` where 1 != 1", - "OrderBy": "0 DESC", - "Query": "select id from `user` order by id desc limit :__upper_limit", - "Table": "`user`" - } - ] - }, - { - "OperatorType": "Limit", - "Count": 1, - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id from music where 1 != 1", - "OrderBy": "0 DESC", - "Query": "select id from music order by id desc limit :__upper_limit", - "Table": "music" - } - ] - } - ] - } -} -{ - "QueryType": "SELECT", - "Original": "(SELECT id FROM user ORDER BY id DESC LIMIT 1) UNION ALL (SELECT id FROM music ORDER BY id DESC LIMIT 1)", - "Instructions": { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Limit", - "Count": 1, - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id from `user` where 1 != 1", + "FieldQuery": "select id, weight_string(id) from `user` where 1 != 1", "OrderBy": "0 DESC", - "Query": "select id from `user` order by id desc limit :__upper_limit", + "Query": "select id, weight_string(id) from `user` order by id desc limit :__upper_limit", "Table": "`user`" } ] @@ -269,9 +137,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select id from music where 1 != 1", + "FieldQuery": "select id, weight_string(id) from music where 1 != 1", "OrderBy": "0 DESC", - "Query": "select id from music order by id desc limit :__upper_limit", + "Query": "select id, weight_string(id) from music order by id desc limit :__upper_limit", "Table": "music" } ] @@ -279,6 +147,7 @@ ] } } +Gen4 plan same as above # Union all "select col1, col2 from user union all select col1, col2 from user_extra" @@ -297,21 +166,7 @@ "Table": "`user`" } } -{ - "QueryType": "SELECT", - "Original": "select col1, col2 from user union all select col1, col2 from user_extra", - "Instructions": { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select col1, col2 from `user` where 1 != 1 union all select col1, col2 from user_extra where 1 != 1", - "Query": "select col1, col2 from `user` union all select col1, col2 from user_extra", - "Table": "`user`" - } -} +Gen4 plan same as above # union operations in subqueries (FROM) "select * from (select * from user union all select * from user_extra) as t" @@ -350,54 +205,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select id from `user` where 1 != 1", - "OrderBy": "0 ASC", - "Query": "select id from `user` order by id asc limit :__upper_limit", - "Table": "`user`" - } - ] - }, - { - "OperatorType": "Limit", - "Count": 5, - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id from music where 1 != 1", - "OrderBy": "0 DESC", - "Query": "select id from music order by id desc limit :__upper_limit", - "Table": "music" - } - ] - } - ] - } -} -{ - "QueryType": "SELECT", - "Original": "(select id from user order by id limit 5) union all (select id from music order by id desc limit 5)", - "Instructions": { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Limit", - "Count": 5, - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id from `user` where 1 != 1", + "FieldQuery": "select id, weight_string(id) from `user` where 1 != 1", "OrderBy": "0 ASC", - "Query": "select id from `user` order by id asc limit :__upper_limit", + "Query": "select id, weight_string(id) from `user` order by id asc limit :__upper_limit", "Table": "`user`" } ] @@ -413,9 +223,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select id from music where 1 != 1", + "FieldQuery": "select id, weight_string(id) from music where 1 != 1", "OrderBy": "0 DESC", - "Query": "select id from music order by id desc limit :__upper_limit", + "Query": "select id, weight_string(id) from music order by id desc limit :__upper_limit", "Table": "music" } ] @@ -423,6 +233,7 @@ ] } } +Gen4 plan same as above # union all on scatter and single route "select id from user where id = 1 union select id from user where id = 1 union all select id from user" @@ -461,41 +272,7 @@ ] } } -{ - "QueryType": "SELECT", - "Original": "select id from user where id = 1 union select id from user where id = 1 union all select id from user", - "Instructions": { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectEqualUnique", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id from `user` where 1 != 1 union select id from `user` where 1 != 1", - "Query": "select id from `user` where id = 1 union select id from `user` where id = 1", - "Table": "`user`", - "Values": [ - 1 - ], - "Vindex": "user_index" - }, - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id from `user` where 1 != 1", - "Query": "select id from `user`", - "Table": "`user`" - } - ] - } -} +Gen4 plan same as above # union of information_schema with normal table "select * from information_schema.a union select * from unsharded" @@ -634,9 +411,13 @@ Gen4 plan same as above ] } } +Gen4 plan same as above + +# multi-shard union +"select 1 from music union (select id from user union all select name from unsharded)" { "QueryType": "SELECT", - "Original": "(select id from user union select id from music) union select 1 from dual", + "Original": "select 1 from music union (select id from user union all select name from unsharded)", "Instructions": { "OperatorType": "Distinct", "Inputs": [ @@ -644,130 +425,18 @@ Gen4 plan same as above "OperatorType": "Concatenate", "Inputs": [ { - "OperatorType": "Distinct", - "Inputs": [ - { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id from `user` where 1 != 1", - "Query": "select id from `user`", - "Table": "`user`" - }, - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id from music where 1 != 1", - "Query": "select id from music", - "Table": "music" - } - ] - } - ] - }, - { - "OperatorType": "Route", - "Variant": "SelectReference", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "FieldQuery": "select 1 from dual where 1 != 1", - "Query": "select 1 from dual", - "Table": "dual" - } - ] - } - ] - } -} - -# multi-shard union -"select 1 from music union (select id from user union all select name from unsharded)" -{ - "QueryType": "SELECT", - "Original": "select 1 from music union (select id from user union all select name from unsharded)", - "Instructions": { - "OperatorType": "Distinct", - "Inputs": [ - { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select 1 from music where 1 != 1", - "Query": "select 1 from music", - "Table": "music" - }, - { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id from `user` where 1 != 1", - "Query": "select id from `user`", - "Table": "`user`" - }, - { - "OperatorType": "Route", - "Variant": "SelectUnsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "FieldQuery": "select `name` from unsharded where 1 != 1", - "Query": "select `name` from unsharded", - "Table": "unsharded" - } - ] - } - ] - } - ] - } -} -{ - "QueryType": "SELECT", - "Original": "select 1 from music union (select id from user union all select name from unsharded)", - "Instructions": { - "OperatorType": "Distinct", - "Inputs": [ - { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select 1 from music where 1 != 1", - "Query": "select 1 from music", - "Table": "music" - }, - { - "OperatorType": "Concatenate", + "OperatorType": "Route", + "Variant": "SelectScatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select 1 from music where 1 != 1", + "Query": "select 1 from music", + "Table": "music" + }, + { + "OperatorType": "Concatenate", "Inputs": [ { "OperatorType": "Route", @@ -798,6 +467,7 @@ Gen4 plan same as above ] } } +Gen4 plan same as above # multi-shard union "select 1 from music union (select id from user union select name from unsharded)" @@ -858,63 +528,7 @@ Gen4 plan same as above ] } } -{ - "QueryType": "SELECT", - "Original": "select 1 from music union (select id from user union select name from unsharded)", - "Instructions": { - "OperatorType": "Distinct", - "Inputs": [ - { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select 1 from music where 1 != 1", - "Query": "select 1 from music", - "Table": "music" - }, - { - "OperatorType": "Distinct", - "Inputs": [ - { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id from `user` where 1 != 1", - "Query": "select id from `user`", - "Table": "`user`" - }, - { - "OperatorType": "Route", - "Variant": "SelectUnsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "FieldQuery": "select `name` from unsharded where 1 != 1", - "Query": "select `name` from unsharded", - "Table": "unsharded" - } - ] - } - ] - } - ] - } - ] - } -} +Gen4 plan same as above # union with the same target shard because of vindex "select * from music where id = 1 union select * from user where id = 1" @@ -962,50 +576,7 @@ Gen4 plan same as above ] } } -{ - "QueryType": "SELECT", - "Original": "select * from music where id = 1 union select * from user where id = 1", - "Instructions": { - "OperatorType": "Distinct", - "Inputs": [ - { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectEqualUnique", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select * from music where 1 != 1", - "Query": "select * from music where id = 1", - "Table": "music", - "Values": [ - 1 - ], - "Vindex": "music_user_map" - }, - { - "OperatorType": "Route", - "Variant": "SelectEqualUnique", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select * from `user` where 1 != 1", - "Query": "select * from `user` where id = 1", - "Table": "`user`", - "Values": [ - 1 - ], - "Vindex": "user_index" - } - ] - } - ] - } -} +Gen4 plan same as above # union with different target shards "select 1 from music where id = 1 union select 1 from music where id = 2" @@ -1073,9 +644,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select 1 from `user` where 1 != 1", + "FieldQuery": "select 1, weight_string(1) from `user` where 1 != 1", "OrderBy": "0 DESC", - "Query": "select 1 from `user` order by 1 desc", + "Query": "select 1, weight_string(1) from `user` order by 1 desc", "Table": "`user`" }, { @@ -1085,47 +656,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select 1 from `user` where 1 != 1", + "FieldQuery": "select 1, weight_string(1) from `user` where 1 != 1", "OrderBy": "0 ASC", - "Query": "select 1 from `user` order by 1 asc", - "Table": "`user`" - } - ] - } - ] - } -} -{ - "QueryType": "SELECT", - "Original": "(select 1 from user order by 1 desc) union (select 1 from user order by 1 asc)", - "Instructions": { - "OperatorType": "Distinct", - "Inputs": [ - { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select 1 from `user` where 1 != 1", - "OrderBy": "0 DESC", - "Query": "select 1 from `user` order by 1 desc", - "Table": "`user`" - }, - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select 1 from `user` where 1 != 1", - "OrderBy": "0 ASC", - "Query": "select 1 from `user` order by 1 asc", + "Query": "select 1, weight_string(1) from `user` order by 1 asc", "Table": "`user`" } ] @@ -1133,6 +666,7 @@ Gen4 plan same as above ] } } +Gen4 plan same as above # multiple unions "select 1 union select null union select 1.0 union select '1' union select 2 union select 2.0 from user" @@ -1172,42 +706,7 @@ Gen4 plan same as above ] } } -{ - "QueryType": "SELECT", - "Original": "select 1 union select null union select 1.0 union select '1' union select 2 union select 2.0 from user", - "Instructions": { - "OperatorType": "Distinct", - "Inputs": [ - { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectReference", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "FieldQuery": "select 1 from dual where 1 != 1 union select null from dual where 1 != 1 union select 1.0 from dual where 1 != 1 union select '1' from dual where 1 != 1 union select 2 from dual where 1 != 1", - "Query": "select 1 from dual union select null from dual union select 1.0 from dual union select '1' from dual union select 2 from dual", - "Table": "dual" - }, - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select 2.0 from `user` where 1 != 1", - "Query": "select 2.0 from `user`", - "Table": "`user`" - } - ] - } - ] - } -} +Gen4 plan same as above # union distinct between a scatter query and a join (other side) "(select user.id, user.name from user join user_extra where user_extra.extra = 'asdf') union select 'b','c' from user" @@ -1266,61 +765,7 @@ Gen4 plan same as above ] } } -{ - "QueryType": "SELECT", - "Original": "(select user.id, user.name from user join user_extra where user_extra.extra = 'asdf') union select 'b','c' from user", - "Instructions": { - "OperatorType": "Distinct", - "Inputs": [ - { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Join", - "Variant": "Join", - "JoinColumnIndexes": "-1,-2", - "TableName": "`user`_user_extra", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select `user`.id, `user`.`name` from `user` where 1 != 1", - "Query": "select `user`.id, `user`.`name` from `user`", - "Table": "`user`" - }, - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select 1 from user_extra where 1 != 1", - "Query": "select 1 from user_extra where user_extra.extra = 'asdf'", - "Table": "user_extra" - } - ] - }, - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select 'b', 'c' from `user` where 1 != 1", - "Query": "select 'b', 'c' from `user`", - "Table": "`user`" - } - ] - } - ] - } -} +Gen4 plan same as above # union distinct between a scatter query and a join (other side) "select 'b','c' from user union (select user.id, user.name from user join user_extra where user_extra.extra = 'asdf')" @@ -1379,61 +824,7 @@ Gen4 plan same as above ] } } -{ - "QueryType": "SELECT", - "Original": "select 'b','c' from user union (select user.id, user.name from user join user_extra where user_extra.extra = 'asdf')", - "Instructions": { - "OperatorType": "Distinct", - "Inputs": [ - { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select 'b', 'c' from `user` where 1 != 1", - "Query": "select 'b', 'c' from `user`", - "Table": "`user`" - }, - { - "OperatorType": "Join", - "Variant": "Join", - "JoinColumnIndexes": "-1,-2", - "TableName": "`user`_user_extra", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select `user`.id, `user`.`name` from `user` where 1 != 1", - "Query": "select `user`.id, `user`.`name` from `user`", - "Table": "`user`" - }, - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select 1 from user_extra where 1 != 1", - "Query": "select 1 from user_extra where user_extra.extra = 'asdf'", - "Table": "user_extra" - } - ] - } - ] - } - ] - } -} +Gen4 plan same as above # ambiguous LIMIT "select id from user limit 1 union all select id from music limit 1" diff --git a/go/vt/vtgate/planbuilder/vindex_func.go b/go/vt/vtgate/planbuilder/vindex_func.go index 1c0073c39aa..8d9a6d0a74a 100644 --- a/go/vt/vtgate/planbuilder/vindex_func.go +++ b/go/vt/vtgate/planbuilder/vindex_func.go @@ -17,7 +17,7 @@ limitations under the License. package planbuilder import ( - "errors" + "fmt" "vitess.io/vitess/go/vt/vtgate/semantics" @@ -131,9 +131,19 @@ func (vf *vindexFunc) SupplyCol(col *sqlparser.ColName) (rc *resultColumn, colNu return rc, len(vf.resultColumns) - 1 } +// UnsupportedSupplyWeightString represents the error where the supplying a weight string is not supported +type UnsupportedSupplyWeightString struct { + Type string +} + +// Error function implements the error interface +func (err UnsupportedSupplyWeightString) Error() string { + return fmt.Sprintf("cannot do collation on %s", err.Type) +} + // SupplyWeightString implements the logicalPlan interface func (vf *vindexFunc) SupplyWeightString(colNumber int) (weightcolNumber int, err error) { - return 0, errors.New("cannot do collation on vindex function") + return 0, UnsupportedSupplyWeightString{Type: "vindex function"} } // Rewrite implements the logicalPlan interface diff --git a/go/vt/wrangler/vdiff.go b/go/vt/wrangler/vdiff.go index 9d32aa04717..c64a9a4a2a1 100644 --- a/go/vt/wrangler/vdiff.go +++ b/go/vt/wrangler/vdiff.go @@ -492,7 +492,7 @@ func newMergeSorter(participants map[string]*shardStreamer, comparePKs []int) *e } ob := make([]engine.OrderbyParams, 0, len(comparePKs)) for _, cpk := range comparePKs { - ob = append(ob, engine.OrderbyParams{Col: cpk}) + ob = append(ob, engine.OrderbyParams{Col: cpk, WeightStringCol: -1}) } return &engine.MergeSort{ Primitives: prims,