From 74e2943c0b42eeb04c5beadd3b01541a9cff8bdb Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Sun, 14 Mar 2021 12:23:11 +0530 Subject: [PATCH 01/19] adding weight strings is now the default unless the order by column type is known Signed-off-by: GuptaManan100 --- go/vt/vtgate/planbuilder/merge_sort.go | 2 +- .../planbuilder/testdata/aggr_cases.txt | 156 +++++++++--------- .../testdata/memory_sort_cases.txt | 42 ++--- .../testdata/postprocess_cases.txt | 36 ++-- .../planbuilder/testdata/select_cases.txt | 12 +- .../planbuilder/testdata/union_cases.txt | 72 ++++---- 6 files changed, 160 insertions(+), 160 deletions(-) diff --git a/go/vt/vtgate/planbuilder/merge_sort.go b/go/vt/vtgate/planbuilder/merge_sort.go index ee28f7fec71..e76a4379aea 100644 --- a/go/vt/vtgate/planbuilder/merge_sort.go +++ b/go/vt/vtgate/planbuilder/merge_sort.go @@ -66,7 +66,7 @@ func (ms *mergeSort) Wireup(plan logicalPlan, jt *jointab) error { rb := ms.input.(*route) for i, orderby := range rb.eroute.OrderBy { rc := ms.resultColumns[orderby.Col] - if sqltypes.IsText(rc.column.typ) { + if sqltypes.IsText(rc.column.typ) || rc.column.typ == sqltypes.Null { // If a weight string was previously requested, reuse it. if colNumber, ok := ms.weightStrings[rc]; ok { rb.eroute.OrderBy[i].Col = colNumber diff --git a/go/vt/vtgate/planbuilder/testdata/aggr_cases.txt b/go/vt/vtgate/planbuilder/testdata/aggr_cases.txt index 747d50f67ac..ee755d77f06 100644 --- a/go/vt/vtgate/planbuilder/testdata/aggr_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/aggr_cases.txt @@ -116,9 +116,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select count(*), a, textcol1, b, weight_string(textcol1) from `user` where 1 != 1 group by a, textcol1, b", - "OrderBy": "1 ASC, 4 ASC, 3 ASC", - "Query": "select count(*), a, textcol1, b, weight_string(textcol1) from `user` group by a, textcol1, b order by a asc, textcol1 asc, b asc", + "FieldQuery": "select count(*), a, textcol1, b, weight_string(textcol1), weight_string(a), weight_string(b) from `user` where 1 != 1 group by a, textcol1, b", + "OrderBy": "5 ASC, 4 ASC, 6 ASC", + "Query": "select count(*), a, textcol1, b, weight_string(textcol1), weight_string(a), weight_string(b) from `user` group by a, textcol1, b order by a asc, textcol1 asc, b asc", "Table": "`user`" } ] @@ -149,9 +149,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select count(*) as k, a, textcol1, b, weight_string(textcol1) from `user` where 1 != 1 group by a, textcol1, b", - "OrderBy": "4 ASC, 1 ASC, 3 ASC", - "Query": "select count(*) as k, a, textcol1, b, weight_string(textcol1) from `user` group by a, textcol1, b order by textcol1 asc, a asc, b asc", + "FieldQuery": "select count(*) as k, a, textcol1, b, weight_string(textcol1), weight_string(a), weight_string(b) from `user` where 1 != 1 group by a, textcol1, b", + "OrderBy": "4 ASC, 5 ASC, 6 ASC", + "Query": "select count(*) as k, a, textcol1, b, weight_string(textcol1), weight_string(a), weight_string(b) from `user` group by a, textcol1, b order by textcol1 asc, a asc, b asc", "Table": "`user`" } ] @@ -282,9 +282,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col1, col2 from `user` where 1 != 1 group by col1", - "OrderBy": "0 ASC, 1 ASC, 0 ASC", - "Query": "select distinct col1, col2 from `user` group by col1 order by col1 asc, col2 asc, col1 asc", + "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1", + "OrderBy": "2 ASC, 3 ASC, 2 ASC", + "Query": "select distinct col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1 order by col1 asc, col2 asc, col1 asc", "Table": "`user`" } ] @@ -383,9 +383,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col, count(*) from `user` where 1 != 1 group by col", - "OrderBy": "0 ASC", - "Query": "select col, count(*) from `user` group by col order by col asc", + "FieldQuery": "select col, count(*), weight_string(col) from `user` where 1 != 1 group by col", + "OrderBy": "2 ASC", + "Query": "select col, count(*), weight_string(col) from `user` group by col order by col asc", "Table": "`user`" } ] @@ -415,9 +415,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select `name`, count(*) from `user` where 1 != 1 group by `name`", - "OrderBy": "0 ASC", - "Query": "select `name`, count(*) from `user` group by `name` order by `name` asc", + "FieldQuery": "select `name`, count(*), weight_string(`name`) from `user` where 1 != 1 group by `name`", + "OrderBy": "2 ASC", + "Query": "select `name`, count(*), weight_string(`name`) from `user` group by `name` order by `name` asc", "Table": "`user`" } ] @@ -596,9 +596,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col from `user` where 1 != 1", - "OrderBy": "0 ASC", - "Query": "select distinct col from `user` order by col asc", + "FieldQuery": "select col, weight_string(col) from `user` where 1 != 1", + "OrderBy": "1 ASC", + "Query": "select distinct col, weight_string(col) from `user` order by col asc", "Table": "`user`" } ] @@ -623,9 +623,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col from `user` where 1 != 1 group by col", - "OrderBy": "0 ASC", - "Query": "select col from `user` group by col order by col asc", + "FieldQuery": "select col, weight_string(col) from `user` where 1 != 1 group by col", + "OrderBy": "1 ASC", + "Query": "select col, weight_string(col) from `user` group by col order by col asc", "Table": "`user`" } ] @@ -669,9 +669,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col, count(distinct id) from `user` where 1 != 1 group by col", - "OrderBy": "0 ASC", - "Query": "select col, count(distinct id) from `user` group by col order by col asc", + "FieldQuery": "select col, count(distinct id), weight_string(col) from `user` where 1 != 1 group by col", + "OrderBy": "2 ASC", + "Query": "select col, count(distinct id), weight_string(col) from `user` group by col order by col asc", "Table": "`user`" } ] @@ -697,9 +697,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col1, col2 from `user` where 1 != 1 group by col1, col2", - "OrderBy": "0 ASC, 1 ASC", - "Query": "select col1, col2 from `user` group by col1, col2 order by col1 asc, col2 asc", + "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, col2", + "OrderBy": "2 ASC, 3 ASC", + "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, col2 order by col1 asc, col2 asc", "Table": "`user`" } ] @@ -724,9 +724,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col2 from `user` where 1 != 1 group by col2", - "OrderBy": "0 ASC", - "Query": "select col2 from `user` group by col2 order by col2 asc", + "FieldQuery": "select col2, weight_string(col2) from `user` where 1 != 1 group by col2", + "OrderBy": "1 ASC", + "Query": "select col2, weight_string(col2) from `user` group by col2 order by col2 asc", "Table": "`user`" } ] @@ -752,9 +752,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col1, col2 from `user` where 1 != 1 group by col1, col2", - "OrderBy": "0 ASC, 1 ASC", - "Query": "select col1, col2 from `user` group by col1, col2 order by col1 asc, col2 asc", + "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, col2", + "OrderBy": "2 ASC, 3 ASC", + "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, col2 order by col1 asc, col2 asc", "Table": "`user`" } ] @@ -780,9 +780,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col1, col2 from `user` where 1 != 1 group by col1, col2", - "OrderBy": "0 ASC, 1 ASC", - "Query": "select col1, col2 from `user` group by col1, col2 order by col1 asc, col2 asc", + "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, col2", + "OrderBy": "2 ASC, 3 ASC", + "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, col2 order by col1 asc, col2 asc", "Table": "`user`" } ] @@ -808,9 +808,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col1, min(distinct col2) from `user` where 1 != 1 group by col1", - "OrderBy": "0 ASC", - "Query": "select col1, min(distinct col2) from `user` group by col1 order by col1 asc", + "FieldQuery": "select col1, min(distinct col2), weight_string(col1) from `user` where 1 != 1 group by col1", + "OrderBy": "2 ASC", + "Query": "select col1, min(distinct col2), weight_string(col1) from `user` group by col1 order by col1 asc", "Table": "`user`" } ] @@ -841,9 +841,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col1, col2 from `user` where 1 != 1 group by col1, col2", - "OrderBy": "0 ASC, 1 ASC", - "Query": "select col1, col2 from `user` group by col1, col2 order by col1 asc, col2 asc", + "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, col2", + "OrderBy": "2 ASC, 3 ASC", + "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, col2 order by col1 asc, col2 asc", "Table": "`user`" } ] @@ -875,9 +875,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, count(*) from `user` where 1 != 1 group by b, a", - "OrderBy": "1 ASC, 0 ASC", - "Query": "select a, b, count(*) from `user` group by b, a order by b asc, a asc", + "FieldQuery": "select a, b, count(*), weight_string(b), weight_string(a) from `user` where 1 != 1 group by b, a", + "OrderBy": "3 ASC, 4 ASC", + "Query": "select a, b, count(*), weight_string(b), weight_string(a) from `user` group by b, a order by b asc, a asc", "Table": "`user`" } ] @@ -903,9 +903,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, count(*) from `user` where 1 != 1 group by 2, 1", - "OrderBy": "1 ASC, 0 ASC", - "Query": "select a, b, count(*) from `user` group by 2, 1 order by b asc, a asc", + "FieldQuery": "select a, b, count(*), weight_string(b), weight_string(a) from `user` where 1 != 1 group by 2, 1", + "OrderBy": "3 ASC, 4 ASC", + "Query": "select a, b, count(*), weight_string(b), weight_string(a) from `user` group by 2, 1 order by b asc, a asc", "Table": "`user`" } ] @@ -931,9 +931,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, count(*) from `user` where 1 != 1 group by b, a", - "OrderBy": "1 ASC, 0 ASC", - "Query": "select a, b, count(*) from `user` group by b, a order by b asc, a asc", + "FieldQuery": "select a, b, count(*), weight_string(b), weight_string(a) from `user` where 1 != 1 group by b, a", + "OrderBy": "3 ASC, 4 ASC", + "Query": "select a, b, count(*), weight_string(b), weight_string(a) from `user` group by b, a order by b asc, a asc", "Table": "`user`" } ] @@ -958,9 +958,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col from `user` where 1 != 1 group by 1", - "OrderBy": "0 ASC", - "Query": "select col from `user` group by 1 order by col asc", + "FieldQuery": "select col, weight_string(col) from `user` where 1 != 1 group by 1", + "OrderBy": "1 ASC", + "Query": "select col, weight_string(col) from `user` group by 1 order by col asc", "Table": "`user`" } ] @@ -1020,9 +1020,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, c, d, count(*) from `user` where 1 != 1 group by 1, 2, 3", - "OrderBy": "0 ASC, 1 ASC, 2 ASC", - "Query": "select a, b, c, d, count(*) from `user` group by 1, 2, 3 order by 1 asc, 2 asc, 3 asc", + "FieldQuery": "select a, b, c, d, count(*), weight_string(a), weight_string(b), weight_string(c) from `user` where 1 != 1 group by 1, 2, 3", + "OrderBy": "5 ASC, 6 ASC, 7 ASC", + "Query": "select a, b, c, d, count(*), weight_string(a), weight_string(b), weight_string(c) from `user` group by 1, 2, 3 order by 1 asc, 2 asc, 3 asc", "Table": "`user`" } ] @@ -1048,9 +1048,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, c, d, count(*) from `user` where 1 != 1 group by 1, 2, 3", - "OrderBy": "0 ASC, 1 ASC, 2 ASC", - "Query": "select a, b, c, d, count(*) from `user` group by 1, 2, 3 order by a asc, b asc, c asc", + "FieldQuery": "select a, b, c, d, count(*), weight_string(a), weight_string(b), weight_string(c) from `user` where 1 != 1 group by 1, 2, 3", + "OrderBy": "5 ASC, 6 ASC, 7 ASC", + "Query": "select a, b, c, d, count(*), weight_string(a), weight_string(b), weight_string(c) from `user` group by 1, 2, 3 order by a asc, b asc, c asc", "Table": "`user`" } ] @@ -1076,9 +1076,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, c, d, count(*) from `user` where 1 != 1 group by 1, 2, 3, 4", - "OrderBy": "3 ASC, 1 ASC, 0 ASC, 2 ASC", - "Query": "select a, b, c, d, count(*) from `user` group by 1, 2, 3, 4 order by d asc, b asc, a asc, c asc", + "FieldQuery": "select a, b, c, d, count(*), weight_string(d), weight_string(b), weight_string(a), weight_string(c) from `user` where 1 != 1 group by 1, 2, 3, 4", + "OrderBy": "5 ASC, 6 ASC, 7 ASC, 8 ASC", + "Query": "select a, b, c, d, count(*), weight_string(d), weight_string(b), weight_string(a), weight_string(c) from `user` group by 1, 2, 3, 4 order by d asc, b asc, a asc, c asc", "Table": "`user`" } ] @@ -1104,9 +1104,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, c, d, count(*) from `user` where 1 != 1 group by 3, 2, 1, 4", - "OrderBy": "3 ASC, 1 ASC, 0 ASC, 2 ASC", - "Query": "select a, b, c, d, count(*) from `user` group by 3, 2, 1, 4 order by d asc, b asc, a asc, c asc", + "FieldQuery": "select a, b, c, d, count(*), weight_string(d), weight_string(b), weight_string(a), weight_string(c) from `user` where 1 != 1 group by 3, 2, 1, 4", + "OrderBy": "5 ASC, 6 ASC, 7 ASC, 8 ASC", + "Query": "select a, b, c, d, count(*), weight_string(d), weight_string(b), weight_string(a), weight_string(c) from `user` group by 3, 2, 1, 4 order by d asc, b asc, a asc, c asc", "Table": "`user`" } ] @@ -1132,9 +1132,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, c, count(*) from `user` where 1 != 1 group by 3, 2, 1", - "OrderBy": "0 DESC, 2 DESC, 1 ASC", - "Query": "select a, b, c, count(*) from `user` group by 3, 2, 1 order by 1 desc, 3 desc, b asc", + "FieldQuery": "select a, b, c, count(*), weight_string(a), weight_string(c), weight_string(b) from `user` where 1 != 1 group by 3, 2, 1", + "OrderBy": "4 DESC, 5 DESC, 6 ASC", + "Query": "select a, b, c, count(*), weight_string(a), weight_string(c), weight_string(b) from `user` group by 3, 2, 1 order by 1 desc, 3 desc, b asc", "Table": "`user`" } ] @@ -1168,9 +1168,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col, count(*) from `user` where 1 != 1 group by col", - "OrderBy": "0 ASC", - "Query": "select col, count(*) from `user` group by col order by col asc limit :__upper_limit", + "FieldQuery": "select col, count(*), weight_string(col) from `user` where 1 != 1 group by col", + "OrderBy": "2 ASC", + "Query": "select col, count(*), weight_string(col) from `user` group by col order by col asc limit :__upper_limit", "Table": "`user`" } ] @@ -1261,9 +1261,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select a, count(*) from `user` where 1 != 1", - "OrderBy": "0 ASC", - "Query": "select a, count(*) from `user` order by a asc", + "FieldQuery": "select a, count(*), weight_string(a) from `user` where 1 != 1", + "OrderBy": "2 ASC", + "Query": "select a, count(*), weight_string(a) from `user` order by a asc", "Table": "`user`" } ] @@ -1294,9 +1294,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select a, count(*) from `user` where 1 != 1 group by a", - "OrderBy": "0 ASC, 0 ASC", - "Query": "select a, count(*) from `user` group by a order by a asc, a asc", + "FieldQuery": "select a, count(*), weight_string(a) from `user` where 1 != 1 group by a", + "OrderBy": "2 ASC, 2 ASC", + "Query": "select a, count(*), weight_string(a) from `user` group by a order by a asc, a asc", "Table": "`user`" } ] diff --git a/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.txt b/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.txt index cb743923963..1891e3d88ff 100644 --- a/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.txt @@ -23,9 +23,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, count(*) from `user` where 1 != 1 group by a", - "OrderBy": "0 ASC", - "Query": "select a, b, count(*) from `user` group by a order by a asc", + "FieldQuery": "select a, b, count(*), weight_string(a) from `user` where 1 != 1 group by a", + "OrderBy": "3 ASC", + "Query": "select a, b, count(*), weight_string(a) from `user` group by a order by a asc", "Table": "`user`" } ] @@ -58,9 +58,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, count(*) as k from `user` where 1 != 1 group by a", - "OrderBy": "0 ASC", - "Query": "select a, b, count(*) as k from `user` group by a order by a asc", + "FieldQuery": "select a, b, count(*) as k, weight_string(a) from `user` where 1 != 1 group by a", + "OrderBy": "3 ASC", + "Query": "select a, b, count(*) as k, weight_string(a) from `user` group by a order by a asc", "Table": "`user`" } ] @@ -93,9 +93,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, count(*) as k from `user` where 1 != 1 group by a", - "OrderBy": "0 ASC", - "Query": "select a, b, count(*) as k from `user` group by a order by a asc", + "FieldQuery": "select a, b, count(*) as k, weight_string(a) from `user` where 1 != 1 group by a", + "OrderBy": "3 ASC", + "Query": "select a, b, count(*) as k, weight_string(a) from `user` group by a order by a asc", "Table": "`user`" } ] @@ -132,9 +132,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, count(*) as k from `user` where 1 != 1 group by a", - "OrderBy": "0 ASC", - "Query": "select a, b, count(*) as k from `user` group by a order by a asc", + "FieldQuery": "select a, b, count(*) as k, weight_string(a) from `user` where 1 != 1 group by a", + "OrderBy": "3 ASC", + "Query": "select a, b, count(*) as k, weight_string(a) from `user` group by a order by a asc", "Table": "`user`" } ] @@ -169,9 +169,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, count(*) as k from `user` where 1 != 1 group by a", - "OrderBy": "0 ASC", - "Query": "select a, b, count(*) as k from `user` group by a order by 1 asc", + "FieldQuery": "select a, b, count(*) as k, weight_string(a) from `user` where 1 != 1 group by a", + "OrderBy": "3 ASC", + "Query": "select a, b, count(*) as k, weight_string(a) from `user` group by a order by 1 asc", "Table": "`user`" } ] @@ -504,9 +504,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select a from `user` where 1 != 1", - "OrderBy": "0 DESC", - "Query": "select a from `user` order by binary a desc", + "FieldQuery": "select a, weight_string(a) from `user` where 1 != 1", + "OrderBy": "1 DESC", + "Query": "select a, weight_string(a) from `user` order by binary a desc", "Table": "`user`" } } @@ -529,9 +529,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select u.a from `user` as u where 1 != 1", - "OrderBy": "0 DESC", - "Query": "select u.a from `user` as u order by binary a desc", + "FieldQuery": "select u.a, weight_string(u.a) from `user` as u where 1 != 1", + "OrderBy": "1 DESC", + "Query": "select u.a, weight_string(u.a) from `user` as u order by binary a desc", "Table": "`user`" }, { diff --git a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.txt b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.txt index 28263baae24..1104b91bcd1 100644 --- a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.txt @@ -191,9 +191,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col from `user` where 1 != 1", - "OrderBy": "0 ASC", - "Query": "select col from `user` order by col asc", + "FieldQuery": "select col, weight_string(col) from `user` where 1 != 1", + "OrderBy": "1 ASC", + "Query": "select col, weight_string(col) from `user` order by col asc", "Table": "`user`" } } @@ -210,9 +210,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select user_id, col1, col2 from authoritative where 1 != 1", - "OrderBy": "0 ASC", - "Query": "select user_id, col1, col2 from authoritative order by user_id asc", + "FieldQuery": "select user_id, col1, col2, weight_string(user_id) from authoritative where 1 != 1", + "OrderBy": "3 ASC", + "Query": "select user_id, col1, col2, weight_string(user_id) from authoritative order by user_id asc", "Table": "authoritative" } } @@ -248,9 +248,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select a, textcol1, b, weight_string(textcol1) from `user` where 1 != 1", - "OrderBy": "0 ASC, 3 ASC, 2 ASC", - "Query": "select a, textcol1, b, weight_string(textcol1) from `user` order by a asc, textcol1 asc, b asc", + "FieldQuery": "select a, textcol1, b, weight_string(a), weight_string(textcol1), weight_string(b) from `user` where 1 != 1", + "OrderBy": "3 ASC, 4 ASC, 5 ASC", + "Query": "select a, textcol1, b, weight_string(a), weight_string(textcol1), weight_string(b) from `user` order by a asc, textcol1 asc, b asc", "Table": "`user`" } } @@ -267,9 +267,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select a, `user`.textcol1, b, weight_string(`user`.textcol1) from `user` where 1 != 1", - "OrderBy": "0 ASC, 3 ASC, 2 ASC", - "Query": "select a, `user`.textcol1, b, weight_string(`user`.textcol1) from `user` order by a asc, textcol1 asc, b asc", + "FieldQuery": "select a, `user`.textcol1, b, weight_string(a), weight_string(`user`.textcol1), weight_string(b) from `user` where 1 != 1", + "OrderBy": "3 ASC, 4 ASC, 5 ASC", + "Query": "select a, `user`.textcol1, b, weight_string(a), weight_string(`user`.textcol1), weight_string(b) from `user` order by a asc, textcol1 asc, b asc", "Table": "`user`" } } @@ -286,9 +286,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select a, textcol1, b, textcol2, weight_string(textcol1), weight_string(textcol2) from `user` where 1 != 1", - "OrderBy": "0 ASC, 4 ASC, 2 ASC, 5 ASC", - "Query": "select a, textcol1, b, textcol2, weight_string(textcol1), weight_string(textcol2) from `user` order by a asc, textcol1 asc, b asc, textcol2 asc", + "FieldQuery": "select a, textcol1, b, textcol2, weight_string(a), weight_string(textcol1), weight_string(b), weight_string(textcol2) from `user` where 1 != 1", + "OrderBy": "4 ASC, 5 ASC, 6 ASC, 7 ASC", + "Query": "select a, textcol1, b, textcol2, weight_string(a), weight_string(textcol1), weight_string(b), weight_string(textcol2) from `user` order by a asc, textcol1 asc, b asc, textcol2 asc", "Table": "`user`" } } @@ -342,9 +342,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col from `user` where 1 != 1", - "OrderBy": "0 ASC", - "Query": "select col from `user` where :__sq_has_values1 = 1 and col in ::__sq1 order by col asc", + "FieldQuery": "select col, weight_string(col) from `user` where 1 != 1", + "OrderBy": "1 ASC", + "Query": "select col, weight_string(col) from `user` where :__sq_has_values1 = 1 and col in ::__sq1 order by col asc", "Table": "`user`" } ] diff --git a/go/vt/vtgate/planbuilder/testdata/select_cases.txt b/go/vt/vtgate/planbuilder/testdata/select_cases.txt index 93f34a6ab9f..1f50b25f021 100644 --- a/go/vt/vtgate/planbuilder/testdata/select_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/select_cases.txt @@ -849,9 +849,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select user_id from music where 1 != 1", - "OrderBy": "0 ASC", - "Query": "select user_id from music order by user_id asc limit :__upper_limit", + "FieldQuery": "select user_id, weight_string(user_id) from music where 1 != 1", + "OrderBy": "1 ASC", + "Query": "select user_id, weight_string(user_id) from music order by user_id asc limit :__upper_limit", "Table": "music" } ] @@ -1544,9 +1544,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select user_id, count(id) from music where 1 != 1 group by user_id", - "OrderBy": "0 ASC", - "Query": "select user_id, count(id) from music group by user_id having count(user_id) = 1 order by user_id asc limit :__upper_limit", + "FieldQuery": "select user_id, count(id), weight_string(user_id) from music where 1 != 1 group by user_id", + "OrderBy": "2 ASC", + "Query": "select user_id, count(id), weight_string(user_id) from music group by user_id having count(user_id) = 1 order by user_id asc limit :__upper_limit", "Table": "music" } ] diff --git a/go/vt/vtgate/planbuilder/testdata/union_cases.txt b/go/vt/vtgate/planbuilder/testdata/union_cases.txt index bc82ca929f4..4b8ef1aa59c 100644 --- a/go/vt/vtgate/planbuilder/testdata/union_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/union_cases.txt @@ -206,9 +206,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select id from `user` where 1 != 1", - "OrderBy": "0 DESC", - "Query": "select id from `user` order by id desc limit :__upper_limit", + "FieldQuery": "select id, weight_string(id) from `user` where 1 != 1", + "OrderBy": "1 DESC", + "Query": "select id, weight_string(id) from `user` order by id desc limit :__upper_limit", "Table": "`user`" } ] @@ -224,9 +224,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select id from music where 1 != 1", - "OrderBy": "0 DESC", - "Query": "select id from music order by id desc limit :__upper_limit", + "FieldQuery": "select id, weight_string(id) from music where 1 != 1", + "OrderBy": "1 DESC", + "Query": "select id, weight_string(id) from music order by id desc limit :__upper_limit", "Table": "music" } ] @@ -251,9 +251,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select id from `user` where 1 != 1", - "OrderBy": "0 DESC", - "Query": "select id from `user` order by id desc limit :__upper_limit", + "FieldQuery": "select id, weight_string(id) from `user` where 1 != 1", + "OrderBy": "1 DESC", + "Query": "select id, weight_string(id) from `user` order by id desc limit :__upper_limit", "Table": "`user`" } ] @@ -269,9 +269,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select id from music where 1 != 1", - "OrderBy": "0 DESC", - "Query": "select id from music order by id desc limit :__upper_limit", + "FieldQuery": "select id, weight_string(id) from music where 1 != 1", + "OrderBy": "1 DESC", + "Query": "select id, weight_string(id) from music order by id desc limit :__upper_limit", "Table": "music" } ] @@ -350,9 +350,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select id from `user` where 1 != 1", - "OrderBy": "0 ASC", - "Query": "select id from `user` order by id asc limit :__upper_limit", + "FieldQuery": "select id, weight_string(id) from `user` where 1 != 1", + "OrderBy": "1 ASC", + "Query": "select id, weight_string(id) from `user` order by id asc limit :__upper_limit", "Table": "`user`" } ] @@ -368,9 +368,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select id from music where 1 != 1", - "OrderBy": "0 DESC", - "Query": "select id from music order by id desc limit :__upper_limit", + "FieldQuery": "select id, weight_string(id) from music where 1 != 1", + "OrderBy": "1 DESC", + "Query": "select id, weight_string(id) from music order by id desc limit :__upper_limit", "Table": "music" } ] @@ -395,9 +395,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select id from `user` where 1 != 1", - "OrderBy": "0 ASC", - "Query": "select id from `user` order by id asc limit :__upper_limit", + "FieldQuery": "select id, weight_string(id) from `user` where 1 != 1", + "OrderBy": "1 ASC", + "Query": "select id, weight_string(id) from `user` order by id asc limit :__upper_limit", "Table": "`user`" } ] @@ -413,9 +413,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select id from music where 1 != 1", - "OrderBy": "0 DESC", - "Query": "select id from music order by id desc limit :__upper_limit", + "FieldQuery": "select id, weight_string(id) from music where 1 != 1", + "OrderBy": "1 DESC", + "Query": "select id, weight_string(id) from music order by id desc limit :__upper_limit", "Table": "music" } ] @@ -1073,9 +1073,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select 1 from `user` where 1 != 1", - "OrderBy": "0 DESC", - "Query": "select 1 from `user` order by 1 desc", + "FieldQuery": "select 1, weight_string(1) from `user` where 1 != 1", + "OrderBy": "1 DESC", + "Query": "select 1, weight_string(1) from `user` order by 1 desc", "Table": "`user`" }, { @@ -1085,9 +1085,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select 1 from `user` where 1 != 1", - "OrderBy": "0 ASC", - "Query": "select 1 from `user` order by 1 asc", + "FieldQuery": "select 1, weight_string(1) from `user` where 1 != 1", + "OrderBy": "1 ASC", + "Query": "select 1, weight_string(1) from `user` order by 1 asc", "Table": "`user`" } ] @@ -1111,9 +1111,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select 1 from `user` where 1 != 1", - "OrderBy": "0 DESC", - "Query": "select 1 from `user` order by 1 desc", + "FieldQuery": "select 1, weight_string(1) from `user` where 1 != 1", + "OrderBy": "1 DESC", + "Query": "select 1, weight_string(1) from `user` order by 1 desc", "Table": "`user`" }, { @@ -1123,9 +1123,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select 1 from `user` where 1 != 1", - "OrderBy": "0 ASC", - "Query": "select 1 from `user` order by 1 asc", + "FieldQuery": "select 1, weight_string(1) from `user` where 1 != 1", + "OrderBy": "1 ASC", + "Query": "select 1, weight_string(1) from `user` order by 1 asc", "Table": "`user`" } ] From e30e5ba825cae2391ad2855baed3d0d619bafe09 Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Sun, 14 Mar 2021 12:28:25 +0530 Subject: [PATCH 02/19] added a testcase for integer column Signed-off-by: GuptaManan100 --- .../planbuilder/testdata/aggr_cases.txt | 28 +++++++++++++++++++ .../planbuilder/testdata/schema_test.json | 4 +++ 2 files changed, 32 insertions(+) diff --git a/go/vt/vtgate/planbuilder/testdata/aggr_cases.txt b/go/vt/vtgate/planbuilder/testdata/aggr_cases.txt index ee755d77f06..3ce4db9ee91 100644 --- a/go/vt/vtgate/planbuilder/testdata/aggr_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/aggr_cases.txt @@ -125,6 +125,34 @@ Gen4 plan same as above } } +# scatter group by a integer column. Do not add weight strings for this. +"select count(*), intcol from user group by intcol" +{ + "QueryType": "SELECT", + "Original": "select count(*), intcol from user group by intcol", + "Instructions": { + "OperatorType": "Aggregate", + "Variant": "Ordered", + "Aggregates": "count(0)", + "Distinct": "false", + "GroupBy": "1", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "SelectScatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select count(*), intcol from `user` where 1 != 1 group by intcol", + "OrderBy": "1 ASC", + "Query": "select count(*), intcol from `user` group by intcol order by intcol asc", + "Table": "`user`" + } + ] + } +} + # scatter group by a text column, reuse existing weight_string "select count(*) k, a, textcol1, b from user group by a, textcol1, b order by k, textcol1" { diff --git a/go/vt/vtgate/planbuilder/testdata/schema_test.json b/go/vt/vtgate/planbuilder/testdata/schema_test.json index 17f6576dbe4..fd56617707b 100644 --- a/go/vt/vtgate/planbuilder/testdata/schema_test.json +++ b/go/vt/vtgate/planbuilder/testdata/schema_test.json @@ -111,6 +111,10 @@ "name": "textcol1", "type": "VARCHAR" }, + { + "name": "intcol", + "type": "INT16" + }, { "name": "textcol2", "type": "VARCHAR" From 610e4d4cb865721100fd6f1de9e8918f29d57b0c Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Sun, 14 Mar 2021 13:36:36 +0530 Subject: [PATCH 03/19] added an end to end test Signed-off-by: GuptaManan100 --- go/test/endtoend/vtgate/aggr_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/go/test/endtoend/vtgate/aggr_test.go b/go/test/endtoend/vtgate/aggr_test.go index f68a5808c34..e832cc8f55c 100644 --- a/go/test/endtoend/vtgate/aggr_test.go +++ b/go/test/endtoend/vtgate/aggr_test.go @@ -40,3 +40,15 @@ func TestAggregateTypes(t *testing.T) { assertMatches(t, conn, "select val1, count(distinct val2) k, count(*) from aggr_test group by val1 order by k desc, val1 limit 4", `[[VARCHAR("c") INT64(2) INT64(2)] [VARCHAR("a") INT64(1) INT64(2)] [VARCHAR("b") INT64(1) INT64(1)] [VARCHAR("e") INT64(1) INT64(2)]]`) exec(t, conn, "delete from aggr_test") } + +func TestOrderByVarcharColumn(t *testing.T) { + defer cluster.PanicHandler(t) + ctx := context.Background() + conn, err := mysql.Connect(ctx, &vtParams) + require.Nil(t, err) + defer conn.Close() + exec(t, conn, "insert into t4(id1, id2) values(1,'a'), (2,'A'), (3,'b'), (4,'c'), (5,'test')") + exec(t, conn, "insert into t4(id1, id2) values(6,'d'), (7,'e'), (8,'E')") + assertMatches(t, conn, "select id1, id2 from t4 order by id2 desc", `[[INT64(5) VARCHAR("test")] [INT64(8) VARCHAR("E")] [INT64(7) VARCHAR("e")] [INT64(6) VARCHAR("d")] [INT64(4) VARCHAR("c")] [INT64(3) VARCHAR("b")] [INT64(2) VARCHAR("A")] [INT64(1) VARCHAR("a")]]`) + exec(t, conn, "delete from t4") +} From 1e5762e99edaf9599ce74b8a7ceaf3ea4aef49af Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Mon, 15 Mar 2021 15:38:30 +0530 Subject: [PATCH 04/19] added both weighted string and normal column to ordering Signed-off-by: GuptaManan100 --- go/test/endtoend/vtgate/aggr_test.go | 5 +- go/vt/vtgate/engine/memory_sort.go | 59 ++++-- go/vt/vtgate/engine/memory_sort_test.go | 257 ++++++++++++++++-------- go/vt/vtgate/engine/merge_sort.go | 35 +++- go/vt/vtgate/engine/merge_sort_test.go | 110 +++++++--- go/vt/vtgate/engine/route.go | 32 ++- go/vt/vtgate/engine/route_test.go | 225 ++++++++++++--------- go/vt/vtgate/evalengine/arithmetic.go | 16 +- go/vt/vtgate/planbuilder/memory_sort.go | 11 +- go/vt/vtgate/planbuilder/merge_sort.go | 4 +- go/vt/vtgate/planbuilder/ordering.go | 5 +- go/vt/wrangler/vdiff.go | 2 +- 12 files changed, 510 insertions(+), 251 deletions(-) diff --git a/go/test/endtoend/vtgate/aggr_test.go b/go/test/endtoend/vtgate/aggr_test.go index e832cc8f55c..c40b370004c 100644 --- a/go/test/endtoend/vtgate/aggr_test.go +++ b/go/test/endtoend/vtgate/aggr_test.go @@ -41,7 +41,7 @@ func TestAggregateTypes(t *testing.T) { exec(t, conn, "delete from aggr_test") } -func TestOrderByVarcharColumn(t *testing.T) { +func TestOrderBy(t *testing.T) { defer cluster.PanicHandler(t) ctx := context.Background() conn, err := mysql.Connect(ctx, &vtParams) @@ -49,6 +49,9 @@ func TestOrderByVarcharColumn(t *testing.T) { defer conn.Close() exec(t, conn, "insert into t4(id1, id2) values(1,'a'), (2,'A'), (3,'b'), (4,'c'), (5,'test')") exec(t, conn, "insert into t4(id1, id2) values(6,'d'), (7,'e'), (8,'E')") + // test ordering of varchar column assertMatches(t, conn, "select id1, id2 from t4 order by id2 desc", `[[INT64(5) VARCHAR("test")] [INT64(8) VARCHAR("E")] [INT64(7) VARCHAR("e")] [INT64(6) VARCHAR("d")] [INT64(4) VARCHAR("c")] [INT64(3) VARCHAR("b")] [INT64(2) VARCHAR("A")] [INT64(1) VARCHAR("a")]]`) + // test ordering of int column + assertMatches(t, conn, "select id1, id2 from t4 order by id1 desc", `[[INT64(8) VARCHAR("E")] [INT64(7) VARCHAR("e")] [INT64(6) VARCHAR("d")] [INT64(5) VARCHAR("test")] [INT64(4) VARCHAR("c")] [INT64(3) VARCHAR("b")] [INT64(2) VARCHAR("A")] [INT64(1) VARCHAR("a")]]`) exec(t, conn, "delete from t4") } diff --git a/go/vt/vtgate/engine/memory_sort.go b/go/vt/vtgate/engine/memory_sort.go index 9c4a2df4c02..b7685e8dcf6 100644 --- a/go/vt/vtgate/engine/memory_sort.go +++ b/go/vt/vtgate/engine/memory_sort.go @@ -75,9 +75,12 @@ func (ms *MemorySort) Execute(vcursor VCursor, bindVars map[string]*querypb.Bind if err != nil { return nil, err } + orderBy, weightStrings, desc := extractSlices(ms.OrderBy) sh := &sortHeap{ - rows: result.Rows, - orderBy: ms.OrderBy, + rows: result.Rows, + orderBy: orderBy, + weightStrings: weightStrings, + desc: desc, } sort.Sort(sh) if sh.err != nil { @@ -103,9 +106,12 @@ func (ms *MemorySort) StreamExecute(vcursor VCursor, bindVars map[string]*queryp // You have to reverse the ordering because the highest values // must be dropped once the upper limit is reached. + orderBy, weightStrings, desc := extractSlices(ms.OrderBy) sh := &sortHeap{ - orderBy: ms.OrderBy, - reverse: true, + orderBy: orderBy, + weightStrings: weightStrings, + desc: desc, + reverse: true, } err = ms.Input.StreamExecute(vcursor, bindVars, wantfields, func(qr *sqltypes.Result) error { if len(qr.Fields) != 0 { @@ -115,9 +121,9 @@ func (ms *MemorySort) StreamExecute(vcursor VCursor, bindVars map[string]*queryp } for _, row := range qr.Rows { heap.Push(sh, row) - } - for len(sh.rows) > count { - _ = heap.Pop(sh) + for len(sh.rows) > count { + _ = heap.Pop(sh) + } } if vcursor.ExceedsMaxMemoryRows(len(sh.rows)) { return fmt.Errorf("in-memory row count exceeded allowed limit of %d", vcursor.MaxMemoryRows()) @@ -140,6 +146,15 @@ func (ms *MemorySort) StreamExecute(vcursor VCursor, bindVars map[string]*queryp return cb(&sqltypes.Result{Rows: sh.rows}) } +func extractSlices(input []OrderbyParams) (orderBy []int, weightString []int, desc []bool) { + for _, order := range input { + orderBy = append(orderBy, order.Col) + weightString = append(weightString, order.WeightStringCol) + desc = append(desc, order.Desc) + } + return +} + // GetFields satisfies the Primitive interface. func (ms *MemorySort) GetFields(vcursor VCursor, bindVars map[string]*querypb.BindVariable) (*sqltypes.Result, error) { return ms.Input.GetFields(vcursor, bindVars) @@ -215,10 +230,12 @@ func GenericJoin(input interface{}, f func(interface{}) string) string { // sortHeap is sorted based on the orderBy params. // Implementation is similar to scatterHeap type sortHeap struct { - rows [][]sqltypes.Value - orderBy []OrderbyParams - reverse bool - err error + rows [][]sqltypes.Value + orderBy []int + weightStrings []int + desc []bool + reverse bool + err error } // Len satisfies sort.Interface and heap.Interface. @@ -228,14 +245,24 @@ func (sh *sortHeap) Len() int { // Less satisfies sort.Interface and heap.Interface. func (sh *sortHeap) Less(i, j int) bool { - for _, order := range sh.orderBy { + for k := range sh.orderBy { if sh.err != nil { return true } - cmp, err := evalengine.NullsafeCompare(sh.rows[i][order.Col], sh.rows[j][order.Col]) + cmp, err := evalengine.NullsafeCompare(sh.rows[i][sh.orderBy[k]], sh.rows[j][sh.orderBy[k]]) if err != nil { - sh.err = err - return true + _, isComparisonErr := err.(evalengine.UnsupportedComparisonError) + if !(isComparisonErr && sh.weightStrings[k] != -1) { + sh.err = err + return true + } + sh.orderBy[k] = sh.weightStrings[k] + sh.weightStrings[k] = -1 + cmp, err = evalengine.NullsafeCompare(sh.rows[i][sh.orderBy[k]], sh.rows[j][sh.orderBy[k]]) + if err != nil { + sh.err = err + return true + } } if cmp == 0 { continue @@ -250,7 +277,7 @@ func (sh *sortHeap) Less(i, j int) bool { // cmp = -cmp // } //} - if sh.reverse != order.Desc { + if sh.reverse != sh.desc[k] { cmp = -cmp } return cmp < 0 diff --git a/go/vt/vtgate/engine/memory_sort_test.go b/go/vt/vtgate/engine/memory_sort_test.go index dd7fd924a2f..962f1af278a 100644 --- a/go/vt/vtgate/engine/memory_sort_test.go +++ b/go/vt/vtgate/engine/memory_sort_test.go @@ -17,10 +17,10 @@ limitations under the License. package engine import ( - "encoding/json" - "reflect" "testing" + "vitess.io/vitess/go/test/utils" + "github.com/stretchr/testify/require" "vitess.io/vitess/go/sqltypes" @@ -37,7 +37,7 @@ func TestMemorySortExecute(t *testing.T) { results: []*sqltypes.Result{sqltypes.MakeTestResult( fields, "a|1", - "b|2", + "g|2", "a|1", "c|4", "c|3", @@ -46,50 +46,164 @@ func TestMemorySortExecute(t *testing.T) { ms := &MemorySort{ OrderBy: []OrderbyParams{{ - Col: 1, + WeightStringCol: -1, + Col: 1, }}, Input: fp, } result, err := ms.Execute(nil, nil, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) wantResult := sqltypes.MakeTestResult( fields, "a|1", "a|1", - "b|2", + "g|2", "c|3", "c|4", ) - if !reflect.DeepEqual(result, wantResult) { - t.Errorf("oa.Execute:\n%v, want\n%v", result, wantResult) - } + utils.MustMatch(t, wantResult, result) fp.rewind() upperlimit, err := sqlparser.NewPlanValue(sqlparser.NewArgument(":__upper_limit")) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) ms.UpperLimit = upperlimit bv := map[string]*querypb.BindVariable{"__upper_limit": sqltypes.Int64BindVariable(3)} result, err = ms.Execute(nil, bv, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) wantResult = sqltypes.MakeTestResult( fields, "a|1", "a|1", - "b|2", + "g|2", + ) + utils.MustMatch(t, wantResult, result) +} + +func TestMemorySortStreamExecuteWeightString(t *testing.T) { + fields := sqltypes.MakeTestFields( + "weightString|normal", + "varbinary|varchar", ) - if !reflect.DeepEqual(result, wantResult) { - t.Errorf("oa.Execute:\n%v, want\n%v", result, wantResult) + fp := &fakePrimitive{ + results: []*sqltypes.Result{sqltypes.MakeTestResult( + fields, + "v|x", + "g|d", + "a|a", + "c|t", + "f|p", + )}, + } + + ms := &MemorySort{ + OrderBy: []OrderbyParams{{ + WeightStringCol: 0, + Col: 1, + }}, + Input: fp, + } + var results []*sqltypes.Result + + t.Run("order by weight string", func(t *testing.T) { + + err := ms.StreamExecute(&noopVCursor{}, nil, false, func(qr *sqltypes.Result) error { + results = append(results, qr) + return nil + }) + require.NoError(t, err) + + wantResults := sqltypes.MakeTestStreamingResults( + fields, + "a|a", + "c|t", + "f|p", + "g|d", + "v|x", + ) + utils.MustMatch(t, wantResults, results) + }) + + t.Run("Limit test", func(t *testing.T) { + fp.rewind() + upperlimit, err := sqlparser.NewPlanValue(sqlparser.NewArgument(":__upper_limit")) + require.NoError(t, err) + ms.UpperLimit = upperlimit + bv := map[string]*querypb.BindVariable{"__upper_limit": sqltypes.Int64BindVariable(3)} + + results = nil + err = ms.StreamExecute(&noopVCursor{}, bv, false, func(qr *sqltypes.Result) error { + results = append(results, qr) + return nil + }) + require.NoError(t, err) + + wantResults := sqltypes.MakeTestStreamingResults( + fields, + "a|a", + "c|t", + "f|p", + ) + utils.MustMatch(t, wantResults, results) + }) +} + +func TestMemorySortExecuteWeightString(t *testing.T) { + fields := sqltypes.MakeTestFields( + "c1|c2", + "varchar|varbinary", + ) + fp := &fakePrimitive{ + results: []*sqltypes.Result{sqltypes.MakeTestResult( + fields, + "a|1", + "g|2", + "a|1", + "c|4", + "c|3", + )}, + } + + ms := &MemorySort{ + OrderBy: []OrderbyParams{{ + WeightStringCol: 1, + Col: 0, + }}, + Input: fp, } + + result, err := ms.Execute(nil, nil, false) + require.NoError(t, err) + + wantResult := sqltypes.MakeTestResult( + fields, + "a|1", + "a|1", + "g|2", + "c|3", + "c|4", + ) + utils.MustMatch(t, wantResult, result) + + fp.rewind() + upperlimit, err := sqlparser.NewPlanValue(sqlparser.NewArgument(":__upper_limit")) + require.NoError(t, err) + ms.UpperLimit = upperlimit + bv := map[string]*querypb.BindVariable{"__upper_limit": sqltypes.Int64BindVariable(3)} + + result, err = ms.Execute(nil, bv, false) + require.NoError(t, err) + + wantResult = sqltypes.MakeTestResult( + fields, + "a|1", + "a|1", + "g|2", + ) + utils.MustMatch(t, wantResult, result) } func TestMemorySortStreamExecute(t *testing.T) { @@ -101,7 +215,7 @@ func TestMemorySortStreamExecute(t *testing.T) { results: []*sqltypes.Result{sqltypes.MakeTestResult( fields, "a|1", - "b|2", + "g|2", "a|1", "c|4", "c|3", @@ -110,7 +224,8 @@ func TestMemorySortStreamExecute(t *testing.T) { ms := &MemorySort{ OrderBy: []OrderbyParams{{ - Col: 1, + WeightStringCol: -1, + Col: 1, }}, Input: fp, } @@ -120,27 +235,21 @@ func TestMemorySortStreamExecute(t *testing.T) { results = append(results, qr) return nil }) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) wantResults := sqltypes.MakeTestStreamingResults( fields, "a|1", "a|1", - "b|2", + "g|2", "c|3", "c|4", ) - if !reflect.DeepEqual(results, wantResults) { - t.Errorf("oa.Execute:\n%v, want\n%v", results, wantResults) - } + utils.MustMatch(t, wantResults, results) fp.rewind() upperlimit, err := sqlparser.NewPlanValue(sqlparser.NewArgument(":__upper_limit")) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) ms.UpperLimit = upperlimit bv := map[string]*querypb.BindVariable{"__upper_limit": sqltypes.Int64BindVariable(3)} @@ -149,21 +258,15 @@ func TestMemorySortStreamExecute(t *testing.T) { results = append(results, qr) return nil }) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) wantResults = sqltypes.MakeTestStreamingResults( fields, "a|1", "a|1", - "b|2", + "g|2", ) - if !reflect.DeepEqual(results, wantResults) { - r, _ := json.Marshal(results) - w, _ := json.Marshal(wantResults) - t.Errorf("oa.Execute:\n%s, want\n%s", r, w) - } + utils.MustMatch(t, wantResults, results) } func TestMemorySortGetFields(t *testing.T) { @@ -178,12 +281,8 @@ func TestMemorySortGetFields(t *testing.T) { ms := &MemorySort{Input: fp} got, err := ms.GetFields(nil, nil) - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(got, result) { - t.Errorf("l.GetFields:\n%v, want\n%v", got, result) - } + require.NoError(t, err) + utils.MustMatch(t, result, got) } func TestMemorySortExecuteTruncate(t *testing.T) { @@ -195,7 +294,7 @@ func TestMemorySortExecuteTruncate(t *testing.T) { results: []*sqltypes.Result{sqltypes.MakeTestResult( fields, "a|1|1", - "b|2|1", + "g|2|1", "a|1|1", "c|4|1", "c|3|1", @@ -204,28 +303,25 @@ func TestMemorySortExecuteTruncate(t *testing.T) { ms := &MemorySort{ OrderBy: []OrderbyParams{{ - Col: 1, + WeightStringCol: -1, + Col: 1, }}, Input: fp, TruncateColumnCount: 2, } result, err := ms.Execute(nil, nil, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) wantResult := sqltypes.MakeTestResult( fields[:2], "a|1", "a|1", - "b|2", + "g|2", "c|3", "c|4", ) - if !reflect.DeepEqual(result, wantResult) { - t.Errorf("oa.Execute:\n%v, want\n%v", result, wantResult) - } + utils.MustMatch(t, wantResult, result) } func TestMemorySortStreamExecuteTruncate(t *testing.T) { @@ -237,7 +333,7 @@ func TestMemorySortStreamExecuteTruncate(t *testing.T) { results: []*sqltypes.Result{sqltypes.MakeTestResult( fields, "a|1|1", - "b|2|1", + "g|2|1", "a|1|1", "c|4|1", "c|3|1", @@ -246,7 +342,8 @@ func TestMemorySortStreamExecuteTruncate(t *testing.T) { ms := &MemorySort{ OrderBy: []OrderbyParams{{ - Col: 1, + WeightStringCol: -1, + Col: 1, }}, Input: fp, TruncateColumnCount: 2, @@ -257,21 +354,17 @@ func TestMemorySortStreamExecuteTruncate(t *testing.T) { results = append(results, qr) return nil }) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) wantResults := sqltypes.MakeTestStreamingResults( fields[:2], "a|1", "a|1", - "b|2", + "g|2", "c|3", "c|4", ) - if !reflect.DeepEqual(results, wantResults) { - t.Errorf("oa.Execute:\n%v, want\n%v", results, wantResults) - } + utils.MustMatch(t, wantResults, results) } func TestMemorySortMultiColumn(t *testing.T) { @@ -292,18 +385,18 @@ func TestMemorySortMultiColumn(t *testing.T) { ms := &MemorySort{ OrderBy: []OrderbyParams{{ - Col: 1, + Col: 1, + WeightStringCol: -1, }, { - Col: 0, - Desc: true, + Col: 0, + WeightStringCol: -1, + Desc: true, }}, Input: fp, } result, err := ms.Execute(nil, nil, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) wantResult := sqltypes.MakeTestResult( fields, @@ -313,22 +406,16 @@ func TestMemorySortMultiColumn(t *testing.T) { "c|3", "c|4", ) - if !reflect.DeepEqual(result, wantResult) { - t.Errorf("oa.Execute:\n%v, want\n%v", result, wantResult) - } + utils.MustMatch(t, wantResult, result) fp.rewind() upperlimit, err := sqlparser.NewPlanValue(sqlparser.NewArgument(":__upper_limit")) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) ms.UpperLimit = upperlimit bv := map[string]*querypb.BindVariable{"__upper_limit": sqltypes.Int64BindVariable(3)} result, err = ms.Execute(nil, bv, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) wantResult = sqltypes.MakeTestResult( fields, @@ -336,9 +423,7 @@ func TestMemorySortMultiColumn(t *testing.T) { "a|1", "b|2", ) - if !reflect.DeepEqual(result, wantResult) { - t.Errorf("oa.Execute:\n%v, want\n%v", result, wantResult) - } + utils.MustMatch(t, wantResult, result) } func TestMemorySortMaxMemoryRows(t *testing.T) { @@ -375,7 +460,8 @@ func TestMemorySortMaxMemoryRows(t *testing.T) { ms := &MemorySort{ OrderBy: []OrderbyParams{{ - Col: 1, + WeightStringCol: -1, + Col: 1, }}, Input: fp, } @@ -410,7 +496,8 @@ func TestMemorySortExecuteNoVarChar(t *testing.T) { ms := &MemorySort{ OrderBy: []OrderbyParams{{ - Col: 0, + WeightStringCol: -1, + Col: 0, }}, Input: fp, } diff --git a/go/vt/vtgate/engine/merge_sort.go b/go/vt/vtgate/engine/merge_sort.go index 90a33313ea0..bcd80b974bd 100644 --- a/go/vt/vtgate/engine/merge_sort.go +++ b/go/vt/vtgate/engine/merge_sort.go @@ -96,9 +96,12 @@ func (ms *MergeSort) StreamExecute(vcursor VCursor, bindVars map[string]*querypb return err } + orderBy, weightStrings, desc := extractSlices(ms.OrderBy) sh := &scatterHeap{ - rows: make([]streamRow, 0, len(handles)), - orderBy: ms.OrderBy, + rows: make([]streamRow, 0, len(handles)), + orderBy: orderBy, + weightStrings: weightStrings, + desc: desc, } // Prime the heap. One element must be pulled from @@ -236,9 +239,11 @@ type streamRow struct { // yielded an error, err is set. This must be checked // after every heap operation. type scatterHeap struct { - rows []streamRow - orderBy []OrderbyParams - err error + rows []streamRow + orderBy []int + weightStrings []int + desc []bool + err error } // Len satisfies sort.Interface and heap.Interface. @@ -248,19 +253,29 @@ func (sh *scatterHeap) Len() int { // Less satisfies sort.Interface and heap.Interface. func (sh *scatterHeap) Less(i, j int) bool { - for _, order := range sh.orderBy { + for k := range sh.orderBy { if sh.err != nil { return true } - cmp, err := evalengine.NullsafeCompare(sh.rows[i].row[order.Col], sh.rows[j].row[order.Col]) + cmp, err := evalengine.NullsafeCompare(sh.rows[i].row[sh.orderBy[k]], sh.rows[j].row[sh.orderBy[k]]) if err != nil { - sh.err = err - return true + _, isComparisonErr := err.(evalengine.UnsupportedComparisonError) + if !(isComparisonErr && sh.weightStrings[k] != -1) { + sh.err = err + return true + } + sh.orderBy[k] = sh.weightStrings[k] + sh.weightStrings[k] = -1 + cmp, err = evalengine.NullsafeCompare(sh.rows[i].row[sh.orderBy[k]], sh.rows[j].row[sh.orderBy[k]]) + if err != nil { + sh.err = err + return true + } } if cmp == 0 { continue } - if order.Desc { + if sh.desc[k] { cmp = -cmp } return cmp < 0 diff --git a/go/vt/vtgate/engine/merge_sort_test.go b/go/vt/vtgate/engine/merge_sort_test.go index 1b37af94fa0..ede93d784b2 100644 --- a/go/vt/vtgate/engine/merge_sort_test.go +++ b/go/vt/vtgate/engine/merge_sort_test.go @@ -18,9 +18,10 @@ package engine import ( "errors" - "reflect" "testing" + "vitess.io/vitess/go/test/utils" + "github.com/stretchr/testify/require" "vitess.io/vitess/go/sqltypes" @@ -56,7 +57,8 @@ func TestMergeSortNormal(t *testing.T) { ), }} orderBy := []OrderbyParams{{ - Col: 0, + WeightStringCol: -1, + Col: 0, }} var results []*sqltypes.Result @@ -84,9 +86,65 @@ func TestMergeSortNormal(t *testing.T) { "---", "8|h", ) - if !reflect.DeepEqual(results, wantResults) { - t.Errorf("MergeSort:\n%s, want\n%s", sqltypes.PrintResults(results), sqltypes.PrintResults(wantResults)) - } + utils.MustMatch(t, wantResults, results) +} + +func TestMergeSortWeightString(t *testing.T) { + idColFields := sqltypes.MakeTestFields("id|col", "varbinary|varchar") + shardResults := []*shardResult{{ + results: sqltypes.MakeTestStreamingResults(idColFields, + "1|a", + "7|g", + ), + }, { + results: sqltypes.MakeTestStreamingResults(idColFields, + "2|b", + "---", + "3|c", + ), + }, { + results: sqltypes.MakeTestStreamingResults(idColFields, + "4|d", + "6|f", + ), + }, { + results: sqltypes.MakeTestStreamingResults(idColFields, + "4|d", + "---", + "8|h", + ), + }} + orderBy := []OrderbyParams{{ + WeightStringCol: 0, + Col: 1, + }} + + var results []*sqltypes.Result + err := testMergeSort(shardResults, orderBy, func(qr *sqltypes.Result) error { + results = append(results, qr) + return nil + }) + require.NoError(t, err) + + // Results are returned one row at a time. + wantResults := sqltypes.MakeTestStreamingResults(idColFields, + "1|a", + "---", + "2|b", + "---", + "3|c", + "---", + "4|d", + "---", + "4|d", + "---", + "6|f", + "---", + "7|g", + "---", + "8|h", + ) + utils.MustMatch(t, wantResults, results) } // TestMergeSortDescending tests the normal flow of a merge @@ -117,8 +175,9 @@ func TestMergeSortDescending(t *testing.T) { ), }} orderBy := []OrderbyParams{{ - Col: 0, - Desc: true, + WeightStringCol: -1, + Col: 0, + Desc: true, }} var results []*sqltypes.Result @@ -146,9 +205,7 @@ func TestMergeSortDescending(t *testing.T) { "---", "1|a", ) - if !reflect.DeepEqual(results, wantResults) { - t.Errorf("MergeSort:\n%s, want\n%s", sqltypes.PrintResults(results), sqltypes.PrintResults(wantResults)) - } + utils.MustMatch(t, wantResults, results) } func TestMergeSortEmptyResults(t *testing.T) { @@ -169,7 +226,8 @@ func TestMergeSortEmptyResults(t *testing.T) { results: sqltypes.MakeTestStreamingResults(idColFields), }} orderBy := []OrderbyParams{{ - Col: 0, + WeightStringCol: -1, + Col: 0, }} var results []*sqltypes.Result @@ -189,16 +247,15 @@ func TestMergeSortEmptyResults(t *testing.T) { "---", "7|g", ) - if !reflect.DeepEqual(results, wantResults) { - t.Errorf("MergeSort:\n%s, want\n%s", sqltypes.PrintResults(results), sqltypes.PrintResults(wantResults)) - } + utils.MustMatch(t, wantResults, results) } // TestMergeSortResultFailures tests failures at various // stages of result return. func TestMergeSortResultFailures(t *testing.T) { orderBy := []OrderbyParams{{ - Col: 0, + WeightStringCol: -1, + Col: 0, }} // Test early error. @@ -207,9 +264,7 @@ func TestMergeSortResultFailures(t *testing.T) { }} err := testMergeSort(shardResults, orderBy, func(qr *sqltypes.Result) error { return nil }) want := "early error" - if err == nil || err.Error() != want { - t.Errorf("MergeSort(): %v, want %v", err, want) - } + require.EqualError(t, err, want) // Test fail after fields. idFields := sqltypes.MakeTestFields("id", "int32") @@ -219,9 +274,7 @@ func TestMergeSortResultFailures(t *testing.T) { }} err = testMergeSort(shardResults, orderBy, func(qr *sqltypes.Result) error { return nil }) want = "fail after fields" - if err == nil || err.Error() != want { - t.Errorf("MergeSort(): %v, want %v", err, want) - } + require.EqualError(t, err, want) // Test fail after first row. shardResults = []*shardResult{{ @@ -230,9 +283,7 @@ func TestMergeSortResultFailures(t *testing.T) { }} err = testMergeSort(shardResults, orderBy, func(qr *sqltypes.Result) error { return nil }) want = "fail after first row" - if err == nil || err.Error() != want { - t.Errorf("MergeSort(): %v, want %v", err, want) - } + require.EqualError(t, err, want) } func TestMergeSortDataFailures(t *testing.T) { @@ -249,14 +300,13 @@ func TestMergeSortDataFailures(t *testing.T) { ), }} orderBy := []OrderbyParams{{ - Col: 0, + WeightStringCol: -1, + Col: 0, }} err := testMergeSort(shardResults, orderBy, func(qr *sqltypes.Result) error { return nil }) want := `strconv.ParseInt: parsing "2.1": invalid syntax` - if err == nil || err.Error() != want { - t.Errorf("MergeSort(): %v, want %v", err, want) - } + require.EqualError(t, err, want) // Create a new VCursor because the previous MergeSort will still // have lingering goroutines that can cause data race. @@ -272,9 +322,7 @@ func TestMergeSortDataFailures(t *testing.T) { }} err = testMergeSort(shardResults, orderBy, func(qr *sqltypes.Result) error { return nil }) want = `strconv.ParseInt: parsing "1.1": invalid syntax` - if err == nil || err.Error() != want { - t.Errorf("MergeSort(): %v, want %v", err, want) - } + require.EqualError(t, err, want) } func testMergeSort(shardResults []*shardResult, orderBy []OrderbyParams, callback func(qr *sqltypes.Result) error) error { diff --git a/go/vt/vtgate/engine/route.go b/go/vt/vtgate/engine/route.go index cb11e980e58..8914a700f8b 100644 --- a/go/vt/vtgate/engine/route.go +++ b/go/vt/vtgate/engine/route.go @@ -125,8 +125,9 @@ func NewRoute(opcode RouteOpcode, keyspace *vindexes.Keyspace, query, fieldQuery // OrderbyParams specifies the parameters for ordering. // This is used for merge-sorting scatter queries. type OrderbyParams struct { - Col int - Desc bool + Col int + WeightStringCol int + Desc bool } func (obp OrderbyParams) String() string { @@ -136,6 +137,14 @@ func (obp OrderbyParams) String() string { } else { val += " ASC" } + if obp.WeightStringCol != -1 { + val += " " + strconv.Itoa(obp.WeightStringCol) + if obp.Desc { + val += " DESC" + } else { + val += " ASC" + } + } return val } @@ -584,25 +593,36 @@ func (route *Route) sort(in *sqltypes.Result) (*sqltypes.Result, error) { InsertID: in.InsertID, } + orderBy, weightStrings, desc := extractSlices(route.OrderBy) + sort.Slice(out.Rows, func(i, j int) bool { // If there are any errors below, the function sets // the external err and returns true. Once err is set, // all subsequent calls return true. This will make // Slice think that all elements are in the correct // order and return more quickly. - for _, order := range route.OrderBy { + for k := range orderBy { if err != nil { return true } var cmp int - cmp, err = evalengine.NullsafeCompare(out.Rows[i][order.Col], out.Rows[j][order.Col]) + cmp, err = evalengine.NullsafeCompare(out.Rows[i][orderBy[k]], out.Rows[j][orderBy[k]]) if err != nil { - return true + _, isComparisonErr := err.(evalengine.UnsupportedComparisonError) + if !(isComparisonErr && weightStrings[k] != -1) { + return true + } + orderBy[k] = weightStrings[k] + weightStrings[k] = -1 + cmp, err = evalengine.NullsafeCompare(out.Rows[i][orderBy[k]], out.Rows[j][orderBy[k]]) + if err != nil { + return true + } } if cmp == 0 { continue } - if order.Desc { + if desc[k] { cmp = -cmp } return cmp < 0 diff --git a/go/vt/vtgate/engine/route_test.go b/go/vt/vtgate/engine/route_test.go index b0c4788bd9f..67afadce4c4 100644 --- a/go/vt/vtgate/engine/route_test.go +++ b/go/vt/vtgate/engine/route_test.go @@ -59,9 +59,7 @@ func TestSelectUnsharded(t *testing.T) { results: []*sqltypes.Result{defaultSelectResult}, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAnyShard()`, `ExecuteMultiShard ks.0: dummy_select {} false false`, @@ -70,9 +68,7 @@ func TestSelectUnsharded(t *testing.T) { vc.Rewind() result, err = wrapStreamExecute(sel, vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAnyShard()`, `StreamExecuteMulti dummy_select ks.0: {} `, @@ -188,9 +184,7 @@ func TestSelectScatter(t *testing.T) { results: []*sqltypes.Result{defaultSelectResult}, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAllShards()`, `ExecuteMultiShard ks.-20: dummy_select {} ks.20-: dummy_select {} false false`, @@ -199,9 +193,7 @@ func TestSelectScatter(t *testing.T) { vc.Rewind() result, err = wrapStreamExecute(sel, vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAllShards()`, `StreamExecuteMulti dummy_select ks.-20: {} ks.20-: {} `, @@ -228,9 +220,7 @@ func TestSelectEqualUnique(t *testing.T) { results: []*sqltypes.Result{defaultSelectResult}, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [type:INT64 value:"1" ] Destinations:DestinationKeyspaceID(166b40b44aba4bd6)`, `ExecuteMultiShard ks.-20: dummy_select {} false false`, @@ -239,9 +229,7 @@ func TestSelectEqualUnique(t *testing.T) { vc.Rewind() result, err = wrapStreamExecute(sel, vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [type:INT64 value:"1" ] Destinations:DestinationKeyspaceID(166b40b44aba4bd6)`, `StreamExecuteMulti dummy_select ks.-20: {} `, @@ -303,9 +291,7 @@ func TestSelectEqualUniqueScatter(t *testing.T) { results: []*sqltypes.Result{defaultSelectResult}, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [type:INT64 value:"1" ] Destinations:DestinationKeyRange(-)`, `ExecuteMultiShard ks.-20: dummy_select {} ks.20-: dummy_select {} false false`, @@ -314,9 +300,7 @@ func TestSelectEqualUniqueScatter(t *testing.T) { vc.Rewind() result, err = wrapStreamExecute(sel, vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [type:INT64 value:"1" ] Destinations:DestinationKeyRange(-)`, `StreamExecuteMulti dummy_select ks.-20: {} ks.20-: {} `, @@ -357,9 +341,7 @@ func TestSelectEqual(t *testing.T) { }, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `Execute select from, toc from lkp where from in ::from from: type:TUPLE values: false`, `ResolveDestinations ks [type:INT64 value:"1" ] Destinations:DestinationKeyspaceIDs(00,80)`, @@ -369,9 +351,7 @@ func TestSelectEqual(t *testing.T) { vc.Rewind() result, err = wrapStreamExecute(sel, vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `Execute select from, toc from lkp where from in ::from from: type:TUPLE values: false`, `ResolveDestinations ks [type:INT64 value:"1" ] Destinations:DestinationKeyspaceIDs(00,80)`, @@ -400,9 +380,7 @@ func TestSelectEqualNoRoute(t *testing.T) { vc := &loggingVCursor{shards: []string{"-20", "20-"}} result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `Execute select from, toc from lkp where from in ::from from: type:TUPLE values: false`, `ResolveDestinations ks [type:INT64 value:"1" ] Destinations:DestinationNone()`, @@ -411,9 +389,7 @@ func TestSelectEqualNoRoute(t *testing.T) { vc.Rewind() result, err = wrapStreamExecute(sel, vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `Execute select from, toc from lkp where from in ::from from: type:TUPLE values: false`, `ResolveDestinations ks [type:INT64 value:"1" ] Destinations:DestinationNone()`, @@ -449,9 +425,7 @@ func TestSelectINUnique(t *testing.T) { results: []*sqltypes.Result{defaultSelectResult}, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [type:INT64 value:"1" type:INT64 value:"2" type:INT64 value:"4" ] Destinations:DestinationKeyspaceID(166b40b44aba4bd6),DestinationKeyspaceID(06e7ea22ce92708f),DestinationKeyspaceID(d2fd8867d50d2dfe)`, `ExecuteMultiShard ` + @@ -463,9 +437,7 @@ func TestSelectINUnique(t *testing.T) { vc.Rewind() result, err = wrapStreamExecute(sel, vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [type:INT64 value:"1" type:INT64 value:"2" type:INT64 value:"4" ] Destinations:DestinationKeyspaceID(166b40b44aba4bd6),DestinationKeyspaceID(06e7ea22ce92708f),DestinationKeyspaceID(d2fd8867d50d2dfe)`, `StreamExecuteMulti dummy_select ks.-20: {__vals: type:TUPLE values: values: } ks.20-: {__vals: type:TUPLE values: } `, @@ -520,9 +492,7 @@ func TestSelectINNonUnique(t *testing.T) { }, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `Execute select from, toc from lkp where from in ::from from: type:TUPLE values: values: values: false`, `ResolveDestinations ks [type:INT64 value:"1" type:INT64 value:"2" type:INT64 value:"4" ] Destinations:DestinationKeyspaceIDs(00,80),DestinationKeyspaceIDs(00),DestinationKeyspaceIDs(80)`, @@ -535,9 +505,7 @@ func TestSelectINNonUnique(t *testing.T) { vc.Rewind() result, err = wrapStreamExecute(sel, vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `Execute select from, toc from lkp where from in ::from from: type:TUPLE values: values: values: false`, `ResolveDestinations ks [type:INT64 value:"1" type:INT64 value:"2" type:INT64 value:"4" ] Destinations:DestinationKeyspaceIDs(00,80),DestinationKeyspaceIDs(00),DestinationKeyspaceIDs(80)`, @@ -574,9 +542,7 @@ func TestSelectMultiEqual(t *testing.T) { results: []*sqltypes.Result{defaultSelectResult}, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [type:INT64 value:"1" type:INT64 value:"2" type:INT64 value:"4" ] Destinations:DestinationKeyspaceID(166b40b44aba4bd6),DestinationKeyspaceID(06e7ea22ce92708f),DestinationKeyspaceID(d2fd8867d50d2dfe)`, `ExecuteMultiShard ks.-20: dummy_select {} ks.20-: dummy_select {} false false`, @@ -585,9 +551,7 @@ func TestSelectMultiEqual(t *testing.T) { vc.Rewind() result, err = wrapStreamExecute(sel, vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [type:INT64 value:"1" type:INT64 value:"2" type:INT64 value:"4" ] Destinations:DestinationKeyspaceID(166b40b44aba4bd6),DestinationKeyspaceID(06e7ea22ce92708f),DestinationKeyspaceID(d2fd8867d50d2dfe)`, `StreamExecuteMulti dummy_select ks.-20: {} ks.20-: {} `, @@ -611,9 +575,7 @@ func TestSelectNext(t *testing.T) { results: []*sqltypes.Result{defaultSelectResult}, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAnyShard()`, `ExecuteMultiShard ks.-20: dummy_select {} false false`, @@ -645,9 +607,7 @@ func TestSelectDBA(t *testing.T) { results: []*sqltypes.Result{defaultSelectResult}, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAnyShard()`, `ExecuteMultiShard ks.-20: dummy_select {} false false`, @@ -679,9 +639,7 @@ func TestSelectReference(t *testing.T) { results: []*sqltypes.Result{defaultSelectResult}, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAnyShard()`, `ExecuteMultiShard ks.-20: dummy_select {} false false`, @@ -717,9 +675,7 @@ func TestRouteGetFields(t *testing.T) { vc := &loggingVCursor{shards: []string{"-20", "20-"}} result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, true) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `Execute select from, toc from lkp where from in ::from from: type:TUPLE values: false`, `ResolveDestinations ks [type:INT64 value:"1" ] Destinations:DestinationNone()`, @@ -730,9 +686,7 @@ func TestRouteGetFields(t *testing.T) { vc.Rewind() result, err = wrapStreamExecute(sel, vc, map[string]*querypb.BindVariable{}, true) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `Execute select from, toc from lkp where from in ::from from: type:TUPLE values: false`, `ResolveDestinations ks [type:INT64 value:"1" ] Destinations:DestinationNone()`, @@ -753,7 +707,8 @@ func TestRouteSort(t *testing.T) { "dummy_select_field", ) sel.OrderBy = []OrderbyParams{{ - Col: 0, + Col: 0, + WeightStringCol: -1, }} vc := &loggingVCursor{ @@ -772,9 +727,7 @@ func TestRouteSort(t *testing.T) { }, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAnyShard()`, `ExecuteMultiShard ks.0: dummy_select {} false false`, @@ -794,9 +747,7 @@ func TestRouteSort(t *testing.T) { sel.OrderBy[0].Desc = true vc.Rewind() result, err = sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) wantResult = sqltypes.MakeTestResult( sqltypes.MakeTestFields( "id", @@ -827,6 +778,108 @@ func TestRouteSort(t *testing.T) { require.EqualError(t, err, `types are not comparable: VARCHAR vs VARCHAR`) } +func TestRouteSortWeightStrings(t *testing.T) { + sel := NewRoute( + SelectUnsharded, + &vindexes.Keyspace{ + Name: "ks", + Sharded: false, + }, + "dummy_select", + "dummy_select_field", + ) + sel.OrderBy = []OrderbyParams{{ + Col: 1, + WeightStringCol: 0, + }} + + vc := &loggingVCursor{ + shards: []string{"0"}, + results: []*sqltypes.Result{ + sqltypes.MakeTestResult( + sqltypes.MakeTestFields( + "weightString|normal", + "varbinary|varchar", + ), + "v|x", + "g|d", + "a|a", + "c|t", + "f|p", + ), + }, + } + + var result *sqltypes.Result + var wantResult *sqltypes.Result + var err error + t.Run("Sort using Weight Strings", func(t *testing.T) { + result, err = sel.Execute(vc, map[string]*querypb.BindVariable{}, false) + require.NoError(t, err) + vc.ExpectLog(t, []string{ + `ResolveDestinations ks [] Destinations:DestinationAnyShard()`, + `ExecuteMultiShard ks.0: dummy_select {} false false`, + }) + wantResult = sqltypes.MakeTestResult( + sqltypes.MakeTestFields( + "weightString|normal", + "varbinary|varchar", + ), + "a|a", + "c|t", + "f|p", + "g|d", + "v|x", + ) + expectResult(t, "sel.Execute", result, wantResult) + }) + + t.Run("Descending ordering using weighted strings", func(t *testing.T) { + sel.OrderBy[0].Desc = true + vc.Rewind() + result, err = sel.Execute(vc, map[string]*querypb.BindVariable{}, false) + require.NoError(t, err) + wantResult = sqltypes.MakeTestResult( + sqltypes.MakeTestFields( + "weightString|normal", + "varbinary|varchar", + ), + "v|x", + "g|d", + "f|p", + "c|t", + "a|a", + ) + expectResult(t, "sel.Execute", result, wantResult) + }) + + t.Run("Error when no weight string set", func(t *testing.T) { + sel.OrderBy = []OrderbyParams{{ + Col: 1, + WeightStringCol: -1, + }} + + vc = &loggingVCursor{ + shards: []string{"0"}, + results: []*sqltypes.Result{ + sqltypes.MakeTestResult( + sqltypes.MakeTestFields( + "weightString|normal", + "varbinary|varchar", + ), + "v|x", + "g|d", + "a|a", + "c|t", + "f|p", + ), + }, + } + _, err = sel.Execute(vc, map[string]*querypb.BindVariable{}, false) + require.EqualError(t, err, `types are not comparable: VARCHAR vs VARCHAR`) + }) +} + func TestRouteSortTruncate(t *testing.T) { sel := NewRoute( SelectUnsharded, @@ -858,9 +911,7 @@ func TestRouteSortTruncate(t *testing.T) { }, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAnyShard()`, `ExecuteMultiShard ks.0: dummy_select {} false false`, @@ -904,9 +955,7 @@ func TestRouteStreamTruncate(t *testing.T) { }, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAnyShard()`, `ExecuteMultiShard ks.0: dummy_select {} false false`, @@ -951,9 +1000,7 @@ func TestRouteStreamSortTruncate(t *testing.T) { }, } result, err := wrapStreamExecute(sel, vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAnyShard()`, `StreamExecuteMulti dummy_select ks.0: {} `, @@ -1062,9 +1109,7 @@ func TestExecFail(t *testing.T) { }, } _, err = sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Errorf("unexpected ScatterErrorsAsWarnings error %v", err) - } + require.NoError(t, err, "unexpected ScatterErrorsAsWarnings error %v", err) // Ensure that the error code is preserved from SQLErrors and that it // turns into ERUnknownError for all others @@ -1100,9 +1145,7 @@ func TestExecFail(t *testing.T) { }, } result, err := sel.Execute(vc, map[string]*querypb.BindVariable{}, false) - if err != nil { - t.Errorf("unexpected ScatterErrorsAsWarnings error %v", err) - } + require.NoError(t, err, "unexpected ScatterErrorsAsWarnings error %v", err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAllShards()`, `ExecuteMultiShard ks.-20: dummy_select {} ks.20-: dummy_select {} false false`, diff --git a/go/vt/vtgate/evalengine/arithmetic.go b/go/vt/vtgate/evalengine/arithmetic.go index 7ad75e6159b..3a887d77763 100644 --- a/go/vt/vtgate/evalengine/arithmetic.go +++ b/go/vt/vtgate/evalengine/arithmetic.go @@ -34,6 +34,17 @@ import ( // a Value, used for arithmetic operations. var zeroBytes = []byte("0") +// UnsupportedComparisonError represents the error where the comparison between the two types is unsupported on vitess +type UnsupportedComparisonError struct { + Type1 querypb.Type + Type2 querypb.Type +} + +// Error function implements the error interface +func (err UnsupportedComparisonError) Error() string { + return fmt.Sprintf("types are not comparable: %v vs %v", err.Type1, err.Type2) +} + // Add adds two values together // if v1 or v2 is null, then it returns null func Add(v1, v2 sqltypes.Value) (sqltypes.Value, error) { @@ -201,7 +212,10 @@ func NullsafeCompare(v1, v2 sqltypes.Value) (int, error) { if isByteComparable(v1) && isByteComparable(v2) { return bytes.Compare(v1.ToBytes(), v2.ToBytes()), nil } - return 0, fmt.Errorf("types are not comparable: %v vs %v", v1.Type(), v2.Type()) + return 0, UnsupportedComparisonError{ + Type1: v1.Type(), + Type2: v2.Type(), + } } // NullsafeHashcode returns an int64 hashcode that is guaranteed to be the same diff --git a/go/vt/vtgate/planbuilder/memory_sort.go b/go/vt/vtgate/planbuilder/memory_sort.go index 053c5ad5e87..81167cedcdf 100644 --- a/go/vt/vtgate/planbuilder/memory_sort.go +++ b/go/vt/vtgate/planbuilder/memory_sort.go @@ -83,8 +83,9 @@ func newMemorySort(plan logicalPlan, orderBy sqlparser.OrderBy) (*memorySort, er return nil, fmt.Errorf("unsupported: memory sort: order by must reference a column in the select list: %s", sqlparser.String(order)) } ob := engine.OrderbyParams{ - Col: colNumber, - Desc: order.Direction == sqlparser.DescOrder, + Col: colNumber, + WeightStringCol: -1, + Desc: order.Direction == sqlparser.DescOrder, } ms.eMemorySort.OrderBy = append(ms.eMemorySort.OrderBy, ob) } @@ -110,10 +111,10 @@ func (ms *memorySort) SetLimit(limit *sqlparser.Limit) error { func (ms *memorySort) Wireup(plan logicalPlan, jt *jointab) error { for i, orderby := range ms.eMemorySort.OrderBy { rc := ms.resultColumns[orderby.Col] - if sqltypes.IsText(rc.column.typ) { + if sqltypes.IsText(rc.column.typ) || rc.column.typ == sqltypes.Null { // If a weight string was previously requested, reuse it. if weightcolNumber, ok := ms.weightStrings[rc]; ok { - ms.eMemorySort.OrderBy[i].Col = weightcolNumber + ms.eMemorySort.OrderBy[i].WeightStringCol = weightcolNumber continue } weightcolNumber, err := ms.input.SupplyWeightString(orderby.Col) @@ -121,7 +122,7 @@ func (ms *memorySort) Wireup(plan logicalPlan, jt *jointab) error { return err } ms.weightStrings[rc] = weightcolNumber - ms.eMemorySort.OrderBy[i].Col = weightcolNumber + ms.eMemorySort.OrderBy[i].WeightStringCol = weightcolNumber ms.eMemorySort.TruncateColumnCount = len(ms.resultColumns) } } diff --git a/go/vt/vtgate/planbuilder/merge_sort.go b/go/vt/vtgate/planbuilder/merge_sort.go index e76a4379aea..eec6d629aab 100644 --- a/go/vt/vtgate/planbuilder/merge_sort.go +++ b/go/vt/vtgate/planbuilder/merge_sort.go @@ -69,11 +69,11 @@ func (ms *mergeSort) Wireup(plan logicalPlan, jt *jointab) error { if sqltypes.IsText(rc.column.typ) || rc.column.typ == sqltypes.Null { // If a weight string was previously requested, reuse it. if colNumber, ok := ms.weightStrings[rc]; ok { - rb.eroute.OrderBy[i].Col = colNumber + rb.eroute.OrderBy[i].WeightStringCol = colNumber continue } var err error - rb.eroute.OrderBy[i].Col, err = rb.SupplyWeightString(orderby.Col) + rb.eroute.OrderBy[i].WeightStringCol, err = rb.SupplyWeightString(orderby.Col) if err != nil { return err } diff --git a/go/vt/vtgate/planbuilder/ordering.go b/go/vt/vtgate/planbuilder/ordering.go index 1e4c87bba46..c8c52739090 100644 --- a/go/vt/vtgate/planbuilder/ordering.go +++ b/go/vt/vtgate/planbuilder/ordering.go @@ -290,8 +290,9 @@ func planRouteOrdering(orderBy sqlparser.OrderBy, node *route) (logicalPlan, err return nil, fmt.Errorf("unsupported: in scatter query: order by must reference a column in the select list: %s", sqlparser.String(order)) } ob := engine.OrderbyParams{ - Col: colNumber, - Desc: order.Direction == sqlparser.DescOrder, + Col: colNumber, + WeightStringCol: -1, + Desc: order.Direction == sqlparser.DescOrder, } node.eroute.OrderBy = append(node.eroute.OrderBy, ob) diff --git a/go/vt/wrangler/vdiff.go b/go/vt/wrangler/vdiff.go index 9d32aa04717..c64a9a4a2a1 100644 --- a/go/vt/wrangler/vdiff.go +++ b/go/vt/wrangler/vdiff.go @@ -492,7 +492,7 @@ func newMergeSorter(participants map[string]*shardStreamer, comparePKs []int) *e } ob := make([]engine.OrderbyParams, 0, len(comparePKs)) for _, cpk := range comparePKs { - ob = append(ob, engine.OrderbyParams{Col: cpk}) + ob = append(ob, engine.OrderbyParams{Col: cpk, WeightStringCol: -1}) } return &engine.MergeSort{ Primitives: prims, From 7305da9927dbdafb04fee368159bc12f1f3b3ca0 Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Tue, 16 Mar 2021 17:31:47 +0530 Subject: [PATCH 05/19] fixed vindex_func test Signed-off-by: GuptaManan100 --- go/vt/vtgate/planbuilder/concatenate.go | 2 +- go/vt/vtgate/planbuilder/join.go | 16 +- go/vt/vtgate/planbuilder/join2.go | 2 +- go/vt/vtgate/planbuilder/logical_plan.go | 19 +- go/vt/vtgate/planbuilder/memory_sort.go | 5 +- go/vt/vtgate/planbuilder/merge_sort.go | 2 +- go/vt/vtgate/planbuilder/ordered_aggregate.go | 5 +- go/vt/vtgate/planbuilder/plan_test.go | 4 +- go/vt/vtgate/planbuilder/pullout_subquery.go | 2 +- go/vt/vtgate/planbuilder/route.go | 13 +- .../vtgate/planbuilder/sql_calc_found_rows.go | 4 +- .../planbuilder/testdata/aggr_cases.txt | 66 +- .../testdata/memory_sort_cases.txt | 96 +-- .../testdata/postprocess_cases.txt | 14 +- .../planbuilder/testdata/select_cases.txt | 4 +- .../planbuilder/testdata/union_cases.txt | 683 +----------------- go/vt/vtgate/planbuilder/vindex_func.go | 6 +- 17 files changed, 160 insertions(+), 783 deletions(-) diff --git a/go/vt/vtgate/planbuilder/concatenate.go b/go/vt/vtgate/planbuilder/concatenate.go index b482e795845..71fd2405827 100644 --- a/go/vt/vtgate/planbuilder/concatenate.go +++ b/go/vt/vtgate/planbuilder/concatenate.go @@ -70,7 +70,7 @@ func (c *concatenate) SupplyCol(col *sqlparser.ColName) (rc *resultColumn, colNu panic("implement me") } -func (c *concatenate) SupplyWeightString(colNumber int) (weightcolNumber int, err error) { +func (c *concatenate) SupplyWeightString(colNumber int) (weightcolNumber int) { panic("implement me") } diff --git a/go/vt/vtgate/planbuilder/join.go b/go/vt/vtgate/planbuilder/join.go index d8e7429a900..a6410bfcc69 100644 --- a/go/vt/vtgate/planbuilder/join.go +++ b/go/vt/vtgate/planbuilder/join.go @@ -212,28 +212,22 @@ func (jb *join) SupplyCol(col *sqlparser.ColName) (rc *resultColumn, colNumber i } // SupplyWeightString implements the logicalPlan interface -func (jb *join) SupplyWeightString(colNumber int) (weightcolNumber int, err error) { +func (jb *join) SupplyWeightString(colNumber int) (weightcolNumber int) { rc := jb.resultColumns[colNumber] if weightcolNumber, ok := jb.weightStrings[rc]; ok { - return weightcolNumber, nil + return weightcolNumber } routeNumber := rc.column.Origin().Order() if jb.isOnLeft(routeNumber) { - sourceCol, err := jb.Left.SupplyWeightString(-jb.ejoin.Cols[colNumber] - 1) - if err != nil { - return 0, err - } + sourceCol := jb.Left.SupplyWeightString(-jb.ejoin.Cols[colNumber] - 1) jb.ejoin.Cols = append(jb.ejoin.Cols, -sourceCol-1) } else { - sourceCol, err := jb.Right.SupplyWeightString(jb.ejoin.Cols[colNumber] - 1) - if err != nil { - return 0, err - } + sourceCol := jb.Right.SupplyWeightString(jb.ejoin.Cols[colNumber] - 1) jb.ejoin.Cols = append(jb.ejoin.Cols, sourceCol+1) } jb.resultColumns = append(jb.resultColumns, rc) jb.weightStrings[rc] = len(jb.ejoin.Cols) - 1 - return len(jb.ejoin.Cols) - 1, nil + return len(jb.ejoin.Cols) - 1 } // Rewrite implements the logicalPlan interface diff --git a/go/vt/vtgate/planbuilder/join2.go b/go/vt/vtgate/planbuilder/join2.go index 7b273c88865..c2a6063d5ed 100644 --- a/go/vt/vtgate/planbuilder/join2.go +++ b/go/vt/vtgate/planbuilder/join2.go @@ -73,7 +73,7 @@ func (j *joinV4) SupplyCol(col *sqlparser.ColName) (rc *resultColumn, colNumber } // SupplyWeightString implements the logicalPlan interface -func (j *joinV4) SupplyWeightString(colNumber int) (weightcolNumber int, err error) { +func (j *joinV4) SupplyWeightString(colNumber int) (weightcolNumber int) { panic("implement me") } diff --git a/go/vt/vtgate/planbuilder/logical_plan.go b/go/vt/vtgate/planbuilder/logical_plan.go index bf17fca74c9..822e68d8497 100644 --- a/go/vt/vtgate/planbuilder/logical_plan.go +++ b/go/vt/vtgate/planbuilder/logical_plan.go @@ -69,8 +69,8 @@ type logicalPlan interface { SupplyCol(col *sqlparser.ColName) (rc *resultColumn, colNumber int) // SupplyWeightString must supply a weight_string expression of the - // specified column. - SupplyWeightString(colNumber int) (weightcolNumber int, err error) + // specified column. It returns -1 if we cannot supply a weight column for it. + SupplyWeightString(colNumber int) (weightcolNumber int) // Primitive returns the underlying primitive. // This function should only be called after Wireup is finished. @@ -170,7 +170,7 @@ func (bc *logicalPlanCommon) SupplyCol(col *sqlparser.ColName) (rc *resultColumn return bc.input.SupplyCol(col) } -func (bc *logicalPlanCommon) SupplyWeightString(colNumber int) (weightcolNumber int, err error) { +func (bc *logicalPlanCommon) SupplyWeightString(colNumber int) (weightcolNumber int) { return bc.input.SupplyWeightString(colNumber) } @@ -239,25 +239,22 @@ func (rsb *resultsBuilder) SupplyCol(col *sqlparser.ColName) (rc *resultColumn, return rc, colNumber } -func (rsb *resultsBuilder) SupplyWeightString(colNumber int) (weightcolNumber int, err error) { +func (rsb *resultsBuilder) SupplyWeightString(colNumber int) (weightcolNumber int) { rc := rsb.resultColumns[colNumber] if weightcolNumber, ok := rsb.weightStrings[rc]; ok { - return weightcolNumber, nil - } - weightcolNumber, err = rsb.input.SupplyWeightString(colNumber) - if err != nil { - return 0, nil + return weightcolNumber } + weightcolNumber = rsb.input.SupplyWeightString(colNumber) rsb.weightStrings[rc] = weightcolNumber if weightcolNumber < len(rsb.resultColumns) { - return weightcolNumber, nil + return weightcolNumber } // Add result columns from input until weightcolNumber is reached. for weightcolNumber >= len(rsb.resultColumns) { rsb.resultColumns = append(rsb.resultColumns, rsb.input.ResultColumns()[len(rsb.resultColumns)]) } rsb.truncater.SetTruncateColumnCount(len(rsb.resultColumns)) - return weightcolNumber, nil + return weightcolNumber } //------------------------------------------------------------------------- diff --git a/go/vt/vtgate/planbuilder/memory_sort.go b/go/vt/vtgate/planbuilder/memory_sort.go index 81167cedcdf..1ab655979a1 100644 --- a/go/vt/vtgate/planbuilder/memory_sort.go +++ b/go/vt/vtgate/planbuilder/memory_sort.go @@ -117,10 +117,7 @@ func (ms *memorySort) Wireup(plan logicalPlan, jt *jointab) error { ms.eMemorySort.OrderBy[i].WeightStringCol = weightcolNumber continue } - weightcolNumber, err := ms.input.SupplyWeightString(orderby.Col) - if err != nil { - return err - } + weightcolNumber := ms.input.SupplyWeightString(orderby.Col) ms.weightStrings[rc] = weightcolNumber ms.eMemorySort.OrderBy[i].WeightStringCol = weightcolNumber ms.eMemorySort.TruncateColumnCount = len(ms.resultColumns) diff --git a/go/vt/vtgate/planbuilder/merge_sort.go b/go/vt/vtgate/planbuilder/merge_sort.go index eec6d629aab..83d4e38df73 100644 --- a/go/vt/vtgate/planbuilder/merge_sort.go +++ b/go/vt/vtgate/planbuilder/merge_sort.go @@ -73,7 +73,7 @@ func (ms *mergeSort) Wireup(plan logicalPlan, jt *jointab) error { continue } var err error - rb.eroute.OrderBy[i].WeightStringCol, err = rb.SupplyWeightString(orderby.Col) + rb.eroute.OrderBy[i].WeightStringCol = rb.SupplyWeightString(orderby.Col) if err != nil { return err } diff --git a/go/vt/vtgate/planbuilder/ordered_aggregate.go b/go/vt/vtgate/planbuilder/ordered_aggregate.go index 7e2065b7f64..36c4461b154 100644 --- a/go/vt/vtgate/planbuilder/ordered_aggregate.go +++ b/go/vt/vtgate/planbuilder/ordered_aggregate.go @@ -338,10 +338,7 @@ func (oa *orderedAggregate) Wireup(plan logicalPlan, jt *jointab) error { oa.eaggr.Keys[i] = weightcolNumber continue } - weightcolNumber, err := oa.input.SupplyWeightString(colNumber) - if err != nil { - return err - } + weightcolNumber := oa.input.SupplyWeightString(colNumber) oa.weightStrings[rc] = weightcolNumber oa.eaggr.Keys[i] = weightcolNumber oa.eaggr.TruncateColumnCount = len(oa.resultColumns) diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go index 0250f4215f5..56441311b31 100644 --- a/go/vt/vtgate/planbuilder/plan_test.go +++ b/go/vt/vtgate/planbuilder/plan_test.go @@ -401,6 +401,7 @@ func testFile(t *testing.T, filename, tempDir string, vschema *vschemaWrapper, c t.Run(filename, func(t *testing.T) { expected := &strings.Builder{} fail := checkAllTests + var outFirstPlanner string for tcase := range iterateExecFile(filename) { t.Run(fmt.Sprintf("%d V3: %s", tcase.lineno, tcase.comments), func(t *testing.T) { vschema.version = V3 @@ -414,6 +415,7 @@ func testFile(t *testing.T, filename, tempDir string, vschema *vschemaWrapper, c if err != nil { out = `"` + out + `"` } + outFirstPlanner = out expected.WriteString(fmt.Sprintf("%s\"%s\"\n%s\n", tcase.comments, escapeNewLines(tcase.input), out)) }) @@ -445,7 +447,7 @@ func testFile(t *testing.T, filename, tempDir string, vschema *vschemaWrapper, c out = `"` + out + `"` } - if tcase.output == out { + if outFirstPlanner == out { expected.WriteString(samePlanMarker) } else { expected.WriteString(fmt.Sprintf("%s\n", out)) diff --git a/go/vt/vtgate/planbuilder/pullout_subquery.go b/go/vt/vtgate/planbuilder/pullout_subquery.go index ec0862bcf0d..e85d90eba40 100644 --- a/go/vt/vtgate/planbuilder/pullout_subquery.go +++ b/go/vt/vtgate/planbuilder/pullout_subquery.go @@ -110,7 +110,7 @@ func (ps *pulloutSubquery) SupplyCol(col *sqlparser.ColName) (rc *resultColumn, } // SupplyWeightString implements the logicalPlan interface -func (ps *pulloutSubquery) SupplyWeightString(colNumber int) (weightcolNumber int, err error) { +func (ps *pulloutSubquery) SupplyWeightString(colNumber int) (weightcolNumber int) { return ps.underlying.SupplyWeightString(colNumber) } diff --git a/go/vt/vtgate/planbuilder/route.go b/go/vt/vtgate/planbuilder/route.go index 43ac4b1ba93..2793ee055c2 100644 --- a/go/vt/vtgate/planbuilder/route.go +++ b/go/vt/vtgate/planbuilder/route.go @@ -333,19 +333,19 @@ func (rb *route) SupplyCol(col *sqlparser.ColName) (rc *resultColumn, colNumber } // SupplyWeightString implements the logicalPlan interface -func (rb *route) SupplyWeightString(colNumber int) (weightcolNumber int, err error) { +func (rb *route) SupplyWeightString(colNumber int) (weightcolNumber int) { rc := rb.resultColumns[colNumber] if weightcolNumber, ok := rb.weightStrings[rc]; ok { - return weightcolNumber, nil + return weightcolNumber } s, ok := rb.Select.(*sqlparser.Select) if !ok { - return 0, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "unexpected AST struct for query") + return 0 } aliasExpr, ok := s.SelectExprs[colNumber].(*sqlparser.AliasedExpr) if !ok { - return 0, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "unexpected AST struct for query %T", s.SelectExprs[colNumber]) + return 0 } expr := &sqlparser.AliasedExpr{ Expr: &sqlparser.FuncExpr{ @@ -359,12 +359,13 @@ func (rb *route) SupplyWeightString(colNumber int) (weightcolNumber int, err err } // It's ok to pass nil for pb and logicalPlan because PushSelect doesn't use them. // TODO: we are ignoring a potential error here. need to clean this up + var err error _, _, weightcolNumber, err = planProjection(nil, rb, expr, nil) if err != nil { - return 0, err + return 0 } rb.weightStrings[rc] = weightcolNumber - return weightcolNumber, nil + return weightcolNumber } // Rewrite implements the logicalPlan interface diff --git a/go/vt/vtgate/planbuilder/sql_calc_found_rows.go b/go/vt/vtgate/planbuilder/sql_calc_found_rows.go index 2729dbfbc3a..bcef9fce59f 100644 --- a/go/vt/vtgate/planbuilder/sql_calc_found_rows.go +++ b/go/vt/vtgate/planbuilder/sql_calc_found_rows.go @@ -90,8 +90,8 @@ func (s *sqlCalcFoundRows) SupplyCol(col *sqlparser.ColName) (*resultColumn, int } //SupplyWeightString implements the logicalPlan interface -func (s *sqlCalcFoundRows) SupplyWeightString(int) (weightcolNumber int, err error) { - return 0, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "[BUG] unreachable: sqlCalcFoundRows.SupplyWeightString") +func (s *sqlCalcFoundRows) SupplyWeightString(int) (weightcolNumber int) { + return -1 } // Rewrite implements the logicalPlan interface diff --git a/go/vt/vtgate/planbuilder/testdata/aggr_cases.txt b/go/vt/vtgate/planbuilder/testdata/aggr_cases.txt index 3ce4db9ee91..17a78b2746c 100644 --- a/go/vt/vtgate/planbuilder/testdata/aggr_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/aggr_cases.txt @@ -117,7 +117,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select count(*), a, textcol1, b, weight_string(textcol1), weight_string(a), weight_string(b) from `user` where 1 != 1 group by a, textcol1, b", - "OrderBy": "5 ASC, 4 ASC, 6 ASC", + "OrderBy": "1 ASC 5 ASC, 2 ASC 4 ASC, 3 ASC 6 ASC", "Query": "select count(*), a, textcol1, b, weight_string(textcol1), weight_string(a), weight_string(b) from `user` group by a, textcol1, b order by a asc, textcol1 asc, b asc", "Table": "`user`" } @@ -161,14 +161,14 @@ Gen4 plan same as above "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "0 ASC, 4 ASC", + "OrderBy": "0 ASC 4 ASC, 2 ASC 5 ASC", "Inputs": [ { "OperatorType": "Aggregate", "Variant": "Ordered", "Aggregates": "count(0)", "Distinct": "false", - "GroupBy": "1, 4, 3", + "GroupBy": "1, 5, 3", "Inputs": [ { "OperatorType": "Route", @@ -177,9 +177,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select count(*) as k, a, textcol1, b, weight_string(textcol1), weight_string(a), weight_string(b) from `user` where 1 != 1 group by a, textcol1, b", - "OrderBy": "4 ASC, 5 ASC, 6 ASC", - "Query": "select count(*) as k, a, textcol1, b, weight_string(textcol1), weight_string(a), weight_string(b) from `user` group by a, textcol1, b order by textcol1 asc, a asc, b asc", + "FieldQuery": "select count(*) as k, a, textcol1, b, weight_string(count(*)), weight_string(textcol1), weight_string(a), weight_string(b) from `user` where 1 != 1 group by a, textcol1, b", + "OrderBy": "2 ASC 5 ASC, 1 ASC 6 ASC, 3 ASC 7 ASC", + "Query": "select count(*) as k, a, textcol1, b, weight_string(count(*)), weight_string(textcol1), weight_string(a), weight_string(b) from `user` group by a, textcol1, b order by textcol1 asc, a asc, b asc", "Table": "`user`" } ] @@ -311,7 +311,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1", - "OrderBy": "2 ASC, 3 ASC, 2 ASC", + "OrderBy": "0 ASC 2 ASC, 1 ASC 3 ASC, 0 ASC 2 ASC", "Query": "select distinct col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1 order by col1 asc, col2 asc, col1 asc", "Table": "`user`" } @@ -412,7 +412,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col, count(*), weight_string(col) from `user` where 1 != 1 group by col", - "OrderBy": "2 ASC", + "OrderBy": "0 ASC 2 ASC", "Query": "select col, count(*), weight_string(col) from `user` group by col order by col asc", "Table": "`user`" } @@ -444,7 +444,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select `name`, count(*), weight_string(`name`) from `user` where 1 != 1 group by `name`", - "OrderBy": "2 ASC", + "OrderBy": "0 ASC 2 ASC", "Query": "select `name`, count(*), weight_string(`name`) from `user` group by `name` order by `name` asc", "Table": "`user`" } @@ -625,7 +625,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col, weight_string(col) from `user` where 1 != 1", - "OrderBy": "1 ASC", + "OrderBy": "0 ASC 1 ASC", "Query": "select distinct col, weight_string(col) from `user` order by col asc", "Table": "`user`" } @@ -652,7 +652,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col, weight_string(col) from `user` where 1 != 1 group by col", - "OrderBy": "1 ASC", + "OrderBy": "0 ASC 1 ASC", "Query": "select col, weight_string(col) from `user` group by col order by col asc", "Table": "`user`" } @@ -698,7 +698,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col, count(distinct id), weight_string(col) from `user` where 1 != 1 group by col", - "OrderBy": "2 ASC", + "OrderBy": "0 ASC 2 ASC", "Query": "select col, count(distinct id), weight_string(col) from `user` group by col order by col asc", "Table": "`user`" } @@ -726,7 +726,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, col2", - "OrderBy": "2 ASC, 3 ASC", + "OrderBy": "0 ASC 2 ASC, 1 ASC 3 ASC", "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, col2 order by col1 asc, col2 asc", "Table": "`user`" } @@ -753,7 +753,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col2, weight_string(col2) from `user` where 1 != 1 group by col2", - "OrderBy": "1 ASC", + "OrderBy": "0 ASC 1 ASC", "Query": "select col2, weight_string(col2) from `user` group by col2 order by col2 asc", "Table": "`user`" } @@ -781,7 +781,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, col2", - "OrderBy": "2 ASC, 3 ASC", + "OrderBy": "0 ASC 2 ASC, 1 ASC 3 ASC", "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, col2 order by col1 asc, col2 asc", "Table": "`user`" } @@ -809,7 +809,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, col2", - "OrderBy": "2 ASC, 3 ASC", + "OrderBy": "0 ASC 2 ASC, 1 ASC 3 ASC", "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, col2 order by col1 asc, col2 asc", "Table": "`user`" } @@ -837,7 +837,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col1, min(distinct col2), weight_string(col1) from `user` where 1 != 1 group by col1", - "OrderBy": "2 ASC", + "OrderBy": "0 ASC 2 ASC", "Query": "select col1, min(distinct col2), weight_string(col1) from `user` group by col1 order by col1 asc", "Table": "`user`" } @@ -853,7 +853,7 @@ Gen4 plan same as above "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "1 ASC", + "OrderBy": "1 ASC 2 ASC", "Inputs": [ { "OperatorType": "Aggregate", @@ -869,9 +869,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, col2", - "OrderBy": "2 ASC, 3 ASC", - "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, col2 order by col1 asc, col2 asc", + "FieldQuery": "select col1, col2, weight_string(col2), weight_string(col1) from `user` where 1 != 1 group by col1, col2", + "OrderBy": "0 ASC 3 ASC, 1 ASC 2 ASC", + "Query": "select col1, col2, weight_string(col2), weight_string(col1) from `user` group by col1, col2 order by col1 asc, col2 asc", "Table": "`user`" } ] @@ -904,7 +904,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select a, b, count(*), weight_string(b), weight_string(a) from `user` where 1 != 1 group by b, a", - "OrderBy": "3 ASC, 4 ASC", + "OrderBy": "1 ASC 3 ASC, 0 ASC 4 ASC", "Query": "select a, b, count(*), weight_string(b), weight_string(a) from `user` group by b, a order by b asc, a asc", "Table": "`user`" } @@ -932,7 +932,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select a, b, count(*), weight_string(b), weight_string(a) from `user` where 1 != 1 group by 2, 1", - "OrderBy": "3 ASC, 4 ASC", + "OrderBy": "1 ASC 3 ASC, 0 ASC 4 ASC", "Query": "select a, b, count(*), weight_string(b), weight_string(a) from `user` group by 2, 1 order by b asc, a asc", "Table": "`user`" } @@ -960,7 +960,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select a, b, count(*), weight_string(b), weight_string(a) from `user` where 1 != 1 group by b, a", - "OrderBy": "3 ASC, 4 ASC", + "OrderBy": "1 ASC 3 ASC, 0 ASC 4 ASC", "Query": "select a, b, count(*), weight_string(b), weight_string(a) from `user` group by b, a order by b asc, a asc", "Table": "`user`" } @@ -987,7 +987,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col, weight_string(col) from `user` where 1 != 1 group by 1", - "OrderBy": "1 ASC", + "OrderBy": "0 ASC 1 ASC", "Query": "select col, weight_string(col) from `user` group by 1 order by col asc", "Table": "`user`" } @@ -1049,7 +1049,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select a, b, c, d, count(*), weight_string(a), weight_string(b), weight_string(c) from `user` where 1 != 1 group by 1, 2, 3", - "OrderBy": "5 ASC, 6 ASC, 7 ASC", + "OrderBy": "0 ASC 5 ASC, 1 ASC 6 ASC, 2 ASC 7 ASC", "Query": "select a, b, c, d, count(*), weight_string(a), weight_string(b), weight_string(c) from `user` group by 1, 2, 3 order by 1 asc, 2 asc, 3 asc", "Table": "`user`" } @@ -1077,7 +1077,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select a, b, c, d, count(*), weight_string(a), weight_string(b), weight_string(c) from `user` where 1 != 1 group by 1, 2, 3", - "OrderBy": "5 ASC, 6 ASC, 7 ASC", + "OrderBy": "0 ASC 5 ASC, 1 ASC 6 ASC, 2 ASC 7 ASC", "Query": "select a, b, c, d, count(*), weight_string(a), weight_string(b), weight_string(c) from `user` group by 1, 2, 3 order by a asc, b asc, c asc", "Table": "`user`" } @@ -1105,7 +1105,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select a, b, c, d, count(*), weight_string(d), weight_string(b), weight_string(a), weight_string(c) from `user` where 1 != 1 group by 1, 2, 3, 4", - "OrderBy": "5 ASC, 6 ASC, 7 ASC, 8 ASC", + "OrderBy": "3 ASC 5 ASC, 1 ASC 6 ASC, 0 ASC 7 ASC, 2 ASC 8 ASC", "Query": "select a, b, c, d, count(*), weight_string(d), weight_string(b), weight_string(a), weight_string(c) from `user` group by 1, 2, 3, 4 order by d asc, b asc, a asc, c asc", "Table": "`user`" } @@ -1133,7 +1133,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select a, b, c, d, count(*), weight_string(d), weight_string(b), weight_string(a), weight_string(c) from `user` where 1 != 1 group by 3, 2, 1, 4", - "OrderBy": "5 ASC, 6 ASC, 7 ASC, 8 ASC", + "OrderBy": "3 ASC 5 ASC, 1 ASC 6 ASC, 0 ASC 7 ASC, 2 ASC 8 ASC", "Query": "select a, b, c, d, count(*), weight_string(d), weight_string(b), weight_string(a), weight_string(c) from `user` group by 3, 2, 1, 4 order by d asc, b asc, a asc, c asc", "Table": "`user`" } @@ -1161,7 +1161,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select a, b, c, count(*), weight_string(a), weight_string(c), weight_string(b) from `user` where 1 != 1 group by 3, 2, 1", - "OrderBy": "4 DESC, 5 DESC, 6 ASC", + "OrderBy": "0 DESC 4 DESC, 2 DESC 5 DESC, 1 ASC 6 ASC", "Query": "select a, b, c, count(*), weight_string(a), weight_string(c), weight_string(b) from `user` group by 3, 2, 1 order by 1 desc, 3 desc, b asc", "Table": "`user`" } @@ -1197,7 +1197,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col, count(*), weight_string(col) from `user` where 1 != 1 group by col", - "OrderBy": "2 ASC", + "OrderBy": "0 ASC 2 ASC", "Query": "select col, count(*), weight_string(col) from `user` group by col order by col asc limit :__upper_limit", "Table": "`user`" } @@ -1290,7 +1290,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select a, count(*), weight_string(a) from `user` where 1 != 1", - "OrderBy": "2 ASC", + "OrderBy": "0 ASC 2 ASC", "Query": "select a, count(*), weight_string(a) from `user` order by a asc", "Table": "`user`" } @@ -1323,7 +1323,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select a, count(*), weight_string(a) from `user` where 1 != 1 group by a", - "OrderBy": "2 ASC, 2 ASC", + "OrderBy": "0 ASC 2 ASC, 0 ASC 2 ASC", "Query": "select a, count(*), weight_string(a) from `user` group by a order by a asc, a asc", "Table": "`user`" } diff --git a/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.txt b/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.txt index 1891e3d88ff..43b4f42462e 100644 --- a/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.txt @@ -7,7 +7,7 @@ "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "1 ASC", + "OrderBy": "1 ASC 3 ASC", "Inputs": [ { "OperatorType": "Aggregate", @@ -23,9 +23,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, count(*), weight_string(a) from `user` where 1 != 1 group by a", - "OrderBy": "3 ASC", - "Query": "select a, b, count(*), weight_string(a) from `user` group by a order by a asc", + "FieldQuery": "select a, b, count(*), weight_string(b), weight_string(a) from `user` where 1 != 1 group by a", + "OrderBy": "0 ASC 4 ASC", + "Query": "select a, b, count(*), weight_string(b), weight_string(a) from `user` group by a order by a asc", "Table": "`user`" } ] @@ -42,7 +42,7 @@ "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "2 ASC", + "OrderBy": "2 ASC 3 ASC", "Inputs": [ { "OperatorType": "Aggregate", @@ -58,9 +58,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, count(*) as k, weight_string(a) from `user` where 1 != 1 group by a", - "OrderBy": "3 ASC", - "Query": "select a, b, count(*) as k, weight_string(a) from `user` group by a order by a asc", + "FieldQuery": "select a, b, count(*) as k, weight_string(count(*)), weight_string(a) from `user` where 1 != 1 group by a", + "OrderBy": "0 ASC 4 ASC", + "Query": "select a, b, count(*) as k, weight_string(count(*)), weight_string(a) from `user` group by a order by a asc", "Table": "`user`" } ] @@ -77,7 +77,7 @@ "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "1 ASC, 0 ASC, 2 ASC", + "OrderBy": "1 ASC 3 ASC, 0 ASC 4 ASC, 2 ASC 5 ASC", "Inputs": [ { "OperatorType": "Aggregate", @@ -93,9 +93,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, count(*) as k, weight_string(a) from `user` where 1 != 1 group by a", - "OrderBy": "3 ASC", - "Query": "select a, b, count(*) as k, weight_string(a) from `user` group by a order by a asc", + "FieldQuery": "select a, b, count(*) as k, weight_string(b), weight_string(a), weight_string(count(*)) from `user` where 1 != 1 group by a", + "OrderBy": "0 ASC 4 ASC", + "Query": "select a, b, count(*) as k, weight_string(b), weight_string(a), weight_string(count(*)) from `user` group by a order by a asc", "Table": "`user`" } ] @@ -116,7 +116,7 @@ { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "2 DESC", + "OrderBy": "2 DESC 3 DESC", "Inputs": [ { "OperatorType": "Aggregate", @@ -132,9 +132,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, count(*) as k, weight_string(a) from `user` where 1 != 1 group by a", - "OrderBy": "3 ASC", - "Query": "select a, b, count(*) as k, weight_string(a) from `user` group by a order by a asc", + "FieldQuery": "select a, b, count(*) as k, weight_string(count(*)), weight_string(a) from `user` where 1 != 1 group by a", + "OrderBy": "0 ASC 4 ASC", + "Query": "select a, b, count(*) as k, weight_string(count(*)), weight_string(a) from `user` group by a order by a asc", "Table": "`user`" } ] @@ -153,7 +153,7 @@ "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "0 ASC, 2 ASC", + "OrderBy": "0 ASC 3 ASC, 2 ASC 4 ASC", "Inputs": [ { "OperatorType": "Aggregate", @@ -169,9 +169,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, count(*) as k, weight_string(a) from `user` where 1 != 1 group by a", - "OrderBy": "3 ASC", - "Query": "select a, b, count(*) as k, weight_string(a) from `user` group by a order by 1 asc", + "FieldQuery": "select a, b, count(*) as k, weight_string(a), weight_string(count(*)) from `user` where 1 != 1 group by a", + "OrderBy": "0 ASC 3 ASC", + "Query": "select a, b, count(*) as k, weight_string(a), weight_string(count(*)) from `user` group by a order by 1 asc", "Table": "`user`" } ] @@ -189,7 +189,7 @@ "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "2 ASC, 1 ASC, 2 ASC", + "OrderBy": "0 ASC 2 ASC, 1 ASC 3 ASC, 0 ASC 2 ASC", "Inputs": [ { "OperatorType": "Aggregate", @@ -205,9 +205,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select textcol1 as t, count(*) as k, weight_string(textcol1) from `user` where 1 != 1 group by textcol1", - "OrderBy": "2 ASC, 2 ASC", - "Query": "select textcol1 as t, count(*) as k, weight_string(textcol1) from `user` group by textcol1 order by textcol1 asc, textcol1 asc", + "FieldQuery": "select textcol1 as t, count(*) as k, weight_string(textcol1), weight_string(count(*)) from `user` where 1 != 1 group by textcol1", + "OrderBy": "0 ASC 2 ASC, 0 ASC 2 ASC", + "Query": "select textcol1 as t, count(*) as k, weight_string(textcol1), weight_string(count(*)) from `user` group by textcol1 order by textcol1 asc, textcol1 asc", "Table": "`user`" } ] @@ -224,7 +224,7 @@ "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "0 ASC", + "OrderBy": "0 ASC 2 ASC", "Inputs": [ { "OperatorType": "Subquery", @@ -235,7 +235,7 @@ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "-1,-2", + "JoinColumnIndexes": "-1,-2,-3", "TableName": "`user`_user_extra", "Inputs": [ { @@ -245,8 +245,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select `user`.id, `user`.col from `user` where 1 != 1", - "Query": "select `user`.id, `user`.col from `user`", + "FieldQuery": "select `user`.id, `user`.col, weight_string(`user`.id) from `user` where 1 != 1", + "Query": "select `user`.id, `user`.col, weight_string(`user`.id) from `user`", "Table": "`user`" }, { @@ -276,12 +276,12 @@ "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "2 ASC", + "OrderBy": "2 ASC 3 ASC", "Inputs": [ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "-1,-2,1", + "JoinColumnIndexes": "-1,-2,1,2", "TableName": "`user`_music", "Inputs": [ { @@ -306,8 +306,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select music.col3 as c from music where 1 != 1", - "Query": "select music.col3 as c from music where music.id = :user_id", + "FieldQuery": "select music.col3 as c, weight_string(music.col3) from music where 1 != 1", + "Query": "select music.col3 as c, weight_string(music.col3) from music where music.id = :user_id", "Table": "music", "Values": [ ":user_id" @@ -328,12 +328,12 @@ "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "0 ASC, 2 DESC, 1 ASC", + "OrderBy": "0 ASC 3 ASC, 2 DESC 4 DESC, 1 ASC 5 ASC", "Inputs": [ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "-1,-2,1", + "JoinColumnIndexes": "-1,-2,1,-3,2,-4", "TableName": "`user`_music", "Inputs": [ { @@ -343,8 +343,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select `user`.col1 as a, `user`.col2, `user`.id from `user` where 1 != 1", - "Query": "select `user`.col1 as a, `user`.col2, `user`.id from `user` where `user`.id = 1", + "FieldQuery": "select `user`.col1 as a, `user`.col2, weight_string(`user`.col1), weight_string(`user`.col2), `user`.id from `user` where 1 != 1", + "Query": "select `user`.col1 as a, `user`.col2, weight_string(`user`.col1), weight_string(`user`.col2), `user`.id from `user` where `user`.id = 1", "Table": "`user`", "Values": [ 1 @@ -358,8 +358,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select music.col3 from music where 1 != 1", - "Query": "select music.col3 from music where music.id = :user_id", + "FieldQuery": "select music.col3, weight_string(music.col3) from music where 1 != 1", + "Query": "select music.col3, weight_string(music.col3) from music where music.id = :user_id", "Table": "music", "Values": [ ":user_id" @@ -380,12 +380,12 @@ "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "3 ASC, 2 ASC", + "OrderBy": "1 ASC 3 ASC, 2 ASC 4 ASC", "Inputs": [ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "-1,-2,1,-3", + "JoinColumnIndexes": "-1,-2,1,-3,2", "TableName": "`user`_unsharded", "Inputs": [ { @@ -406,8 +406,8 @@ "Name": "main", "Sharded": false }, - "FieldQuery": "select un.col2 from unsharded as un where 1 != 1", - "Query": "select un.col2 from unsharded as un", + "FieldQuery": "select un.col2, weight_string(un.col2) from unsharded as un where 1 != 1", + "Query": "select un.col2, weight_string(un.col2) from unsharded as un", "Table": "unsharded" } ] @@ -424,12 +424,12 @@ "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "3 ASC, 2 ASC", + "OrderBy": "1 ASC 3 ASC, 2 ASC 4 ASC", "Inputs": [ { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "1,2,-1,3", + "JoinColumnIndexes": "1,2,-1,3,-2", "TableName": "unsharded_`user`", "Inputs": [ { @@ -439,8 +439,8 @@ "Name": "main", "Sharded": false }, - "FieldQuery": "select un.col2 from unsharded as un where 1 != 1", - "Query": "select un.col2 from unsharded as un", + "FieldQuery": "select un.col2, weight_string(un.col2) from unsharded as un where 1 != 1", + "Query": "select un.col2, weight_string(un.col2) from unsharded as un", "Table": "unsharded" }, { @@ -505,7 +505,7 @@ "Sharded": true }, "FieldQuery": "select a, weight_string(a) from `user` where 1 != 1", - "OrderBy": "1 DESC", + "OrderBy": "0 DESC 1 DESC", "Query": "select a, weight_string(a) from `user` order by binary a desc", "Table": "`user`" } @@ -530,7 +530,7 @@ "Sharded": true }, "FieldQuery": "select u.a, weight_string(u.a) from `user` as u where 1 != 1", - "OrderBy": "1 DESC", + "OrderBy": "0 DESC 1 DESC", "Query": "select u.a, weight_string(u.a) from `user` as u order by binary a desc", "Table": "`user`" }, diff --git a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.txt b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.txt index 1104b91bcd1..c2e355ae6cc 100644 --- a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.txt @@ -192,7 +192,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col, weight_string(col) from `user` where 1 != 1", - "OrderBy": "1 ASC", + "OrderBy": "0 ASC 1 ASC", "Query": "select col, weight_string(col) from `user` order by col asc", "Table": "`user`" } @@ -211,7 +211,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select user_id, col1, col2, weight_string(user_id) from authoritative where 1 != 1", - "OrderBy": "3 ASC", + "OrderBy": "0 ASC 3 ASC", "Query": "select user_id, col1, col2, weight_string(user_id) from authoritative order by user_id asc", "Table": "authoritative" } @@ -230,7 +230,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select user_id, col1, col2, weight_string(col1) from authoritative where 1 != 1", - "OrderBy": "3 ASC", + "OrderBy": "1 ASC 3 ASC", "Query": "select user_id, col1, col2, weight_string(col1) from authoritative order by col1 asc", "Table": "authoritative" } @@ -249,7 +249,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select a, textcol1, b, weight_string(a), weight_string(textcol1), weight_string(b) from `user` where 1 != 1", - "OrderBy": "3 ASC, 4 ASC, 5 ASC", + "OrderBy": "0 ASC 3 ASC, 1 ASC 4 ASC, 2 ASC 5 ASC", "Query": "select a, textcol1, b, weight_string(a), weight_string(textcol1), weight_string(b) from `user` order by a asc, textcol1 asc, b asc", "Table": "`user`" } @@ -268,7 +268,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select a, `user`.textcol1, b, weight_string(a), weight_string(`user`.textcol1), weight_string(b) from `user` where 1 != 1", - "OrderBy": "3 ASC, 4 ASC, 5 ASC", + "OrderBy": "0 ASC 3 ASC, 1 ASC 4 ASC, 2 ASC 5 ASC", "Query": "select a, `user`.textcol1, b, weight_string(a), weight_string(`user`.textcol1), weight_string(b) from `user` order by a asc, textcol1 asc, b asc", "Table": "`user`" } @@ -287,7 +287,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select a, textcol1, b, textcol2, weight_string(a), weight_string(textcol1), weight_string(b), weight_string(textcol2) from `user` where 1 != 1", - "OrderBy": "4 ASC, 5 ASC, 6 ASC, 7 ASC", + "OrderBy": "0 ASC 4 ASC, 1 ASC 5 ASC, 2 ASC 6 ASC, 3 ASC 7 ASC", "Query": "select a, textcol1, b, textcol2, weight_string(a), weight_string(textcol1), weight_string(b), weight_string(textcol2) from `user` order by a asc, textcol1 asc, b asc, textcol2 asc", "Table": "`user`" } @@ -343,7 +343,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col, weight_string(col) from `user` where 1 != 1", - "OrderBy": "1 ASC", + "OrderBy": "0 ASC 1 ASC", "Query": "select col, weight_string(col) from `user` where :__sq_has_values1 = 1 and col in ::__sq1 order by col asc", "Table": "`user`" } diff --git a/go/vt/vtgate/planbuilder/testdata/select_cases.txt b/go/vt/vtgate/planbuilder/testdata/select_cases.txt index 1f50b25f021..a7bc0946823 100644 --- a/go/vt/vtgate/planbuilder/testdata/select_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/select_cases.txt @@ -850,7 +850,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select user_id, weight_string(user_id) from music where 1 != 1", - "OrderBy": "1 ASC", + "OrderBy": "0 ASC 1 ASC", "Query": "select user_id, weight_string(user_id) from music order by user_id asc limit :__upper_limit", "Table": "music" } @@ -1545,7 +1545,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select user_id, count(id), weight_string(user_id) from music where 1 != 1 group by user_id", - "OrderBy": "2 ASC", + "OrderBy": "0 ASC 2 ASC", "Query": "select user_id, count(id), weight_string(user_id) from music group by user_id having count(user_id) = 1 order by user_id asc limit :__upper_limit", "Table": "music" } diff --git a/go/vt/vtgate/planbuilder/testdata/union_cases.txt b/go/vt/vtgate/planbuilder/testdata/union_cases.txt index 4b8ef1aa59c..f0354dbfd29 100644 --- a/go/vt/vtgate/planbuilder/testdata/union_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/union_cases.txt @@ -15,21 +15,7 @@ "Table": "`user`" } } -{ - "QueryType": "SELECT", - "Original": "select id from user union all select id from music", - "Instructions": { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id from `user` where 1 != 1 union all select id from music where 1 != 1", - "Query": "select id from `user` union all select id from music", - "Table": "`user`" - } -} +Gen4 plan same as above # union distinct between two scatter selects "select id from user union select id from music" @@ -69,42 +55,7 @@ ] } } -{ - "QueryType": "SELECT", - "Original": "select id from user union select id from music", - "Instructions": { - "OperatorType": "Distinct", - "Inputs": [ - { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id from `user` where 1 != 1", - "Query": "select id from `user`", - "Table": "`user`" - }, - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id from music where 1 != 1", - "Query": "select id from music", - "Table": "music" - } - ] - } - ] - } -} +Gen4 plan same as above # union all between two SelectEqualUnique "select id from user where id = 1 union all select id from user where id = 5" @@ -147,45 +98,7 @@ ] } } -{ - "QueryType": "SELECT", - "Original": "select id from user where id = 1 union all select id from user where id = 5", - "Instructions": { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectEqualUnique", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id from `user` where 1 != 1", - "Query": "select id from `user` where id = 1", - "Table": "`user`", - "Values": [ - 1 - ], - "Vindex": "user_index" - }, - { - "OperatorType": "Route", - "Variant": "SelectEqualUnique", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id from `user` where 1 != 1", - "Query": "select id from `user` where id = 5", - "Table": "`user`", - "Values": [ - 5 - ], - "Vindex": "user_index" - } - ] - } -} +Gen4 plan same as above #almost dereks query - two queries with order by and limit being scattered to two different sets of tablets "(SELECT id FROM user ORDER BY id DESC LIMIT 1) UNION ALL (SELECT id FROM music ORDER BY id DESC LIMIT 1)" @@ -207,52 +120,7 @@ "Sharded": true }, "FieldQuery": "select id, weight_string(id) from `user` where 1 != 1", - "OrderBy": "1 DESC", - "Query": "select id, weight_string(id) from `user` order by id desc limit :__upper_limit", - "Table": "`user`" - } - ] - }, - { - "OperatorType": "Limit", - "Count": 1, - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id, weight_string(id) from music where 1 != 1", - "OrderBy": "1 DESC", - "Query": "select id, weight_string(id) from music order by id desc limit :__upper_limit", - "Table": "music" - } - ] - } - ] - } -} -{ - "QueryType": "SELECT", - "Original": "(SELECT id FROM user ORDER BY id DESC LIMIT 1) UNION ALL (SELECT id FROM music ORDER BY id DESC LIMIT 1)", - "Instructions": { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Limit", - "Count": 1, - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id, weight_string(id) from `user` where 1 != 1", - "OrderBy": "1 DESC", + "OrderBy": "0 DESC 1 DESC", "Query": "select id, weight_string(id) from `user` order by id desc limit :__upper_limit", "Table": "`user`" } @@ -270,7 +138,7 @@ "Sharded": true }, "FieldQuery": "select id, weight_string(id) from music where 1 != 1", - "OrderBy": "1 DESC", + "OrderBy": "0 DESC 1 DESC", "Query": "select id, weight_string(id) from music order by id desc limit :__upper_limit", "Table": "music" } @@ -279,6 +147,7 @@ ] } } +Gen4 plan same as above # Union all "select col1, col2 from user union all select col1, col2 from user_extra" @@ -297,21 +166,7 @@ "Table": "`user`" } } -{ - "QueryType": "SELECT", - "Original": "select col1, col2 from user union all select col1, col2 from user_extra", - "Instructions": { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select col1, col2 from `user` where 1 != 1 union all select col1, col2 from user_extra where 1 != 1", - "Query": "select col1, col2 from `user` union all select col1, col2 from user_extra", - "Table": "`user`" - } -} +Gen4 plan same as above # union operations in subqueries (FROM) "select * from (select * from user union all select * from user_extra) as t" @@ -351,52 +206,7 @@ "Sharded": true }, "FieldQuery": "select id, weight_string(id) from `user` where 1 != 1", - "OrderBy": "1 ASC", - "Query": "select id, weight_string(id) from `user` order by id asc limit :__upper_limit", - "Table": "`user`" - } - ] - }, - { - "OperatorType": "Limit", - "Count": 5, - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id, weight_string(id) from music where 1 != 1", - "OrderBy": "1 DESC", - "Query": "select id, weight_string(id) from music order by id desc limit :__upper_limit", - "Table": "music" - } - ] - } - ] - } -} -{ - "QueryType": "SELECT", - "Original": "(select id from user order by id limit 5) union all (select id from music order by id desc limit 5)", - "Instructions": { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Limit", - "Count": 5, - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id, weight_string(id) from `user` where 1 != 1", - "OrderBy": "1 ASC", + "OrderBy": "0 ASC 1 ASC", "Query": "select id, weight_string(id) from `user` order by id asc limit :__upper_limit", "Table": "`user`" } @@ -414,7 +224,7 @@ "Sharded": true }, "FieldQuery": "select id, weight_string(id) from music where 1 != 1", - "OrderBy": "1 DESC", + "OrderBy": "0 DESC 1 DESC", "Query": "select id, weight_string(id) from music order by id desc limit :__upper_limit", "Table": "music" } @@ -423,6 +233,7 @@ ] } } +Gen4 plan same as above # union all on scatter and single route "select id from user where id = 1 union select id from user where id = 1 union all select id from user" @@ -461,41 +272,7 @@ ] } } -{ - "QueryType": "SELECT", - "Original": "select id from user where id = 1 union select id from user where id = 1 union all select id from user", - "Instructions": { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectEqualUnique", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id from `user` where 1 != 1 union select id from `user` where 1 != 1", - "Query": "select id from `user` where id = 1 union select id from `user` where id = 1", - "Table": "`user`", - "Values": [ - 1 - ], - "Vindex": "user_index" - }, - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id from `user` where 1 != 1", - "Query": "select id from `user`", - "Table": "`user`" - } - ] - } -} +Gen4 plan same as above # union of information_schema with normal table "select * from information_schema.a union select * from unsharded" @@ -634,9 +411,13 @@ Gen4 plan same as above ] } } +Gen4 plan same as above + +# multi-shard union +"select 1 from music union (select id from user union all select name from unsharded)" { "QueryType": "SELECT", - "Original": "(select id from user union select id from music) union select 1 from dual", + "Original": "select 1 from music union (select id from user union all select name from unsharded)", "Instructions": { "OperatorType": "Distinct", "Inputs": [ @@ -644,130 +425,18 @@ Gen4 plan same as above "OperatorType": "Concatenate", "Inputs": [ { - "OperatorType": "Distinct", - "Inputs": [ - { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id from `user` where 1 != 1", - "Query": "select id from `user`", - "Table": "`user`" - }, - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id from music where 1 != 1", - "Query": "select id from music", - "Table": "music" - } - ] - } - ] - }, - { - "OperatorType": "Route", - "Variant": "SelectReference", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "FieldQuery": "select 1 from dual where 1 != 1", - "Query": "select 1 from dual", - "Table": "dual" - } - ] - } - ] - } -} - -# multi-shard union -"select 1 from music union (select id from user union all select name from unsharded)" -{ - "QueryType": "SELECT", - "Original": "select 1 from music union (select id from user union all select name from unsharded)", - "Instructions": { - "OperatorType": "Distinct", - "Inputs": [ - { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select 1 from music where 1 != 1", - "Query": "select 1 from music", - "Table": "music" - }, - { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id from `user` where 1 != 1", - "Query": "select id from `user`", - "Table": "`user`" - }, - { - "OperatorType": "Route", - "Variant": "SelectUnsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "FieldQuery": "select `name` from unsharded where 1 != 1", - "Query": "select `name` from unsharded", - "Table": "unsharded" - } - ] - } - ] - } - ] - } -} -{ - "QueryType": "SELECT", - "Original": "select 1 from music union (select id from user union all select name from unsharded)", - "Instructions": { - "OperatorType": "Distinct", - "Inputs": [ - { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select 1 from music where 1 != 1", - "Query": "select 1 from music", - "Table": "music" - }, - { - "OperatorType": "Concatenate", + "OperatorType": "Route", + "Variant": "SelectScatter", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "FieldQuery": "select 1 from music where 1 != 1", + "Query": "select 1 from music", + "Table": "music" + }, + { + "OperatorType": "Concatenate", "Inputs": [ { "OperatorType": "Route", @@ -798,6 +467,7 @@ Gen4 plan same as above ] } } +Gen4 plan same as above # multi-shard union "select 1 from music union (select id from user union select name from unsharded)" @@ -858,63 +528,7 @@ Gen4 plan same as above ] } } -{ - "QueryType": "SELECT", - "Original": "select 1 from music union (select id from user union select name from unsharded)", - "Instructions": { - "OperatorType": "Distinct", - "Inputs": [ - { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select 1 from music where 1 != 1", - "Query": "select 1 from music", - "Table": "music" - }, - { - "OperatorType": "Distinct", - "Inputs": [ - { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select id from `user` where 1 != 1", - "Query": "select id from `user`", - "Table": "`user`" - }, - { - "OperatorType": "Route", - "Variant": "SelectUnsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "FieldQuery": "select `name` from unsharded where 1 != 1", - "Query": "select `name` from unsharded", - "Table": "unsharded" - } - ] - } - ] - } - ] - } - ] - } -} +Gen4 plan same as above # union with the same target shard because of vindex "select * from music where id = 1 union select * from user where id = 1" @@ -962,50 +576,7 @@ Gen4 plan same as above ] } } -{ - "QueryType": "SELECT", - "Original": "select * from music where id = 1 union select * from user where id = 1", - "Instructions": { - "OperatorType": "Distinct", - "Inputs": [ - { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectEqualUnique", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select * from music where 1 != 1", - "Query": "select * from music where id = 1", - "Table": "music", - "Values": [ - 1 - ], - "Vindex": "music_user_map" - }, - { - "OperatorType": "Route", - "Variant": "SelectEqualUnique", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select * from `user` where 1 != 1", - "Query": "select * from `user` where id = 1", - "Table": "`user`", - "Values": [ - 1 - ], - "Vindex": "user_index" - } - ] - } - ] - } -} +Gen4 plan same as above # union with different target shards "select 1 from music where id = 1 union select 1 from music where id = 2" @@ -1074,45 +645,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select 1, weight_string(1) from `user` where 1 != 1", - "OrderBy": "1 DESC", - "Query": "select 1, weight_string(1) from `user` order by 1 desc", - "Table": "`user`" - }, - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select 1, weight_string(1) from `user` where 1 != 1", - "OrderBy": "1 ASC", - "Query": "select 1, weight_string(1) from `user` order by 1 asc", - "Table": "`user`" - } - ] - } - ] - } -} -{ - "QueryType": "SELECT", - "Original": "(select 1 from user order by 1 desc) union (select 1 from user order by 1 asc)", - "Instructions": { - "OperatorType": "Distinct", - "Inputs": [ - { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select 1, weight_string(1) from `user` where 1 != 1", - "OrderBy": "1 DESC", + "OrderBy": "0 DESC 1 DESC", "Query": "select 1, weight_string(1) from `user` order by 1 desc", "Table": "`user`" }, @@ -1124,7 +657,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select 1, weight_string(1) from `user` where 1 != 1", - "OrderBy": "1 ASC", + "OrderBy": "0 ASC 1 ASC", "Query": "select 1, weight_string(1) from `user` order by 1 asc", "Table": "`user`" } @@ -1133,6 +666,7 @@ Gen4 plan same as above ] } } +Gen4 plan same as above # multiple unions "select 1 union select null union select 1.0 union select '1' union select 2 union select 2.0 from user" @@ -1172,42 +706,7 @@ Gen4 plan same as above ] } } -{ - "QueryType": "SELECT", - "Original": "select 1 union select null union select 1.0 union select '1' union select 2 union select 2.0 from user", - "Instructions": { - "OperatorType": "Distinct", - "Inputs": [ - { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectReference", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "FieldQuery": "select 1 from dual where 1 != 1 union select null from dual where 1 != 1 union select 1.0 from dual where 1 != 1 union select '1' from dual where 1 != 1 union select 2 from dual where 1 != 1", - "Query": "select 1 from dual union select null from dual union select 1.0 from dual union select '1' from dual union select 2 from dual", - "Table": "dual" - }, - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select 2.0 from `user` where 1 != 1", - "Query": "select 2.0 from `user`", - "Table": "`user`" - } - ] - } - ] - } -} +Gen4 plan same as above # union distinct between a scatter query and a join (other side) "(select user.id, user.name from user join user_extra where user_extra.extra = 'asdf') union select 'b','c' from user" @@ -1266,61 +765,7 @@ Gen4 plan same as above ] } } -{ - "QueryType": "SELECT", - "Original": "(select user.id, user.name from user join user_extra where user_extra.extra = 'asdf') union select 'b','c' from user", - "Instructions": { - "OperatorType": "Distinct", - "Inputs": [ - { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Join", - "Variant": "Join", - "JoinColumnIndexes": "-1,-2", - "TableName": "`user`_user_extra", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select `user`.id, `user`.`name` from `user` where 1 != 1", - "Query": "select `user`.id, `user`.`name` from `user`", - "Table": "`user`" - }, - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select 1 from user_extra where 1 != 1", - "Query": "select 1 from user_extra where user_extra.extra = 'asdf'", - "Table": "user_extra" - } - ] - }, - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select 'b', 'c' from `user` where 1 != 1", - "Query": "select 'b', 'c' from `user`", - "Table": "`user`" - } - ] - } - ] - } -} +Gen4 plan same as above # union distinct between a scatter query and a join (other side) "select 'b','c' from user union (select user.id, user.name from user join user_extra where user_extra.extra = 'asdf')" @@ -1379,61 +824,7 @@ Gen4 plan same as above ] } } -{ - "QueryType": "SELECT", - "Original": "select 'b','c' from user union (select user.id, user.name from user join user_extra where user_extra.extra = 'asdf')", - "Instructions": { - "OperatorType": "Distinct", - "Inputs": [ - { - "OperatorType": "Concatenate", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select 'b', 'c' from `user` where 1 != 1", - "Query": "select 'b', 'c' from `user`", - "Table": "`user`" - }, - { - "OperatorType": "Join", - "Variant": "Join", - "JoinColumnIndexes": "-1,-2", - "TableName": "`user`_user_extra", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select `user`.id, `user`.`name` from `user` where 1 != 1", - "Query": "select `user`.id, `user`.`name` from `user`", - "Table": "`user`" - }, - { - "OperatorType": "Route", - "Variant": "SelectScatter", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "FieldQuery": "select 1 from user_extra where 1 != 1", - "Query": "select 1 from user_extra where user_extra.extra = 'asdf'", - "Table": "user_extra" - } - ] - } - ] - } - ] - } -} +Gen4 plan same as above # ambiguous LIMIT "select id from user limit 1 union all select id from music limit 1" diff --git a/go/vt/vtgate/planbuilder/vindex_func.go b/go/vt/vtgate/planbuilder/vindex_func.go index 1c0073c39aa..da145ded96b 100644 --- a/go/vt/vtgate/planbuilder/vindex_func.go +++ b/go/vt/vtgate/planbuilder/vindex_func.go @@ -17,8 +17,6 @@ limitations under the License. package planbuilder import ( - "errors" - "vitess.io/vitess/go/vt/vtgate/semantics" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" @@ -132,8 +130,8 @@ func (vf *vindexFunc) SupplyCol(col *sqlparser.ColName) (rc *resultColumn, colNu } // SupplyWeightString implements the logicalPlan interface -func (vf *vindexFunc) SupplyWeightString(colNumber int) (weightcolNumber int, err error) { - return 0, errors.New("cannot do collation on vindex function") +func (vf *vindexFunc) SupplyWeightString(colNumber int) (weightcolNumber int) { + return -1 } // Rewrite implements the logicalPlan interface From 47989fcc0a624826ba413bc8e52080f8c4b20ffb Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Wed, 17 Mar 2021 11:00:06 +0530 Subject: [PATCH 06/19] added a function to get the return type of a column Signed-off-by: GuptaManan100 --- go/vt/sqlparser/ast_funcs.go | 24 +++++++++++++ go/vt/vtgate/planbuilder/symtab.go | 1 + .../planbuilder/testdata/aggr_cases.txt | 18 +++++----- .../testdata/memory_sort_cases.txt | 34 +++++++++---------- 4 files changed, 51 insertions(+), 26 deletions(-) diff --git a/go/vt/sqlparser/ast_funcs.go b/go/vt/sqlparser/ast_funcs.go index 7924815f2f4..f209ad47a75 100644 --- a/go/vt/sqlparser/ast_funcs.go +++ b/go/vt/sqlparser/ast_funcs.go @@ -1355,3 +1355,27 @@ func handleUnaryMinus(expr Expr) Expr { } return &UnaryExpr{Operator: UMinusOp, Expr: expr} } + +// GetReturnType returns the type of the select expression that MySQL will return +func GetReturnType(input SQLNode) querypb.Type { + switch node := input.(type) { + case *Nextval: + return querypb.Type_INT64 + case *AliasedExpr: + return GetReturnType(node.Expr) + case *FuncExpr: + functionName := strings.ToUpper(node.Name.String()) + switch functionName { + case "ABS": + if len(node.Exprs) == 1 { + expr := node.Exprs[0] + return GetReturnType(expr) + } + case "COUNT": + return querypb.Type_INT64 + } + case *StarExpr: + return querypb.Type_NULL_TYPE + } + return querypb.Type_NULL_TYPE +} diff --git a/go/vt/vtgate/planbuilder/symtab.go b/go/vt/vtgate/planbuilder/symtab.go index 61cc1d331cf..4702548149a 100644 --- a/go/vt/vtgate/planbuilder/symtab.go +++ b/go/vt/vtgate/planbuilder/symtab.go @@ -566,6 +566,7 @@ func newResultColumn(expr *sqlparser.AliasedExpr, origin logicalPlan) *resultCol // Just to be safe, generate an anonymous column for the expression. rc.column = &column{ origin: origin, + typ: sqlparser.GetReturnType(expr), } } return rc diff --git a/go/vt/vtgate/planbuilder/testdata/aggr_cases.txt b/go/vt/vtgate/planbuilder/testdata/aggr_cases.txt index 17a78b2746c..73b795768b4 100644 --- a/go/vt/vtgate/planbuilder/testdata/aggr_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/aggr_cases.txt @@ -161,14 +161,14 @@ Gen4 plan same as above "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "0 ASC 4 ASC, 2 ASC 5 ASC", + "OrderBy": "0 ASC, 2 ASC 4 ASC", "Inputs": [ { "OperatorType": "Aggregate", "Variant": "Ordered", "Aggregates": "count(0)", "Distinct": "false", - "GroupBy": "1, 5, 3", + "GroupBy": "1, 4, 3", "Inputs": [ { "OperatorType": "Route", @@ -177,9 +177,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select count(*) as k, a, textcol1, b, weight_string(count(*)), weight_string(textcol1), weight_string(a), weight_string(b) from `user` where 1 != 1 group by a, textcol1, b", - "OrderBy": "2 ASC 5 ASC, 1 ASC 6 ASC, 3 ASC 7 ASC", - "Query": "select count(*) as k, a, textcol1, b, weight_string(count(*)), weight_string(textcol1), weight_string(a), weight_string(b) from `user` group by a, textcol1, b order by textcol1 asc, a asc, b asc", + "FieldQuery": "select count(*) as k, a, textcol1, b, weight_string(textcol1), weight_string(a), weight_string(b) from `user` where 1 != 1 group by a, textcol1, b", + "OrderBy": "2 ASC 4 ASC, 1 ASC 5 ASC, 3 ASC 6 ASC", + "Query": "select count(*) as k, a, textcol1, b, weight_string(textcol1), weight_string(a), weight_string(b) from `user` group by a, textcol1, b order by textcol1 asc, a asc, b asc", "Table": "`user`" } ] @@ -853,7 +853,7 @@ Gen4 plan same as above "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "1 ASC 2 ASC", + "OrderBy": "1 ASC", "Inputs": [ { "OperatorType": "Aggregate", @@ -869,9 +869,9 @@ Gen4 plan same as above "Name": "user", "Sharded": true }, - "FieldQuery": "select col1, col2, weight_string(col2), weight_string(col1) from `user` where 1 != 1 group by col1, col2", - "OrderBy": "0 ASC 3 ASC, 1 ASC 2 ASC", - "Query": "select col1, col2, weight_string(col2), weight_string(col1) from `user` group by col1, col2 order by col1 asc, col2 asc", + "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, col2", + "OrderBy": "0 ASC 2 ASC, 1 ASC 3 ASC", + "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, col2 order by col1 asc, col2 asc", "Table": "`user`" } ] diff --git a/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.txt b/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.txt index 43b4f42462e..985011e910b 100644 --- a/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.txt @@ -42,7 +42,7 @@ "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "2 ASC 3 ASC", + "OrderBy": "2 ASC", "Inputs": [ { "OperatorType": "Aggregate", @@ -58,9 +58,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, count(*) as k, weight_string(count(*)), weight_string(a) from `user` where 1 != 1 group by a", - "OrderBy": "0 ASC 4 ASC", - "Query": "select a, b, count(*) as k, weight_string(count(*)), weight_string(a) from `user` group by a order by a asc", + "FieldQuery": "select a, b, count(*) as k, weight_string(a) from `user` where 1 != 1 group by a", + "OrderBy": "0 ASC 3 ASC", + "Query": "select a, b, count(*) as k, weight_string(a) from `user` group by a order by a asc", "Table": "`user`" } ] @@ -77,7 +77,7 @@ "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "1 ASC 3 ASC, 0 ASC 4 ASC, 2 ASC 5 ASC", + "OrderBy": "1 ASC 3 ASC, 0 ASC 4 ASC, 2 ASC", "Inputs": [ { "OperatorType": "Aggregate", @@ -93,9 +93,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, count(*) as k, weight_string(b), weight_string(a), weight_string(count(*)) from `user` where 1 != 1 group by a", + "FieldQuery": "select a, b, count(*) as k, weight_string(b), weight_string(a) from `user` where 1 != 1 group by a", "OrderBy": "0 ASC 4 ASC", - "Query": "select a, b, count(*) as k, weight_string(b), weight_string(a), weight_string(count(*)) from `user` group by a order by a asc", + "Query": "select a, b, count(*) as k, weight_string(b), weight_string(a) from `user` group by a order by a asc", "Table": "`user`" } ] @@ -116,7 +116,7 @@ { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "2 DESC 3 DESC", + "OrderBy": "2 DESC", "Inputs": [ { "OperatorType": "Aggregate", @@ -132,9 +132,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, count(*) as k, weight_string(count(*)), weight_string(a) from `user` where 1 != 1 group by a", - "OrderBy": "0 ASC 4 ASC", - "Query": "select a, b, count(*) as k, weight_string(count(*)), weight_string(a) from `user` group by a order by a asc", + "FieldQuery": "select a, b, count(*) as k, weight_string(a) from `user` where 1 != 1 group by a", + "OrderBy": "0 ASC 3 ASC", + "Query": "select a, b, count(*) as k, weight_string(a) from `user` group by a order by a asc", "Table": "`user`" } ] @@ -153,7 +153,7 @@ "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "0 ASC 3 ASC, 2 ASC 4 ASC", + "OrderBy": "0 ASC 3 ASC, 2 ASC", "Inputs": [ { "OperatorType": "Aggregate", @@ -169,9 +169,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select a, b, count(*) as k, weight_string(a), weight_string(count(*)) from `user` where 1 != 1 group by a", + "FieldQuery": "select a, b, count(*) as k, weight_string(a) from `user` where 1 != 1 group by a", "OrderBy": "0 ASC 3 ASC", - "Query": "select a, b, count(*) as k, weight_string(a), weight_string(count(*)) from `user` group by a order by 1 asc", + "Query": "select a, b, count(*) as k, weight_string(a) from `user` group by a order by 1 asc", "Table": "`user`" } ] @@ -189,7 +189,7 @@ "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "0 ASC 2 ASC, 1 ASC 3 ASC, 0 ASC 2 ASC", + "OrderBy": "0 ASC 2 ASC, 1 ASC, 0 ASC 2 ASC", "Inputs": [ { "OperatorType": "Aggregate", @@ -205,9 +205,9 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select textcol1 as t, count(*) as k, weight_string(textcol1), weight_string(count(*)) from `user` where 1 != 1 group by textcol1", + "FieldQuery": "select textcol1 as t, count(*) as k, weight_string(textcol1) from `user` where 1 != 1 group by textcol1", "OrderBy": "0 ASC 2 ASC, 0 ASC 2 ASC", - "Query": "select textcol1 as t, count(*) as k, weight_string(textcol1), weight_string(count(*)) from `user` group by textcol1 order by textcol1 asc, textcol1 asc", + "Query": "select textcol1 as t, count(*) as k, weight_string(textcol1) from `user` group by textcol1 order by textcol1 asc, textcol1 asc", "Table": "`user`" } ] From dc4f008bf2eae3e6f4ec3355b21ed21d2a758e9a Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Wed, 17 Mar 2021 11:08:26 +0530 Subject: [PATCH 07/19] fixed a test Signed-off-by: GuptaManan100 --- go/vt/vtgate/executor_select_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/go/vt/vtgate/executor_select_test.go b/go/vt/vtgate/executor_select_test.go index 09bf6ee8ff2..bd0cbc098da 100644 --- a/go/vt/vtgate/executor_select_test.go +++ b/go/vt/vtgate/executor_select_test.go @@ -221,13 +221,16 @@ func TestStreamLimitOffset(t *testing.T) { Fields: []*querypb.Field{ {Name: "id", Type: sqltypes.Int32}, {Name: "textcol", Type: sqltypes.VarChar}, + {Name: "weight_string(id)", Type: sqltypes.VarBinary}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewInt32(1), sqltypes.NewVarChar("1234"), + sqltypes.NULL, }, { sqltypes.NewInt32(4), sqltypes.NewVarChar("4567"), + sqltypes.NULL, }}, }}) @@ -235,10 +238,12 @@ func TestStreamLimitOffset(t *testing.T) { Fields: []*querypb.Field{ {Name: "id", Type: sqltypes.Int32}, {Name: "textcol", Type: sqltypes.VarChar}, + {Name: "weight_string(id)", Type: sqltypes.VarBinary}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewInt32(2), sqltypes.NewVarChar("2345"), + sqltypes.NULL, }}, }}) From 08091fabc5f3a8783a44869fa5f39128f5fb401d Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Wed, 17 Mar 2021 15:59:34 +0530 Subject: [PATCH 08/19] added comments and fixed tests Signed-off-by: GuptaManan100 --- go/vt/sqlparser/ast_funcs.go | 2 ++ go/vt/vtgate/engine/memory_sort.go | 5 +++++ go/vt/vtgate/engine/merge_sort.go | 2 ++ go/vt/vtgate/engine/route.go | 6 +++++- go/vt/vtgate/executor_select_test.go | 24 ++++++++++++++++++------ go/vt/vtgate/planbuilder/memory_sort.go | 1 + go/vt/vtgate/planbuilder/merge_sort.go | 1 + go/vt/vtgate/planbuilder/route.go | 6 +++--- 8 files changed, 37 insertions(+), 10 deletions(-) diff --git a/go/vt/sqlparser/ast_funcs.go b/go/vt/sqlparser/ast_funcs.go index dd00f5a932a..e702208da12 100644 --- a/go/vt/sqlparser/ast_funcs.go +++ b/go/vt/sqlparser/ast_funcs.go @@ -1372,6 +1372,7 @@ func GetReturnType(input SQLNode) querypb.Type { functionName := strings.ToUpper(node.Name.String()) switch functionName { case "ABS": + // Returned value depends on the return type of the input if len(node.Exprs) == 1 { expr := node.Exprs[0] return GetReturnType(expr) @@ -1380,6 +1381,7 @@ func GetReturnType(input SQLNode) querypb.Type { return querypb.Type_INT64 } case *StarExpr: + // return null type when we do not know the type return querypb.Type_NULL_TYPE } return querypb.Type_NULL_TYPE diff --git a/go/vt/vtgate/engine/memory_sort.go b/go/vt/vtgate/engine/memory_sort.go index b7685e8dcf6..d752b281d68 100644 --- a/go/vt/vtgate/engine/memory_sort.go +++ b/go/vt/vtgate/engine/memory_sort.go @@ -121,6 +121,8 @@ func (ms *MemorySort) StreamExecute(vcursor VCursor, bindVars map[string]*queryp } for _, row := range qr.Rows { heap.Push(sh, row) + // Remove the highest element from the heap if the size is more than the count + // This optimization means that the maximum size of the heap is going to be (count + 1) for len(sh.rows) > count { _ = heap.Pop(sh) } @@ -146,6 +148,7 @@ func (ms *MemorySort) StreamExecute(vcursor VCursor, bindVars map[string]*queryp return cb(&sqltypes.Result{Rows: sh.rows}) } +// extractSlices extracts the three fields of OrderbyParams into 3 slices func extractSlices(input []OrderbyParams) (orderBy []int, weightString []int, desc []bool) { for _, order := range input { orderBy = append(orderBy, order.Col) @@ -249,6 +252,7 @@ func (sh *sortHeap) Less(i, j int) bool { if sh.err != nil { return true } + // First try to compare the columns that we want to order cmp, err := evalengine.NullsafeCompare(sh.rows[i][sh.orderBy[k]], sh.rows[j][sh.orderBy[k]]) if err != nil { _, isComparisonErr := err.(evalengine.UnsupportedComparisonError) @@ -256,6 +260,7 @@ func (sh *sortHeap) Less(i, j int) bool { sh.err = err return true } + // in case of a comparison error switch to using the weight string column for ordering sh.orderBy[k] = sh.weightStrings[k] sh.weightStrings[k] = -1 cmp, err = evalengine.NullsafeCompare(sh.rows[i][sh.orderBy[k]], sh.rows[j][sh.orderBy[k]]) diff --git a/go/vt/vtgate/engine/merge_sort.go b/go/vt/vtgate/engine/merge_sort.go index bcd80b974bd..be7b36afe54 100644 --- a/go/vt/vtgate/engine/merge_sort.go +++ b/go/vt/vtgate/engine/merge_sort.go @@ -257,6 +257,7 @@ func (sh *scatterHeap) Less(i, j int) bool { if sh.err != nil { return true } + // First try to compare the columns that we want to order cmp, err := evalengine.NullsafeCompare(sh.rows[i].row[sh.orderBy[k]], sh.rows[j].row[sh.orderBy[k]]) if err != nil { _, isComparisonErr := err.(evalengine.UnsupportedComparisonError) @@ -264,6 +265,7 @@ func (sh *scatterHeap) Less(i, j int) bool { sh.err = err return true } + // in case of a comparison error switch to using the weight string column for ordering sh.orderBy[k] = sh.weightStrings[k] sh.weightStrings[k] = -1 cmp, err = evalengine.NullsafeCompare(sh.rows[i].row[sh.orderBy[k]], sh.rows[j].row[sh.orderBy[k]]) diff --git a/go/vt/vtgate/engine/route.go b/go/vt/vtgate/engine/route.go index 8914a700f8b..a06627615c3 100644 --- a/go/vt/vtgate/engine/route.go +++ b/go/vt/vtgate/engine/route.go @@ -125,7 +125,9 @@ func NewRoute(opcode RouteOpcode, keyspace *vindexes.Keyspace, query, fieldQuery // OrderbyParams specifies the parameters for ordering. // This is used for merge-sorting scatter queries. type OrderbyParams struct { - Col int + Col int + // WeightStringCol is the weight_string column that will be used for sorting. + // It is set to -1 if such a column is not added to the query WeightStringCol int Desc bool } @@ -606,12 +608,14 @@ func (route *Route) sort(in *sqltypes.Result) (*sqltypes.Result, error) { return true } var cmp int + // First try to compare the columns that we want to order cmp, err = evalengine.NullsafeCompare(out.Rows[i][orderBy[k]], out.Rows[j][orderBy[k]]) if err != nil { _, isComparisonErr := err.(evalengine.UnsupportedComparisonError) if !(isComparisonErr && weightStrings[k] != -1) { return true } + // in case of a comparison error switch to using the weight string column for ordering orderBy[k] = weightStrings[k] weightStrings[k] = -1 cmp, err = evalengine.NullsafeCompare(out.Rows[i][orderBy[k]], out.Rows[j][orderBy[k]]) diff --git a/go/vt/vtgate/executor_select_test.go b/go/vt/vtgate/executor_select_test.go index bd0cbc098da..f57b37b6ae8 100644 --- a/go/vt/vtgate/executor_select_test.go +++ b/go/vt/vtgate/executor_select_test.go @@ -1164,6 +1164,7 @@ func TestSelectScatterOrderBy(t *testing.T) { Fields: []*querypb.Field{ {Name: "col1", Type: sqltypes.Int32}, {Name: "col2", Type: sqltypes.Int32}, + {Name: "weight_string(col2)", Type: sqltypes.VarBinary}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ @@ -1172,6 +1173,7 @@ func TestSelectScatterOrderBy(t *testing.T) { // This will allow us to test that cross-shard ordering // still works correctly. sqltypes.NewInt32(int32(i % 4)), + sqltypes.NULL, }}, }}) conns = append(conns, sbc) @@ -1183,7 +1185,7 @@ func TestSelectScatterOrderBy(t *testing.T) { require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "select col1, col2 from `user` order by col2 desc", + Sql: "select col1, col2, weight_string(col2) from `user` order by col2 desc", BindVariables: map[string]*querypb.BindVariable{}, }} for _, conn := range conns { @@ -1292,11 +1294,13 @@ func TestStreamSelectScatterOrderBy(t *testing.T) { Fields: []*querypb.Field{ {Name: "id", Type: sqltypes.Int32}, {Name: "col", Type: sqltypes.Int32}, + {Name: "weight_string(col)", Type: sqltypes.VarBinary}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ sqltypes.NewInt32(1), sqltypes.NewInt32(int32(i % 4)), + sqltypes.NULL, }}, }}) conns = append(conns, sbc) @@ -1308,7 +1312,7 @@ func TestStreamSelectScatterOrderBy(t *testing.T) { require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "select id, col from `user` order by col desc", + Sql: "select id, col, weight_string(col) from `user` order by col desc", BindVariables: map[string]*querypb.BindVariable{}, }} for _, conn := range conns { @@ -1406,11 +1410,13 @@ func TestSelectScatterAggregate(t *testing.T) { Fields: []*querypb.Field{ {Name: "col", Type: sqltypes.Int32}, {Name: "sum(foo)", Type: sqltypes.Int32}, + {Name: "weight_string(col)", Type: sqltypes.VarBinary}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ sqltypes.NewInt32(int32(i % 4)), sqltypes.NewInt32(int32(i)), + sqltypes.NULL, }}, }}) conns = append(conns, sbc) @@ -1422,7 +1428,7 @@ func TestSelectScatterAggregate(t *testing.T) { require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "select col, sum(foo) from `user` group by col order by col asc", + Sql: "select col, sum(foo), weight_string(col) from `user` group by col order by col asc", BindVariables: map[string]*querypb.BindVariable{}, }} for _, conn := range conns { @@ -1463,11 +1469,13 @@ func TestStreamSelectScatterAggregate(t *testing.T) { Fields: []*querypb.Field{ {Name: "col", Type: sqltypes.Int32}, {Name: "sum(foo)", Type: sqltypes.Int32}, + {Name: "weight_string(col)", Type: sqltypes.VarBinary}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ sqltypes.NewInt32(int32(i % 4)), sqltypes.NewInt32(int32(i)), + sqltypes.NULL, }}, }}) conns = append(conns, sbc) @@ -1479,7 +1487,7 @@ func TestStreamSelectScatterAggregate(t *testing.T) { require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "select col, sum(foo) from `user` group by col order by col asc", + Sql: "select col, sum(foo), weight_string(col) from `user` group by col order by col asc", BindVariables: map[string]*querypb.BindVariable{}, }} for _, conn := range conns { @@ -1521,11 +1529,13 @@ func TestSelectScatterLimit(t *testing.T) { Fields: []*querypb.Field{ {Name: "col1", Type: sqltypes.Int32}, {Name: "col2", Type: sqltypes.Int32}, + {Name: "weight_string(col2)", Type: sqltypes.VarBinary}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ sqltypes.NewInt32(1), sqltypes.NewInt32(int32(i % 4)), + sqltypes.NULL, }}, }}) conns = append(conns, sbc) @@ -1537,7 +1547,7 @@ func TestSelectScatterLimit(t *testing.T) { require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "select col1, col2 from `user` order by col2 desc limit :__upper_limit", + Sql: "select col1, col2, weight_string(col2) from `user` order by col2 desc limit :__upper_limit", BindVariables: map[string]*querypb.BindVariable{"__upper_limit": sqltypes.Int64BindVariable(3)}, }} for _, conn := range conns { @@ -1587,11 +1597,13 @@ func TestStreamSelectScatterLimit(t *testing.T) { Fields: []*querypb.Field{ {Name: "col1", Type: sqltypes.Int32}, {Name: "col2", Type: sqltypes.Int32}, + {Name: "weight_string(col2)", Type: sqltypes.VarBinary}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ sqltypes.NewInt32(1), sqltypes.NewInt32(int32(i % 4)), + sqltypes.NULL, }}, }}) conns = append(conns, sbc) @@ -1603,7 +1615,7 @@ func TestStreamSelectScatterLimit(t *testing.T) { require.NoError(t, err) wantQueries := []*querypb.BoundQuery{{ - Sql: "select col1, col2 from `user` order by col2 desc limit :__upper_limit", + Sql: "select col1, col2, weight_string(col2) from `user` order by col2 desc limit :__upper_limit", BindVariables: map[string]*querypb.BindVariable{"__upper_limit": sqltypes.Int64BindVariable(3)}, }} for _, conn := range conns { diff --git a/go/vt/vtgate/planbuilder/memory_sort.go b/go/vt/vtgate/planbuilder/memory_sort.go index 1ab655979a1..7501a85bda0 100644 --- a/go/vt/vtgate/planbuilder/memory_sort.go +++ b/go/vt/vtgate/planbuilder/memory_sort.go @@ -111,6 +111,7 @@ func (ms *memorySort) SetLimit(limit *sqlparser.Limit) error { func (ms *memorySort) Wireup(plan logicalPlan, jt *jointab) error { for i, orderby := range ms.eMemorySort.OrderBy { rc := ms.resultColumns[orderby.Col] + // Add a weight_string column if we know that the column is a textual column or if its type is unknown if sqltypes.IsText(rc.column.typ) || rc.column.typ == sqltypes.Null { // If a weight string was previously requested, reuse it. if weightcolNumber, ok := ms.weightStrings[rc]; ok { diff --git a/go/vt/vtgate/planbuilder/merge_sort.go b/go/vt/vtgate/planbuilder/merge_sort.go index 83d4e38df73..77fb17bfc53 100644 --- a/go/vt/vtgate/planbuilder/merge_sort.go +++ b/go/vt/vtgate/planbuilder/merge_sort.go @@ -66,6 +66,7 @@ func (ms *mergeSort) Wireup(plan logicalPlan, jt *jointab) error { rb := ms.input.(*route) for i, orderby := range rb.eroute.OrderBy { rc := ms.resultColumns[orderby.Col] + // Add a weight_string column if we know that the column is a textual column or if its type is unknown if sqltypes.IsText(rc.column.typ) || rc.column.typ == sqltypes.Null { // If a weight string was previously requested, reuse it. if colNumber, ok := ms.weightStrings[rc]; ok { diff --git a/go/vt/vtgate/planbuilder/route.go b/go/vt/vtgate/planbuilder/route.go index 2793ee055c2..e8ecc129034 100644 --- a/go/vt/vtgate/planbuilder/route.go +++ b/go/vt/vtgate/planbuilder/route.go @@ -340,12 +340,12 @@ func (rb *route) SupplyWeightString(colNumber int) (weightcolNumber int) { } s, ok := rb.Select.(*sqlparser.Select) if !ok { - return 0 + return -1 } aliasExpr, ok := s.SelectExprs[colNumber].(*sqlparser.AliasedExpr) if !ok { - return 0 + return -1 } expr := &sqlparser.AliasedExpr{ Expr: &sqlparser.FuncExpr{ @@ -362,7 +362,7 @@ func (rb *route) SupplyWeightString(colNumber int) (weightcolNumber int) { var err error _, _, weightcolNumber, err = planProjection(nil, rb, expr, nil) if err != nil { - return 0 + return -1 } rb.weightStrings[rc] = weightcolNumber return weightcolNumber From f8e5471f15172ad8e55d286bfbdcb94cf2e01023 Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Wed, 17 Mar 2021 16:02:41 +0530 Subject: [PATCH 09/19] removed weight_string col from planner string output Signed-off-by: GuptaManan100 --- go/vt/vtgate/engine/route.go | 8 --- .../planbuilder/testdata/aggr_cases.txt | 54 +++++++++---------- .../testdata/memory_sort_cases.txt | 34 ++++++------ .../testdata/postprocess_cases.txt | 14 ++--- .../planbuilder/testdata/select_cases.txt | 4 +- .../planbuilder/testdata/union_cases.txt | 12 ++--- 6 files changed, 59 insertions(+), 67 deletions(-) diff --git a/go/vt/vtgate/engine/route.go b/go/vt/vtgate/engine/route.go index a06627615c3..1f909947ddf 100644 --- a/go/vt/vtgate/engine/route.go +++ b/go/vt/vtgate/engine/route.go @@ -139,14 +139,6 @@ func (obp OrderbyParams) String() string { } else { val += " ASC" } - if obp.WeightStringCol != -1 { - val += " " + strconv.Itoa(obp.WeightStringCol) - if obp.Desc { - val += " DESC" - } else { - val += " ASC" - } - } return val } diff --git a/go/vt/vtgate/planbuilder/testdata/aggr_cases.txt b/go/vt/vtgate/planbuilder/testdata/aggr_cases.txt index 73b795768b4..7c76066eec1 100644 --- a/go/vt/vtgate/planbuilder/testdata/aggr_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/aggr_cases.txt @@ -117,7 +117,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select count(*), a, textcol1, b, weight_string(textcol1), weight_string(a), weight_string(b) from `user` where 1 != 1 group by a, textcol1, b", - "OrderBy": "1 ASC 5 ASC, 2 ASC 4 ASC, 3 ASC 6 ASC", + "OrderBy": "1 ASC, 2 ASC, 3 ASC", "Query": "select count(*), a, textcol1, b, weight_string(textcol1), weight_string(a), weight_string(b) from `user` group by a, textcol1, b order by a asc, textcol1 asc, b asc", "Table": "`user`" } @@ -161,7 +161,7 @@ Gen4 plan same as above "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "0 ASC, 2 ASC 4 ASC", + "OrderBy": "0 ASC, 2 ASC", "Inputs": [ { "OperatorType": "Aggregate", @@ -178,7 +178,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select count(*) as k, a, textcol1, b, weight_string(textcol1), weight_string(a), weight_string(b) from `user` where 1 != 1 group by a, textcol1, b", - "OrderBy": "2 ASC 4 ASC, 1 ASC 5 ASC, 3 ASC 6 ASC", + "OrderBy": "2 ASC, 1 ASC, 3 ASC", "Query": "select count(*) as k, a, textcol1, b, weight_string(textcol1), weight_string(a), weight_string(b) from `user` group by a, textcol1, b order by textcol1 asc, a asc, b asc", "Table": "`user`" } @@ -311,7 +311,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1", - "OrderBy": "0 ASC 2 ASC, 1 ASC 3 ASC, 0 ASC 2 ASC", + "OrderBy": "0 ASC, 1 ASC, 0 ASC", "Query": "select distinct col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1 order by col1 asc, col2 asc, col1 asc", "Table": "`user`" } @@ -412,7 +412,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col, count(*), weight_string(col) from `user` where 1 != 1 group by col", - "OrderBy": "0 ASC 2 ASC", + "OrderBy": "0 ASC", "Query": "select col, count(*), weight_string(col) from `user` group by col order by col asc", "Table": "`user`" } @@ -444,7 +444,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select `name`, count(*), weight_string(`name`) from `user` where 1 != 1 group by `name`", - "OrderBy": "0 ASC 2 ASC", + "OrderBy": "0 ASC", "Query": "select `name`, count(*), weight_string(`name`) from `user` group by `name` order by `name` asc", "Table": "`user`" } @@ -625,7 +625,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col, weight_string(col) from `user` where 1 != 1", - "OrderBy": "0 ASC 1 ASC", + "OrderBy": "0 ASC", "Query": "select distinct col, weight_string(col) from `user` order by col asc", "Table": "`user`" } @@ -652,7 +652,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col, weight_string(col) from `user` where 1 != 1 group by col", - "OrderBy": "0 ASC 1 ASC", + "OrderBy": "0 ASC", "Query": "select col, weight_string(col) from `user` group by col order by col asc", "Table": "`user`" } @@ -698,7 +698,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col, count(distinct id), weight_string(col) from `user` where 1 != 1 group by col", - "OrderBy": "0 ASC 2 ASC", + "OrderBy": "0 ASC", "Query": "select col, count(distinct id), weight_string(col) from `user` group by col order by col asc", "Table": "`user`" } @@ -726,7 +726,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, col2", - "OrderBy": "0 ASC 2 ASC, 1 ASC 3 ASC", + "OrderBy": "0 ASC, 1 ASC", "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, col2 order by col1 asc, col2 asc", "Table": "`user`" } @@ -753,7 +753,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col2, weight_string(col2) from `user` where 1 != 1 group by col2", - "OrderBy": "0 ASC 1 ASC", + "OrderBy": "0 ASC", "Query": "select col2, weight_string(col2) from `user` group by col2 order by col2 asc", "Table": "`user`" } @@ -781,7 +781,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, col2", - "OrderBy": "0 ASC 2 ASC, 1 ASC 3 ASC", + "OrderBy": "0 ASC, 1 ASC", "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, col2 order by col1 asc, col2 asc", "Table": "`user`" } @@ -809,7 +809,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, col2", - "OrderBy": "0 ASC 2 ASC, 1 ASC 3 ASC", + "OrderBy": "0 ASC, 1 ASC", "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, col2 order by col1 asc, col2 asc", "Table": "`user`" } @@ -837,7 +837,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col1, min(distinct col2), weight_string(col1) from `user` where 1 != 1 group by col1", - "OrderBy": "0 ASC 2 ASC", + "OrderBy": "0 ASC", "Query": "select col1, min(distinct col2), weight_string(col1) from `user` group by col1 order by col1 asc", "Table": "`user`" } @@ -870,7 +870,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col1, col2, weight_string(col1), weight_string(col2) from `user` where 1 != 1 group by col1, col2", - "OrderBy": "0 ASC 2 ASC, 1 ASC 3 ASC", + "OrderBy": "0 ASC, 1 ASC", "Query": "select col1, col2, weight_string(col1), weight_string(col2) from `user` group by col1, col2 order by col1 asc, col2 asc", "Table": "`user`" } @@ -904,7 +904,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select a, b, count(*), weight_string(b), weight_string(a) from `user` where 1 != 1 group by b, a", - "OrderBy": "1 ASC 3 ASC, 0 ASC 4 ASC", + "OrderBy": "1 ASC, 0 ASC", "Query": "select a, b, count(*), weight_string(b), weight_string(a) from `user` group by b, a order by b asc, a asc", "Table": "`user`" } @@ -932,7 +932,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select a, b, count(*), weight_string(b), weight_string(a) from `user` where 1 != 1 group by 2, 1", - "OrderBy": "1 ASC 3 ASC, 0 ASC 4 ASC", + "OrderBy": "1 ASC, 0 ASC", "Query": "select a, b, count(*), weight_string(b), weight_string(a) from `user` group by 2, 1 order by b asc, a asc", "Table": "`user`" } @@ -960,7 +960,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select a, b, count(*), weight_string(b), weight_string(a) from `user` where 1 != 1 group by b, a", - "OrderBy": "1 ASC 3 ASC, 0 ASC 4 ASC", + "OrderBy": "1 ASC, 0 ASC", "Query": "select a, b, count(*), weight_string(b), weight_string(a) from `user` group by b, a order by b asc, a asc", "Table": "`user`" } @@ -987,7 +987,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col, weight_string(col) from `user` where 1 != 1 group by 1", - "OrderBy": "0 ASC 1 ASC", + "OrderBy": "0 ASC", "Query": "select col, weight_string(col) from `user` group by 1 order by col asc", "Table": "`user`" } @@ -1049,7 +1049,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select a, b, c, d, count(*), weight_string(a), weight_string(b), weight_string(c) from `user` where 1 != 1 group by 1, 2, 3", - "OrderBy": "0 ASC 5 ASC, 1 ASC 6 ASC, 2 ASC 7 ASC", + "OrderBy": "0 ASC, 1 ASC, 2 ASC", "Query": "select a, b, c, d, count(*), weight_string(a), weight_string(b), weight_string(c) from `user` group by 1, 2, 3 order by 1 asc, 2 asc, 3 asc", "Table": "`user`" } @@ -1077,7 +1077,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select a, b, c, d, count(*), weight_string(a), weight_string(b), weight_string(c) from `user` where 1 != 1 group by 1, 2, 3", - "OrderBy": "0 ASC 5 ASC, 1 ASC 6 ASC, 2 ASC 7 ASC", + "OrderBy": "0 ASC, 1 ASC, 2 ASC", "Query": "select a, b, c, d, count(*), weight_string(a), weight_string(b), weight_string(c) from `user` group by 1, 2, 3 order by a asc, b asc, c asc", "Table": "`user`" } @@ -1105,7 +1105,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select a, b, c, d, count(*), weight_string(d), weight_string(b), weight_string(a), weight_string(c) from `user` where 1 != 1 group by 1, 2, 3, 4", - "OrderBy": "3 ASC 5 ASC, 1 ASC 6 ASC, 0 ASC 7 ASC, 2 ASC 8 ASC", + "OrderBy": "3 ASC, 1 ASC, 0 ASC, 2 ASC", "Query": "select a, b, c, d, count(*), weight_string(d), weight_string(b), weight_string(a), weight_string(c) from `user` group by 1, 2, 3, 4 order by d asc, b asc, a asc, c asc", "Table": "`user`" } @@ -1133,7 +1133,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select a, b, c, d, count(*), weight_string(d), weight_string(b), weight_string(a), weight_string(c) from `user` where 1 != 1 group by 3, 2, 1, 4", - "OrderBy": "3 ASC 5 ASC, 1 ASC 6 ASC, 0 ASC 7 ASC, 2 ASC 8 ASC", + "OrderBy": "3 ASC, 1 ASC, 0 ASC, 2 ASC", "Query": "select a, b, c, d, count(*), weight_string(d), weight_string(b), weight_string(a), weight_string(c) from `user` group by 3, 2, 1, 4 order by d asc, b asc, a asc, c asc", "Table": "`user`" } @@ -1161,7 +1161,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select a, b, c, count(*), weight_string(a), weight_string(c), weight_string(b) from `user` where 1 != 1 group by 3, 2, 1", - "OrderBy": "0 DESC 4 DESC, 2 DESC 5 DESC, 1 ASC 6 ASC", + "OrderBy": "0 DESC, 2 DESC, 1 ASC", "Query": "select a, b, c, count(*), weight_string(a), weight_string(c), weight_string(b) from `user` group by 3, 2, 1 order by 1 desc, 3 desc, b asc", "Table": "`user`" } @@ -1197,7 +1197,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col, count(*), weight_string(col) from `user` where 1 != 1 group by col", - "OrderBy": "0 ASC 2 ASC", + "OrderBy": "0 ASC", "Query": "select col, count(*), weight_string(col) from `user` group by col order by col asc limit :__upper_limit", "Table": "`user`" } @@ -1290,7 +1290,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select a, count(*), weight_string(a) from `user` where 1 != 1", - "OrderBy": "0 ASC 2 ASC", + "OrderBy": "0 ASC", "Query": "select a, count(*), weight_string(a) from `user` order by a asc", "Table": "`user`" } @@ -1323,7 +1323,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select a, count(*), weight_string(a) from `user` where 1 != 1 group by a", - "OrderBy": "0 ASC 2 ASC, 0 ASC 2 ASC", + "OrderBy": "0 ASC, 0 ASC", "Query": "select a, count(*), weight_string(a) from `user` group by a order by a asc, a asc", "Table": "`user`" } diff --git a/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.txt b/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.txt index 985011e910b..53b22207510 100644 --- a/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/memory_sort_cases.txt @@ -7,7 +7,7 @@ "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "1 ASC 3 ASC", + "OrderBy": "1 ASC", "Inputs": [ { "OperatorType": "Aggregate", @@ -24,7 +24,7 @@ "Sharded": true }, "FieldQuery": "select a, b, count(*), weight_string(b), weight_string(a) from `user` where 1 != 1 group by a", - "OrderBy": "0 ASC 4 ASC", + "OrderBy": "0 ASC", "Query": "select a, b, count(*), weight_string(b), weight_string(a) from `user` group by a order by a asc", "Table": "`user`" } @@ -59,7 +59,7 @@ "Sharded": true }, "FieldQuery": "select a, b, count(*) as k, weight_string(a) from `user` where 1 != 1 group by a", - "OrderBy": "0 ASC 3 ASC", + "OrderBy": "0 ASC", "Query": "select a, b, count(*) as k, weight_string(a) from `user` group by a order by a asc", "Table": "`user`" } @@ -77,7 +77,7 @@ "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "1 ASC 3 ASC, 0 ASC 4 ASC, 2 ASC", + "OrderBy": "1 ASC, 0 ASC, 2 ASC", "Inputs": [ { "OperatorType": "Aggregate", @@ -94,7 +94,7 @@ "Sharded": true }, "FieldQuery": "select a, b, count(*) as k, weight_string(b), weight_string(a) from `user` where 1 != 1 group by a", - "OrderBy": "0 ASC 4 ASC", + "OrderBy": "0 ASC", "Query": "select a, b, count(*) as k, weight_string(b), weight_string(a) from `user` group by a order by a asc", "Table": "`user`" } @@ -133,7 +133,7 @@ "Sharded": true }, "FieldQuery": "select a, b, count(*) as k, weight_string(a) from `user` where 1 != 1 group by a", - "OrderBy": "0 ASC 3 ASC", + "OrderBy": "0 ASC", "Query": "select a, b, count(*) as k, weight_string(a) from `user` group by a order by a asc", "Table": "`user`" } @@ -153,7 +153,7 @@ "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "0 ASC 3 ASC, 2 ASC", + "OrderBy": "0 ASC, 2 ASC", "Inputs": [ { "OperatorType": "Aggregate", @@ -170,7 +170,7 @@ "Sharded": true }, "FieldQuery": "select a, b, count(*) as k, weight_string(a) from `user` where 1 != 1 group by a", - "OrderBy": "0 ASC 3 ASC", + "OrderBy": "0 ASC", "Query": "select a, b, count(*) as k, weight_string(a) from `user` group by a order by 1 asc", "Table": "`user`" } @@ -189,7 +189,7 @@ "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "0 ASC 2 ASC, 1 ASC, 0 ASC 2 ASC", + "OrderBy": "0 ASC, 1 ASC, 0 ASC", "Inputs": [ { "OperatorType": "Aggregate", @@ -206,7 +206,7 @@ "Sharded": true }, "FieldQuery": "select textcol1 as t, count(*) as k, weight_string(textcol1) from `user` where 1 != 1 group by textcol1", - "OrderBy": "0 ASC 2 ASC, 0 ASC 2 ASC", + "OrderBy": "0 ASC, 0 ASC", "Query": "select textcol1 as t, count(*) as k, weight_string(textcol1) from `user` group by textcol1 order by textcol1 asc, textcol1 asc", "Table": "`user`" } @@ -224,7 +224,7 @@ "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "0 ASC 2 ASC", + "OrderBy": "0 ASC", "Inputs": [ { "OperatorType": "Subquery", @@ -276,7 +276,7 @@ "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "2 ASC 3 ASC", + "OrderBy": "2 ASC", "Inputs": [ { "OperatorType": "Join", @@ -328,7 +328,7 @@ "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "0 ASC 3 ASC, 2 DESC 4 DESC, 1 ASC 5 ASC", + "OrderBy": "0 ASC, 2 DESC, 1 ASC", "Inputs": [ { "OperatorType": "Join", @@ -380,7 +380,7 @@ "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "1 ASC 3 ASC, 2 ASC 4 ASC", + "OrderBy": "1 ASC, 2 ASC", "Inputs": [ { "OperatorType": "Join", @@ -424,7 +424,7 @@ "Instructions": { "OperatorType": "Sort", "Variant": "Memory", - "OrderBy": "1 ASC 3 ASC, 2 ASC 4 ASC", + "OrderBy": "1 ASC, 2 ASC", "Inputs": [ { "OperatorType": "Join", @@ -505,7 +505,7 @@ "Sharded": true }, "FieldQuery": "select a, weight_string(a) from `user` where 1 != 1", - "OrderBy": "0 DESC 1 DESC", + "OrderBy": "0 DESC", "Query": "select a, weight_string(a) from `user` order by binary a desc", "Table": "`user`" } @@ -530,7 +530,7 @@ "Sharded": true }, "FieldQuery": "select u.a, weight_string(u.a) from `user` as u where 1 != 1", - "OrderBy": "0 DESC 1 DESC", + "OrderBy": "0 DESC", "Query": "select u.a, weight_string(u.a) from `user` as u order by binary a desc", "Table": "`user`" }, diff --git a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.txt b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.txt index c2e355ae6cc..a8e9497f8e4 100644 --- a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.txt @@ -192,7 +192,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col, weight_string(col) from `user` where 1 != 1", - "OrderBy": "0 ASC 1 ASC", + "OrderBy": "0 ASC", "Query": "select col, weight_string(col) from `user` order by col asc", "Table": "`user`" } @@ -211,7 +211,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select user_id, col1, col2, weight_string(user_id) from authoritative where 1 != 1", - "OrderBy": "0 ASC 3 ASC", + "OrderBy": "0 ASC", "Query": "select user_id, col1, col2, weight_string(user_id) from authoritative order by user_id asc", "Table": "authoritative" } @@ -230,7 +230,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select user_id, col1, col2, weight_string(col1) from authoritative where 1 != 1", - "OrderBy": "1 ASC 3 ASC", + "OrderBy": "1 ASC", "Query": "select user_id, col1, col2, weight_string(col1) from authoritative order by col1 asc", "Table": "authoritative" } @@ -249,7 +249,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select a, textcol1, b, weight_string(a), weight_string(textcol1), weight_string(b) from `user` where 1 != 1", - "OrderBy": "0 ASC 3 ASC, 1 ASC 4 ASC, 2 ASC 5 ASC", + "OrderBy": "0 ASC, 1 ASC, 2 ASC", "Query": "select a, textcol1, b, weight_string(a), weight_string(textcol1), weight_string(b) from `user` order by a asc, textcol1 asc, b asc", "Table": "`user`" } @@ -268,7 +268,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select a, `user`.textcol1, b, weight_string(a), weight_string(`user`.textcol1), weight_string(b) from `user` where 1 != 1", - "OrderBy": "0 ASC 3 ASC, 1 ASC 4 ASC, 2 ASC 5 ASC", + "OrderBy": "0 ASC, 1 ASC, 2 ASC", "Query": "select a, `user`.textcol1, b, weight_string(a), weight_string(`user`.textcol1), weight_string(b) from `user` order by a asc, textcol1 asc, b asc", "Table": "`user`" } @@ -287,7 +287,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select a, textcol1, b, textcol2, weight_string(a), weight_string(textcol1), weight_string(b), weight_string(textcol2) from `user` where 1 != 1", - "OrderBy": "0 ASC 4 ASC, 1 ASC 5 ASC, 2 ASC 6 ASC, 3 ASC 7 ASC", + "OrderBy": "0 ASC, 1 ASC, 2 ASC, 3 ASC", "Query": "select a, textcol1, b, textcol2, weight_string(a), weight_string(textcol1), weight_string(b), weight_string(textcol2) from `user` order by a asc, textcol1 asc, b asc, textcol2 asc", "Table": "`user`" } @@ -343,7 +343,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select col, weight_string(col) from `user` where 1 != 1", - "OrderBy": "0 ASC 1 ASC", + "OrderBy": "0 ASC", "Query": "select col, weight_string(col) from `user` where :__sq_has_values1 = 1 and col in ::__sq1 order by col asc", "Table": "`user`" } diff --git a/go/vt/vtgate/planbuilder/testdata/select_cases.txt b/go/vt/vtgate/planbuilder/testdata/select_cases.txt index a4846f0d100..d0e5f0629a8 100644 --- a/go/vt/vtgate/planbuilder/testdata/select_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/select_cases.txt @@ -850,7 +850,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select user_id, weight_string(user_id) from music where 1 != 1", - "OrderBy": "0 ASC 1 ASC", + "OrderBy": "0 ASC", "Query": "select user_id, weight_string(user_id) from music order by user_id asc limit :__upper_limit", "Table": "music" } @@ -1545,7 +1545,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select user_id, count(id), weight_string(user_id) from music where 1 != 1 group by user_id", - "OrderBy": "0 ASC 2 ASC", + "OrderBy": "0 ASC", "Query": "select user_id, count(id), weight_string(user_id) from music group by user_id having count(user_id) = 1 order by user_id asc limit :__upper_limit", "Table": "music" } diff --git a/go/vt/vtgate/planbuilder/testdata/union_cases.txt b/go/vt/vtgate/planbuilder/testdata/union_cases.txt index f0354dbfd29..c7572346b1d 100644 --- a/go/vt/vtgate/planbuilder/testdata/union_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/union_cases.txt @@ -120,7 +120,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select id, weight_string(id) from `user` where 1 != 1", - "OrderBy": "0 DESC 1 DESC", + "OrderBy": "0 DESC", "Query": "select id, weight_string(id) from `user` order by id desc limit :__upper_limit", "Table": "`user`" } @@ -138,7 +138,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select id, weight_string(id) from music where 1 != 1", - "OrderBy": "0 DESC 1 DESC", + "OrderBy": "0 DESC", "Query": "select id, weight_string(id) from music order by id desc limit :__upper_limit", "Table": "music" } @@ -206,7 +206,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select id, weight_string(id) from `user` where 1 != 1", - "OrderBy": "0 ASC 1 ASC", + "OrderBy": "0 ASC", "Query": "select id, weight_string(id) from `user` order by id asc limit :__upper_limit", "Table": "`user`" } @@ -224,7 +224,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select id, weight_string(id) from music where 1 != 1", - "OrderBy": "0 DESC 1 DESC", + "OrderBy": "0 DESC", "Query": "select id, weight_string(id) from music order by id desc limit :__upper_limit", "Table": "music" } @@ -645,7 +645,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select 1, weight_string(1) from `user` where 1 != 1", - "OrderBy": "0 DESC 1 DESC", + "OrderBy": "0 DESC", "Query": "select 1, weight_string(1) from `user` order by 1 desc", "Table": "`user`" }, @@ -657,7 +657,7 @@ Gen4 plan same as above "Sharded": true }, "FieldQuery": "select 1, weight_string(1) from `user` where 1 != 1", - "OrderBy": "0 ASC 1 ASC", + "OrderBy": "0 ASC", "Query": "select 1, weight_string(1) from `user` order by 1 asc", "Table": "`user`" } From 8667c1e3e386fff8b1deb85d5660beafb5700cd6 Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Wed, 17 Mar 2021 16:29:48 +0530 Subject: [PATCH 10/19] move GetRouteType to symtab and augmented it Signed-off-by: GuptaManan100 --- go/vt/sqlparser/ast_funcs.go | 26 ------------ go/vt/vtgate/planbuilder/symtab.go | 31 +++++++++++++- go/vt/vtgate/planbuilder/symtab_test.go | 55 +++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 27 deletions(-) diff --git a/go/vt/sqlparser/ast_funcs.go b/go/vt/sqlparser/ast_funcs.go index e702208da12..d20c4dae2cd 100644 --- a/go/vt/sqlparser/ast_funcs.go +++ b/go/vt/sqlparser/ast_funcs.go @@ -1360,29 +1360,3 @@ func handleUnaryMinus(expr Expr) Expr { func encodeSQLString(val string) string { return sqltypes.EncodeStringSQL(val) } - -// GetReturnType returns the type of the select expression that MySQL will return -func GetReturnType(input SQLNode) querypb.Type { - switch node := input.(type) { - case *Nextval: - return querypb.Type_INT64 - case *AliasedExpr: - return GetReturnType(node.Expr) - case *FuncExpr: - functionName := strings.ToUpper(node.Name.String()) - switch functionName { - case "ABS": - // Returned value depends on the return type of the input - if len(node.Exprs) == 1 { - expr := node.Exprs[0] - return GetReturnType(expr) - } - case "COUNT": - return querypb.Type_INT64 - } - case *StarExpr: - // return null type when we do not know the type - return querypb.Type_NULL_TYPE - } - return querypb.Type_NULL_TYPE -} diff --git a/go/vt/vtgate/planbuilder/symtab.go b/go/vt/vtgate/planbuilder/symtab.go index 4702548149a..0f83ad87eeb 100644 --- a/go/vt/vtgate/planbuilder/symtab.go +++ b/go/vt/vtgate/planbuilder/symtab.go @@ -566,8 +566,37 @@ func newResultColumn(expr *sqlparser.AliasedExpr, origin logicalPlan) *resultCol // Just to be safe, generate an anonymous column for the expression. rc.column = &column{ origin: origin, - typ: sqlparser.GetReturnType(expr), + typ: GetReturnType(expr), } } return rc } + +// GetReturnType returns the type of the select expression that MySQL will return +func GetReturnType(input sqlparser.SQLNode) querypb.Type { + switch node := input.(type) { + case *sqlparser.Nextval: + return querypb.Type_INT64 + case *sqlparser.AliasedExpr: + return GetReturnType(node.Expr) + case *sqlparser.FuncExpr: + functionName := strings.ToUpper(node.Name.String()) + switch functionName { + case "ABS": + // Returned value depends on the return type of the input + if len(node.Exprs) == 1 { + expr := node.Exprs[0] + return GetReturnType(expr) + } + case "COUNT": + return querypb.Type_INT64 + } + case *sqlparser.ColName: + col := node.Metadata.(*column) + return col.typ + case *sqlparser.StarExpr: + // return null type when we do not know the type + return querypb.Type_NULL_TYPE + } + return querypb.Type_NULL_TYPE +} diff --git a/go/vt/vtgate/planbuilder/symtab_test.go b/go/vt/vtgate/planbuilder/symtab_test.go index bd9ac8a6a90..a528cadccbc 100644 --- a/go/vt/vtgate/planbuilder/symtab_test.go +++ b/go/vt/vtgate/planbuilder/symtab_test.go @@ -16,6 +16,15 @@ limitations under the License. package planbuilder +import ( + "testing" + + "github.com/stretchr/testify/require" + + querypb "vitess.io/vitess/go/vt/proto/query" + "vitess.io/vitess/go/vt/sqlparser" +) + /* func TestSymtabAddVSchemaTable(t *testing.T) { tname := sqlparser.TableName{Name: sqlparser.NewTableIdent("t")} @@ -178,3 +187,49 @@ func TestSymtabAddVSchemaTable(t *testing.T) { } } */ + +func TestGetReturnType(t *testing.T) { + tests := []struct { + input sqlparser.SQLNode + output querypb.Type + }{{ + input: &sqlparser.FuncExpr{Name: sqlparser.NewColIdent("Abs"), Exprs: sqlparser.SelectExprs{ + &sqlparser.AliasedExpr{ + Expr: &sqlparser.ColName{ + Name: sqlparser.NewColIdent("A"), + Metadata: &column{ + typ: querypb.Type_DECIMAL, + }, + }, + }, + }}, + output: querypb.Type_DECIMAL, + }, { + input: &sqlparser.StarExpr{}, + output: querypb.Type_NULL_TYPE, + }, { + input: &sqlparser.FuncExpr{Name: sqlparser.NewColIdent("Count"), Exprs: sqlparser.SelectExprs{ + &sqlparser.StarExpr{}, + }}, + output: querypb.Type_INT64, + }, { + input: &sqlparser.FuncExpr{Name: sqlparser.NewColIdent("cOunt"), Exprs: sqlparser.SelectExprs{ + &sqlparser.StarExpr{}, + }}, + output: querypb.Type_INT64, + }, { + input: &sqlparser.Nextval{ + Expr: &sqlparser.ColName{ + Name: sqlparser.NewColIdent("A"), + }, + }, + output: querypb.Type_INT64, + }} + + for _, test := range tests { + t.Run(sqlparser.String(test.input), func(t *testing.T) { + got := GetReturnType(test.input) + require.Equal(t, test.output, got) + }) + } +} From 005dc63787abf167508d4b4c4ccb17b57f7be5e6 Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Wed, 17 Mar 2021 16:33:49 +0530 Subject: [PATCH 11/19] ran make sizegen Signed-off-by: GuptaManan100 --- go/vt/vtgate/engine/cached_size.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/go/vt/vtgate/engine/cached_size.go b/go/vt/vtgate/engine/cached_size.go index 18e3a530d19..1d0b4304d0e 100644 --- a/go/vt/vtgate/engine/cached_size.go +++ b/go/vt/vtgate/engine/cached_size.go @@ -307,7 +307,7 @@ func (cached *MemorySort) CachedSize(alloc bool) int64 { size += cached.UpperLimit.CachedSize(false) // field OrderBy []vitess.io/vitess/go/vt/vtgate/engine.OrderbyParams { - size += int64(cap(cached.OrderBy)) * int64(9) + size += int64(cap(cached.OrderBy)) * int64(17) } // field Input vitess.io/vitess/go/vt/vtgate/engine.Primitive if cc, ok := cached.Input.(cachedObject); ok { @@ -334,7 +334,7 @@ func (cached *MergeSort) CachedSize(alloc bool) int64 { } // field OrderBy []vitess.io/vitess/go/vt/vtgate/engine.OrderbyParams { - size += int64(cap(cached.OrderBy)) * int64(9) + size += int64(cap(cached.OrderBy)) * int64(17) } return size } @@ -527,7 +527,7 @@ func (cached *Route) CachedSize(alloc bool) int64 { } // field OrderBy []vitess.io/vitess/go/vt/vtgate/engine.OrderbyParams { - size += int64(cap(cached.OrderBy)) * int64(9) + size += int64(cap(cached.OrderBy)) * int64(17) } // field SysTableTableSchema vitess.io/vitess/go/vt/vtgate/evalengine.Expr if cc, ok := cached.SysTableTableSchema.(cachedObject); ok { From c6924d70a6a1c0cc1a8765b4c7a9961a8437d3c0 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Mon, 22 Mar 2021 08:00:12 +0100 Subject: [PATCH 12/19] extract fallback comparisons into a type Signed-off-by: Andres Taylor --- go/vt/vtgate/engine/memory_sort.go | 82 +++++++++++++------------ go/vt/vtgate/engine/memory_sort_test.go | 6 +- go/vt/vtgate/engine/merge_sort.go | 39 ++++-------- go/vt/vtgate/engine/route.go | 27 +++----- 4 files changed, 66 insertions(+), 88 deletions(-) diff --git a/go/vt/vtgate/engine/memory_sort.go b/go/vt/vtgate/engine/memory_sort.go index d752b281d68..25953bfa7ff 100644 --- a/go/vt/vtgate/engine/memory_sort.go +++ b/go/vt/vtgate/engine/memory_sort.go @@ -75,12 +75,9 @@ func (ms *MemorySort) Execute(vcursor VCursor, bindVars map[string]*querypb.Bind if err != nil { return nil, err } - orderBy, weightStrings, desc := extractSlices(ms.OrderBy) sh := &sortHeap{ - rows: result.Rows, - orderBy: orderBy, - weightStrings: weightStrings, - desc: desc, + rows: result.Rows, + comparers: extractSlices(ms.OrderBy), } sort.Sort(sh) if sh.err != nil { @@ -106,12 +103,9 @@ func (ms *MemorySort) StreamExecute(vcursor VCursor, bindVars map[string]*queryp // You have to reverse the ordering because the highest values // must be dropped once the upper limit is reached. - orderBy, weightStrings, desc := extractSlices(ms.OrderBy) sh := &sortHeap{ - orderBy: orderBy, - weightStrings: weightStrings, - desc: desc, - reverse: true, + comparers: extractSlices(ms.OrderBy), + reverse: true, } err = ms.Input.StreamExecute(vcursor, bindVars, wantfields, func(qr *sqltypes.Result) error { if len(qr.Fields) != 0 { @@ -148,14 +142,40 @@ func (ms *MemorySort) StreamExecute(vcursor VCursor, bindVars map[string]*queryp return cb(&sqltypes.Result{Rows: sh.rows}) } +type comparer struct { + orderBy, weightString int + desc bool +} + +func (c *comparer) compare(r1, r2 []sqltypes.Value) (int, error) { + cmp, err := evalengine.NullsafeCompare(r1[c.orderBy], r2[c.orderBy]) + if err != nil { + _, isComparisonErr := err.(evalengine.UnsupportedComparisonError) + if !(isComparisonErr && c.weightString != -1) { + return 0, err + } + // in case of a comparison error switch to using the weight string column for ordering + c.orderBy = c.weightString + c.weightString = -1 + cmp, err = evalengine.NullsafeCompare(r1[c.orderBy], r2[c.orderBy]) + if err != nil { + return 0, err + } + } + return cmp, nil +} + // extractSlices extracts the three fields of OrderbyParams into 3 slices -func extractSlices(input []OrderbyParams) (orderBy []int, weightString []int, desc []bool) { +func extractSlices(input []OrderbyParams) []*comparer { + var result []*comparer for _, order := range input { - orderBy = append(orderBy, order.Col) - weightString = append(weightString, order.WeightStringCol) - desc = append(desc, order.Desc) + result = append(result, &comparer{ + orderBy: order.Col, + weightString: order.WeightStringCol, + desc: order.Desc, + }) } - return + return result } // GetFields satisfies the Primitive interface. @@ -233,12 +253,10 @@ func GenericJoin(input interface{}, f func(interface{}) string) string { // sortHeap is sorted based on the orderBy params. // Implementation is similar to scatterHeap type sortHeap struct { - rows [][]sqltypes.Value - orderBy []int - weightStrings []int - desc []bool - reverse bool - err error + rows [][]sqltypes.Value + comparers []*comparer + reverse bool + err error } // Len satisfies sort.Interface and heap.Interface. @@ -248,26 +266,14 @@ func (sh *sortHeap) Len() int { // Less satisfies sort.Interface and heap.Interface. func (sh *sortHeap) Less(i, j int) bool { - for k := range sh.orderBy { + for _, c := range sh.comparers { if sh.err != nil { return true } - // First try to compare the columns that we want to order - cmp, err := evalengine.NullsafeCompare(sh.rows[i][sh.orderBy[k]], sh.rows[j][sh.orderBy[k]]) + cmp, err := c.compare(sh.rows[i], sh.rows[j]) if err != nil { - _, isComparisonErr := err.(evalengine.UnsupportedComparisonError) - if !(isComparisonErr && sh.weightStrings[k] != -1) { - sh.err = err - return true - } - // in case of a comparison error switch to using the weight string column for ordering - sh.orderBy[k] = sh.weightStrings[k] - sh.weightStrings[k] = -1 - cmp, err = evalengine.NullsafeCompare(sh.rows[i][sh.orderBy[k]], sh.rows[j][sh.orderBy[k]]) - if err != nil { - sh.err = err - return true - } + sh.err = err + return true } if cmp == 0 { continue @@ -282,7 +288,7 @@ func (sh *sortHeap) Less(i, j int) bool { // cmp = -cmp // } //} - if sh.reverse != sh.desc[k] { + if sh.reverse != c.desc { cmp = -cmp } return cmp < 0 diff --git a/go/vt/vtgate/engine/memory_sort_test.go b/go/vt/vtgate/engine/memory_sort_test.go index 962f1af278a..1a39208f83c 100644 --- a/go/vt/vtgate/engine/memory_sort_test.go +++ b/go/vt/vtgate/engine/memory_sort_test.go @@ -91,7 +91,7 @@ func TestMemorySortStreamExecuteWeightString(t *testing.T) { fp := &fakePrimitive{ results: []*sqltypes.Result{sqltypes.MakeTestResult( fields, - "v|x", + "null|x", "g|d", "a|a", "c|t", @@ -118,11 +118,11 @@ func TestMemorySortStreamExecuteWeightString(t *testing.T) { wantResults := sqltypes.MakeTestStreamingResults( fields, + "null|x", "a|a", "c|t", "f|p", "g|d", - "v|x", ) utils.MustMatch(t, wantResults, results) }) @@ -143,9 +143,9 @@ func TestMemorySortStreamExecuteWeightString(t *testing.T) { wantResults := sqltypes.MakeTestStreamingResults( fields, + "null|x", "a|a", "c|t", - "f|p", ) utils.MustMatch(t, wantResults, results) }) diff --git a/go/vt/vtgate/engine/merge_sort.go b/go/vt/vtgate/engine/merge_sort.go index be7b36afe54..3f359c0f44b 100644 --- a/go/vt/vtgate/engine/merge_sort.go +++ b/go/vt/vtgate/engine/merge_sort.go @@ -20,8 +20,6 @@ import ( "container/heap" "io" - "vitess.io/vitess/go/vt/vtgate/evalengine" - "context" "vitess.io/vitess/go/sqltypes" @@ -96,12 +94,10 @@ func (ms *MergeSort) StreamExecute(vcursor VCursor, bindVars map[string]*querypb return err } - orderBy, weightStrings, desc := extractSlices(ms.OrderBy) + comparers := extractSlices(ms.OrderBy) sh := &scatterHeap{ - rows: make([]streamRow, 0, len(handles)), - orderBy: orderBy, - weightStrings: weightStrings, - desc: desc, + rows: make([]streamRow, 0, len(handles)), + comparers: comparers, } // Prime the heap. One element must be pulled from @@ -239,11 +235,9 @@ type streamRow struct { // yielded an error, err is set. This must be checked // after every heap operation. type scatterHeap struct { - rows []streamRow - orderBy []int - weightStrings []int - desc []bool - err error + rows []streamRow + err error + comparers []*comparer } // Len satisfies sort.Interface and heap.Interface. @@ -253,31 +247,20 @@ func (sh *scatterHeap) Len() int { // Less satisfies sort.Interface and heap.Interface. func (sh *scatterHeap) Less(i, j int) bool { - for k := range sh.orderBy { + for _, c := range sh.comparers { if sh.err != nil { return true } // First try to compare the columns that we want to order - cmp, err := evalengine.NullsafeCompare(sh.rows[i].row[sh.orderBy[k]], sh.rows[j].row[sh.orderBy[k]]) + cmp, err := c.compare(sh.rows[i].row, sh.rows[j].row) if err != nil { - _, isComparisonErr := err.(evalengine.UnsupportedComparisonError) - if !(isComparisonErr && sh.weightStrings[k] != -1) { - sh.err = err - return true - } - // in case of a comparison error switch to using the weight string column for ordering - sh.orderBy[k] = sh.weightStrings[k] - sh.weightStrings[k] = -1 - cmp, err = evalengine.NullsafeCompare(sh.rows[i].row[sh.orderBy[k]], sh.rows[j].row[sh.orderBy[k]]) - if err != nil { - sh.err = err - return true - } + sh.err = err + return true } if cmp == 0 { continue } - if sh.desc[k] { + if c.desc { cmp = -cmp } return cmp < 0 diff --git a/go/vt/vtgate/engine/route.go b/go/vt/vtgate/engine/route.go index 1f909947ddf..56d5979c4a5 100644 --- a/go/vt/vtgate/engine/route.go +++ b/go/vt/vtgate/engine/route.go @@ -587,38 +587,27 @@ func (route *Route) sort(in *sqltypes.Result) (*sqltypes.Result, error) { InsertID: in.InsertID, } - orderBy, weightStrings, desc := extractSlices(route.OrderBy) + comparers := extractSlices(route.OrderBy) sort.Slice(out.Rows, func(i, j int) bool { + var cmp int + if err != nil { + return true + } // If there are any errors below, the function sets // the external err and returns true. Once err is set, // all subsequent calls return true. This will make // Slice think that all elements are in the correct // order and return more quickly. - for k := range orderBy { + for _, c := range comparers { + cmp, err = c.compare(out.Rows[i], out.Rows[j]) if err != nil { return true } - var cmp int - // First try to compare the columns that we want to order - cmp, err = evalengine.NullsafeCompare(out.Rows[i][orderBy[k]], out.Rows[j][orderBy[k]]) - if err != nil { - _, isComparisonErr := err.(evalengine.UnsupportedComparisonError) - if !(isComparisonErr && weightStrings[k] != -1) { - return true - } - // in case of a comparison error switch to using the weight string column for ordering - orderBy[k] = weightStrings[k] - weightStrings[k] = -1 - cmp, err = evalengine.NullsafeCompare(out.Rows[i][orderBy[k]], out.Rows[j][orderBy[k]]) - if err != nil { - return true - } - } if cmp == 0 { continue } - if desc[k] { + if c.desc { cmp = -cmp } return cmp < 0 From ed25dcc61ef5d578db55087644a614ef2b45ff3e Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Mon, 22 Mar 2021 14:29:36 +0530 Subject: [PATCH 13/19] moved comparer to its own file and added tests for it Signed-off-by: GuptaManan100 --- go/vt/vtgate/engine/comparer.go | 67 ++++++++++++++++ go/vt/vtgate/engine/comparer_test.go | 114 +++++++++++++++++++++++++++ go/vt/vtgate/engine/memory_sort.go | 48 +---------- go/vt/vtgate/engine/merge_sort.go | 3 - go/vt/vtgate/engine/route.go | 3 - 5 files changed, 182 insertions(+), 53 deletions(-) create mode 100644 go/vt/vtgate/engine/comparer.go create mode 100644 go/vt/vtgate/engine/comparer_test.go diff --git a/go/vt/vtgate/engine/comparer.go b/go/vt/vtgate/engine/comparer.go new file mode 100644 index 00000000000..a0a0a528421 --- /dev/null +++ b/go/vt/vtgate/engine/comparer.go @@ -0,0 +1,67 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package engine + +import ( + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/vtgate/evalengine" +) + +// comparer is the struct that has the logic for comparing two rows in the result set +type comparer struct { + orderBy, weightString int + desc bool +} + +// compare compares two rows given the comparer and returns which one should be earlier in the result set +// -1 if the first row should be earlier +// 1 is the second row should be earlier +// 0 if both the rows have equal ordering +func (c *comparer) compare(r1, r2 []sqltypes.Value) (int, error) { + cmp, err := evalengine.NullsafeCompare(r1[c.orderBy], r2[c.orderBy]) + if err != nil { + _, isComparisonErr := err.(evalengine.UnsupportedComparisonError) + if !(isComparisonErr && c.weightString != -1) { + return 0, err + } + // in case of a comparison error switch to using the weight string column for ordering + c.orderBy = c.weightString + c.weightString = -1 + cmp, err = evalengine.NullsafeCompare(r1[c.orderBy], r2[c.orderBy]) + if err != nil { + return 0, err + } + } + // change the result if descending ordering is required + if c.desc { + cmp = -cmp + } + return cmp, nil +} + +// extractSlices extracts the three fields of OrderbyParams into 3 slices +func extractSlices(input []OrderbyParams) []*comparer { + var result []*comparer + for _, order := range input { + result = append(result, &comparer{ + orderBy: order.Col, + weightString: order.WeightStringCol, + desc: order.Desc, + }) + } + return result +} diff --git a/go/vt/vtgate/engine/comparer_test.go b/go/vt/vtgate/engine/comparer_test.go new file mode 100644 index 00000000000..c1be2c25e82 --- /dev/null +++ b/go/vt/vtgate/engine/comparer_test.go @@ -0,0 +1,114 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package engine + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/sqltypes" +) + +func TestComparer(t *testing.T) { + tests := []struct { + comparer comparer + row1 []sqltypes.Value + row2 []sqltypes.Value + output int + }{ + { + comparer: comparer{ + orderBy: 0, + weightString: -1, + desc: true, + }, + row1: []sqltypes.Value{ + sqltypes.NewInt64(23), + }, + row2: []sqltypes.Value{ + sqltypes.NewInt64(34), + }, + output: 1, + }, { + comparer: comparer{ + orderBy: 0, + weightString: -1, + desc: false, + }, + row1: []sqltypes.Value{ + sqltypes.NewInt64(23), + }, + row2: []sqltypes.Value{ + sqltypes.NewInt64(23), + }, + output: 0, + }, { + comparer: comparer{ + orderBy: 0, + weightString: -1, + desc: false, + }, + row1: []sqltypes.Value{ + sqltypes.NewInt64(23), + }, + row2: []sqltypes.Value{ + sqltypes.NewInt64(12), + }, + output: 1, + }, { + comparer: comparer{ + orderBy: 1, + weightString: 0, + desc: false, + }, + row1: []sqltypes.Value{ + sqltypes.NewInt64(23), + sqltypes.NewVarChar("b"), + }, + row2: []sqltypes.Value{ + sqltypes.NewInt64(34), + sqltypes.NewVarChar("a"), + }, + output: -1, + }, { + comparer: comparer{ + orderBy: 1, + weightString: 0, + desc: true, + }, + row1: []sqltypes.Value{ + sqltypes.NewInt64(23), + sqltypes.NewVarChar("A"), + }, + row2: []sqltypes.Value{ + sqltypes.NewInt64(23), + sqltypes.NewVarChar("a"), + }, + output: 0, + }, + } + + for i, test := range tests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + got, err := test.comparer.compare(test.row1, test.row2) + require.NoError(t, err) + require.Equal(t, test.output, got) + }) + } +} diff --git a/go/vt/vtgate/engine/memory_sort.go b/go/vt/vtgate/engine/memory_sort.go index 25953bfa7ff..560b029356e 100644 --- a/go/vt/vtgate/engine/memory_sort.go +++ b/go/vt/vtgate/engine/memory_sort.go @@ -142,42 +142,6 @@ func (ms *MemorySort) StreamExecute(vcursor VCursor, bindVars map[string]*queryp return cb(&sqltypes.Result{Rows: sh.rows}) } -type comparer struct { - orderBy, weightString int - desc bool -} - -func (c *comparer) compare(r1, r2 []sqltypes.Value) (int, error) { - cmp, err := evalengine.NullsafeCompare(r1[c.orderBy], r2[c.orderBy]) - if err != nil { - _, isComparisonErr := err.(evalengine.UnsupportedComparisonError) - if !(isComparisonErr && c.weightString != -1) { - return 0, err - } - // in case of a comparison error switch to using the weight string column for ordering - c.orderBy = c.weightString - c.weightString = -1 - cmp, err = evalengine.NullsafeCompare(r1[c.orderBy], r2[c.orderBy]) - if err != nil { - return 0, err - } - } - return cmp, nil -} - -// extractSlices extracts the three fields of OrderbyParams into 3 slices -func extractSlices(input []OrderbyParams) []*comparer { - var result []*comparer - for _, order := range input { - result = append(result, &comparer{ - orderBy: order.Col, - weightString: order.WeightStringCol, - desc: order.Desc, - }) - } - return result -} - // GetFields satisfies the Primitive interface. func (ms *MemorySort) GetFields(vcursor VCursor, bindVars map[string]*querypb.BindVariable) (*sqltypes.Result, error) { return ms.Input.GetFields(vcursor, bindVars) @@ -278,17 +242,7 @@ func (sh *sortHeap) Less(i, j int) bool { if cmp == 0 { continue } - // This is equivalent to: - //if !sh.reverse { - // if order.Desc { - // cmp = -cmp - // } - //} else { - // if !order.Desc { - // cmp = -cmp - // } - //} - if sh.reverse != c.desc { + if sh.reverse { cmp = -cmp } return cmp < 0 diff --git a/go/vt/vtgate/engine/merge_sort.go b/go/vt/vtgate/engine/merge_sort.go index 3f359c0f44b..b87aa981765 100644 --- a/go/vt/vtgate/engine/merge_sort.go +++ b/go/vt/vtgate/engine/merge_sort.go @@ -260,9 +260,6 @@ func (sh *scatterHeap) Less(i, j int) bool { if cmp == 0 { continue } - if c.desc { - cmp = -cmp - } return cmp < 0 } return true diff --git a/go/vt/vtgate/engine/route.go b/go/vt/vtgate/engine/route.go index 56d5979c4a5..b77d8c115b3 100644 --- a/go/vt/vtgate/engine/route.go +++ b/go/vt/vtgate/engine/route.go @@ -607,9 +607,6 @@ func (route *Route) sort(in *sqltypes.Result) (*sqltypes.Result, error) { if cmp == 0 { continue } - if c.desc { - cmp = -cmp - } return cmp < 0 } return true From 43361564c50c8d35dffa337c77b4cdb247bca3a2 Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Mon, 22 Mar 2021 16:00:47 +0530 Subject: [PATCH 14/19] added a test for group by with order by Signed-off-by: GuptaManan100 --- go/test/endtoend/vtgate/aggr_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/go/test/endtoend/vtgate/aggr_test.go b/go/test/endtoend/vtgate/aggr_test.go index c40b370004c..f99e18f718f 100644 --- a/go/test/endtoend/vtgate/aggr_test.go +++ b/go/test/endtoend/vtgate/aggr_test.go @@ -55,3 +55,15 @@ func TestOrderBy(t *testing.T) { assertMatches(t, conn, "select id1, id2 from t4 order by id1 desc", `[[INT64(8) VARCHAR("E")] [INT64(7) VARCHAR("e")] [INT64(6) VARCHAR("d")] [INT64(5) VARCHAR("test")] [INT64(4) VARCHAR("c")] [INT64(3) VARCHAR("b")] [INT64(2) VARCHAR("A")] [INT64(1) VARCHAR("a")]]`) exec(t, conn, "delete from t4") } + +func TestGroupBy(t *testing.T) { + defer cluster.PanicHandler(t) + ctx := context.Background() + conn, err := mysql.Connect(ctx, &vtParams) + require.Nil(t, err) + defer conn.Close() + exec(t, conn, "insert into t3(id5, id6, id7) values(1,1,2), (2,2,4), (3,2,4), (4,1,2), (5,1,2), (6,3,6)") + // test ordering and group by int column + assertMatches(t, conn, "select id6, id7, count(*) k from t3 group by id6, id7 order by k", `[[INT64(3) INT64(6) INT64(1)] [INT64(2) INT64(4) INT64(2)] [INT64(1) INT64(2) INT64(3)]]`) + exec(t, conn, "delete from t3") +} From 6e5625726ea46d0f288651e23f4abe08d55cb79b Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Mon, 22 Mar 2021 17:20:40 +0530 Subject: [PATCH 15/19] added tests for streamExecute Signed-off-by: GuptaManan100 --- go/test/endtoend/vtgate/aggr_test.go | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/go/test/endtoend/vtgate/aggr_test.go b/go/test/endtoend/vtgate/aggr_test.go index f99e18f718f..8018a52476d 100644 --- a/go/test/endtoend/vtgate/aggr_test.go +++ b/go/test/endtoend/vtgate/aggr_test.go @@ -47,13 +47,21 @@ func TestOrderBy(t *testing.T) { conn, err := mysql.Connect(ctx, &vtParams) require.Nil(t, err) defer conn.Close() - exec(t, conn, "insert into t4(id1, id2) values(1,'a'), (2,'A'), (3,'b'), (4,'c'), (5,'test')") - exec(t, conn, "insert into t4(id1, id2) values(6,'d'), (7,'e'), (8,'E')") + exec(t, conn, "insert into t4(id1, id2) values(1,'a'), (2,'Abc'), (3,'b'), (4,'c'), (5,'test')") + exec(t, conn, "insert into t4(id1, id2) values(6,'d'), (7,'e'), (8,'F')") // test ordering of varchar column - assertMatches(t, conn, "select id1, id2 from t4 order by id2 desc", `[[INT64(5) VARCHAR("test")] [INT64(8) VARCHAR("E")] [INT64(7) VARCHAR("e")] [INT64(6) VARCHAR("d")] [INT64(4) VARCHAR("c")] [INT64(3) VARCHAR("b")] [INT64(2) VARCHAR("A")] [INT64(1) VARCHAR("a")]]`) + assertMatches(t, conn, "select id1, id2 from t4 order by id2 desc", `[[INT64(5) VARCHAR("test")] [INT64(8) VARCHAR("F")] [INT64(7) VARCHAR("e")] [INT64(6) VARCHAR("d")] [INT64(4) VARCHAR("c")] [INT64(3) VARCHAR("b")] [INT64(2) VARCHAR("Abc")] [INT64(1) VARCHAR("a")]]`) // test ordering of int column - assertMatches(t, conn, "select id1, id2 from t4 order by id1 desc", `[[INT64(8) VARCHAR("E")] [INT64(7) VARCHAR("e")] [INT64(6) VARCHAR("d")] [INT64(5) VARCHAR("test")] [INT64(4) VARCHAR("c")] [INT64(3) VARCHAR("b")] [INT64(2) VARCHAR("A")] [INT64(1) VARCHAR("a")]]`) - exec(t, conn, "delete from t4") + assertMatches(t, conn, "select id1, id2 from t4 order by id1 desc", `[[INT64(8) VARCHAR("F")] [INT64(7) VARCHAR("e")] [INT64(6) VARCHAR("d")] [INT64(5) VARCHAR("test")] [INT64(4) VARCHAR("c")] [INT64(3) VARCHAR("b")] [INT64(2) VARCHAR("Abc")] [INT64(1) VARCHAR("a")]]`) + + defer func() { + exec(t, conn, "set workload = oltp") + exec(t, conn, "delete from t4") + }() + // Test the same queries in streaming mode + exec(t, conn, "set workload = olap") + assertMatches(t, conn, "select id1, id2 from t4 order by id2 desc", `[[INT64(5) VARCHAR("test")] [INT64(8) VARCHAR("F")] [INT64(7) VARCHAR("e")] [INT64(6) VARCHAR("d")] [INT64(4) VARCHAR("c")] [INT64(3) VARCHAR("b")] [INT64(2) VARCHAR("Abc")] [INT64(1) VARCHAR("a")]]`) + assertMatches(t, conn, "select id1, id2 from t4 order by id1 desc", `[[INT64(8) VARCHAR("F")] [INT64(7) VARCHAR("e")] [INT64(6) VARCHAR("d")] [INT64(5) VARCHAR("test")] [INT64(4) VARCHAR("c")] [INT64(3) VARCHAR("b")] [INT64(2) VARCHAR("Abc")] [INT64(1) VARCHAR("a")]]`) } func TestGroupBy(t *testing.T) { @@ -65,5 +73,12 @@ func TestGroupBy(t *testing.T) { exec(t, conn, "insert into t3(id5, id6, id7) values(1,1,2), (2,2,4), (3,2,4), (4,1,2), (5,1,2), (6,3,6)") // test ordering and group by int column assertMatches(t, conn, "select id6, id7, count(*) k from t3 group by id6, id7 order by k", `[[INT64(3) INT64(6) INT64(1)] [INT64(2) INT64(4) INT64(2)] [INT64(1) INT64(2) INT64(3)]]`) - exec(t, conn, "delete from t3") + + defer func() { + exec(t, conn, "set workload = oltp") + exec(t, conn, "delete from t3") + }() + // Test the same queries in streaming mode + exec(t, conn, "set workload = olap") + assertMatches(t, conn, "select id6, id7, count(*) k from t3 group by id6, id7 order by k", `[[INT64(3) INT64(6) INT64(1)] [INT64(2) INT64(4) INT64(2)] [INT64(1) INT64(2) INT64(3)]]`) } From 316be68f19f8c0a322f56a4906da836fbea61899 Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Mon, 22 Mar 2021 19:56:45 +0530 Subject: [PATCH 16/19] decouple planner and executor by returning errors from supply weight string Signed-off-by: GuptaManan100 --- go/vt/vtgate/planbuilder/concatenate.go | 2 +- go/vt/vtgate/planbuilder/join.go | 16 +++++++++++----- go/vt/vtgate/planbuilder/join2.go | 2 +- go/vt/vtgate/planbuilder/logical_plan.go | 19 +++++++++++-------- go/vt/vtgate/planbuilder/memory_sort.go | 9 ++++++++- go/vt/vtgate/planbuilder/merge_sort.go | 6 +++++- go/vt/vtgate/planbuilder/ordered_aggregate.go | 9 ++++++++- go/vt/vtgate/planbuilder/pullout_subquery.go | 2 +- go/vt/vtgate/planbuilder/route.go | 13 ++++++------- .../vtgate/planbuilder/sql_calc_found_rows.go | 4 ++-- go/vt/vtgate/planbuilder/vindex_func.go | 16 ++++++++++++++-- 11 files changed, 68 insertions(+), 30 deletions(-) diff --git a/go/vt/vtgate/planbuilder/concatenate.go b/go/vt/vtgate/planbuilder/concatenate.go index 71fd2405827..b482e795845 100644 --- a/go/vt/vtgate/planbuilder/concatenate.go +++ b/go/vt/vtgate/planbuilder/concatenate.go @@ -70,7 +70,7 @@ func (c *concatenate) SupplyCol(col *sqlparser.ColName) (rc *resultColumn, colNu panic("implement me") } -func (c *concatenate) SupplyWeightString(colNumber int) (weightcolNumber int) { +func (c *concatenate) SupplyWeightString(colNumber int) (weightcolNumber int, err error) { panic("implement me") } diff --git a/go/vt/vtgate/planbuilder/join.go b/go/vt/vtgate/planbuilder/join.go index a6410bfcc69..d8e7429a900 100644 --- a/go/vt/vtgate/planbuilder/join.go +++ b/go/vt/vtgate/planbuilder/join.go @@ -212,22 +212,28 @@ func (jb *join) SupplyCol(col *sqlparser.ColName) (rc *resultColumn, colNumber i } // SupplyWeightString implements the logicalPlan interface -func (jb *join) SupplyWeightString(colNumber int) (weightcolNumber int) { +func (jb *join) SupplyWeightString(colNumber int) (weightcolNumber int, err error) { rc := jb.resultColumns[colNumber] if weightcolNumber, ok := jb.weightStrings[rc]; ok { - return weightcolNumber + return weightcolNumber, nil } routeNumber := rc.column.Origin().Order() if jb.isOnLeft(routeNumber) { - sourceCol := jb.Left.SupplyWeightString(-jb.ejoin.Cols[colNumber] - 1) + sourceCol, err := jb.Left.SupplyWeightString(-jb.ejoin.Cols[colNumber] - 1) + if err != nil { + return 0, err + } jb.ejoin.Cols = append(jb.ejoin.Cols, -sourceCol-1) } else { - sourceCol := jb.Right.SupplyWeightString(jb.ejoin.Cols[colNumber] - 1) + sourceCol, err := jb.Right.SupplyWeightString(jb.ejoin.Cols[colNumber] - 1) + if err != nil { + return 0, err + } jb.ejoin.Cols = append(jb.ejoin.Cols, sourceCol+1) } jb.resultColumns = append(jb.resultColumns, rc) jb.weightStrings[rc] = len(jb.ejoin.Cols) - 1 - return len(jb.ejoin.Cols) - 1 + return len(jb.ejoin.Cols) - 1, nil } // Rewrite implements the logicalPlan interface diff --git a/go/vt/vtgate/planbuilder/join2.go b/go/vt/vtgate/planbuilder/join2.go index c2a6063d5ed..7b273c88865 100644 --- a/go/vt/vtgate/planbuilder/join2.go +++ b/go/vt/vtgate/planbuilder/join2.go @@ -73,7 +73,7 @@ func (j *joinV4) SupplyCol(col *sqlparser.ColName) (rc *resultColumn, colNumber } // SupplyWeightString implements the logicalPlan interface -func (j *joinV4) SupplyWeightString(colNumber int) (weightcolNumber int) { +func (j *joinV4) SupplyWeightString(colNumber int) (weightcolNumber int, err error) { panic("implement me") } diff --git a/go/vt/vtgate/planbuilder/logical_plan.go b/go/vt/vtgate/planbuilder/logical_plan.go index 822e68d8497..81e4fae0aa2 100644 --- a/go/vt/vtgate/planbuilder/logical_plan.go +++ b/go/vt/vtgate/planbuilder/logical_plan.go @@ -69,8 +69,8 @@ type logicalPlan interface { SupplyCol(col *sqlparser.ColName) (rc *resultColumn, colNumber int) // SupplyWeightString must supply a weight_string expression of the - // specified column. It returns -1 if we cannot supply a weight column for it. - SupplyWeightString(colNumber int) (weightcolNumber int) + // specified column. It returns an error if we cannot supply a weight column for it. + SupplyWeightString(colNumber int) (weightcolNumber int, err error) // Primitive returns the underlying primitive. // This function should only be called after Wireup is finished. @@ -170,7 +170,7 @@ func (bc *logicalPlanCommon) SupplyCol(col *sqlparser.ColName) (rc *resultColumn return bc.input.SupplyCol(col) } -func (bc *logicalPlanCommon) SupplyWeightString(colNumber int) (weightcolNumber int) { +func (bc *logicalPlanCommon) SupplyWeightString(colNumber int) (weightcolNumber int, err error) { return bc.input.SupplyWeightString(colNumber) } @@ -239,22 +239,25 @@ func (rsb *resultsBuilder) SupplyCol(col *sqlparser.ColName) (rc *resultColumn, return rc, colNumber } -func (rsb *resultsBuilder) SupplyWeightString(colNumber int) (weightcolNumber int) { +func (rsb *resultsBuilder) SupplyWeightString(colNumber int) (weightcolNumber int, err error) { rc := rsb.resultColumns[colNumber] if weightcolNumber, ok := rsb.weightStrings[rc]; ok { - return weightcolNumber + return weightcolNumber, nil + } + weightcolNumber, err = rsb.input.SupplyWeightString(colNumber) + if err != nil { + return 0, nil } - weightcolNumber = rsb.input.SupplyWeightString(colNumber) rsb.weightStrings[rc] = weightcolNumber if weightcolNumber < len(rsb.resultColumns) { - return weightcolNumber + return weightcolNumber, nil } // Add result columns from input until weightcolNumber is reached. for weightcolNumber >= len(rsb.resultColumns) { rsb.resultColumns = append(rsb.resultColumns, rsb.input.ResultColumns()[len(rsb.resultColumns)]) } rsb.truncater.SetTruncateColumnCount(len(rsb.resultColumns)) - return weightcolNumber + return weightcolNumber, nil } //------------------------------------------------------------------------- diff --git a/go/vt/vtgate/planbuilder/memory_sort.go b/go/vt/vtgate/planbuilder/memory_sort.go index 7501a85bda0..1f96ed47d5e 100644 --- a/go/vt/vtgate/planbuilder/memory_sort.go +++ b/go/vt/vtgate/planbuilder/memory_sort.go @@ -118,7 +118,14 @@ func (ms *memorySort) Wireup(plan logicalPlan, jt *jointab) error { ms.eMemorySort.OrderBy[i].WeightStringCol = weightcolNumber continue } - weightcolNumber := ms.input.SupplyWeightString(orderby.Col) + weightcolNumber, err := ms.input.SupplyWeightString(orderby.Col) + if err != nil { + _, isUnsupportedErr := err.(UnsupportedSupplyWeightString) + if isUnsupportedErr { + continue + } + return err + } ms.weightStrings[rc] = weightcolNumber ms.eMemorySort.OrderBy[i].WeightStringCol = weightcolNumber ms.eMemorySort.TruncateColumnCount = len(ms.resultColumns) diff --git a/go/vt/vtgate/planbuilder/merge_sort.go b/go/vt/vtgate/planbuilder/merge_sort.go index 77fb17bfc53..1fea7894cc4 100644 --- a/go/vt/vtgate/planbuilder/merge_sort.go +++ b/go/vt/vtgate/planbuilder/merge_sort.go @@ -74,8 +74,12 @@ func (ms *mergeSort) Wireup(plan logicalPlan, jt *jointab) error { continue } var err error - rb.eroute.OrderBy[i].WeightStringCol = rb.SupplyWeightString(orderby.Col) + rb.eroute.OrderBy[i].WeightStringCol, err = rb.SupplyWeightString(orderby.Col) if err != nil { + _, isUnsupportedErr := err.(UnsupportedSupplyWeightString) + if isUnsupportedErr { + continue + } return err } ms.truncateColumnCount = len(ms.resultColumns) diff --git a/go/vt/vtgate/planbuilder/ordered_aggregate.go b/go/vt/vtgate/planbuilder/ordered_aggregate.go index 36c4461b154..82d06035b2b 100644 --- a/go/vt/vtgate/planbuilder/ordered_aggregate.go +++ b/go/vt/vtgate/planbuilder/ordered_aggregate.go @@ -338,7 +338,14 @@ func (oa *orderedAggregate) Wireup(plan logicalPlan, jt *jointab) error { oa.eaggr.Keys[i] = weightcolNumber continue } - weightcolNumber := oa.input.SupplyWeightString(colNumber) + weightcolNumber, err := oa.input.SupplyWeightString(colNumber) + if err != nil { + _, isUnsupportedErr := err.(UnsupportedSupplyWeightString) + if isUnsupportedErr { + continue + } + return err + } oa.weightStrings[rc] = weightcolNumber oa.eaggr.Keys[i] = weightcolNumber oa.eaggr.TruncateColumnCount = len(oa.resultColumns) diff --git a/go/vt/vtgate/planbuilder/pullout_subquery.go b/go/vt/vtgate/planbuilder/pullout_subquery.go index e85d90eba40..ec0862bcf0d 100644 --- a/go/vt/vtgate/planbuilder/pullout_subquery.go +++ b/go/vt/vtgate/planbuilder/pullout_subquery.go @@ -110,7 +110,7 @@ func (ps *pulloutSubquery) SupplyCol(col *sqlparser.ColName) (rc *resultColumn, } // SupplyWeightString implements the logicalPlan interface -func (ps *pulloutSubquery) SupplyWeightString(colNumber int) (weightcolNumber int) { +func (ps *pulloutSubquery) SupplyWeightString(colNumber int) (weightcolNumber int, err error) { return ps.underlying.SupplyWeightString(colNumber) } diff --git a/go/vt/vtgate/planbuilder/route.go b/go/vt/vtgate/planbuilder/route.go index e8ecc129034..43ac4b1ba93 100644 --- a/go/vt/vtgate/planbuilder/route.go +++ b/go/vt/vtgate/planbuilder/route.go @@ -333,19 +333,19 @@ func (rb *route) SupplyCol(col *sqlparser.ColName) (rc *resultColumn, colNumber } // SupplyWeightString implements the logicalPlan interface -func (rb *route) SupplyWeightString(colNumber int) (weightcolNumber int) { +func (rb *route) SupplyWeightString(colNumber int) (weightcolNumber int, err error) { rc := rb.resultColumns[colNumber] if weightcolNumber, ok := rb.weightStrings[rc]; ok { - return weightcolNumber + return weightcolNumber, nil } s, ok := rb.Select.(*sqlparser.Select) if !ok { - return -1 + return 0, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "unexpected AST struct for query") } aliasExpr, ok := s.SelectExprs[colNumber].(*sqlparser.AliasedExpr) if !ok { - return -1 + return 0, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "unexpected AST struct for query %T", s.SelectExprs[colNumber]) } expr := &sqlparser.AliasedExpr{ Expr: &sqlparser.FuncExpr{ @@ -359,13 +359,12 @@ func (rb *route) SupplyWeightString(colNumber int) (weightcolNumber int) { } // It's ok to pass nil for pb and logicalPlan because PushSelect doesn't use them. // TODO: we are ignoring a potential error here. need to clean this up - var err error _, _, weightcolNumber, err = planProjection(nil, rb, expr, nil) if err != nil { - return -1 + return 0, err } rb.weightStrings[rc] = weightcolNumber - return weightcolNumber + return weightcolNumber, nil } // Rewrite implements the logicalPlan interface diff --git a/go/vt/vtgate/planbuilder/sql_calc_found_rows.go b/go/vt/vtgate/planbuilder/sql_calc_found_rows.go index bcef9fce59f..21a7bae3d29 100644 --- a/go/vt/vtgate/planbuilder/sql_calc_found_rows.go +++ b/go/vt/vtgate/planbuilder/sql_calc_found_rows.go @@ -90,8 +90,8 @@ func (s *sqlCalcFoundRows) SupplyCol(col *sqlparser.ColName) (*resultColumn, int } //SupplyWeightString implements the logicalPlan interface -func (s *sqlCalcFoundRows) SupplyWeightString(int) (weightcolNumber int) { - return -1 +func (s *sqlCalcFoundRows) SupplyWeightString(int) (weightcolNumber int, err error) { + return 0, UnsupportedSupplyWeightString{Type: "sqlCalcFoundRows"} } // Rewrite implements the logicalPlan interface diff --git a/go/vt/vtgate/planbuilder/vindex_func.go b/go/vt/vtgate/planbuilder/vindex_func.go index da145ded96b..8d9a6d0a74a 100644 --- a/go/vt/vtgate/planbuilder/vindex_func.go +++ b/go/vt/vtgate/planbuilder/vindex_func.go @@ -17,6 +17,8 @@ limitations under the License. package planbuilder import ( + "fmt" + "vitess.io/vitess/go/vt/vtgate/semantics" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" @@ -129,9 +131,19 @@ func (vf *vindexFunc) SupplyCol(col *sqlparser.ColName) (rc *resultColumn, colNu return rc, len(vf.resultColumns) - 1 } +// UnsupportedSupplyWeightString represents the error where the supplying a weight string is not supported +type UnsupportedSupplyWeightString struct { + Type string +} + +// Error function implements the error interface +func (err UnsupportedSupplyWeightString) Error() string { + return fmt.Sprintf("cannot do collation on %s", err.Type) +} + // SupplyWeightString implements the logicalPlan interface -func (vf *vindexFunc) SupplyWeightString(colNumber int) (weightcolNumber int) { - return -1 +func (vf *vindexFunc) SupplyWeightString(colNumber int) (weightcolNumber int, err error) { + return 0, UnsupportedSupplyWeightString{Type: "vindex function"} } // Rewrite implements the logicalPlan interface From 5e43cffde7aa24abe201b979f7de1f621f64fff4 Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Tue, 23 Mar 2021 12:24:57 +0530 Subject: [PATCH 17/19] refactor code Signed-off-by: GuptaManan100 --- go/test/endtoend/vtgate/aggr_test.go | 23 ---------------------- go/test/endtoend/vtgate/misc_test.go | 23 ++++++++++++++++++++++ go/vt/vtgate/engine/comparer.go | 2 +- go/vt/vtgate/planbuilder/symtab.go | 26 ++++++++++++------------- go/vt/vtgate/planbuilder/symtab_test.go | 15 +++----------- 5 files changed, 39 insertions(+), 50 deletions(-) diff --git a/go/test/endtoend/vtgate/aggr_test.go b/go/test/endtoend/vtgate/aggr_test.go index 8018a52476d..adc8c93a84c 100644 --- a/go/test/endtoend/vtgate/aggr_test.go +++ b/go/test/endtoend/vtgate/aggr_test.go @@ -41,29 +41,6 @@ func TestAggregateTypes(t *testing.T) { exec(t, conn, "delete from aggr_test") } -func TestOrderBy(t *testing.T) { - defer cluster.PanicHandler(t) - ctx := context.Background() - conn, err := mysql.Connect(ctx, &vtParams) - require.Nil(t, err) - defer conn.Close() - exec(t, conn, "insert into t4(id1, id2) values(1,'a'), (2,'Abc'), (3,'b'), (4,'c'), (5,'test')") - exec(t, conn, "insert into t4(id1, id2) values(6,'d'), (7,'e'), (8,'F')") - // test ordering of varchar column - assertMatches(t, conn, "select id1, id2 from t4 order by id2 desc", `[[INT64(5) VARCHAR("test")] [INT64(8) VARCHAR("F")] [INT64(7) VARCHAR("e")] [INT64(6) VARCHAR("d")] [INT64(4) VARCHAR("c")] [INT64(3) VARCHAR("b")] [INT64(2) VARCHAR("Abc")] [INT64(1) VARCHAR("a")]]`) - // test ordering of int column - assertMatches(t, conn, "select id1, id2 from t4 order by id1 desc", `[[INT64(8) VARCHAR("F")] [INT64(7) VARCHAR("e")] [INT64(6) VARCHAR("d")] [INT64(5) VARCHAR("test")] [INT64(4) VARCHAR("c")] [INT64(3) VARCHAR("b")] [INT64(2) VARCHAR("Abc")] [INT64(1) VARCHAR("a")]]`) - - defer func() { - exec(t, conn, "set workload = oltp") - exec(t, conn, "delete from t4") - }() - // Test the same queries in streaming mode - exec(t, conn, "set workload = olap") - assertMatches(t, conn, "select id1, id2 from t4 order by id2 desc", `[[INT64(5) VARCHAR("test")] [INT64(8) VARCHAR("F")] [INT64(7) VARCHAR("e")] [INT64(6) VARCHAR("d")] [INT64(4) VARCHAR("c")] [INT64(3) VARCHAR("b")] [INT64(2) VARCHAR("Abc")] [INT64(1) VARCHAR("a")]]`) - assertMatches(t, conn, "select id1, id2 from t4 order by id1 desc", `[[INT64(8) VARCHAR("F")] [INT64(7) VARCHAR("e")] [INT64(6) VARCHAR("d")] [INT64(5) VARCHAR("test")] [INT64(4) VARCHAR("c")] [INT64(3) VARCHAR("b")] [INT64(2) VARCHAR("Abc")] [INT64(1) VARCHAR("a")]]`) -} - func TestGroupBy(t *testing.T) { defer cluster.PanicHandler(t) ctx := context.Background() diff --git a/go/test/endtoend/vtgate/misc_test.go b/go/test/endtoend/vtgate/misc_test.go index 38570ef03ed..3dc1f55aa5a 100644 --- a/go/test/endtoend/vtgate/misc_test.go +++ b/go/test/endtoend/vtgate/misc_test.go @@ -558,6 +558,29 @@ func TestShowVariables(t *testing.T) { require.True(t, found, "Expected a row for version in show query") } +func TestOrderBy(t *testing.T) { + defer cluster.PanicHandler(t) + ctx := context.Background() + conn, err := mysql.Connect(ctx, &vtParams) + require.Nil(t, err) + defer conn.Close() + exec(t, conn, "insert into t4(id1, id2) values(1,'a'), (2,'Abc'), (3,'b'), (4,'c'), (5,'test')") + exec(t, conn, "insert into t4(id1, id2) values(6,'d'), (7,'e'), (8,'F')") + // test ordering of varchar column + assertMatches(t, conn, "select id1, id2 from t4 order by id2 desc", `[[INT64(5) VARCHAR("test")] [INT64(8) VARCHAR("F")] [INT64(7) VARCHAR("e")] [INT64(6) VARCHAR("d")] [INT64(4) VARCHAR("c")] [INT64(3) VARCHAR("b")] [INT64(2) VARCHAR("Abc")] [INT64(1) VARCHAR("a")]]`) + // test ordering of int column + assertMatches(t, conn, "select id1, id2 from t4 order by id1 desc", `[[INT64(8) VARCHAR("F")] [INT64(7) VARCHAR("e")] [INT64(6) VARCHAR("d")] [INT64(5) VARCHAR("test")] [INT64(4) VARCHAR("c")] [INT64(3) VARCHAR("b")] [INT64(2) VARCHAR("Abc")] [INT64(1) VARCHAR("a")]]`) + + defer func() { + exec(t, conn, "set workload = oltp") + exec(t, conn, "delete from t4") + }() + // Test the same queries in streaming mode + exec(t, conn, "set workload = olap") + assertMatches(t, conn, "select id1, id2 from t4 order by id2 desc", `[[INT64(5) VARCHAR("test")] [INT64(8) VARCHAR("F")] [INT64(7) VARCHAR("e")] [INT64(6) VARCHAR("d")] [INT64(4) VARCHAR("c")] [INT64(3) VARCHAR("b")] [INT64(2) VARCHAR("Abc")] [INT64(1) VARCHAR("a")]]`) + assertMatches(t, conn, "select id1, id2 from t4 order by id1 desc", `[[INT64(8) VARCHAR("F")] [INT64(7) VARCHAR("e")] [INT64(6) VARCHAR("d")] [INT64(5) VARCHAR("test")] [INT64(4) VARCHAR("c")] [INT64(3) VARCHAR("b")] [INT64(2) VARCHAR("Abc")] [INT64(1) VARCHAR("a")]]`) +} + func assertMatches(t *testing.T, conn *mysql.Conn, query, expected string) { t.Helper() qr := exec(t, conn, query) diff --git a/go/vt/vtgate/engine/comparer.go b/go/vt/vtgate/engine/comparer.go index a0a0a528421..a685c4816a2 100644 --- a/go/vt/vtgate/engine/comparer.go +++ b/go/vt/vtgate/engine/comparer.go @@ -53,7 +53,7 @@ func (c *comparer) compare(r1, r2 []sqltypes.Value) (int, error) { return cmp, nil } -// extractSlices extracts the three fields of OrderbyParams into 3 slices +// extractSlices extracts the three fields of OrderbyParams into a slice of comparers func extractSlices(input []OrderbyParams) []*comparer { var result []*comparer for _, order := range input { diff --git a/go/vt/vtgate/planbuilder/symtab.go b/go/vt/vtgate/planbuilder/symtab.go index 0f83ad87eeb..5a0ad71cac6 100644 --- a/go/vt/vtgate/planbuilder/symtab.go +++ b/go/vt/vtgate/planbuilder/symtab.go @@ -564,39 +564,37 @@ func newResultColumn(expr *sqlparser.AliasedExpr, origin logicalPlan) *resultCol } else { // We don't generate an alias if the expression is non-trivial. // Just to be safe, generate an anonymous column for the expression. + typ, err := GetReturnType(expr.Expr) rc.column = &column{ origin: origin, - typ: GetReturnType(expr), + } + if err == nil { + rc.column.typ = typ } } return rc } // GetReturnType returns the type of the select expression that MySQL will return -func GetReturnType(input sqlparser.SQLNode) querypb.Type { +func GetReturnType(input sqlparser.Expr) (querypb.Type, error) { switch node := input.(type) { - case *sqlparser.Nextval: - return querypb.Type_INT64 - case *sqlparser.AliasedExpr: - return GetReturnType(node.Expr) case *sqlparser.FuncExpr: functionName := strings.ToUpper(node.Name.String()) switch functionName { case "ABS": // Returned value depends on the return type of the input if len(node.Exprs) == 1 { - expr := node.Exprs[0] - return GetReturnType(expr) + expr, isAliasedExpr := node.Exprs[0].(*sqlparser.AliasedExpr) + if isAliasedExpr { + return GetReturnType(expr.Expr) + } } case "COUNT": - return querypb.Type_INT64 + return querypb.Type_INT64, nil } case *sqlparser.ColName: col := node.Metadata.(*column) - return col.typ - case *sqlparser.StarExpr: - // return null type when we do not know the type - return querypb.Type_NULL_TYPE + return col.typ, nil } - return querypb.Type_NULL_TYPE + return 0, fmt.Errorf("cannot evaluate return type for %T", input) } diff --git a/go/vt/vtgate/planbuilder/symtab_test.go b/go/vt/vtgate/planbuilder/symtab_test.go index a528cadccbc..e8189d55dce 100644 --- a/go/vt/vtgate/planbuilder/symtab_test.go +++ b/go/vt/vtgate/planbuilder/symtab_test.go @@ -190,7 +190,7 @@ func TestSymtabAddVSchemaTable(t *testing.T) { func TestGetReturnType(t *testing.T) { tests := []struct { - input sqlparser.SQLNode + input sqlparser.Expr output querypb.Type }{{ input: &sqlparser.FuncExpr{Name: sqlparser.NewColIdent("Abs"), Exprs: sqlparser.SelectExprs{ @@ -204,9 +204,6 @@ func TestGetReturnType(t *testing.T) { }, }}, output: querypb.Type_DECIMAL, - }, { - input: &sqlparser.StarExpr{}, - output: querypb.Type_NULL_TYPE, }, { input: &sqlparser.FuncExpr{Name: sqlparser.NewColIdent("Count"), Exprs: sqlparser.SelectExprs{ &sqlparser.StarExpr{}, @@ -217,18 +214,12 @@ func TestGetReturnType(t *testing.T) { &sqlparser.StarExpr{}, }}, output: querypb.Type_INT64, - }, { - input: &sqlparser.Nextval{ - Expr: &sqlparser.ColName{ - Name: sqlparser.NewColIdent("A"), - }, - }, - output: querypb.Type_INT64, }} for _, test := range tests { t.Run(sqlparser.String(test.input), func(t *testing.T) { - got := GetReturnType(test.input) + got, err := GetReturnType(test.input) + require.NoError(t, err) require.Equal(t, test.output, got) }) } From 5397668b918f05a860f8977d1dd2e3314ac11f5a Mon Sep 17 00:00:00 2001 From: GuptaManan100 Date: Tue, 23 Mar 2021 12:31:29 +0530 Subject: [PATCH 18/19] added a test for error in return type as well Signed-off-by: GuptaManan100 --- go/vt/vtgate/planbuilder/symtab_test.go | 28 ++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/go/vt/vtgate/planbuilder/symtab_test.go b/go/vt/vtgate/planbuilder/symtab_test.go index e8189d55dce..c327ad9c385 100644 --- a/go/vt/vtgate/planbuilder/symtab_test.go +++ b/go/vt/vtgate/planbuilder/symtab_test.go @@ -17,6 +17,7 @@ limitations under the License. package planbuilder import ( + "fmt" "testing" "github.com/stretchr/testify/require" @@ -190,8 +191,9 @@ func TestSymtabAddVSchemaTable(t *testing.T) { func TestGetReturnType(t *testing.T) { tests := []struct { - input sqlparser.Expr - output querypb.Type + input sqlparser.Expr + output querypb.Type + expectedErr error }{{ input: &sqlparser.FuncExpr{Name: sqlparser.NewColIdent("Abs"), Exprs: sqlparser.SelectExprs{ &sqlparser.AliasedExpr{ @@ -203,24 +205,36 @@ func TestGetReturnType(t *testing.T) { }, }, }}, - output: querypb.Type_DECIMAL, + output: querypb.Type_DECIMAL, + expectedErr: nil, }, { input: &sqlparser.FuncExpr{Name: sqlparser.NewColIdent("Count"), Exprs: sqlparser.SelectExprs{ &sqlparser.StarExpr{}, }}, - output: querypb.Type_INT64, + output: querypb.Type_INT64, + expectedErr: nil, }, { input: &sqlparser.FuncExpr{Name: sqlparser.NewColIdent("cOunt"), Exprs: sqlparser.SelectExprs{ &sqlparser.StarExpr{}, }}, - output: querypb.Type_INT64, + output: querypb.Type_INT64, + expectedErr: nil, + }, { + input: &sqlparser.FuncExpr{Name: sqlparser.NewColIdent("Abs"), Exprs: sqlparser.SelectExprs{ + &sqlparser.StarExpr{}, + }}, + expectedErr: fmt.Errorf("cannot evaluate return type for *sqlparser.FuncExpr"), }} for _, test := range tests { t.Run(sqlparser.String(test.input), func(t *testing.T) { got, err := GetReturnType(test.input) - require.NoError(t, err) - require.Equal(t, test.output, got) + if test.expectedErr != nil { + require.EqualError(t, err, test.expectedErr.Error()) + } else { + require.NoError(t, err) + require.Equal(t, test.output, got) + } }) } } From 1982137a9268530ee74a359afec2c58eb51a3bdd Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Tue, 23 Mar 2021 13:08:11 +0100 Subject: [PATCH 19/19] use vterrors instead of fmt.error Signed-off-by: Andres Taylor --- go/vt/vtgate/planbuilder/symtab.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/go/vt/vtgate/planbuilder/symtab.go b/go/vt/vtgate/planbuilder/symtab.go index 5a0ad71cac6..afaffea1f97 100644 --- a/go/vt/vtgate/planbuilder/symtab.go +++ b/go/vt/vtgate/planbuilder/symtab.go @@ -22,6 +22,9 @@ import ( "strconv" "strings" + vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/vterrors" + "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vtgate/vindexes" @@ -596,5 +599,5 @@ func GetReturnType(input sqlparser.Expr) (querypb.Type, error) { col := node.Metadata.(*column) return col.typ, nil } - return 0, fmt.Errorf("cannot evaluate return type for %T", input) + return 0, vterrors.Errorf(vtrpcpb.Code_UNIMPLEMENTED, "cannot evaluate return type for %T", input) }