From c9c337ec8ba842ef9dc352dbed214adfb01674e4 Mon Sep 17 00:00:00 2001 From: hejiayua Date: Mon, 11 Nov 2019 17:59:18 +0800 Subject: [PATCH 1/3] vec QuoteSig (#13312) --- expression/builtin_string_vec.go | 23 +++++++++++++++++++++-- expression/builtin_string_vec_test.go | 4 +++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/expression/builtin_string_vec.go b/expression/builtin_string_vec.go index e6eed321a320e..98043fac83e04 100644 --- a/expression/builtin_string_vec.go +++ b/expression/builtin_string_vec.go @@ -503,11 +503,30 @@ func (b *builtinFieldStringSig) vecEvalInt(input *chunk.Chunk, result *chunk.Col } func (b *builtinQuoteSig) vectorized() bool { - return false + return true } func (b *builtinQuoteSig) vecEvalString(input *chunk.Chunk, result *chunk.Column) error { - return errors.Errorf("not implemented") + n := input.NumRows() + buf, err := b.bufAllocator.get(types.ETString, n) + if err != nil { + return err + } + defer b.bufAllocator.put(buf) + if err := b.args[0].VecEvalString(b.ctx, input, buf); err != nil { + return err + } + + result.ReserveString(n) + for i := 0; i < n; i++ { + if buf.IsNull(i) { + result.AppendString("NULL") + continue + } + str := buf.GetString(i) + result.AppendString(Quote(str)) + } + return nil } func (b *builtinInsertBinarySig) vectorized() bool { diff --git a/expression/builtin_string_vec_test.go b/expression/builtin_string_vec_test.go index 3c18a6414a5f7..bd08d0ab9161c 100644 --- a/expression/builtin_string_vec_test.go +++ b/expression/builtin_string_vec_test.go @@ -154,10 +154,12 @@ var vecBuiltinStringCases = map[string][]vecExprBenchCase{ {retEvalType: types.ETString, childrenTypes: []types.EvalType{types.ETInt}}, {retEvalType: types.ETString, childrenTypes: []types.EvalType{types.ETString}, geners: []dataGenerator{&numStrGener{rangeInt64Gener{-10, 10}}}}, }, + ast.Quote: { + {retEvalType: types.ETString, childrenTypes: []types.EvalType{types.ETString}}, + }, ast.Ord: { {retEvalType: types.ETInt, childrenTypes: []types.EvalType{types.ETString}}, }, - ast.Quote: {}, ast.Bin: { {retEvalType: types.ETString, childrenTypes: []types.EvalType{types.ETInt}}, }, From 838481a67630b0a0cb54524089c50d570fa67b6f Mon Sep 17 00:00:00 2001 From: Eugene Kalinin Date: Mon, 11 Nov 2019 13:18:03 +0300 Subject: [PATCH 2/3] expression: implement vectorized evaluation for builtinSysDateWithoutFspSig (#13348) --- expression/builtin_time_vec.go | 21 +++++++++++++++++++-- expression/builtin_time_vec_test.go | 3 +++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/expression/builtin_time_vec.go b/expression/builtin_time_vec.go index 5a839b22a324c..23b8bc7fa4008 100644 --- a/expression/builtin_time_vec.go +++ b/expression/builtin_time_vec.go @@ -136,11 +136,28 @@ func (b *builtinFromUnixTime2ArgSig) vecEvalString(input *chunk.Chunk, result *c } func (b *builtinSysDateWithoutFspSig) vectorized() bool { - return false + return true } func (b *builtinSysDateWithoutFspSig) vecEvalTime(input *chunk.Chunk, result *chunk.Column) error { - return errors.Errorf("not implemented") + n := input.NumRows() + loc := b.ctx.GetSessionVars().Location() + nowTs, err := getStmtTimestamp(b.ctx) + if err != nil { + return err + } + + now := nowTs.In(loc) + result.ResizeTime(n, false) + times := result.Times() + t, err := convertTimeToMysqlTime(now, 0, types.ModeHalfEven) + if err != nil { + return err + } + for i := 0; i < n; i++ { + times[i] = t + } + return nil } func (b *builtinExtractDatetimeSig) vectorized() bool { diff --git a/expression/builtin_time_vec_test.go b/expression/builtin_time_vec_test.go index 4e3f466d90f8c..5f25d77204452 100644 --- a/expression/builtin_time_vec_test.go +++ b/expression/builtin_time_vec_test.go @@ -209,6 +209,9 @@ var vecBuiltinTimeCases = map[string][]vecExprBenchCase{ geners: []dataGenerator{&formatGener{0.2}, &locationGener{0.2}}, }, }, + ast.Sysdate: { + {retEvalType: types.ETDatetime}, + }, } func (s *testEvaluatorSuite) TestVectorizedBuiltinTimeEvalOneVec(c *C) { From 60bfdf5ba4bd5a5c1f012c2d4869daee568480d3 Mon Sep 17 00:00:00 2001 From: Zhuhe Fang Date: Mon, 11 Nov 2019 18:30:34 +0800 Subject: [PATCH 3/3] Revert "executor, util: rename original List to ListInMemory and add a new interface List (#13353)" (#13363) This reverts commit 2b5f81add11497b5b5595b3ba16ac88cddf33a65. --- executor/executor.go | 10 ++-- executor/hash_table.go | 11 ++--- executor/index_lookup_join.go | 16 ++----- executor/index_lookup_merge_join.go | 9 ++-- executor/join.go | 6 +-- executor/pkg_test.go | 2 +- executor/sort.go | 18 +++----- util/chunk/disk.go | 9 +--- util/chunk/iterator.go | 10 ++-- util/chunk/iterator_test.go | 6 +-- util/chunk/list.go | 72 ++++++++++++----------------- util/chunk/list_test.go | 16 +++---- 12 files changed, 75 insertions(+), 110 deletions(-) diff --git a/executor/executor.go b/executor/executor.go index a10ecf2e636ee..482119db8e0a9 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -132,10 +132,10 @@ func newFirstChunk(e Executor) *chunk.Chunk { return chunk.New(base.retFieldTypes, base.initCap, base.maxChunkSize) } -// newList creates a new ListInMemory to buffer current executor's result. -func newList(e Executor) *chunk.ListInMemory { +// newList creates a new List to buffer current executor's result. +func newList(e Executor) *chunk.List { base := e.base() - return chunk.NewListInMemory(base.retFieldTypes, base.initCap, base.maxChunkSize) + return chunk.NewList(base.retFieldTypes, base.initCap, base.maxChunkSize) } // retTypes returns all output column types. @@ -1115,7 +1115,7 @@ type TableScanExec struct { iter kv.Iterator columns []*model.ColumnInfo isVirtualTable bool - virtualTableChunkList *chunk.ListInMemory + virtualTableChunkList *chunk.List virtualTableChunkIdx int } @@ -1146,7 +1146,7 @@ func (e *TableScanExec) Next(ctx context.Context, req *chunk.Chunk) error { func (e *TableScanExec) nextChunk4InfoSchema(ctx context.Context, chk *chunk.Chunk) error { chk.GrowAndReset(e.maxChunkSize) if e.virtualTableChunkList == nil { - e.virtualTableChunkList = chunk.NewListInMemory(retTypes(e), e.initCap, e.maxChunkSize) + e.virtualTableChunkList = chunk.NewList(retTypes(e), e.initCap, e.maxChunkSize) columns := make([]*table.Column, e.schema.Len()) for i, colInfo := range e.columns { columns[i] = table.ToColumn(colInfo) diff --git a/executor/hash_table.go b/executor/hash_table.go index 875fc3d38f1bf..256be4bfcbf32 100644 --- a/executor/hash_table.go +++ b/executor/hash_table.go @@ -91,7 +91,7 @@ type hashRowContainer struct { memTracker *memory.Tracker // records stores the chunks in memory. - records *chunk.ListInMemory + records *chunk.List // recordsInDisk stores the chunks in disk. recordsInDisk *chunk.ListInDisk @@ -117,7 +117,7 @@ func newHashRowContainer(sCtx sessionctx.Context, estCount int, hCtx *hashContex if estCount < maxChunkSize*estCountMinFactor { estCount = 0 } - initList := chunk.NewListInMemory(hCtx.allTypes, maxChunkSize, maxChunkSize) + initList := chunk.NewList(hCtx.allTypes, maxChunkSize, maxChunkSize) c := &hashRowContainer{ sc: sCtx.GetSessionVars().StmtCtx, hCtx: hCtx, @@ -202,10 +202,7 @@ func (c *hashRowContainer) PutChunk(chk *chunk.Chunk) error { } } else { chkIdx = uint32(c.records.NumChunks()) - err := c.records.Add(chk) - if err != nil { - return err - } + c.records.Add(chk) if atomic.LoadUint32(&c.exceeded) != 0 { err := c.spillToDisk() if err != nil { @@ -272,7 +269,7 @@ func (c *hashRowContainer) ActionSpill() memory.ActionOnExceed { return &spillDiskAction{c: c} } -// spillDiskAction implements memory.ActionOnExceed for chunk.ListInMemory. If +// spillDiskAction implements memory.ActionOnExceed for chunk.List. If // the memory quota of a query is exceeded, spillDiskAction.Action is // triggered. type spillDiskAction struct { diff --git a/executor/index_lookup_join.go b/executor/index_lookup_join.go index dfd349e8834fa..57f669e375e26 100644 --- a/executor/index_lookup_join.go +++ b/executor/index_lookup_join.go @@ -94,10 +94,10 @@ type innerCtx struct { } type lookUpJoinTask struct { - outerResult *chunk.ListInMemory + outerResult *chunk.List outerMatch [][]bool - innerResult *chunk.ListInMemory + innerResult *chunk.List encodedLookUpKeys []*chunk.Chunk lookupMap *mvmap.MVMap matchedInners []chunk.Row @@ -392,10 +392,7 @@ func (ow *outerWorker) buildTask(ctx context.Context) (*lookUpJoinTask, error) { break } - err = task.outerResult.Add(chk) - if err != nil { - return nil, err - } + task.outerResult.Add(chk) } if task.outerResult.Len() == 0 { return nil, nil @@ -604,7 +601,7 @@ func (iw *innerWorker) fetchInnerResults(ctx context.Context, task *lookUpJoinTa return err } defer terror.Call(innerExec.Close) - innerResult := chunk.NewListInMemory(retTypes(innerExec), iw.ctx.GetSessionVars().MaxChunkSize, iw.ctx.GetSessionVars().MaxChunkSize) + innerResult := chunk.NewList(retTypes(innerExec), iw.ctx.GetSessionVars().MaxChunkSize, iw.ctx.GetSessionVars().MaxChunkSize) innerResult.GetMemTracker().SetLabel(buildSideResultLabel) innerResult.GetMemTracker().AttachTo(task.memTracker) for { @@ -620,10 +617,7 @@ func (iw *innerWorker) fetchInnerResults(ctx context.Context, task *lookUpJoinTa if iw.executorChk.NumRows() == 0 { break } - err = innerResult.Add(iw.executorChk) - if err != nil { - return err - } + innerResult.Add(iw.executorChk) iw.executorChk = newFirstChunk(innerExec) } task.innerResult = innerResult diff --git a/executor/index_lookup_merge_join.go b/executor/index_lookup_merge_join.go index dbf541aa6898b..e4dbe7369d3b2 100644 --- a/executor/index_lookup_merge_join.go +++ b/executor/index_lookup_merge_join.go @@ -86,7 +86,7 @@ type innerMergeCtx struct { } type lookUpMergeJoinTask struct { - outerResult *chunk.ListInMemory + outerResult *chunk.List outerOrderIdx []chunk.RowPtr innerResult *chunk.Chunk @@ -323,7 +323,7 @@ func (omw *outerMergeWorker) pushToChan(ctx context.Context, task *lookUpMergeJo func (omw *outerMergeWorker) buildTask(ctx context.Context) (*lookUpMergeJoinTask, error) { task := &lookUpMergeJoinTask{ results: make(chan *indexMergeJoinResult, numResChkHold), - outerResult: chunk.NewListInMemory(omw.rowTypes, omw.executor.base().initCap, omw.executor.base().maxChunkSize), + outerResult: chunk.NewList(omw.rowTypes, omw.executor.base().initCap, omw.executor.base().maxChunkSize), } task.memTracker = memory.NewTracker(stringutil.MemoizeStr(func() string { return fmt.Sprintf("lookup join task %p", task) }), -1) task.memTracker.AttachTo(omw.parentMemTracker) @@ -346,10 +346,7 @@ func (omw *outerMergeWorker) buildTask(ctx context.Context) (*lookUpMergeJoinTas break } - err = task.outerResult.Add(execChk) - if err != nil { - return task, err - } + task.outerResult.Add(execChk) requiredRows -= execChk.NumRows() task.memTracker.Consume(execChk.MemoryUsage()) } diff --git a/executor/join.go b/executor/join.go index 33010f5f6da3d..61d7386cdce44 100644 --- a/executor/join.go +++ b/executor/join.go @@ -555,7 +555,7 @@ type NestedLoopApplyExec struct { outerChunk *chunk.Chunk outerChunkCursor int outerSelected []bool - innerList *chunk.ListInMemory + innerList *chunk.List innerChunk *chunk.Chunk innerSelected []bool innerIter chunk.Iterator @@ -586,7 +586,7 @@ func (e *NestedLoopApplyExec) Open(ctx context.Context) error { e.innerRows = e.innerRows[:0] e.outerChunk = newFirstChunk(e.outerExec) e.innerChunk = newFirstChunk(e.innerExec) - e.innerList = chunk.NewListInMemory(retTypes(e.innerExec), e.initCap, e.maxChunkSize) + e.innerList = chunk.NewList(retTypes(e.innerExec), e.initCap, e.maxChunkSize) e.memTracker = memory.NewTracker(e.id, e.ctx.GetSessionVars().MemQuotaNestedLoopApply) e.memTracker.AttachTo(e.ctx.GetSessionVars().StmtCtx.MemTracker) @@ -628,7 +628,7 @@ func (e *NestedLoopApplyExec) fetchSelectedOuterRow(ctx context.Context, chk *ch } } -// fetchAllInners reads all data from the inner table and stores them in a ListInMemory. +// fetchAllInners reads all data from the inner table and stores them in a List. func (e *NestedLoopApplyExec) fetchAllInners(ctx context.Context) error { err := e.innerExec.Open(ctx) defer terror.Call(e.innerExec.Close) diff --git a/executor/pkg_test.go b/executor/pkg_test.go index 22a361293a89f..bdd7a91772bed 100644 --- a/executor/pkg_test.go +++ b/executor/pkg_test.go @@ -61,7 +61,7 @@ func (s *pkgTestSuite) TestNestedLoopApply(c *C) { innerFilter: []expression.Expression{innerFilter}, joiner: joiner, } - join.innerList = chunk.NewListInMemory(retTypes(innerExec), innerExec.initCap, innerExec.maxChunkSize) + join.innerList = chunk.NewList(retTypes(innerExec), innerExec.initCap, innerExec.maxChunkSize) join.innerChunk = newFirstChunk(innerExec) join.outerChunk = newFirstChunk(outerExec) joinChk := newFirstChunk(join) diff --git a/executor/sort.go b/executor/sort.go index a9b378b41452b..aa895015b1eec 100644 --- a/executor/sort.go +++ b/executor/sort.go @@ -45,7 +45,7 @@ type SortExec struct { // keyCmpFuncs is used to compare each ByItem. keyCmpFuncs []chunk.CompareFunc // rowChunks is the chunks to store row values. - rowChunks *chunk.ListInMemory + rowChunks *chunk.List // rowPointer store the chunk index and row index for each row. rowPtrs []chunk.RowPtr @@ -95,7 +95,7 @@ func (e *SortExec) Next(ctx context.Context, req *chunk.Chunk) error { func (e *SortExec) fetchRowChunks(ctx context.Context) error { fields := retTypes(e) - e.rowChunks = chunk.NewListInMemory(fields, e.initCap, e.maxChunkSize) + e.rowChunks = chunk.NewList(fields, e.initCap, e.maxChunkSize) e.rowChunks.GetMemTracker().AttachTo(e.memTracker) e.rowChunks.GetMemTracker().SetLabel(rowChunksLabel) for { @@ -108,10 +108,7 @@ func (e *SortExec) fetchRowChunks(ctx context.Context) error { if rowCount == 0 { break } - err = e.rowChunks.Add(chk) - if err != nil { - return err - } + e.rowChunks.Add(chk) } return nil } @@ -259,7 +256,7 @@ func (e *TopNExec) Next(ctx context.Context, req *chunk.Chunk) error { func (e *TopNExec) loadChunksUntilTotalLimit(ctx context.Context) error { e.chkHeap = &topNChunkHeap{e} - e.rowChunks = chunk.NewListInMemory(retTypes(e), e.initCap, e.maxChunkSize) + e.rowChunks = chunk.NewList(retTypes(e), e.initCap, e.maxChunkSize) e.rowChunks.GetMemTracker().AttachTo(e.memTracker) e.rowChunks.GetMemTracker().SetLabel(rowChunksLabel) for uint64(e.rowChunks.Len()) < e.totalLimit { @@ -273,10 +270,7 @@ func (e *TopNExec) loadChunksUntilTotalLimit(ctx context.Context) error { if srcChk.NumRows() == 0 { break } - err = e.rowChunks.Add(srcChk) - if err != nil { - return err - } + e.rowChunks.Add(srcChk) } e.initPointers() e.initCompareFuncs() @@ -336,7 +330,7 @@ func (e *TopNExec) processChildChk(childRowChk *chunk.Chunk) error { // but we want descending top N, then we will keep all data in memory. // But if data is distributed randomly, this function will be called log(n) times. func (e *TopNExec) doCompaction() error { - newRowChunks := chunk.NewListInMemory(retTypes(e), e.initCap, e.maxChunkSize) + newRowChunks := chunk.NewList(retTypes(e), e.initCap, e.maxChunkSize) newRowPtrs := make([]chunk.RowPtr, 0, e.rowChunks.Len()) for _, rowPtr := range e.rowPtrs { newRowPtr := newRowChunks.AppendRow(e.rowChunks.GetRow(rowPtr)) diff --git a/util/chunk/disk.go b/util/chunk/disk.go index bc20be59d89d8..a374a2307c954 100644 --- a/util/chunk/disk.go +++ b/util/chunk/disk.go @@ -83,7 +83,7 @@ func NewListInDisk(fieldTypes []*types.FieldType) *ListInDisk { return l } -// GetDiskTracker returns the memory tracker of this ListInMemory. +// GetDiskTracker returns the memory tracker of this List. func (l *ListInDisk) GetDiskTracker() *disk.Tracker { return l.diskTracker } @@ -92,7 +92,7 @@ func (l *ListInDisk) GetDiskTracker() *disk.Tracker { // is not empty and not used any more and has the same field types. func (l *ListInDisk) Add(chk *Chunk) (err error) { if chk.NumRows() == 0 { - return errors.New("chunk appended to ListInMemory should have at least 1 row") + return errors.New("chunk appended to List should have at least 1 row") } if l.disk == nil { l.disk, err = ioutil.TempFile(tmpDir, l.diskTracker.Label().String()) @@ -138,11 +138,6 @@ func (l *ListInDisk) NumChunks() int { return len(l.offsets) } -// NumRowsOfChunk returns the number of rows of a chunk in the ListInDisk. -func (l *ListInDisk) NumRowsOfChunk(chkID int) int { - return len(l.offsets[chkID]) -} - // Close releases the disk resource. func (l *ListInDisk) Close() error { if l.disk != nil { diff --git a/util/chunk/iterator.go b/util/chunk/iterator.go index 071a3087922ff..c6a3bc58ff303 100644 --- a/util/chunk/iterator.go +++ b/util/chunk/iterator.go @@ -159,13 +159,13 @@ func (it *Iterator4Chunk) GetChunk() *Chunk { return it.chk } -// NewIterator4List returns a Iterator for ListInMemory. -func NewIterator4List(li *ListInMemory) Iterator { +// NewIterator4List returns a Iterator for List. +func NewIterator4List(li *List) Iterator { return &iterator4List{li: li} } type iterator4List struct { - li *ListInMemory + li *List chkCursor int rowCursor int } @@ -232,12 +232,12 @@ func (it *iterator4List) Len() int { } // NewIterator4RowPtr returns a Iterator for RowPtrs. -func NewIterator4RowPtr(li *ListInMemory, ptrs []RowPtr) Iterator { +func NewIterator4RowPtr(li *List, ptrs []RowPtr) Iterator { return &iterator4RowPtr{li: li, ptrs: ptrs} } type iterator4RowPtr struct { - li *ListInMemory + li *List ptrs []RowPtr cursor int } diff --git a/util/chunk/iterator_test.go b/util/chunk/iterator_test.go index 7125f24d194fd..2c937447b1909 100644 --- a/util/chunk/iterator_test.go +++ b/util/chunk/iterator_test.go @@ -49,8 +49,8 @@ func (s *testChunkSuite) TestIterator(c *check.C) { expected = append(expected, int64(i)) } var rows []Row - li := NewListInMemory(fields, 1, 2) - li2 := NewListInMemory(fields, 8, 16) + li := NewList(fields, 1, 2) + li2 := NewList(fields, 8, 16) var ptrs []RowPtr var ptrs2 []RowPtr for i := 0; i < n; i++ { @@ -120,7 +120,7 @@ func (s *testChunkSuite) TestIterator(c *check.C) { c.Assert(it.Begin(), check.Equals, it.End()) it = NewIterator4Chunk(new(Chunk)) c.Assert(it.Begin(), check.Equals, it.End()) - it = NewIterator4List(new(ListInMemory)) + it = NewIterator4List(new(List)) c.Assert(it.Begin(), check.Equals, it.End()) it = NewIterator4RowPtr(li, nil) c.Assert(it.Begin(), check.Equals, it.End()) diff --git a/util/chunk/list.go b/util/chunk/list.go index 4826b229d82ae..2f71febb1179d 100644 --- a/util/chunk/list.go +++ b/util/chunk/list.go @@ -22,16 +22,8 @@ import ( "github.com/pingcap/tidb/util/stringutil" ) -// List is interface for ListInMemory and ListInDisk -type List interface { - NumRowsOfChunk(int) - NumChunks() - GetRow(RowPtr) - Add(chk *Chunk) (err error) -} - -// ListInMemory holds a slice of chunks, use to append rows with max chunk size properly handled. -type ListInMemory struct { +// List holds a slice of chunks, use to append rows with max chunk size properly handled. +type List struct { fieldTypes []*types.FieldType initChunkSize int maxChunkSize int @@ -50,11 +42,11 @@ type RowPtr struct { RowIdx uint32 } -var chunkListLabel fmt.Stringer = stringutil.StringerStr("chunk.ListInMemory") +var chunkListLabel fmt.Stringer = stringutil.StringerStr("chunk.List") -// NewListInMemory creates a new ListInMemory with field types, init chunk size and max chunk size. -func NewListInMemory(fieldTypes []*types.FieldType, initChunkSize, maxChunkSize int) *ListInMemory { - l := &ListInMemory{ +// NewList creates a new List with field types, init chunk size and max chunk size. +func NewList(fieldTypes []*types.FieldType, initChunkSize, maxChunkSize int) *List { + l := &List{ fieldTypes: fieldTypes, initChunkSize: initChunkSize, maxChunkSize: maxChunkSize, @@ -64,28 +56,28 @@ func NewListInMemory(fieldTypes []*types.FieldType, initChunkSize, maxChunkSize return l } -// GetMemTracker returns the memory tracker of this ListInMemory. -func (l *ListInMemory) GetMemTracker() *memory.Tracker { +// GetMemTracker returns the memory tracker of this List. +func (l *List) GetMemTracker() *memory.Tracker { return l.memTracker } -// Len returns the length of the ListInMemory. -func (l *ListInMemory) Len() int { +// Len returns the length of the List. +func (l *List) Len() int { return l.length } -// NumChunks returns the number of chunks in the ListInMemory. -func (l *ListInMemory) NumChunks() int { +// NumChunks returns the number of chunks in the List. +func (l *List) NumChunks() int { return len(l.chunks) } // GetChunk gets the Chunk by ChkIdx. -func (l *ListInMemory) GetChunk(chkIdx int) *Chunk { +func (l *List) GetChunk(chkIdx int) *Chunk { return l.chunks[chkIdx] } -// AppendRow appends a row to the ListInMemory, the row is copied to the ListInMemory. -func (l *ListInMemory) AppendRow(row Row) RowPtr { +// AppendRow appends a row to the List, the row is copied to the List. +func (l *List) AppendRow(row Row) RowPtr { chkIdx := len(l.chunks) - 1 if chkIdx == -1 || l.chunks[chkIdx].NumRows() >= l.chunks[chkIdx].Capacity() || chkIdx == l.consumedIdx { newChk := l.allocChunk() @@ -103,12 +95,13 @@ func (l *ListInMemory) AppendRow(row Row) RowPtr { return RowPtr{ChkIdx: uint32(chkIdx), RowIdx: uint32(rowIdx)} } -// Add adds a chunk to the ListInMemory, the chunk may be modified later by the list. +// Add adds a chunk to the List, the chunk may be modified later by the list. // Caller must make sure the input chk is not empty and not used any more and has the same field types. -func (l *ListInMemory) Add(chk *Chunk) (err error) { +func (l *List) Add(chk *Chunk) { // FixMe: we should avoid add a Chunk that chk.NumRows() > list.maxChunkSize. if chk.NumRows() == 0 { - return errors.New("chunk appended to ListInMemory should have at least 1 row") + // TODO: return error here. + panic("chunk appended to List should have at least 1 row") } if chkIdx := len(l.chunks) - 1; l.consumedIdx != chkIdx { l.memTracker.Consume(l.chunks[chkIdx].MemoryUsage()) @@ -118,10 +111,10 @@ func (l *ListInMemory) Add(chk *Chunk) (err error) { l.consumedIdx++ l.chunks = append(l.chunks, chk) l.length += chk.NumRows() - return nil + return } -func (l *ListInMemory) allocChunk() (chk *Chunk) { +func (l *List) allocChunk() (chk *Chunk) { if len(l.freelist) > 0 { lastIdx := len(l.freelist) - 1 chk = l.freelist[lastIdx] @@ -137,18 +130,13 @@ func (l *ListInMemory) allocChunk() (chk *Chunk) { } // GetRow gets a Row from the list by RowPtr. -func (l *ListInMemory) GetRow(ptr RowPtr) Row { +func (l *List) GetRow(ptr RowPtr) Row { chk := l.chunks[ptr.ChkIdx] return chk.GetRow(int(ptr.RowIdx)) } -// NumRowsOfChunk returns the number of rows of a chunk. -func (l *ListInMemory) NumRowsOfChunk(chkID int) int { - return l.GetChunk(chkID).NumRows() -} - -// Reset resets the ListInMemory. -func (l *ListInMemory) Reset() { +// Reset resets the List. +func (l *List) Reset() { if lastIdx := len(l.chunks) - 1; lastIdx != l.consumedIdx { l.memTracker.Consume(l.chunks[lastIdx].MemoryUsage()) } @@ -160,11 +148,11 @@ func (l *ListInMemory) Reset() { // preAlloc4Row pre-allocates the storage memory for a Row. // NOTE: only used in test -// 1. The ListInMemory must be empty or holds no useful data. -// 2. The schema of the Row must be the same with the ListInMemory. +// 1. The List must be empty or holds no useful data. +// 2. The schema of the Row must be the same with the List. // 3. This API is paired with the `Insert()` function, which inserts all the -// rows data into the ListInMemory after the pre-allocation. -func (l *ListInMemory) preAlloc4Row(row Row) (ptr RowPtr) { +// rows data into the List after the pre-allocation. +func (l *List) preAlloc4Row(row Row) (ptr RowPtr) { chkIdx := len(l.chunks) - 1 if chkIdx == -1 || l.chunks[chkIdx].NumRows() >= l.chunks[chkIdx].Capacity() { newChk := l.allocChunk() @@ -184,7 +172,7 @@ func (l *ListInMemory) preAlloc4Row(row Row) (ptr RowPtr) { // Insert inserts `row` on the position specified by `ptr`. // Note: Insert will cover the origin data, it should be called after // PreAlloc. -func (l *ListInMemory) Insert(ptr RowPtr, row Row) { +func (l *List) Insert(ptr RowPtr, row Row) { l.chunks[ptr.ChkIdx].insert(int(ptr.RowIdx), row) } @@ -193,7 +181,7 @@ func (l *ListInMemory) Insert(ptr RowPtr, row Row) { type ListWalkFunc = func(row Row) error // Walk iterate the list and call walkFunc for each row. -func (l *ListInMemory) Walk(walkFunc ListWalkFunc) error { +func (l *List) Walk(walkFunc ListWalkFunc) error { for i := 0; i < len(l.chunks); i++ { chk := l.chunks[i] for j := 0; j < chk.NumRows(); j++ { diff --git a/util/chunk/list_test.go b/util/chunk/list_test.go index c71bd712a1875..b7d7780ffa2f2 100644 --- a/util/chunk/list_test.go +++ b/util/chunk/list_test.go @@ -32,7 +32,7 @@ func (s *testChunkSuite) TestList(c *check.C) { fields := []*types.FieldType{ types.NewFieldType(mysql.TypeLonglong), } - l := NewListInMemory(fields, 2, 2) + l := NewList(fields, 2, 2) srcChunk := NewChunkWithCapacity(fields, 32) srcChunk.AppendInt64(0, 1) srcRow := srcChunk.GetRow(0) @@ -104,7 +104,7 @@ func (s *testChunkSuite) TestListMemoryUsage(c *check.C) { srcChk.AppendTime(3, timeObj) srcChk.AppendDuration(4, durationObj) - list := NewListInMemory(fieldTypes, maxChunkSize, maxChunkSize*2) + list := NewList(fieldTypes, maxChunkSize, maxChunkSize*2) c.Assert(list.GetMemTracker().BytesConsumed(), check.Equals, int64(0)) list.AppendRow(srcChk.GetRow(0)) @@ -133,8 +133,8 @@ func (s *testChunkSuite) TestListPrePreAlloc4RowAndInsert(c *check.C) { srcChk.AppendString(3, strings.Repeat(strconv.FormatInt(i, 10), int(i))) } - srcList := NewListInMemory(fieldTypes, 3, 3) - destList := NewListInMemory(fieldTypes, 5, 5) + srcList := NewList(fieldTypes, 3, 3) + destList := NewList(fieldTypes, 5, 5) destRowPtr := make([]RowPtr, srcChk.NumRows()) for i := 0; i < srcChk.NumRows(); i++ { srcList.AppendRow(srcChk.GetRow(i)) @@ -176,7 +176,7 @@ func BenchmarkListMemoryUsage(b *testing.B) { row := chk.GetRow(0) initCap := 50 - list := NewListInMemory(fieldTypes, 2, 8) + list := NewList(fieldTypes, 2, 8) for i := 0; i < initCap; i++ { list.AppendRow(row) } @@ -194,7 +194,7 @@ func BenchmarkPreAllocList(b *testing.B) { row := chk.GetRow(0) b.ResetTimer() - list := NewListInMemory(fieldTypes, 1024, 1024) + list := NewList(fieldTypes, 1024, 1024) for i := 0; i < b.N; i++ { list.Reset() // 32768 indicates the number of int64 rows to fill 256KB L2 cache. @@ -225,7 +225,7 @@ func BenchmarkListAdd(b *testing.B) { numChk, numRow := 1, 2 chks, fields := initChunks(numChk, numRow) chk := chks[0] - l := NewListInMemory(fields, numRow, numRow) + l := NewList(fields, numRow, numRow) b.ResetTimer() for i := 0; i < b.N; i++ { @@ -236,7 +236,7 @@ func BenchmarkListAdd(b *testing.B) { func BenchmarkListGetRow(b *testing.B) { numChk, numRow := 10000, 2 chks, fields := initChunks(numChk, numRow) - l := NewListInMemory(fields, numRow, numRow) + l := NewList(fields, numRow, numRow) for _, chk := range chks { l.Add(chk) }