From c945afb637959b6766ddab8b0a5fd55b1413fa76 Mon Sep 17 00:00:00 2001 From: Haibin Xie Date: Fri, 6 Sep 2019 15:09:54 +0800 Subject: [PATCH 1/2] stats: support display invalid key in stats buckets --- executor/analyze.go | 2 +- statistics/feedback.go | 6 +++--- statistics/histogram.go | 7 ++++--- statistics/histogram_test.go | 11 +++++++++++ util/codec/codec.go | 10 +++++----- util/codec/codec_test.go | 6 +++--- 6 files changed, 27 insertions(+), 15 deletions(-) diff --git a/executor/analyze.go b/executor/analyze.go index b0568abefd789..2e55fde99b43b 100644 --- a/executor/analyze.go +++ b/executor/analyze.go @@ -1183,7 +1183,7 @@ type analyzeIndexIncrementalExec struct { func analyzeIndexIncremental(idxExec *analyzeIndexIncrementalExec) analyzeResult { startPos := idxExec.oldHist.GetUpper(idxExec.oldHist.Len() - 1) - values, err := codec.DecodeRange(startPos.GetBytes(), len(idxExec.idxInfo.Columns)) + values, _, err := codec.DecodeRange(startPos.GetBytes(), len(idxExec.idxInfo.Columns)) if err != nil { return analyzeResult{Err: err, job: idxExec.job} } diff --git a/statistics/feedback.go b/statistics/feedback.go index 99a1596bd4e1b..992bb3922325a 100644 --- a/statistics/feedback.go +++ b/statistics/feedback.go @@ -121,11 +121,11 @@ func (q *QueryFeedback) DecodeToRanges(isIndex bool) ([]*ranger.Range, error) { if isIndex { var err error // As we do not know the origin length, just use a custom value here. - lowVal, err = codec.DecodeRange(low.GetBytes(), 4) + lowVal, _, err = codec.DecodeRange(low.GetBytes(), 4) if err != nil { return nil, errors.Trace(err) } - highVal, err = codec.DecodeRange(high.GetBytes(), 4) + highVal, _, err = codec.DecodeRange(high.GetBytes(), 4) if err != nil { return nil, errors.Trace(err) } @@ -837,7 +837,7 @@ func ConvertDatumsType(vals []types.Datum, ft *types.FieldType, loc *time.Locati } func decodeColumnBounds(data []byte, ft *types.FieldType) ([]types.Datum, error) { - vals, err := codec.DecodeRange(data, 1) + vals, _, err := codec.DecodeRange(data, 1) if err != nil { return nil, err } diff --git a/statistics/histogram.go b/statistics/histogram.go index 73adeee757bac..7ff3868bdf33e 100644 --- a/statistics/histogram.go +++ b/statistics/histogram.go @@ -216,9 +216,10 @@ func ValueToString(value *types.Datum, idxCols int) (string, error) { if idxCols == 0 { return value.ToString() } - decodedVals, err := codec.DecodeRange(value.GetBytes(), idxCols) - if err != nil { - return "", errors.Trace(err) + // Ignore the error and treat remaining part that cannot decode successfully as bytes. + decodedVals, remained, _ := codec.DecodeRange(value.GetBytes(), idxCols) + if len(remained) > 0 { + decodedVals = append(decodedVals, types.NewBytesDatum(remained)) } str, err := types.DatumsToString(decodedVals, true) if err != nil { diff --git a/statistics/histogram_test.go b/statistics/histogram_test.go index e131b143e7306..68b8c55f2ce33 100644 --- a/statistics/histogram_test.go +++ b/statistics/histogram_test.go @@ -120,3 +120,14 @@ num: 30 lower_bound: 12 upper_bound: 14 repeats: 10` c.Assert(err, IsNil, Commentf("Test failed: %v", err)) c.Assert(newIdx.String(), Equals, idxResult) } + +func (s *testStatisticsSuite) TestValueToString4InvalidKey(c *C) { + bytes, err := codec.EncodeKey(nil, nil, types.NewDatum(1), types.NewDatum(0.5)) + c.Assert(err, IsNil) + // Append invalid flag. + bytes = append(bytes, 20) + datum := types.NewDatum(bytes) + res, err := ValueToString(&datum, 3) + c.Assert(err, IsNil) + c.Assert(res, Equals, "(1, 0.5, \x14)") +} diff --git a/util/codec/codec.go b/util/codec/codec.go index 19cbfff6c7977..f2a9b323b3f5b 100644 --- a/util/codec/codec.go +++ b/util/codec/codec.go @@ -312,9 +312,9 @@ func Decode(b []byte, size int) ([]types.Datum, error) { // DecodeRange decodes the range values from a byte slice that generated by EncodeKey. // It handles some special values like `MinNotNull` and `MaxValueDatum`. -func DecodeRange(b []byte, size int) ([]types.Datum, error) { +func DecodeRange(b []byte, size int) ([]types.Datum, []byte, error) { if len(b) < 1 { - return nil, errors.New("invalid encoded key: length of key is zero") + return nil, b, errors.New("invalid encoded key: length of key is zero") } var ( @@ -326,7 +326,7 @@ func DecodeRange(b []byte, size int) ([]types.Datum, error) { var d types.Datum b, d, err = DecodeOne(b) if err != nil { - return nil, errors.Trace(err) + return values, b, errors.Trace(err) } values = append(values, d) } @@ -341,10 +341,10 @@ func DecodeRange(b []byte, size int) ([]types.Datum, error) { case maxFlag, maxFlag + 1: values = append(values, types.MaxValueDatum()) default: - return nil, errors.Errorf("invalid encoded key flag %v", b[0]) + return values, b, errors.Errorf("invalid encoded key flag %v", b[0]) } } - return values, nil + return values, nil, nil } // DecodeOne decodes on datum from a byte slice generated with EncodeKey or EncodeValue. diff --git a/util/codec/codec_test.go b/util/codec/codec_test.go index b54a6ae8adeb4..90c84869ad18e 100644 --- a/util/codec/codec_test.go +++ b/util/codec/codec_test.go @@ -985,14 +985,14 @@ func chunkForTest(c *C, sc *stmtctx.StatementContext, datums []types.Datum, tps } func (s *testCodecSuite) TestDecodeRange(c *C) { - _, err := DecodeRange(nil, 0) + _, _, err := DecodeRange(nil, 0) c.Assert(err, NotNil) datums := types.MakeDatums(1, "abc", 1.1, []byte("def")) rowData, err := EncodeValue(nil, nil, datums...) c.Assert(err, IsNil) - datums1, err := DecodeRange(rowData, len(datums)) + datums1, _, err := DecodeRange(rowData, len(datums)) c.Assert(err, IsNil) for i, datum := range datums1 { cmp, err := datum.CompareDatum(nil, &datums[i]) @@ -1002,7 +1002,7 @@ func (s *testCodecSuite) TestDecodeRange(c *C) { for _, b := range []byte{NilFlag, bytesFlag, maxFlag, maxFlag + 1} { newData := append(rowData, b) - _, err := DecodeRange(newData, len(datums)+1) + _, _, err := DecodeRange(newData, len(datums)+1) c.Assert(err, IsNil) } } From e485866ca05680effcaa252eaa53c505b1705c0b Mon Sep 17 00:00:00 2001 From: Haibin Xie Date: Mon, 9 Sep 2019 18:42:14 +0800 Subject: [PATCH 2/2] fix error check --- statistics/histogram.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/statistics/histogram.go b/statistics/histogram.go index 7ff3868bdf33e..659793dfaa3b4 100644 --- a/statistics/histogram.go +++ b/statistics/histogram.go @@ -216,9 +216,9 @@ func ValueToString(value *types.Datum, idxCols int) (string, error) { if idxCols == 0 { return value.ToString() } - // Ignore the error and treat remaining part that cannot decode successfully as bytes. - decodedVals, remained, _ := codec.DecodeRange(value.GetBytes(), idxCols) - if len(remained) > 0 { + // Treat remaining part that cannot decode successfully as bytes. + decodedVals, remained, err := codec.DecodeRange(value.GetBytes(), idxCols) + if err != nil && len(remained) > 0 { decodedVals = append(decodedVals, types.NewBytesDatum(remained)) } str, err := types.DatumsToString(decodedVals, true)