bbp support 12KB buffer

Change-Id: I3374b35892d9950980d1a8e7999eef61adbbcd8b

perf/bbp/sized.go

@@ -13,9 +13,13 @@ const (
 	minBufSize = 1 << minShift
 	maxBufSize = 1 << maxShift
 
-	bigIdx     = 14 // 16KB
+	idx4KB   = 12       // 4KB
+	idx8KB   = 13       // 8KB
+	size12KB = 12 << 10 // 8KB + 4KB
+	size16KB = 16 << 10 // 8KB + (2 * 4KB)
+
 	minPoolIdx = 6
-	maxPoolIdx = maxShift + (maxShift-bigIdx)*3
+	maxPoolIdx = maxShift + (maxShift-idx8KB)*3 - 2
 	poolSize   = maxPoolIdx + 1
 )
 
@@ -59,7 +63,7 @@ func Grow(buf []byte, capacity int, reuseBuf bool) []byte {
 var (
 	powerOfTwoIdxTable = [maxShift + 1]int{
 		0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, // 0 - 13 [1B, 8KB]
-		14, 18, 22, 26, 30, 34, 38, 42, 46, 50, 54, 58, // 14 - 26 [16KB, 32MB]
+		15, 19, 23, 27, 31, 35, 39, 43, 47, 51, 55, 59, // 14 - 26 [16KB, 32MB]
 	}
 	bufSizeTable [poolSize]int
 	sizedPools   [poolSize]sync.Pool
@@ -68,10 +72,12 @@ var (
 func init() {
 	for i := 0; i < poolSize; i++ {
 		var size int
-		if i < bigIdx {
+		if i <= 13 { // <= 8KB (idx8KB)
 			size = 1 << i
-		} else {
-			j, k := bigIdx+(i-bigIdx)/4, (i-bigIdx)%4
+		} else if i == 14 {
+			size = size12KB // 12KB
+		} else { // i >= 15, size >= 16KB
+			j, k := 14+(i-15)/4, (i-15)%4
 			quarter := (1 << j) / 4
 			size = 1<<j + quarter*k
 		}
@@ -128,30 +134,52 @@ func growWithOptions(buf []byte, capacity int, reuseBuf bool) []byte {
 // indexGet finds the pool index for the given size to get buffer from,
 // if size not equals to a predefined size, it returns the index of the
 // next predefined size.
+//
+// See indexGet_readable for a more readable version of this function.
 func indexGet(size int) int {
 	if size <= minBufSize {
 		return minPoolIdx
 	}
 
-	/*
-		idx := bsr(size)
-		if isPowerOfTow(size) {
-		    return powerOfTwoIdxTable[idx]
+	// For better performance, bsr and isPowerOfTwo are inlined here
+	// to ensure that this function can be inlined.
+
+	p2i := bits.Len32(uint32(size)) - 1
+	idx := powerOfTwoIdxTable[p2i]
+
+	if size&(size-1) != 0 { // not power of two
+		if size > size16KB {
+			mod := size & (1<<p2i - 1)    // size % (2^p2i)
+			idx += (mod - 1) >> (p2i - 2) // (mod - 1) / (2^p2i / 4)
+		} else if size > size12KB {
+			idx = 14
 		}
-	*/
+		idx += 1
+	}
+	return idx
+}
 
-	// The following code is equivalent to the above commented lines,
-	// they are manually inlined here to ensure that this function
-	// will be inlined, for better performance.
-	idx := bits.Len32(uint32(size)) - 1
-	if size&(size-1) == 0 {
+func indexGet_readable(size int) int {
+	if size <= minBufSize {
+		return minPoolIdx
+	}
+
+	idx := bsr(size)
+	if isPowerOfTwo(size) {
 		return powerOfTwoIdxTable[idx]
 	}
 
-	if idx < bigIdx {
+	if idx < idx8KB {
 		return powerOfTwoIdxTable[idx] + 1
 	}
 
+	if idx == idx8KB { // (8KB, 16KB)
+		mod := size & (1<<idx - 1) // size % (2^idx)
+		half := 1 << idx4KB
+		return powerOfTwoIdxTable[idx] + (mod+half-1)/half
+	}
+
+	// larger than 16KB
 	mod := size & (1<<idx - 1) // size % (2^idx)
 	quarter := 1 << (idx - 2)  // (2^idx) / 4
 	return powerOfTwoIdxTable[idx] + (mod+quarter-1)/quarter
@@ -160,11 +188,39 @@ func indexGet(size int) int {
 // indexPut finds the pool index for the given size to put buffer back,
 // if size not equals to a predefined size, it returns the index of the
 // previous predefined size.
+//
+// See indexPut_readable for a more readable version of this function.
 func indexPut(size int) int {
+
+	// For better performance, bsr and isPowerOfTwo are inlined here
+	// to ensure that this function can be inlined.
+
+	p2i := bits.Len32(uint32(size)) - 1
+	idx := powerOfTwoIdxTable[p2i]
+
+	if size&(size-1) != 0 { // not power of two
+		if size > size16KB {
+			mod := size & (1<<p2i - 1) // size % (2^p2i)
+			idx += mod >> (p2i - 2)    // mod / (2^p2i / 4)
+		} else if size > size12KB {
+			idx = 13
+		}
+	}
+	return idx
+}
+
+func indexPut_readable(size int) int {
 	idx := bsr(size)
-	if isPowerOfTwo(size) || idx < bigIdx {
+	if isPowerOfTwo(size) || idx < idx8KB {
 		return powerOfTwoIdxTable[idx]
 	}
+
+	if idx == idx8KB {
+		mod := size & (1<<idx - 1) // size % (2^idx)
+		half := 1 << (idx - 1)
+		return powerOfTwoIdxTable[idx] + mod/half
+	}
+
 	mod := size & (1<<idx - 1) // size % (2^idx)
 	quarter := 1 << (idx - 2)  // (2^idx) / 4
 	return powerOfTwoIdxTable[idx] + mod/quarter

perf/bbp/sized_test.go

@@ -4,6 +4,8 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+
+	"github.com/jxskiss/gopkg/v2/perf/fastrand"
 )
 
 const (
@@ -20,6 +22,13 @@ func TestGet(t *testing.T) {
 }
 
 func Test_indexGet(t *testing.T) {
+	for i := 0; i < 1000; i++ {
+		size := fastrand.Intn(maxBufSize)
+		idx1 := indexGet_readable(size)
+		idx2 := indexGet(size)
+		assert.Equal(t, idx1, idx2)
+	}
+
 	assert.Equal(t, 6, indexGet(63))
 	assert.Equal(t, 6, indexGet(64))
 	assert.Equal(t, 7, indexGet(65))
@@ -29,6 +38,13 @@ func Test_indexGet(t *testing.T) {
 }
 
 func Test_indexPut(t *testing.T) {
+	for i := 0; i < 1000; i++ {
+		size := fastrand.Intn(maxBufSize)
+		idx1 := indexPut_readable(size)
+		idx2 := indexPut(size)
+		assert.Equal(t, idx1, idx2)
+	}
+
 	assert.Equal(t, 5, indexPut(63))
 	assert.Equal(t, 6, indexPut(64))
 	assert.Equal(t, 6, indexPut(65))
@@ -47,8 +63,8 @@ func Test_indexGet_quarters(t *testing.T) {
 	}
 	want := []int{
 		11, 11, 11, 12, 12, 12, 13,
-		15, 15, 17, 18,
-		30, 30, 31, 32, 32, 33,
+		16, 16, 18, 19,
+		31, 31, 32, 33, 33, 34,
 	}
 	for i, size := range sizeList {
 		assert.Equal(t, want[i], indexGet(size))
@@ -63,8 +79,8 @@ func Test_indexPut_quarters(t *testing.T) {
 	}
 	want := []int{
 		10, 10, 11, 11, 11, 12, 12,
-		14, 15, 16, 18,
-		29, 30, 30, 31, 32, 32,
+		15, 16, 17, 19,
+		30, 31, 31, 32, 33, 33,
 	}
 	for i, size := range sizeList {
 		assert.Equal(t, want[i], indexPut(size))
@@ -77,17 +93,19 @@ func Test_various_sizes(t *testing.T) {
 		assert.Equal(t, 64, cap(Get(0, size-1)))
 		assert.Equal(t, 64, cap(Get(0, size)))
 	}
-	for i := 7; i <= 13; i++ {
+	for i := 7; i <= 12; i++ {
 		size := 1 << i
 		assert.Equal(t, size, cap(Get(0, size-1)))
 		assert.Equal(t, size, cap(Get(0, size)))
 		assert.Equal(t, size*2, cap(Get(0, size+1)))
 	}
-	for i := 14; i <= 25; i++ {
+	for i := 13; i <= 25; i++ {
 		size := 1 << i
 		assert.Equal(t, size, cap(Get(0, size-1)))
 		assert.Equal(t, size, cap(Get(0, size)))
-		if i < 25 {
+		if i < 14 {
+			assert.Equal(t, size+size/2, cap(Get(0, size+1)))
+		} else if i < 25 {
 			assert.Equal(t, size+size/4, cap(Get(0, size+1)))
 		} else {
 			assert.Equal(t, size+1, cap(Get(0, size+1)))