Update sectorbuilder

.gitignore

@@ -11,3 +11,5 @@ build/.*
 
 *-fuzz.zip
 /chain/types/work_msg/
+bin/ipget
+bin/tmp/*

Makefile

@@ -88,7 +88,6 @@ clean:
 	rm -rf $(CLEAN)
 	-$(MAKE) -C $(BLS_PATH) clean
 	-$(MAKE) -C $(SECTOR_BUILDER_PATH) clean
-	-$(MAKE) -C $(PROOFS_PATH) clean
 .PHONY: clean
 
 dist-clean:

build/proof-params/parameters.json

@@ -1,42 +1,42 @@
 {
-  "v11-vdf-post-6142c5095dcab8adf8698aaea24eac78035e3fefe6503b3e2f80758e4c948639.params": {
-    "cid": "QmYS9NmkfVtZCtQPF18uQBeWfajF5YRKKkDPXpVR6pzzP4",
-    "digest": "43008a8c945076a08aa2e7c4785241de",
+  "v12-proof-of-spacetime-rational-ba14a058a9dea194f68596f8ecf6537074f038a15c8d1a8550e10e31d4728912.params": {
+    "cid": "QmZBvF2F9wTYKLBxWSCQKe34D3M7vkNNc7ou8mxnNhZkZc",
+    "digest": "5d854e0ecfbd12cb7fa1247a6e6a0315",
     "sector_size": 1024
   },
-  "v11-vdf-post-6142c5095dcab8adf8698aaea24eac78035e3fefe6503b3e2f80758e4c948639.vk": {
-    "cid": "QmTXenYSVvEz2yEJ1deCobfQBVgykkigBuvWJCzTM5tmC2",
-    "digest": "73b49d9cf0ba983b3e861da0e0ee2e2a",
+  "v12-proof-of-spacetime-rational-ba14a058a9dea194f68596f8ecf6537074f038a15c8d1a8550e10e31d4728912.vk": {
+    "cid": "QmZfdHrnk2oN3Gx7hhjpRGXu8qY6FcqLjpHQ6jt1BBDg5R",
+    "digest": "aca566faa466f05fb9d622bec39e4b6d",
     "sector_size": 1024
   },
-  "v11-vdf-post-71ca99ebb6212e45b2affa16eb6e1f671c37986e84ee8b9edbf924ab45b5fdfe.params": {
-    "cid": "Qmc3R5xmapcSGKGR9zRC2fyPo3QuAuChtsKwswE6NZ6QMQ",
-    "digest": "8091b735d254982b0b62e712bab82139",
+  "v12-proof-of-spacetime-rational-c2ae2b440e693ee69fd6da9e85c4294c5c70c1a46d5785ca5f2a676d6cd4c8de.params": {
+    "cid": "QmYdGGwQXpaBGTVWXqMFVXUP2CZhtsV29jxPkRm54ArAdT",
+    "digest": "eb2d3c4cb7b32c87ead5326bcbd495f3",
     "sector_size": 268435456
   },
-  "v11-vdf-post-71ca99ebb6212e45b2affa16eb6e1f671c37986e84ee8b9edbf924ab45b5fdfe.vk": {
-    "cid": "QmeD4mdJFPjsWouBPPwvm6iVFYG8bpqNYetd2Ux7PoZawz",
-    "digest": "32784464acc85a662ea16ecf5aba931d",
+  "v12-proof-of-spacetime-rational-c2ae2b440e693ee69fd6da9e85c4294c5c70c1a46d5785ca5f2a676d6cd4c8de.vk": {
+    "cid": "QmeTtWQ2hCUq34BpHTy21jJqVqHbPJdNhQRqW4SF4ZNA7v",
+    "digest": "c83eca165ba94233861227578d658a22",
     "sector_size": 268435456
   },
-  "v11-zigzag-proof-of-replication-5efcf852a15bd74808bc65d6f2df146de817baea96c96e3b752e6a3349957644.params": {
+  "v12-zigzag-proof-of-replication-5efcf852a15bd74808bc65d6f2df146de817baea96c96e3b752e6a3349957644.params": {
     "cid": "QmNSuxq15JPFCTehxVpgJydNZ79rpLoNwnLzQMGA9EziXg",
     "digest": "818cd9cc2e0e47210a05bd073847ab5a",
     "sector_size": 268435456
   },
-  "v11-zigzag-proof-of-replication-5efcf852a15bd74808bc65d6f2df146de817baea96c96e3b752e6a3349957644.vk": {
+  "v12-zigzag-proof-of-replication-5efcf852a15bd74808bc65d6f2df146de817baea96c96e3b752e6a3349957644.vk": {
     "cid": "Qmbc8LcydZXsVqQrkNMeLEu31Vxi1VigQGJ2ehytxWPALH",
     "digest": "a6636e2ee1a176161e022296bc045e79",
     "sector_size": 268435456
   },
-  "v11-zigzag-proof-of-replication-a09b5cf44f640589b1b02cf823fa28269850342bcefa4878189b9b5c9ec4d2bb.params": {
+  "v12-zigzag-proof-of-replication-a09b5cf44f640589b1b02cf823fa28269850342bcefa4878189b9b5c9ec4d2bb.params": {
     "cid": "QmTfhTnkFvbpFfw8UydFdnPCDfxgAxEcw4fRdGsELpcFnh",
     "digest": "906b6c0c9dc5bb581d9641c11b54e197",
     "sector_size": 1024
   },
-  "v11-zigzag-proof-of-replication-a09b5cf44f640589b1b02cf823fa28269850342bcefa4878189b9b5c9ec4d2bb.vk": {
+  "v12-zigzag-proof-of-replication-a09b5cf44f640589b1b02cf823fa28269850342bcefa4878189b9b5c9ec4d2bb.vk": {
     "cid": "QmRDcUxfpPY9a1vR3T4vRgxHtWHyy9m3xuMQtj8P749r4e",
     "digest": "3632776cd23e376694c625390b9a73ea",
     "sector_size": 1024
   }
-}
+}

extern/rleplus/internal/bitvector.go

@@ -0,0 +1,154 @@
+package bitvector
+
+import (
+	"errors"
+	"log"
+)
+
+var (
+	// ErrOutOfRange - the index passed is out of range for the BitVector
+	ErrOutOfRange = errors.New("index out of range")
+)
+
+// BitNumbering indicates the ordering of bits, either
+// least-significant bit in position 0, or most-significant bit
+// in position 0.
+//
+// It it used in 3 ways with BitVector:
+// 1. Ordering of bits within the Buf []byte structure
+// 2. What order to add bits when using Extend()
+// 3. What order to read bits when using Take()
+//
+// https://en.wikipedia.org/wiki/Bit_numbering
+type BitNumbering int
+
+const (
+	// LSB0 - bit ordering starts with the low-order bit
+	LSB0 BitNumbering = iota
+
+	// MSB0 - bit ordering starts with the high-order bit
+	MSB0
+)
+
+// BitVector is used to manipulate ordered collections of bits
+type BitVector struct {
+	Buf []byte
+
+	// BytePacking is the bit ordering within bytes
+	BytePacking BitNumbering
+
+	// Len is the logical number of bits in the vector.
+	// The last byte in Buf may have undefined bits if Len is not a multiple of 8
+	Len uint
+}
+
+// NewBitVector constructs a new BitVector from a slice of bytes.
+//
+// The bytePacking parameter is required to know how to interpret the bit ordering within the bytes.
+func NewBitVector(buf []byte, bytePacking BitNumbering) *BitVector {
+	return &BitVector{
+		BytePacking: bytePacking,
+		Buf:         buf,
+		Len:         uint(len(buf) * 8),
+	}
+}
+
+// Push adds a single bit to the BitVector.
+//
+// Although it takes a byte, only the low-order bit is used, so just use 0 or 1.
+func (v *BitVector) Push(val byte) {
+	if v.Len%8 == 0 {
+		v.Buf = append(v.Buf, 0)
+	}
+	lastIdx := v.Len / 8
+
+	switch v.BytePacking {
+	case LSB0:
+		v.Buf[lastIdx] |= (val & 1) << (v.Len % 8)
+	default:
+		v.Buf[lastIdx] |= (val & 1) << (7 - (v.Len % 8))
+	}
+
+	v.Len++
+}
+
+// Get returns a single bit as a byte -- either 0 or 1
+func (v *BitVector) Get(idx uint) (byte, error) {
+	if idx >= v.Len {
+		return 0, ErrOutOfRange
+	}
+	blockIdx := idx / 8
+
+	switch v.BytePacking {
+	case LSB0:
+		return v.Buf[blockIdx] >> (idx % 8) & 1, nil
+	default:
+		return v.Buf[blockIdx] >> (7 - idx%8) & 1, nil
+	}
+}
+
+// Extend adds up to 8 bits to the receiver
+//
+// Given a byte b == 0b11010101
+// v.Extend(b, 4, LSB0) would add < 1, 0, 1, 0 >
+// v.Extend(b, 4, MSB0) would add < 1, 1, 0, 1 >
+//
+// Panics if count is out of range
+func (v *BitVector) Extend(val byte, count uint, order BitNumbering) {
+	if count > 8 {
+		log.Panicf("invalid count")
+	}
+
+	for i := uint(0); i < count; i++ {
+		switch order {
+		case LSB0:
+			v.Push((val >> i) & 1)
+		default:
+			v.Push((val >> (7 - i)) & 1)
+		}
+	}
+}
+
+// Take reads up to 8 bits at the given index.
+//
+// Given a BitVector < 1, 1, 0, 1, 0, 1, 0, 1 >
+// v.Take(0, 4, LSB0) would return 0b00001011
+// v.Take(0, 4, MSB0) would return 0b11010000
+//
+// Panics if count is out of range
+func (v *BitVector) Take(index uint, count uint, order BitNumbering) (out byte) {
+	if count > 8 {
+		log.Panicf("invalid count")
+	}
+
+	for i := uint(0); i < count; i++ {
+		val, _ := v.Get(index + i)
+
+		switch order {
+		case LSB0:
+			out |= val << i
+		default:
+			out |= val << (7 - i)
+		}
+	}
+	return
+}
+
+// Iterator returns a function, which when invoked, returns the number
+// of bits requested, and increments an internal cursor.
+//
+// When the end of the BitVector is reached, it returns zeroes indefinitely
+//
+// Panics if count is out of range
+func (v *BitVector) Iterator(order BitNumbering) func(uint) byte {
+	cursor := uint(0)
+	return func(count uint) (out byte) {
+		if count > 8 {
+			log.Panicf("invalid count")
+		}
+
+		out = v.Take(cursor, count, order)
+		cursor += count
+		return
+	}
+}

extern/rleplus/internal/bitvector_test.go

@@ -0,0 +1,136 @@
+package bitvector_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	bitvector "github.com/filecoin-project/go-lotus/extern/rleplus/internal"
+)
+
+func TestBitVector(t *testing.T) {
+	t.Run("zero value", func(t *testing.T) {
+		var v bitvector.BitVector
+
+		assert.Equal(t, bitvector.LSB0, v.BytePacking)
+	})
+
+	t.Run("Push", func(t *testing.T) {
+		// MSB0 bit numbering
+		v := bitvector.BitVector{BytePacking: bitvector.MSB0}
+		v.Push(1)
+		v.Push(0)
+		v.Push(1)
+		v.Push(1)
+
+		assert.Equal(t, byte(176), v.Buf[0])
+
+		// LSB0 bit numbering
+		v = bitvector.BitVector{BytePacking: bitvector.LSB0}
+		v.Push(1)
+		v.Push(0)
+		v.Push(1)
+		v.Push(1)
+
+		assert.Equal(t, byte(13), v.Buf[0])
+	})
+
+	t.Run("Get", func(t *testing.T) {
+		bits := []byte{1, 0, 1, 1, 0, 0, 1, 0}
+
+		for _, numbering := range []bitvector.BitNumbering{bitvector.MSB0, bitvector.LSB0} {
+			v := bitvector.BitVector{BytePacking: numbering}
+
+			for _, bit := range bits {
+				v.Push(bit)
+			}
+
+			for idx, expected := range bits {
+				actual, _ := v.Get(uint(idx))
+				assert.Equal(t, expected, actual)
+			}
+		}
+	})
+
+	t.Run("Extend", func(t *testing.T) {
+		val := byte(171) // 0b10101011
+
+		var v bitvector.BitVector
+
+		// MSB0 bit numbering
+		v = bitvector.BitVector{}
+		v.Extend(val, 4, bitvector.MSB0)
+		assertBitVector(t, []byte{1, 0, 1, 0}, v)
+		v.Extend(val, 5, bitvector.MSB0)
+		assertBitVector(t, []byte{1, 0, 1, 0, 1, 0, 1, 0, 1}, v)
+
+		// LSB0 bit numbering
+		v = bitvector.BitVector{}
+		v.Extend(val, 4, bitvector.LSB0)
+		assertBitVector(t, []byte{1, 1, 0, 1}, v)
+		v.Extend(val, 5, bitvector.LSB0)
+		assertBitVector(t, []byte{1, 1, 0, 1, 1, 1, 0, 1, 0}, v)
+	})
+
+	t.Run("invalid counts to Take/Extend/Iterator cause panics", func(t *testing.T) {
+		v := bitvector.BitVector{BytePacking: bitvector.LSB0}
+
+		assert.Panics(t, func() { v.Extend(0xff, 9, bitvector.LSB0) })
+
+		assert.Panics(t, func() { v.Take(0, 9, bitvector.LSB0) })
+
+		next := v.Iterator(bitvector.LSB0)
+		assert.Panics(t, func() { next(9) })
+	})
+
+	t.Run("Take", func(t *testing.T) {
+		var v bitvector.BitVector
+
+		bits := []byte{1, 0, 1, 0, 1, 0, 1, 1}
+		for _, bit := range bits {
+			v.Push(bit)
+		}
+
+		assert.Equal(t, byte(176), v.Take(4, 4, bitvector.MSB0))
+		assert.Equal(t, byte(13), v.Take(4, 4, bitvector.LSB0))
+	})
+
+	t.Run("Iterator", func(t *testing.T) {
+		var buf []byte
+
+		// make a bitvector of 256 sample bits
+		for i := 0; i < 32; i++ {
+			buf = append(buf, 128+32)
+		}
+
+		v := bitvector.NewBitVector(buf, bitvector.LSB0)
+
+		next := v.Iterator(bitvector.LSB0)
+
+		// compare to Get()
+		for i := uint(0); i < v.Len; i++ {
+			expected, _ := v.Get(i)
+			assert.Equal(t, expected, next(1))
+		}
+
+		// out of range should return zero
+		assert.Equal(t, byte(0), next(1))
+		assert.Equal(t, byte(0), next(8))
+
+		// compare to Take()
+		next = v.Iterator(bitvector.LSB0)
+		assert.Equal(t, next(5), v.Take(0, 5, bitvector.LSB0))
+		assert.Equal(t, next(8), v.Take(5, 8, bitvector.LSB0))
+	})
+}
+
+// Note: When using this helper assertion, expectedBits should *only* be 0s and 1s.
+func assertBitVector(t *testing.T, expectedBits []byte, actual bitvector.BitVector) {
+	assert.Equal(t, uint(len(expectedBits)), actual.Len)
+
+	for idx, bit := range expectedBits {
+		actualBit, err := actual.Get(uint(idx))
+		assert.NoError(t, err)
+		assert.Equal(t, bit, actualBit)
+	}
+}