EVM-friendly TipSet and ECChain formats

This patch implements the changes described in
filecoin-project/FIPs#1004.

Unfortunately, go-f3 now needs to be aware of the internal structure of
`TipSet` objects as it sends them over the wire in one format, but signs
another. This effectively reverts #149.

Specific changes:

1. On the wire (and in finality certificates), `TipSet` objects are now
   directly cbor-marshaled within the payload instead of first being
   marshaled to byte strings.
2. Instead of directly covering the entire tipset list with the payload
   signature:
  1. `TipSet` objects are "marshaled for signing" per the FIP change
      proposed above.
  2. These marshaled tipset objects are stuffed into a merkle tree.
  3. The merkle tree root is signed (along with the payload's other
     fields), again according the proposed changes.

fixes #166

gen/main.go

@@ -12,6 +12,7 @@ import (
 
 func main() {
 	err := gen.WriteTupleEncodersToFile("../gpbft/gen.go", "gpbft",
+		gpbft.TipSet{},
 		gpbft.GMessage{},
 		gpbft.Payload{},
 		gpbft.Justification{},

gpbft/chain.go

@@ -5,16 +5,76 @@ import (
 	"encoding/binary"
 	"encoding/hex"
 	"errors"
+	"fmt"
 	"strings"
+
+	cbg "github.com/whyrusleeping/cbor-gen"
+	"golang.org/x/crypto/blake2b"
 )
 
+// TipSetKey is a
+type TipSetKey = []byte
+
+type CID = []byte
+
+var cidPrefix = []byte{0x01, 0x71, 0xA0, 0xE4, 0x02, 0x20}
+
+// Hashes the given data and returns a CBOR + blake2b-256 CID.
+func MakeCid(data []byte) []byte {
+	// TODO: Consider just using go-cid? We implicitly depend on it through cbor-gen anyways.
+	digest := blake2b.Sum256(data)
+
+	out := make([]byte, 0, 38)
+	out = append(out, cidPrefix...)
+	out = append(out, digest[:]...)
+	return out
+}
+
 // Opaque type representing a tipset.
 // This is expected to be:
 // - a canonical sequence of CIDs of block headers identifying a tipset,
 // - a commitment to the resulting power table,
 // - a commitment to additional derived values.
 // However, GossipPBFT doesn't need to know anything about that structure.
-type TipSet = []byte
+type TipSet struct {
+	Epoch       int64
+	TipSet      TipSetKey
+	PowerTable  CID
+	Commitments [32]byte
+}
+
+func (ts *TipSet) IsZero() bool {
+	return len(ts.TipSet) == 0
+}
+
+func (ts *TipSet) Equal(b *TipSet) bool {
+	return ts.Epoch == b.Epoch &&
+		bytes.Equal(ts.TipSet, b.TipSet) &&
+		bytes.Equal(ts.PowerTable, b.PowerTable) &&
+		ts.Commitments == b.Commitments
+}
+
+func (ts *TipSet) MarshalForSigning() []byte {
+	var buf bytes.Buffer
+	cbg.WriteByteArray(&buf, ts.TipSet)
+	tsCid := MakeCid(buf.Bytes())
+	buf.Reset()
+	buf.Grow(len(tsCid) + len(ts.PowerTable) + 32 + 8)
+	// epoch || commitments || tipset || powertable
+	_ = binary.Write(&buf, binary.BigEndian, ts.Epoch)
+	_, _ = buf.Write(ts.Commitments[:])
+	_, _ = buf.Write(tsCid)
+	_, _ = buf.Write(ts.PowerTable)
+	return buf.Bytes()
+}
+
+func (ts *TipSet) String() string {
+	if ts == nil {
+		return "<nil>"
+	}
+
+	return fmt.Sprintf("%d@%s", ts.Epoch, hex.EncodeToString(ts.TipSet))
+}
 
 // A chain of tipsets comprising a base (the last finalised tipset from which the chain extends).
 // and (possibly empty) suffix.
@@ -45,8 +105,8 @@ func (c ECChain) IsZero() bool {
 }
 
 // Returns the base tipset.
-func (c ECChain) Base() TipSet {
-	return c[0]
+func (c ECChain) Base() *TipSet {
+	return &c[0]
 }
 
 // Returns the suffix of the chain after the base.
@@ -61,8 +121,8 @@ func (c ECChain) Suffix() []TipSet {
 // Returns the last tipset in the chain.
 // This could be the base tipset if there is no suffix.
 // This will panic on a zero value.
-func (c ECChain) Head() TipSet {
-	return c[len(c)-1]
+func (c ECChain) Head() *TipSet {
+	return &c[len(c)-1]
 }
 
 // Returns a new chain with the same base and no suffix.
@@ -71,8 +131,15 @@ func (c ECChain) BaseChain() ECChain {
 	return ECChain{c[0]}
 }
 
-func (c ECChain) Extend(tip ...TipSet) ECChain {
-	return append(c, tip...)
+func (c ECChain) Extend(tips ...TipSetKey) ECChain {
+	offset := c.Head().Epoch + 1
+	for i, tip := range tips {
+		c = append(c, TipSet{
+			Epoch:  offset + int64(i),
+			TipSet: tip,
+		})
+	}
+	return c
 }
 
 // Returns a chain with suffix (after the base) truncated to a maximum length.
@@ -88,9 +155,7 @@ func (c ECChain) Eq(other ECChain) bool {
 		return false
 	}
 	for i := range c {
-		if !bytes.Equal(c[i], other[i]) {
-			return false
-		}
+		c[i].Equal(&other[i])
 	}
 	return true
 }
@@ -101,16 +166,13 @@ func (c ECChain) SameBase(other ECChain) bool {
 	if c.IsZero() || other.IsZero() {
 		return false
 	}
-	return bytes.Equal(c.Base(), other.Base())
+	return c.Base().Equal(other.Base())
 }
 
 // Check whether a chain has a specific base tipset.
 // Always false for a zero value.
-func (c ECChain) HasBase(t TipSet) bool {
-	if c.IsZero() || len(t) == 0 {
-		return false
-	}
-	return bytes.Equal(c[0], t)
+func (c ECChain) HasBase(t *TipSet) bool {
+	return !t.IsZero() && !c.IsZero() && c.Base().Equal(t)
 }
 
 // Checks whether a chain has some prefix (including the base).
@@ -123,21 +185,21 @@ func (c ECChain) HasPrefix(other ECChain) bool {
 		return false
 	}
 	for i := range other {
-		if !bytes.Equal(c[i], other[i]) {
+		if !c[i].Equal(&other[i]) {
 			return false
 		}
 	}
 	return true
 }
 
 // Checks whether a chain has some tipset (including as its base).
-func (c ECChain) HasTipset(t TipSet) bool {
-	if len(t) == 0 {
+func (c ECChain) HasTipset(t *TipSet) bool {
+	if t.IsZero() {
 		// Chain can never contain zero-valued TipSet.
 		return false
 	}
-	for _, t2 := range c {
-		if bytes.Equal(t, t2) {
+	for i := range c {
+		if c[i].Equal(t) {
 			return true
 		}
 	}
@@ -147,7 +209,8 @@ func (c ECChain) HasTipset(t TipSet) bool {
 // Validates a chain value, returning an error if it finds any issues.
 // A chain is valid if it meets the following criteria:
 // 1) All contained tipsets are non-empty.
-// 2) The chain is not longer than CHAIN_MAX_LEN.
+// 2) All epochs are >= 0 and increasing.
+// 3) The chain is not longer than CHAIN_MAX_LEN.
 // An entirely zero-valued chain itself is deemed valid. See ECChain.IsZero.
 func (c ECChain) Validate() error {
 	if c.IsZero() {
@@ -156,36 +219,45 @@ func (c ECChain) Validate() error {
 	if len(c) > CHAIN_MAX_LEN {
 		return errors.New("chain too long")
 	}
-	for _, tipSet := range c {
-		if len(tipSet) == 0 {
+	var lastEpoch int64 = -1
+	for i := range c {
+		ts := &c[i]
+		if ts.IsZero() {
 			return errors.New("chain cannot contain zero-valued tip sets")
 		}
+		if ts.Epoch <= lastEpoch {
+			return errors.New("chain must have increasing epochs")
+		}
+		lastEpoch = ts.Epoch
 	}
 	return nil
 }
 
 // Returns an identifier for the chain suitable for use as a map key.
 // This must completely determine the sequence of tipsets in the chain.
 func (c ECChain) Key() ChainKey {
-	var ln int
-	for _, t := range c {
-		ln += 4      // for length
-		ln += len(t) // for data
+	ln := len(c) * (8 + 32 + 4) // epoch + commitement + ts length
+	for i := range c {
+		ln += len(c[i].TipSet) + len(c[i].PowerTable)
 	}
 	var buf bytes.Buffer
 	buf.Grow(ln)
-	for _, t := range c {
-		_ = binary.Write(&buf, binary.BigEndian, uint32(len(t)))
-		buf.Write(t)
+	for i := range c {
+		ts := &c[i]
+		_ = binary.Write(&buf, binary.BigEndian, ts.Epoch)
+		_, _ = buf.Write(ts.Commitments[:])
+		_ = binary.Write(&buf, binary.BigEndian, uint32(len(ts.TipSet)))
+		buf.Write(ts.TipSet)
+		_, _ = buf.Write(ts.PowerTable)
 	}
 	return ChainKey(buf.String())
 }
 
 func (c ECChain) String() string {
 	var b strings.Builder
 	b.WriteString("[")
-	for i, t := range c {
-		b.WriteString(hex.EncodeToString(t))
+	for i := range c {
+		b.WriteString(c[i].String())
 		if i < len(c)-1 {
 			b.WriteString(", ")
 		}

gpbft/chain_test.go

@@ -7,56 +7,17 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestTipSet(t *testing.T) {
-	t.Parallel()
-	tests := []struct {
-		name       string
-		subject    gpbft.TipSet
-		wantZero   bool
-		wantString string
-	}{
-		{
-			name:       "zero-value struct is zero",
-			wantZero:   true,
-			wantString: "",
-		},
-		{
-			name:       "ZeroTipSet is zero",
-			subject:    []byte{},
-			wantZero:   true,
-			wantString: "",
-		},
-		{
-			name:       "NewTipSet with zero values is zero",
-			subject:    nil,
-			wantZero:   true,
-			wantString: "",
-		},
-		{
-			name:       "Non-zero is not zero",
-			subject:    gpbft.TipSet("fish"),
-			wantString: "fish",
-		},
-	}
-	for _, test := range tests {
-		test := test
-		t.Run(test.name, func(t *testing.T) {
-			require.Equal(t, test.wantZero, len(test.subject) == 0)
-			require.Equal(t, test.wantString, string(test.subject))
-		})
-	}
-}
-
 func TestECChain(t *testing.T) {
 	t.Parallel()
 
-	zeroTipSet := []byte{}
+	zeroTipSet := gpbft.TipSet{}
+	oneTipSet := gpbft.TipSet{Epoch: 0, TipSet: []byte{1}}
 	t.Run("zero-value is zero", func(t *testing.T) {
 		var subject gpbft.ECChain
 		require.True(t, subject.IsZero())
-		require.False(t, subject.HasBase(zeroTipSet))
+		require.False(t, subject.HasBase(&zeroTipSet))
 		require.False(t, subject.HasPrefix(subject))
-		require.False(t, subject.HasTipset(zeroTipSet))
+		require.False(t, subject.HasTipset(&zeroTipSet))
 		require.False(t, subject.SameBase(subject))
 		require.True(t, subject.Eq(subject))
 		require.True(t, subject.Eq(*new(gpbft.ECChain)))
@@ -72,39 +33,39 @@ func TestECChain(t *testing.T) {
 		require.Nil(t, subject)
 	})
 	t.Run("extended chain is as expected", func(t *testing.T) {
-		wantBase := []byte("fish")
+		wantBase := gpbft.TipSet{Epoch: 0, TipSet: []byte("fish")}
 		subject, err := gpbft.NewChain(wantBase)
 		require.NoError(t, err)
 		require.Len(t, subject, 1)
-		require.Equal(t, wantBase, subject.Base())
-		require.Equal(t, wantBase, subject.Head())
+		require.Equal(t, &wantBase, subject.Base())
+		require.Equal(t, &wantBase, subject.Head())
 		require.NoError(t, subject.Validate())
 
-		wantNext := []byte("lobster")
-		subjectExtended := subject.Extend(wantNext)
+		wantNext := gpbft.TipSet{Epoch: 1, TipSet: []byte("lobster")}
+		subjectExtended := subject.Extend(wantNext.TipSet)
 		require.Len(t, subjectExtended, 2)
 		require.NoError(t, subjectExtended.Validate())
-		require.Equal(t, wantBase, subjectExtended.Base())
+		require.Equal(t, &wantBase, subjectExtended.Base())
 		require.Equal(t, []gpbft.TipSet{wantNext}, subjectExtended.Suffix())
-		require.Equal(t, wantNext, subjectExtended.Head())
-		require.Equal(t, wantNext, subjectExtended.Prefix(1).Head())
-		require.True(t, subjectExtended.HasTipset(wantBase))
+		require.Equal(t, &wantNext, subjectExtended.Head())
+		require.Equal(t, &wantNext, subjectExtended.Prefix(1).Head())
+		require.True(t, subjectExtended.HasTipset(&wantBase))
 		require.False(t, subject.HasPrefix(subjectExtended))
 		require.True(t, subjectExtended.HasPrefix(subject))
 
-		require.False(t, subject.Extend(wantBase).HasPrefix(subjectExtended.Extend(wantNext)))
+		require.False(t, subject.Extend(wantBase.TipSet).HasPrefix(subjectExtended.Extend(wantNext.TipSet)))
 	})
 	t.Run("SameBase is false when either chain is zero", func(t *testing.T) {
 		var zeroChain gpbft.ECChain
-		nonZeroChain, err := gpbft.NewChain([]byte{1})
+		nonZeroChain, err := gpbft.NewChain(oneTipSet)
 		require.NoError(t, err)
 		require.False(t, nonZeroChain.SameBase(zeroChain))
 		require.False(t, zeroChain.SameBase(nonZeroChain))
 		require.False(t, zeroChain.SameBase(zeroChain))
 	})
 	t.Run("HasPrefix is false when either chain is zero", func(t *testing.T) {
 		var zeroChain gpbft.ECChain
-		nonZeroChain, err := gpbft.NewChain([]byte{1})
+		nonZeroChain, err := gpbft.NewChain(oneTipSet)
 		require.NoError(t, err)
 		require.False(t, nonZeroChain.HasPrefix(zeroChain))
 		require.False(t, zeroChain.HasPrefix(nonZeroChain))
@@ -115,7 +76,7 @@ func TestECChain(t *testing.T) {
 		require.NoError(t, zeroChain.Validate())
 	})
 	t.Run("ordered chain with zero-valued base is invalid", func(t *testing.T) {
-		subject := gpbft.ECChain{zeroTipSet, []byte{1}}
+		subject := gpbft.ECChain{zeroTipSet, oneTipSet}
 		require.Error(t, subject.Validate())
 	})
 }