Basic implementation of simple BFT

Simple BFT is a variant of Castro's PBFT, with as little parallelism as possible. This package also defines a deterministic system interface and a deterministic event based testing framework. Squashed commit of the following: commit f7b20b2f0a65a59e91d332e13e5dd2016264b7d1 Author: Simon Schubert <sis@zurich.ibm.com> Date: Tue Sep 27 15:33:05 2016 +0200 checkpoint: fix comment Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit 3f370c36f61daf0e5845d8e82dacb43ea546377f Author: Simon Schubert <sis@zurich.ibm.com> Date: Mon Sep 26 15:02:22 2016 +0200 add performance benchmark for rsa Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit b247320d7448dc4c6a7985c1b08c011a1eb4d260 Author: Simon Schubert <sis@zurich.ibm.com> Date: Fri Sep 23 15:02:11 2016 +0200 turn null request into empty batch Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit 830ed7b080dc04c2b31c079b36f7a87c4b98d4ae Author: Simon Schubert <sis@zurich.ibm.com> Date: Fri Sep 23 13:10:51 2016 +0200 on restart, remove future events to old instance Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit 2fbbe1d6d25a72bd9a184c0badf66a46df574857 Author: Simon Schubert <sis@zurich.ibm.com> Date: Thu Sep 22 18:06:28 2016 +0200 test for various restart situations Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit 5df10ae03eb861ed68238aa8bafc3b2e0147fa82 Author: Simon Schubert <sis@zurich.ibm.com> Date: Wed Sep 21 17:00:18 2016 +0200 work off existing chain state Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit 1ba6905539fc94411273608a72c689ca6c0afb91 Author: Simon Schubert <sis@zurich.ibm.com> Date: Wed Sep 21 17:00:07 2016 +0200 add crypto test Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit 75cea80716ff56216a418cb65fb70431f9793c09 Author: Simon Schubert <sis@zurich.ibm.com> Date: Wed Sep 21 16:32:13 2016 +0200 remove timerFunc Spotted-by: Gabor Hosszu Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit b5a71f9f98cb7862c1d725639490f8c8ac8912a4 Author: Simon Schubert <sis@zurich.ibm.com> Date: Wed Sep 21 16:31:14 2016 +0200 add ecdsa benchmark Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit d063c4487b4cd6f008480bdee0ef8280775b021f Author: Simon Schubert <sis@zurich.ibm.com> Date: Wed Sep 21 14:50:44 2016 +0200 hash is no method Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit d6988a2b22b02adaa64351482d17c8a5849eec9b Author: Simon Schubert <sis@zurich.ibm.com> Date: Wed Sep 21 12:57:34 2016 +0200 transmit batch instead of separate items, use merkle tree hash Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit a64d3875399b935df3bbbdfdafc4c9e736efc099 Author: Simon Schubert <sis@zurich.ibm.com> Date: Wed Sep 21 11:52:55 2016 +0200 deliver takes batch Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit 8faee7e4bde2471b6f88d50d09fa86f944b4fdfc Author: Simon Schubert <sis@zurich.ibm.com> Date: Tue Sep 20 18:07:48 2016 +0200 sign over batch Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit 865966071d3726f54e46d76d35b1abcd9a725d84 Author: Simon Schubert <sis@zurich.ibm.com> Date: Tue Sep 20 13:32:09 2016 +0200 resend messages when coming back up Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit f22d84eed8cc5f0441dd59ee46539ebcd9c97a5d Author: Simon Schubert <sis@zurich.ibm.com> Date: Tue Sep 20 12:49:06 2016 +0200 deliver on checkpoint Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit 766ee7131cd5a3423258d267a57cbe496c3419c2 Author: Simon Schubert <sis@zurich.ibm.com> Date: Mon Sep 19 15:42:05 2016 +0200 implement signatures in system Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit 4826f7c00c81ccb94582aaa83b69aa0a0b988e5c Author: Simon Schubert <sis@zurich.ibm.com> Date: Fri Sep 16 16:34:36 2016 +0200 convert checkpoint to signed Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit 36c96039d059bd94592aa7741091f288457d5a64 Author: Simon Schubert <sis@zurich.ibm.com> Date: Fri Sep 16 16:12:42 2016 +0200 start adding persistence Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit cb4ce421a3194d7799bba703d062fe06807aab8f Author: Simon Schubert <sis@zurich.ibm.com> Date: Thu Sep 8 11:37:35 2016 +0200 make sure we don't get stuck in backlog Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit ad425301d7a4f926ec91822e5b6f463e107be071 Author: Simon Schubert <sis@zurich.ibm.com> Date: Wed Sep 7 16:16:25 2016 +0200 add comment about DoS Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit c76035a6d2b75f348be26333180f31d5496d55b3 Author: Simon Schubert <sis@zurich.ibm.com> Date: Wed Sep 7 15:16:46 2016 +0200 keep backlog of messages referring to the future Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit 15c6d7260227df829fe84854faefca6b50483cdd Author: Simon Schubert <sis@zurich.ibm.com> Date: Wed Sep 7 13:47:21 2016 +0200 send next batch only after previous round has finished Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit fde05a7aca7a710ab4e1ff02b4d108c9a5794318 Author: Simon Schubert <sis@zurich.ibm.com> Date: Tue Sep 6 10:46:21 2016 +0200 add network viewchange test Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit 3883f44bff742df0ff34ef79fd4183ca81d1db5e Author: Simon Schubert <sis@zurich.ibm.com> Date: Tue Sep 6 10:46:04 2016 +0200 add/fix status logging Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit 99775383247bec76c4d3e700c1b6b8f55ab57614 Author: Simon Schubert <sis@zurich.ibm.com> Date: Mon Sep 5 10:48:04 2016 +0200 newview progress Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit 7ecdf27fb89f14feb166726034b32e91a4132ff6 Author: Simon Schubert <sis@zurich.ibm.com> Date: Mon Sep 5 10:37:48 2016 +0200 move benchmarks to separate file Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit e9f5d6be14a739de154c9454efff4d8a4954f4a7 Author: Simon Schubert <sis@zurich.ibm.com> Date: Mon Sep 5 10:34:50 2016 +0200 switch hash result to slice Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit 1063637861c24e50dc9017f1cf5e50f0b542c68a Author: Simon Schubert <sis@zurich.ibm.com> Date: Fri Sep 2 16:09:45 2016 +0200 properly reset view change timer Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit 927059b467fc51bc5679675e76e411779c1dc584 Author: Simon Schubert <sis@zurich.ibm.com> Date: Fri Sep 2 15:57:29 2016 +0200 reduce logging Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit 884b81998643d2e4310010bb0c404051a06c20f9 Author: Simon Schubert <sis@zurich.ibm.com> Date: Fri Sep 2 15:47:28 2016 +0200 add view change timeout Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit 2e9084b6ef86b8a2bccbfe0017a8970a6c5bdc63 Author: Simon Schubert <sis@zurich.ibm.com> Date: Fri Sep 2 15:35:11 2016 +0200 more new view work Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit cd1e2df54ce4a1739d461963ced2b950b2f5ee00 Author: Simon Schubert <sis@zurich.ibm.com> Date: Fri Sep 2 11:10:42 2016 +0200 start newview Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit 6f3ef498a37baf2b52ccf0ca929f6040ea100b52 Author: Simon Schubert <sis@zurich.ibm.com> Date: Tue Aug 23 16:55:36 2016 +0200 implement calendar queue for simulation Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit df7b4ba25d00aacce677292a0452abdc64b54d8c Author: Simon Schubert <sis@zurich.ibm.com> Date: Tue Aug 23 11:51:23 2016 +0200 more newview, large network benchmark Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit f7ceb7849c94b9c6c7697f599eb6b5c793f0fdb2 Author: Simon Schubert <sis@zurich.ibm.com> Date: Fri Aug 19 18:51:27 2016 +0200 split into smaller files Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit b1cbfe3850f53f297891406cfcd4bb1de421d67c Author: Simon Schubert <sis@zurich.ibm.com> Date: Fri Aug 19 18:09:03 2016 +0200 more simple bft Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit 0e24a238ae574f9aa292d88f278fde6c606eab0d Author: Simon Schubert <sis@zurich.ibm.com> Date: Tue Aug 16 10:09:54 2016 +0200 basic simplebft Signed-off-by: Simon Schubert <sis@zurich.ibm.com> commit f0fb9e212d8eb86f95b20433d06c019d3ac2d8b5 Author: Simon Schubert <sis@zurich.ibm.com> Date: Fri Aug 12 17:00:37 2016 +0200 start simplebft Signed-off-by: Simon Schubert <sis@zurich.ibm.com> Change-Id: Id3e7ff1a67ca3d64011bce4e149e35be09f15f34 Signed-off-by: Simon Schubert <sis@zurich.ibm.com>
hyperledger · Sep 27, 2016 · 52c8407 · 52c8407
1 parent 9617a6e
commit 52c8407
Show file tree

Hide file tree

Showing 22 changed files with 3,192 additions and 0 deletions.
diff --git a/consensus/simplebft/backlog.go b/consensus/simplebft/backlog.go
@@ -0,0 +1,89 @@
+/*
+Copyright IBM Corp. 2016 All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+		 http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package simplebft
+
+func (s *SBFT) testBacklog(m *Msg, src uint64) bool {
+	if len(s.backLog[src]) > 0 {
+		return true
+	}
+
+	return s.testBacklog2(m, src)
+}
+
+func (s *SBFT) testBacklog2(m *Msg, src uint64) bool {
+	record := func(seq uint64) bool {
+		if seq > s.cur.subject.Seq.Seq {
+			return true
+		}
+		return false
+	}
+
+	if pp := m.GetPreprepare(); pp != nil && !s.cur.executed {
+		return true
+	} else if p := m.GetPrepare(); p != nil {
+		return record(p.Seq.Seq)
+	} else if c := m.GetCommit(); c != nil {
+		return record(c.Seq.Seq)
+	} else if cs := m.GetCheckpoint(); cs != nil {
+		c := &Checkpoint{}
+		return record(c.Seq)
+	}
+	return false
+}
+
+func (s *SBFT) recordBacklogMsg(m *Msg, src uint64) {
+	if src == s.id {
+		panic("should never have to backlog my own message")
+	}
+	// TODO prevent DoS by limiting the number of messages per replica
+	s.backLog[src] = append(s.backLog[src], m)
+}
+
+func (s *SBFT) processBacklog() {
+	processed := true
+
+	for processed {
+		processed = false
+		notReady := uint64(0)
+		for src, _ := range s.backLog {
+			for len(s.backLog[src]) > 0 {
+				m, rest := s.backLog[src][0], s.backLog[src][1:]
+				if s.testBacklog2(m, src) {
+					notReady++
+					break
+				}
+				s.backLog[src] = rest
+
+				log.Debugf("processing stored message from %d: %s", src, m)
+
+				s.handleQueueableMessage(m, src)
+				processed = true
+			}
+		}
+
+		// all minus us
+		if notReady >= s.config.N-1 {
+			// This is a problem - we consider all other replicas
+			// too far ahead for us.  We need to do a state transfer
+			// to get out of this rut.
+			for src := range s.backLog {
+				delete(s.backLog, src)
+			}
+			// TODO trigger state transfer
+		}
+	}
+}
diff --git a/consensus/simplebft/batch.go b/consensus/simplebft/batch.go
@@ -0,0 +1,64 @@
+/*
+Copyright IBM Corp. 2016 All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+		 http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package simplebft
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/golang/protobuf/proto"
+)
+
+func (s *SBFT) makeBatch(seq uint64, prevHash []byte, data [][]byte) *Batch {
+	datahash := merkleHashData(data)
+
+	batchhead := &BatchHeader{
+		Seq:      seq,
+		PrevHash: prevHash,
+		DataHash: datahash,
+	}
+	rawHeader, err := proto.Marshal(batchhead)
+	if err != nil {
+		panic(err)
+	}
+	return &Batch{
+		Header:   rawHeader,
+		Payloads: data,
+	}
+}
+
+func (s *SBFT) checkBatch(b *Batch) (*BatchHeader, error) {
+	datahash := merkleHashData(b.Payloads)
+
+	batchheader := &BatchHeader{}
+	err := proto.Unmarshal(b.Header, batchheader)
+	if err != nil {
+		return nil, err
+	}
+
+	if !reflect.DeepEqual(datahash, batchheader.DataHash) {
+		return nil, fmt.Errorf("malformed batch: invalid hash")
+	}
+
+	return batchheader, nil
+}
+
+////////////////////////////////////////
+
+func (b *Batch) Hash() []byte {
+	return hash(b.Header)
+}
diff --git a/consensus/simplebft/calendarqueue_test.go b/consensus/simplebft/calendarqueue_test.go
@@ -0,0 +1,134 @@
+/*
+Copyright IBM Corp. 2016 All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+		 http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package simplebft
+
+import (
+	"sort"
+	"time"
+)
+
+type calendarQueue struct {
+	dayLength  time.Duration
+	yearLength time.Duration
+	today      time.Duration
+	nextYear   time.Duration
+	slots      [][]testElem
+	maxLen     int
+}
+
+func newCalendarQueue(dayLength time.Duration, days int) *calendarQueue {
+	return &calendarQueue{
+		dayLength:  dayLength,
+		yearLength: dayLength * time.Duration(days),
+		nextYear:   dayLength * time.Duration(days),
+		slots:      make([][]testElem, days),
+	}
+}
+
+func (t *calendarQueue) slot(d time.Duration) int {
+	return int(d/t.dayLength) % len(t.slots)
+}
+
+func (t *calendarQueue) Add(e testElem) {
+	sl := t.slot(e.at)
+	t.slots[sl] = append(t.slots[sl], e)
+	l := len(t.slots[sl])
+	if l > t.maxLen {
+		t.maxLen = l
+	}
+	sort.Sort(testElemQueue(t.slots[sl]))
+}
+
+func (t *calendarQueue) Pop() (testElem, bool) {
+	var lowest *time.Duration
+	sl := t.slot(t.today)
+	start := sl
+	today := t.today
+	for ; today < t.nextYear; today, sl = today+t.dayLength, sl+1 {
+		if len(t.slots[sl]) == 0 {
+			continue
+		}
+		e := t.slots[sl][0]
+		if e.at >= t.nextYear {
+			if lowest == nil || *lowest > e.at {
+				lowest = &e.at
+			}
+			continue
+		}
+		t.slots[sl] = t.slots[sl][1:]
+		t.today = today
+		return e, true
+	}
+
+	// next deadline is after this year, but we only
+	// searched part of the calendar so far.  Search the
+	// remaining prefix.
+	for i := 0; i < start; i++ {
+		if len(t.slots[i]) == 0 {
+			continue
+		}
+		e := t.slots[i][0]
+		if e.at >= t.nextYear {
+			if lowest == nil || *lowest > e.at {
+				lowest = &e.at
+			}
+		}
+	}
+
+	if lowest == nil {
+		return testElem{}, false
+	}
+
+	t.today = *lowest / t.dayLength * t.dayLength
+	t.nextYear = (t.today/t.yearLength + 1) * t.yearLength
+	return t.Pop() // retry!
+}
+
+func (t *calendarQueue) filter(fn func(testElem) bool) {
+	for sli, sl := range t.slots {
+		var del []int
+		for i, e := range sl {
+			if !fn(e) {
+				del = append(del, i)
+			}
+		}
+
+		// now delete
+		for i, e := range del {
+			correctedPos := e - i
+			// in-place remove
+			sl = sl[:correctedPos+copy(sl[correctedPos:], sl[correctedPos+1:])]
+		}
+		t.slots[sli] = sl
+	}
+}
+
+/////////////////////////////////////////
+
+type testElemQueue []testElem
+
+func (q testElemQueue) Len() int {
+	return len(q)
+}
+
+func (q testElemQueue) Less(i, j int) bool {
+	return q[i].at < q[j].at
+}
+
+func (q testElemQueue) Swap(i, j int) {
+	q[i], q[j] = q[j], q[i]
+}
diff --git a/consensus/simplebft/checkpoint.go b/consensus/simplebft/checkpoint.go
@@ -0,0 +1,106 @@
+/*
+Copyright IBM Corp. 2016 All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+		 http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package simplebft
+
+import (
+	"fmt"
+	"reflect"
+)
+
+func (s *SBFT) sendCheckpoint() {
+	sig := s.sys.Sign(s.cur.subject.Digest)
+	c := &Checkpoint{
+		Seq:       s.cur.subject.Seq.Seq,
+		Digest:    s.cur.subject.Digest,
+		Signature: sig,
+	}
+	s.broadcast(&Msg{&Msg_Checkpoint{c}})
+}
+
+func (s *SBFT) handleCheckpoint(c *Checkpoint, src uint64) {
+	if s.cur.checkpointDone {
+		return
+	}
+
+	if c.Seq < s.cur.subject.Seq.Seq {
+		// old message
+		return
+	}
+
+	err := s.checkBytesSig(c.Digest, src, c.Signature)
+	if err != nil {
+		log.Infof("checkpoint signature invalid for %d from %d", c.Seq, src)
+		return
+	}
+
+	// TODO should we always accept checkpoints?
+	if c.Seq != s.cur.subject.Seq.Seq {
+		log.Infof("checkpoint does not match expected subject %v, got %v", &s.cur.subject, c)
+		return
+	}
+	if _, ok := s.cur.checkpoint[src]; ok {
+		log.Infof("duplicate checkpoint for %d from %d", c.Seq, src)
+	}
+	s.cur.checkpoint[src] = c
+
+	max := "_"
+	sums := make(map[string][]uint64)
+	for csrc, c := range s.cur.checkpoint {
+		sum := fmt.Sprintf("%x", c.Digest)
+		sums[sum] = append(sums[sum], csrc)
+
+		if len(sums[sum]) >= s.oneCorrectQuorum() {
+			max = sum
+		}
+	}
+
+	replicas, ok := sums[max]
+	if !ok {
+		return
+	}
+
+	// got a weak checkpoint
+
+	cpset := &CheckpointSet{make(map[uint64]*Checkpoint)}
+	var sigs [][]byte
+	for _, r := range replicas {
+		cp := s.cur.checkpoint[r]
+		cpset.CheckpointSet[r] = cp
+		sigs = append(sigs, cp.Signature)
+	}
+	s.cur.checkpointDone = true
+
+	c = s.cur.checkpoint[replicas[0]]
+
+	if !reflect.DeepEqual(c.Digest, s.cur.subject.Digest) {
+		log.Fatalf("weak checkpoint %x does not match our state %x",
+			c.Digest, s.cur.subject.Digest)
+		// NOT REACHED
+	}
+
+	// ignore null requests
+	if s.cur.preprep.Batch != nil {
+		batch := *s.cur.preprep.Batch
+		batch.Signatures = sigs
+		s.sys.Deliver(&batch)
+	}
+	s.cur.timeout.Cancel()
+	log.Infof("request %s %s completed on %d", s.cur.subject.Seq, hash2str(s.cur.subject.Digest), s.id)
+
+	s.maybeSendNextBatch()
+	s.processBacklog()
+}