Merge pull request #323 from lazyledger/ismail/light-mvp

Data Availability Sampling light client (mvp)

.github/workflows/coverage.yml

@@ -84,7 +84,7 @@ jobs:
           go-version: 1.15
       - name: test & coverage report creation
         run: |
-          CGO_ENABLED=0 cat pkgs.txt.part.${{ matrix.part }} | xargs go test -mod=readonly -timeout 8m -race -coverprofile=${{ matrix.part }}profile.out -covermode=atomic
+          CGO_ENABLED=0 cat pkgs.txt.part.${{ matrix.part }} | xargs go test -mod=readonly -timeout 12m -race -coverprofile=${{ matrix.part }}profile.out -covermode=atomic
         if: env.GIT_DIFF
       - uses: actions/upload-artifact@v2
         with:

cmd/tendermint/commands/light.go

@@ -5,16 +5,19 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
 	"time"
 
+	coreiface "github.com/ipfs/interface-go-ipfs-core"
 	"github.com/spf13/cobra"
 
 	"github.com/lazyledger/lazyledger-core/crypto/merkle"
+	"github.com/lazyledger/lazyledger-core/ipfs"
 	dbm "github.com/lazyledger/lazyledger-core/libs/db"
 	"github.com/lazyledger/lazyledger-core/libs/db/badgerdb"
 	"github.com/lazyledger/lazyledger-core/libs/log"
@@ -64,6 +67,8 @@ var (
 	dir                string
 	maxOpenConnections int
 
+	daSampling     bool
+	numSamples     uint32
 	sequential     bool
 	trustingPeriod time.Duration
 	trustedHeight  int64
@@ -101,6 +106,11 @@ func init() {
 	LightCmd.Flags().BoolVar(&sequential, "sequential", false,
 		"sequential verification. Verify all headers sequentially as opposed to using skipping verification",
 	)
+	LightCmd.Flags().BoolVar(&daSampling, "da-sampling", false,
+		"data availability sampling. Verify each header's data availability via sampling",
+	)
+	LightCmd.Flags().Uint32Var(&numSamples, "num-samples", 15,
+		"Number of data availability samples until block data deemed available.")
 }
 
 func runProxy(cmd *cobra.Command, args []string) error {
@@ -169,9 +179,22 @@ func runProxy(cmd *cobra.Command, args []string) error {
 		}),
 	}
 
-	if sequential {
+	var ipfsCloser io.Closer
+	switch {
+	case daSampling:
+		cfg := ipfs.DefaultConfig()
+		cfg.RootDir = dir
+		// TODO(ismail): share badger instance
+		apiProvider := ipfs.Embedded(true, cfg, logger)
+		var coreAPI coreiface.CoreAPI
+		coreAPI, ipfsCloser, err = apiProvider()
+		if err != nil {
+			return fmt.Errorf("could not start ipfs API: %w", err)
+		}
+		options = append(options, light.DataAvailabilitySampling(numSamples, coreAPI))
+	case sequential:
 		options = append(options, light.SequentialVerification())
-	} else {
+	default:
 		options = append(options, light.SkippingVerification(trustLevel))
 	}
 
@@ -215,7 +238,7 @@ func runProxy(cmd *cobra.Command, args []string) error {
 	cfg.MaxOpenConnections = maxOpenConnections
 	// If necessary adjust global WriteTimeout to ensure it's greater than
 	// TimeoutBroadcastTxCommit.
-	// See https://github.com/lazyledger/lazyledger-core/issues/3435
+	// See https://github.com/tendermint/tendermint/issues/3435
 	if cfg.WriteTimeout <= config.RPC.TimeoutBroadcastTxCommit {
 		cfg.WriteTimeout = config.RPC.TimeoutBroadcastTxCommit + 1*time.Second
 	}
@@ -229,6 +252,9 @@ func runProxy(cmd *cobra.Command, args []string) error {
 	// Stop upon receiving SIGTERM or CTRL-C.
 	tmos.TrapSignal(logger, func() {
 		p.Listener.Close()
+		if ipfsCloser != nil {
+			ipfsCloser.Close()
+		}
 	})
 
 	logger.Info("Starting proxy...", "laddr", listenAddr)

config/config.go

@@ -844,17 +844,17 @@ func DefaultConsensusConfig() *ConsensusConfig {
 // TestConsensusConfig returns a configuration for testing the consensus service
 func TestConsensusConfig() *ConsensusConfig {
 	cfg := DefaultConsensusConfig()
-	cfg.TimeoutPropose = 100 * time.Millisecond
-	cfg.TimeoutProposeDelta = 10 * time.Millisecond
-	cfg.TimeoutPrevote = 40 * time.Millisecond
-	cfg.TimeoutPrevoteDelta = 10 * time.Millisecond
-	cfg.TimeoutPrecommit = 40 * time.Millisecond
-	cfg.TimeoutPrecommitDelta = 10 * time.Millisecond
+	cfg.TimeoutPropose = 200 * time.Millisecond
+	cfg.TimeoutProposeDelta = 20 * time.Millisecond
+	cfg.TimeoutPrevote = 80 * time.Millisecond
+	cfg.TimeoutPrevoteDelta = 20 * time.Millisecond
+	cfg.TimeoutPrecommit = 80 * time.Millisecond
+	cfg.TimeoutPrecommitDelta = 20 * time.Millisecond
 	// NOTE: when modifying, make sure to update time_iota_ms (testGenesisFmt) in toml.go
-	cfg.TimeoutCommit = 40 * time.Millisecond
+	cfg.TimeoutCommit = 80 * time.Millisecond
 	cfg.SkipTimeoutCommit = true
-	cfg.PeerGossipSleepDuration = 10 * time.Millisecond
-	cfg.PeerQueryMaj23SleepDuration = 250 * time.Millisecond
+	cfg.PeerGossipSleepDuration = 20 * time.Millisecond
+	cfg.PeerQueryMaj23SleepDuration = 500 * time.Millisecond
 	cfg.DoubleSignCheckHeight = int64(0)
 	return cfg
 }

consensus/byzantine_test.go

@@ -91,7 +91,7 @@ func TestByzantinePrevoteEquivocation(t *testing.T) {
 
 		cs.SetTimeoutTicker(tickerFunc())
 		cs.SetLogger(logger)
-
+		cs.SetIPFSApi(ipfsTestAPI)
 		css[i] = cs
 	}
 
@@ -215,7 +215,12 @@ func TestByzantineConflictingProposalsWithPartition(t *testing.T) {
 	N := 4
 	logger := consensusLogger().With("test", "byzantine")
 	app := newCounter
-	css, cleanup := randConsensusNet(N, "consensus_byzantine_test", newMockTickerFunc(false), app)
+	css, cleanup := randConsensusNet(
+		N,
+		"consensus_byzantine_test",
+		newMockTickerFunc(false),
+		app,
+	)
 	defer cleanup()
 
 	// give the byzantine validator a normal ticker

consensus/common_test.go

@@ -4,25 +4,27 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"sort"
 	"sync"
 	"testing"
 	"time"
 
 	"github.com/go-kit/kit/log/term"
+	iface "github.com/ipfs/interface-go-ipfs-core"
 	"github.com/stretchr/testify/require"
 
-	"path"
-
 	abcicli "github.com/lazyledger/lazyledger-core/abci/client"
 	"github.com/lazyledger/lazyledger-core/abci/example/counter"
 	"github.com/lazyledger/lazyledger-core/abci/example/kvstore"
 	abci "github.com/lazyledger/lazyledger-core/abci/types"
 	cfg "github.com/lazyledger/lazyledger-core/config"
 	cstypes "github.com/lazyledger/lazyledger-core/consensus/types"
+	"github.com/lazyledger/lazyledger-core/ipfs"
 	tmbytes "github.com/lazyledger/lazyledger-core/libs/bytes"
 	dbm "github.com/lazyledger/lazyledger-core/libs/db"
 	"github.com/lazyledger/lazyledger-core/libs/db/memdb"
@@ -48,11 +50,14 @@ const (
 // test.
 type cleanupFunc func()
 
-// genesis, chain_id, priv_val
+// genesis, chain_id, priv_val, ipfsAPI
 var (
 	config                *cfg.Config // NOTE: must be reset for each _test.go file
 	consensusReplayConfig *cfg.Config
-	ensureTimeout         = 1000 * time.Millisecond
+	ensureTimeout         = 2 * time.Second
+
+	ipfsTestAPI iface.CoreAPI
+	ipfsCloser  io.Closer
 )
 
 func ensureDir(dir string, mode os.FileMode) {
@@ -61,6 +66,19 @@ func ensureDir(dir string, mode os.FileMode) {
 	}
 }
 
+func setTestIpfsAPI() (err error) {
+	mockIPFSProvider := ipfs.Mock()
+	if ipfsTestAPI, ipfsCloser, err = mockIPFSProvider(); err != nil {
+		return
+	}
+	return
+}
+
+func teardownTestIpfsAPI() (err error) {
+	err = ipfsCloser.Close()
+	return
+}
+
 func ResetConfig(name string) *cfg.Config {
 	return cfg.ResetTestRoot(name)
 }
@@ -432,6 +450,7 @@ func randState(nValidators int) (*State, []*validatorStub) {
 	vss := make([]*validatorStub, nValidators)
 
 	cs := newState(state, privVals[0], counter.NewApplication(true))
+	cs.SetIPFSApi(ipfsTestAPI)
 
 	for i := 0; i < nValidators; i++ {
 		vss[i] = newValidatorStub(privVals[i], int32(i))
@@ -678,9 +697,15 @@ func consensusLogger() log.Logger {
 	}).With("module", "consensus")
 }
 
-func randConsensusNet(nValidators int, testName string, tickerFunc func() TimeoutTicker,
-	appFunc func() abci.Application, configOpts ...func(*cfg.Config)) ([]*State, cleanupFunc) {
+func randConsensusNet(
+	nValidators int,
+	testName string,
+	tickerFunc func() TimeoutTicker,
+	appFunc func() abci.Application,
+	configOpts ...func(*cfg.Config),
+) ([]*State, cleanupFunc) {
 	genDoc, privVals := randGenesisDoc(nValidators, false, 30)
+
 	css := make([]*State, nValidators)
 	logger := consensusLogger()
 	configRootDirs := make([]string, 0, nValidators)
@@ -701,6 +726,7 @@ func randConsensusNet(nValidators int, testName string, tickerFunc func() Timeou
 		css[i] = newStateWithConfigAndBlockStore(thisConfig, state, privVals[i], app, stateDB)
 		css[i].SetTimeoutTicker(tickerFunc())
 		css[i].SetLogger(logger.With("validator", i, "module", "consensus"))
+		css[i].SetIPFSApi(ipfsTestAPI)
 	}
 	return css, func() {
 		for _, dir := range configRootDirs {
@@ -722,6 +748,7 @@ func randConsensusNetWithPeers(
 	logger := consensusLogger()
 	var peer0Config *cfg.Config
 	configRootDirs := make([]string, 0, nPeers)
+
 	for i := 0; i < nPeers; i++ {
 		stateDB := memdb.NewDB() // each state needs its own db
 		stateStore := sm.NewStore(stateDB)
@@ -763,6 +790,7 @@ func randConsensusNetWithPeers(
 		css[i] = newStateWithConfig(thisConfig, state, privVal, app)
 		css[i].SetTimeoutTicker(tickerFunc())
 		css[i].SetLogger(logger.With("validator", i, "module", "consensus"))
+		css[i].SetIPFSApi(ipfsTestAPI)
 	}
 	return css, genDoc, peer0Config, func() {
 		for _, dir := range configRootDirs {

consensus/invalid_test.go

@@ -18,14 +18,18 @@ import (
 // Ensure a testnet makes blocks
 func TestReactorInvalidPrecommit(t *testing.T) {
 	N := 4
-	css, cleanup := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter)
+	css, cleanup := randConsensusNet(
+		N,
+		"consensus_reactor_test",
+		newMockTickerFunc(true),
+		newCounter,
+	)
 	t.Cleanup(cleanup)
 
 	for i := 0; i < 4; i++ {
 		ticker := NewTimeoutTicker()
 		ticker.SetLogger(css[i].Logger)
 		css[i].SetTimeoutTicker(ticker)
-
 	}
 
 	reactors, blocksSubs, eventBuses := startConsensusNet(t, css, N)

consensus/mempool_test.go

@@ -30,6 +30,7 @@ func TestMempoolNoProgressUntilTxsAvailable(t *testing.T) {
 	config.Consensus.CreateEmptyBlocks = false
 	state, privVals := randGenesisState(1, false, 10)
 	cs := newStateWithConfig(config, state, privVals[0], NewCounterApplication())
+	cs.SetIPFSApi(ipfsTestAPI)
 	assertMempool(cs.txNotifier).EnableTxsAvailable()
 	height, round := cs.Height, cs.Round
 	newBlockCh := subscribe(cs.eventBus, types.EventQueryNewBlock)
@@ -50,6 +51,7 @@ func TestMempoolProgressAfterCreateEmptyBlocksInterval(t *testing.T) {
 	config.Consensus.CreateEmptyBlocksInterval = ensureTimeout
 	state, privVals := randGenesisState(1, false, 10)
 	cs := newStateWithConfig(config, state, privVals[0], NewCounterApplication())
+	cs.SetIPFSApi(ipfsTestAPI)
 
 	assertMempool(cs.txNotifier).EnableTxsAvailable()
 
@@ -68,6 +70,8 @@ func TestMempoolProgressInHigherRound(t *testing.T) {
 	config.Consensus.CreateEmptyBlocks = false
 	state, privVals := randGenesisState(1, false, 10)
 	cs := newStateWithConfig(config, state, privVals[0], NewCounterApplication())
+	cs.SetIPFSApi(ipfsTestAPI)
+
 	assertMempool(cs.txNotifier).EnableTxsAvailable()
 	height, round := cs.Height, cs.Round
 	newBlockCh := subscribe(cs.eventBus, types.EventQueryNewBlock)
@@ -115,7 +119,9 @@ func TestMempoolTxConcurrentWithCommit(t *testing.T) {
 	state, privVals := randGenesisState(1, false, 10)
 	blockDB := memdb.NewDB()
 	stateStore := sm.NewStore(blockDB)
+
 	cs := newStateWithConfigAndBlockStore(config, state, privVals[0], NewCounterApplication(), blockDB)
+	cs.SetIPFSApi(ipfsTestAPI)
 	err := stateStore.Save(state)
 	require.NoError(t, err)
 	newBlockHeaderCh := subscribe(cs.eventBus, types.EventQueryNewBlockHeader)
@@ -139,8 +145,10 @@ func TestMempoolRmBadTx(t *testing.T) {
 	state, privVals := randGenesisState(1, false, 10)
 	app := NewCounterApplication()
 	blockDB := memdb.NewDB()
+
 	stateStore := sm.NewStore(blockDB)
 	cs := newStateWithConfigAndBlockStore(config, state, privVals[0], app, blockDB)
+	cs.SetIPFSApi(ipfsTestAPI)
 	err := stateStore.Save(state)
 	require.NoError(t, err)