Remove flattenNamespacedEDS

consensus/msgs_test.go

@@ -18,6 +18,7 @@ import (
 	tmcons "github.com/lazyledger/lazyledger-core/proto/tendermint/consensus"
 	tmproto "github.com/lazyledger/lazyledger-core/proto/tendermint/types"
 	"github.com/lazyledger/lazyledger-core/types"
+	"github.com/lazyledger/lazyledger-core/types/consts"
 )
 
 func TestMsgToProto(t *testing.T) {
@@ -47,7 +48,7 @@ func TestMsgToProto(t *testing.T) {
 	pbParts, err := parts.ToProto()
 	require.NoError(t, err)
 
-	roots, err := types.NmtRootsFromBytes([][]byte{tmrand.Bytes(2*types.NamespaceSize + tmhash.Size)})
+	roots, err := types.NmtRootsFromBytes([][]byte{tmrand.Bytes(2*consts.NamespaceSize + tmhash.Size)})
 	require.NoError(t, err)
 	proposal := types.Proposal{
 		Type:      tmproto.ProposalType,

ipfs/plugin/nmt.go

@@ -32,7 +32,7 @@ const (
 	// DagParserFormatName can be used when putting into the IPLD Dag
 	DagParserFormatName = "extended-square-row-or-col"
 
-	// FIXME: These are the same as types.ShareSize and types.NamespaceSize.
+	// FIXME: These are the same as types.ShareSize and consts.NamespaceSize.
 	// Repeated here to avoid a dependency to the wrapping repo as this makes
 	// it hard to compile and use the plugin against a local ipfs version.
 	// TODO: plugins have config options; make this configurable instead
@@ -94,7 +94,7 @@ func sumSha256Namespace8Flagged(data []byte, _length int) ([]byte, error) {
 // <share_0>| ... |<share_numOfShares - 1>
 //
 // To determine the share and the namespace size the constants
-// types.ShareSize and types.NamespaceSize are redefined here to avoid
+// types.ShareSize and consts.NamespaceSize are redefined here to avoid
 // lazyledger-core as a dependency.
 //
 // Note while this coredag.DagParser is implemented here so this plugin can be used from

p2p/ipld/read.go

@@ -12,7 +12,9 @@ import (
 	"github.com/lazyledger/rsmt2d"
 
 	"github.com/lazyledger/lazyledger-core/ipfs/plugin"
+	"github.com/lazyledger/lazyledger-core/p2p/ipld/wrapper"
 	"github.com/lazyledger/lazyledger-core/types"
+	"github.com/lazyledger/lazyledger-core/types/consts"
 )
 
 const baseErrorMsg = "failure to retrieve block data:"
@@ -58,7 +60,7 @@ func RetrieveBlockData(
 	// flatten the square
 	flattened := sc.flatten()
 
-	tree := NewErasuredNamespacedMerkleTree(uint64(edsWidth) / 2)
+	tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(edsWidth) / 2)
 
 	// repair the square
 	eds, err := rsmt2d.RepairExtendedDataSquare(rowRoots, colRoots, flattened, codec, tree.Constructor)
@@ -159,7 +161,7 @@ func (sc *shareCounter) retrieveShare(
 		}
 	}
 
-	if len(data) < types.ShareSize {
+	if len(data) < consts.ShareSize {
 		return
 	}
 
@@ -174,7 +176,7 @@ func (sc *shareCounter) retrieveShare(
 	select {
 	case <-sc.ctx.Done():
 	default:
-		sc.shareChan <- indexedShare{data: data[types.NamespaceSize:], index: index{row: rowIdx, col: colIdx}}
+		sc.shareChan <- indexedShare{data: data[consts.NamespaceSize:], index: index{row: rowIdx, col: colIdx}}
 	}
 }
 

p2p/ipld/read_test.go

@@ -25,7 +25,9 @@ import (
 
 	"github.com/lazyledger/lazyledger-core/ipfs/plugin"
 	"github.com/lazyledger/lazyledger-core/p2p/ipld/racedetector"
+	"github.com/lazyledger/lazyledger-core/p2p/ipld/wrapper"
 	"github.com/lazyledger/lazyledger-core/types"
+	"github.com/lazyledger/lazyledger-core/types/consts"
 )
 
 func TestLeafPath(t *testing.T) {
@@ -112,7 +114,7 @@ func TestGetLeafData(t *testing.T) {
 	batch := format.NewBatch(ctx, ipfsAPI.Dag())
 
 	// generate random data for the nmt
-	data := generateRandNamespacedRawData(16, types.NamespaceSize, types.ShareSize)
+	data := generateRandNamespacedRawData(16, consts.NamespaceSize, consts.ShareSize)
 
 	// create a random tree
 	root, err := getNmtRoot(ctx, batch, data)
@@ -150,16 +152,16 @@ func TestGetLeafData(t *testing.T) {
 
 func TestBlockRecovery(t *testing.T) {
 	// adjustedLeafSize describes the size of a leaf that will not get split
-	adjustedLeafSize := types.MsgShareSize
+	adjustedLeafSize := consts.MsgShareSize
 
 	originalSquareWidth := 8
 	shareCount := originalSquareWidth * originalSquareWidth
 	extendedSquareWidth := 2 * originalSquareWidth
 	extendedShareCount := extendedSquareWidth * extendedSquareWidth
 
 	// generate test data
-	quarterShares := generateRandNamespacedRawData(shareCount, types.NamespaceSize, adjustedLeafSize)
-	allShares := generateRandNamespacedRawData(shareCount, types.NamespaceSize, adjustedLeafSize)
+	quarterShares := generateRandNamespacedRawData(shareCount, consts.NamespaceSize, adjustedLeafSize)
+	allShares := generateRandNamespacedRawData(shareCount, consts.NamespaceSize, adjustedLeafSize)
 
 	testCases := []struct {
 		name string
@@ -181,8 +183,8 @@ func TestBlockRecovery(t *testing.T) {
 			squareSize := uint64(math.Sqrt(float64(len(tc.shares))))
 
 			// create trees for creating roots
-			tree := NewErasuredNamespacedMerkleTree(squareSize)
-			recoverTree := NewErasuredNamespacedMerkleTree(squareSize)
+			tree := wrapper.NewErasuredNamespacedMerkleTree(squareSize)
+			recoverTree := wrapper.NewErasuredNamespacedMerkleTree(squareSize)
 
 			eds, err := rsmt2d.ComputeExtendedDataSquare(tc.shares, rsmt2d.NewRSGF8Codec(), tree.Constructor)
 			if err != nil {
@@ -230,14 +232,14 @@ func TestRetrieveBlockData(t *testing.T) {
 	ipfsAPI := mockedIpfsAPI(t)
 
 	// the max size of messages that won't get split
-	adjustedMsgSize := types.MsgShareSize - 2
+	adjustedMsgSize := consts.MsgShareSize - 2
 
 	tests := []test{
 		{"Empty block", 1, false, ""},
 		{"4 KB block", 4, false, ""},
 		{"16 KB block", 8, false, ""},
 		{"16 KB block timeout expected", 8, true, "timeout"},
-		{"max square size", types.MaxSquareSize, false, ""},
+		{"max square size", consts.MaxSquareSize, false, ""},
 	}
 
 	for _, tc := range tests {
@@ -268,7 +270,7 @@ func TestRetrieveBlockData(t *testing.T) {
 			shareData, _ := blockData.ComputeShares()
 			rawData := shareData.RawShares()
 
-			tree := NewErasuredNamespacedMerkleTree(uint64(tc.squareSize))
+			tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(tc.squareSize))
 			eds, err := rsmt2d.ComputeExtendedDataSquare(rawData, rsmt2d.NewRSGF8Codec(), tree.Constructor)
 			if err != nil {
 				t.Fatal(err)
@@ -327,7 +329,7 @@ func getNmtRoot(
 	namespacedData [][]byte,
 ) (namespace.IntervalDigest, error) {
 	na := NewNmtNodeAdder(ctx, batch)
-	tree := nmt.New(sha256.New(), nmt.NamespaceIDSize(types.NamespaceSize), nmt.NodeVisitor(na.Visit))
+	tree := nmt.New(sha256.New(), nmt.NamespaceIDSize(consts.NamespaceSize), nmt.NodeVisitor(na.Visit))
 	for _, leaf := range namespacedData {
 		err := tree.Push(leaf)
 		if err != nil {
@@ -385,7 +387,7 @@ func removeRandShares(data [][]byte, d int) [][]byte {
 func rootsToDigests(roots [][]byte) []namespace.IntervalDigest {
 	out := make([]namespace.IntervalDigest, len(roots))
 	for i, root := range roots {
-		idigest, err := namespace.IntervalDigestFromBytes(types.NamespaceSize, root)
+		idigest, err := namespace.IntervalDigestFromBytes(consts.NamespaceSize, root)
 		if err != nil {
 			panic(err)
 		}
@@ -405,20 +407,20 @@ func generateRandomBlockData(msgCount, msgSize int) types.Data {
 }
 
 func generateRandomMessages(count, msgSize int) types.Messages {
-	shares := generateRandNamespacedRawData(count, types.NamespaceSize, msgSize)
+	shares := generateRandNamespacedRawData(count, consts.NamespaceSize, msgSize)
 	msgs := make([]types.Message, count)
 	for i, s := range shares {
 		msgs[i] = types.Message{
-			Data:        s[types.NamespaceSize:],
-			NamespaceID: s[:types.NamespaceSize],
+			Data:        s[consts.NamespaceSize:],
+			NamespaceID: s[:consts.NamespaceSize],
 		}
 	}
 	return types.Messages{MessagesList: msgs}
 }
 
 func generateRandomContiguousShares(count int) types.Txs {
 	// the size of a length delimited tx that takes up an entire share
-	const adjustedTxSize = types.TxShareSize - 2
+	const adjustedTxSize = consts.TxShareSize - 2
 	txs := make(types.Txs, count)
 	for i := 0; i < count; i++ {
 		tx := make([]byte, adjustedTxSize)

p2p/ipld/validate_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/lazyledger/lazyledger-core/types"
+	"github.com/lazyledger/lazyledger-core/types/consts"
 )
 
 // TODO(@Wondertan): Add test to simulate ErrValidationFailed
@@ -17,7 +18,7 @@ func TestValidateAvailability(t *testing.T) {
 	const (
 		shares          = 15
 		squareSize      = 8
-		adjustedMsgSize = types.MsgShareSize - 2
+		adjustedMsgSize = consts.MsgShareSize - 2
 	)
 
 	ctx, cancel := context.WithCancel(context.Background())

p2p/ipld/nmt_wrapper.go → p2p/ipld/wrapper/nmt_wrapper.go

@@ -1,4 +1,4 @@
-package ipld
+package wrapper
 
 import (
 	"bytes"
@@ -9,7 +9,7 @@ import (
 	"github.com/lazyledger/nmt/namespace"
 	"github.com/lazyledger/rsmt2d"
 
-	"github.com/lazyledger/lazyledger-core/types"
+	"github.com/lazyledger/lazyledger-core/types/consts"
 )
 
 // emptyNamepsaceID occurs when a share is empty and indicates that
@@ -55,7 +55,7 @@ func (w ErasuredNamespacedMerkleTree) Constructor() rsmt2d.Tree {
 // if the tree size is exceeded.
 func (w *ErasuredNamespacedMerkleTree) Push(data []byte, idx rsmt2d.SquareIndex) {
 	// determine the namespace based on where in the tree we're pushing
-	nsID := make(namespace.ID, types.NamespaceSize)
+	nsID := make(namespace.ID, consts.NamespaceSize)
 
 	if idx.Axis+1 > 2*uint(w.squareSize) || idx.Cell+1 > 2*uint(w.squareSize) {
 		panic(fmt.Sprintf("pushed past predetermined square size: boundary at %d index at %+v", 2*w.squareSize, idx))
@@ -65,17 +65,17 @@ func (w *ErasuredNamespacedMerkleTree) Push(data []byte, idx rsmt2d.SquareIndex)
 	// datasquare if the cell is empty it means we got an empty block so we need
 	// to use TailPaddingNamespaceID
 	if idx.Axis+1 > uint(w.squareSize) || idx.Cell+1 > uint(w.squareSize) {
-		copy(nsID, types.ParitySharesNamespaceID)
+		copy(nsID, consts.ParitySharesNamespaceID)
 	} else {
 		// empty shares use the TailPaddingNamespaceID if the data is empty, so
 		// here we check if the share is empty (namepsace == [0,0,0,0,0,0,0,0])
-		if bytes.Equal(data[:types.NamespaceSize], emptyNamespaceID) {
-			copy(nsID, types.TailPaddingNamespaceID)
+		if bytes.Equal(data[:consts.NamespaceSize], emptyNamespaceID) {
+			copy(nsID, consts.TailPaddingNamespaceID)
 		} else {
-			copy(nsID, data[:types.NamespaceSize])
+			copy(nsID, data[:consts.NamespaceSize])
 		}
 	}
-	nidAndData := append(append(make([]byte, 0, types.NamespaceSize+len(data)), nsID...), data...)
+	nidAndData := append(append(make([]byte, 0, consts.NamespaceSize+len(data)), nsID...), data...)
 	// push to the underlying tree
 	err := w.tree.Push(nidAndData)
 	// panic on error

p2p/ipld/nmt_wrapper_test.go → p2p/ipld/wrapper/nmt_wrapper_test.go

@@ -1,10 +1,13 @@
-package ipld
+package wrapper
 
 import (
+	"bytes"
+	"crypto/rand"
 	"crypto/sha256"
+	"sort"
 	"testing"
 
-	"github.com/lazyledger/lazyledger-core/types"
+	"github.com/lazyledger/lazyledger-core/types/consts"
 	"github.com/lazyledger/nmt"
 	"github.com/lazyledger/rsmt2d"
 	"github.com/stretchr/testify/assert"
@@ -36,7 +39,7 @@ func TestRootErasuredNamespacedMerkleTree(t *testing.T) {
 	// the case, because the ErasuredNamespacedMerkleTree should add namespaces
 	// to the second half of the tree
 	size := 8
-	data := generateRandNamespacedRawData(size, types.NamespaceSize, types.MsgShareSize)
+	data := generateRandNamespacedRawData(size, consts.NamespaceSize, consts.MsgShareSize)
 	n := NewErasuredNamespacedMerkleTree(uint64(size))
 	tree := n.Constructor()
 	nmtTree := nmt.New(sha256.New())
@@ -110,8 +113,8 @@ func TestExtendedDataSquare(t *testing.T) {
 	// data for a 4X4 square
 	raw := generateRandNamespacedRawData(
 		squareSize*squareSize,
-		types.NamespaceSize,
-		types.MsgShareSize,
+		consts.NamespaceSize,
+		consts.MsgShareSize,
 	)
 
 	tree := NewErasuredNamespacedMerkleTree(uint64(squareSize))
@@ -125,12 +128,41 @@ func TestExtendedDataSquare(t *testing.T) {
 func generateErasuredData(t *testing.T, numLeaves int, codec rsmt2d.Codec) [][]byte {
 	raw := generateRandNamespacedRawData(
 		numLeaves,
-		types.NamespaceSize,
-		types.MsgShareSize,
+		consts.NamespaceSize,
+		consts.MsgShareSize,
 	)
 	erasuredData, err := codec.Encode(raw)
 	if err != nil {
 		t.Error(err)
 	}
 	return append(raw, erasuredData...)
 }
+
+// this code is copy pasted from the plugin, and should likely be exported in the plugin instead
+func generateRandNamespacedRawData(total int, nidSize int, leafSize int) [][]byte {
+	data := make([][]byte, total)
+	for i := 0; i < total; i++ {
+		nid := make([]byte, nidSize)
+		_, err := rand.Read(nid)
+		if err != nil {
+			panic(err)
+		}
+		data[i] = nid
+	}
+
+	sortByteArrays(data)
+	for i := 0; i < total; i++ {
+		d := make([]byte, leafSize)
+		_, err := rand.Read(d)
+		if err != nil {
+			panic(err)
+		}
+		data[i] = append(data[i], d...)
+	}
+
+	return data
+}
+
+func sortByteArrays(src [][]byte) {
+	sort.Slice(src, func(i, j int) bool { return bytes.Compare(src[i], src[j]) < 0 })
+}

p2p/ipld/write.go

@@ -2,14 +2,15 @@ package ipld
 
 import (
 	"context"
-	"crypto/sha256"
 	"errors"
 	"fmt"
+	"math"
 
 	format "github.com/ipfs/go-ipld-format"
 	"github.com/lazyledger/nmt"
 	"github.com/lazyledger/rsmt2d"
 
+	"github.com/lazyledger/lazyledger-core/p2p/ipld/wrapper"
 	"github.com/lazyledger/lazyledger-core/types"
 )
 
@@ -29,31 +30,23 @@ func PutBlock(ctx context.Context, adder format.NodeAdder, block *types.Block) e
 		return nil
 	}
 
+	// create nmt adder wrapping batch adder
+	batchAdder := NewNmtNodeAdder(ctx, format.NewBatch(ctx, adder))
+
+	// create the nmt wrapper to generate row and col commitments
+	squareSize := uint32(math.Sqrt(float64(len(shares))))
+	tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(squareSize), nmt.NodeVisitor(batchAdder.Visit))
+
 	// recompute the eds
-	eds, err := rsmt2d.ComputeExtendedDataSquare(shares, rsmt2d.NewRSGF8Codec(), rsmt2d.NewDefaultTree)
+	eds, err := rsmt2d.ComputeExtendedDataSquare(shares, rsmt2d.NewRSGF8Codec(), tree.Constructor)
 	if err != nil {
 		return fmt.Errorf("failure to recompute the extended data square: %w", err)
 	}
 
-	// add namespaces to erasured shares and flatten the eds
-	leaves := types.FlattenNamespacedEDS(namespacedShares, eds)
-
-	// create nmt adder wrapping batch adder
-	batchAdder := NewNmtNodeAdder(ctx, format.NewBatch(ctx, adder))
-
-	// iterate through each set of col and row leaves
-	for _, leafSet := range leaves {
-		tree := nmt.New(sha256.New(), nmt.NodeVisitor(batchAdder.Visit))
-		for _, share := range leafSet {
-			err = tree.Push(share)
-			if err != nil {
-				return err
-			}
-		}
-
-		// compute the root in order to collect the ipld.Nodes
-		tree.Root()
-	}
+	// thanks to the batchAdder.Visit func we added to the nmt wrapper,
+	// generating the roots will start adding the data to IPFS
+	eds.RowRoots()
+	eds.ColumnRoots()
 
 	// commit the batch to ipfs
 	return batchAdder.Commit()