Implement GetLeafData as described in ADR 002

go.mod

@@ -16,9 +16,11 @@ require (
 	github.com/gorilla/websocket v1.4.2
 	github.com/gtank/merlin v0.1.1
 	github.com/hdevalence/ed25519consensus v0.0.0-20201207055737-7fde80a9d5ff
+	github.com/ipfs/go-cid v0.0.7
 	github.com/ipfs/go-ipfs v0.8.0
 	github.com/ipfs/go-ipfs-config v0.12.0
 	github.com/ipfs/go-ipld-format v0.2.0
+	github.com/ipfs/go-path v0.0.8
 	github.com/ipfs/interface-go-ipfs-core v0.4.0
 	github.com/lazyledger/lazyledger-core/p2p/ipld/plugin v0.0.0-20210219190522-0eccfb24e2aa
 	github.com/lazyledger/nmt v0.2.0

p2p/ipld/read.go

@@ -0,0 +1,96 @@
+package ipld
+
+import (
+	"context"
+	"errors"
+	"math"
+
+	"github.com/ipfs/go-cid"
+	coreiface "github.com/ipfs/interface-go-ipfs-core"
+	"github.com/ipfs/interface-go-ipfs-core/path"
+)
+
+// /////////////////////////////////////
+//	Get Leaf Data
+// /////////////////////////////////////
+
+// GetLeafData fetches and returns the data for leaf leafIndex of root rootCid.
+// It stops and returns an error if the provided context is cancelled before
+// finishing
+func GetLeafData(
+	ctx context.Context,
+	rootCid cid.Cid,
+	leafIndex uint32,
+	totalLeafs uint32, // this corresponds to the extended square width
+	api coreiface.CoreAPI,
+) ([]byte, error) {
+	// calculate the path to the leaf
+	leafPath, err := calcCIDPath(leafIndex, totalLeafs)
+	if err != nil {
+		return nil, err
+	}
+
+	// use the root cid and the leafPath to create an ipld path
+	p := path.Join(path.IpldPath(rootCid), leafPath...)
+
+	// resolve the path
+	node, err := api.ResolveNode(ctx, p)
+	if err != nil {
+		return nil, err
+	}
+
+	// return the leaf, without the nmt-leaf-or-node byte
+	return node.RawData()[1:], nil
+}
+
+func calcCIDPath(index, total uint32) ([]string, error) {
+	// ensure that the total is a power of two
+	if total != nextPowerOf2(total) {
+		return nil, errors.New("expected total to be a power of 2")
+	}
+
+	if total == 0 {
+		return nil, nil
+	}
+
+	depth := int(math.Log2(float64(total)))
+	cursor := index
+	path := make([]string, depth)
+	for i := depth - 1; i >= 0; i-- {
+		if cursor%2 == 0 {
+			path[i] = "0"
+		} else {
+			path[i] = "1"
+		}
+		cursor /= 2
+	}
+
+	return path, nil
+}
+
+// nextPowerOf2 returns the next lowest power of 2 unless the input is a power
+// of two, in which case it returns the input
+func nextPowerOf2(v uint32) uint32 {
+	if v == 1 {
+		return 1
+	}
+	// keep track of the input
+	i := v
+
+	// find the next highest power using bit mashing
+	v--
+	v |= v >> 1
+	v |= v >> 2
+	v |= v >> 4
+	v |= v >> 8
+	v |= v >> 16
+	v++
+
+	// check if the input was the next highest power
+	if i == v {
+		return v
+	}
+
+	// return the next lowest power
+	return v / 2
+}

p2p/ipld/read_test.go

@@ -0,0 +1,201 @@
+package ipld
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	cid "github.com/ipfs/go-cid"
+	"github.com/ipfs/go-ipfs/core/coreapi"
+
+	coremock "github.com/ipfs/go-ipfs/core/mock"
+	format "github.com/ipfs/go-ipld-format"
+	"github.com/lazyledger/lazyledger-core/p2p/ipld/plugin/nodes"
+	"github.com/lazyledger/lazyledger-core/types"
+	"github.com/lazyledger/nmt"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCalcCIDPath(t *testing.T) {
+	type test struct {
+		name         string
+		index, total uint32
+		expected     []string
+	}
+
+	// test cases
+	tests := []test{
+		{"nil", 0, 0, []string(nil)},
+		{"0 index 16 total leaves", 0, 16, strings.Split("0/0/0/0", "/")},
+		{"1 index 16 total leaves", 1, 16, strings.Split("0/0/0/1", "/")},
+		{"9 index 16 total leaves", 9, 16, strings.Split("1/0/0/1", "/")},
+		{"15 index 16 total leaves", 15, 16, strings.Split("1/1/1/1", "/")},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			result, err := calcCIDPath(tt.index, tt.total)
+			if err != nil {
+				t.Error(err)
+			}
+			assert.Equal(t, tt.expected, result)
+		},
+		)
+	}
+}
+
+func TestNextPowerOf2(t *testing.T) {
+	type test struct {
+		input    uint32
+		expected uint32
+	}
+	tests := []test{
+		{
+			input:    2,
+			expected: 2,
+		},
+		{
+			input:    11,
+			expected: 8,
+		},
+		{
+			input:    511,
+			expected: 256,
+		},
+		{
+			input:    1,
+			expected: 1,
+		},
+		{
+			input:    0,
+			expected: 0,
+		},
+	}
+	for _, tt := range tests {
+		res := nextPowerOf2(tt.input)
+		assert.Equal(t, tt.expected, res)
+	}
+}
+
+func TestGetLeafData(t *testing.T) {
+	type test struct {
+		name    string
+		timeout time.Duration
+		rootCid cid.Cid
+		leaves  [][]byte
+	}
+
+	// create a mock node
+	ipfsNode, err := coremock.NewMockNode()
+	if err != nil {
+		t.Error(err)
+	}
+
+	// issue a new API object
+	ipfsAPI, err := coreapi.NewCoreAPI(ipfsNode)
+	if err != nil {
+		t.Error(err)
+	}
+
+	// create the context and batch needed for node collection from the tree
+	ctx := context.Background()
+	batch := format.NewBatch(ctx, ipfsAPI.Dag().Pinning())
+
+	// generate random data for the nmt
+	data := generateRandNamespacedRawData(16, types.NamespaceSize, types.ShareSize)
+
+	// create a random tree
+	tree, err := createNmtTree(ctx, batch, data)
+	if err != nil {
+		t.Error(err)
+	}
+
+	// calculate the root
+	root := tree.Root()
+
+	// commit the data to IPFS
+	err = batch.Commit()
+	if err != nil {
+		t.Error(err)
+	}
+
+	// compute the root and create a cid for the root hash
+	rootCid, err := nodes.CidFromNamespacedSha256(root.Bytes())
+	if err != nil {
+		t.Error(err)
+	}
+
+	// test cases
+	tests := []test{
+		{"16 leaves", time.Second, rootCid, data},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(context.Background(), tt.timeout)
+			defer cancel()
+			for i, leaf := range tt.leaves {
+				data, err := GetLeafData(ctx, tt.rootCid, uint32(i), uint32(len(tt.leaves)), ipfsAPI)
+				if err != nil {
+					t.Error(err)
+				}
+				assert.Equal(t, leaf, data)
+			}
+		},
+		)
+	}
+}
+
+// nmtcommitment generates the nmt root of some namespaced data
+func createNmtTree(
+	ctx context.Context,
+	batch *format.Batch,
+	namespacedData [][]byte,
+) (*nmt.NamespacedMerkleTree, error) {
+	na := nodes.NewNmtNodeAdder(ctx, batch)
+	tree := nmt.New(sha256.New(), nmt.NamespaceIDSize(types.NamespaceSize), nmt.NodeVisitor(na.Visit))
+	for _, leaf := range namespacedData {
+		err := tree.Push(leaf[:types.NamespaceSize], leaf[types.NamespaceSize:])
+		if err != nil {
+			return tree, err
+		}
+	}
+
+	return tree, nil
+}
+
+// this code is copy pasted from the plugin, and should likely be exported in the plugin instead
+func generateRandNamespacedRawData(total int, nidSize int, leafSize int) [][]byte {
+	data := make([][]byte, total)
+	for i := 0; i < total; i++ {
+		nid := make([]byte, nidSize)
+		_, err := rand.Read(nid)
+		if err != nil {
+			panic(err)
+		}
+		data[i] = nid
+	}
+
+	sortByteArrays(data)
+	for i := 0; i < total; i++ {
+		d := make([]byte, leafSize)
+		_, err := rand.Read(d)
+		if err != nil {
+			panic(err)
+		}
+		data[i] = append(data[i], d...)
+	}
+
+	return data
+}
+
+func sortByteArrays(src [][]byte) {
+	sort.Slice(src, func(i, j int) bool { return bytes.Compare(src[i], src[j]) < 0 })
+}