ipld: implement GetLeafData

use the ResolveNode method of the ipfs api instead of dag get

update CalcCIDPath tests

first draft of RetrieveBlockData implementation

isolate GetLeafData for its own PR

go mod tidy and remove unused functions

linter gods

go.mod

@@ -16,9 +16,11 @@ require (
 	github.com/gorilla/websocket v1.4.2
 	github.com/gtank/merlin v0.1.1
 	github.com/hdevalence/ed25519consensus v0.0.0-20201207055737-7fde80a9d5ff
+	github.com/ipfs/go-cid v0.0.7
 	github.com/ipfs/go-ipfs v0.8.0
 	github.com/ipfs/go-ipfs-config v0.12.0
 	github.com/ipfs/go-ipld-format v0.2.0
+	github.com/ipfs/go-path v0.0.8
 	github.com/ipfs/interface-go-ipfs-core v0.4.0
 	github.com/lazyledger/lazyledger-core/p2p/ipld/plugin v0.0.0-20210219190522-0eccfb24e2aa
 	github.com/lazyledger/nmt v0.2.0

p2p/ipld/read.go

@@ -0,0 +1,96 @@
+package ipld
+
+import (
+	"context"
+	"errors"
+	"math"
+
+	"github.com/ipfs/go-cid"
+	coreiface "github.com/ipfs/interface-go-ipfs-core"
+	"github.com/ipfs/interface-go-ipfs-core/path"
+)
+
+// /////////////////////////////////////
+//	Get Leaf Data
+// /////////////////////////////////////
+
+// GetLeafData fetches and returns the data for leaf leafIndex of root rootCid.
+// It stops and returns an error if the provided context is cancelled before
+// finishing
+func GetLeafData(
+	ctx context.Context,
+	rootCid cid.Cid,
+	leafIndex uint32,
+	totalLeafs uint32, // this corresponds to the extended square width
+	api coreiface.CoreAPI,
+) ([]byte, error) {
+	// calculate the path to the leaf
+	leafPath, err := calcCIDPath(leafIndex, totalLeafs)
+	if err != nil {
+		return nil, err
+	}
+
+	// use the root cid and the leafPath to create an ipld path
+	p := path.Join(path.IpldPath(rootCid), leafPath...)
+
+	// resolve the path
+	node, err := api.ResolveNode(ctx, p)
+	if err != nil {
+		return nil, err
+	}
+
+	// return the leaf, without the nmt-leaf-or-node byte
+	return node.RawData()[1:], nil
+}
+
+func calcCIDPath(index, total uint32) ([]string, error) {
+	// ensure that the total is a power of two
+	if total != nextPowerOf2(total) {
+		return nil, errors.New("expected total to be a power of 2")
+	}
+
+	if total == 0 {
+		return nil, nil
+	}
+
+	depth := int(math.Log2(float64(total)))
+	cursor := index
+	path := make([]string, depth)
+	for i := depth - 1; i >= 0; i-- {
+		if cursor%2 == 0 {
+			path[i] = "0"
+		} else {
+			path[i] = "1"
+		}
+		cursor /= 2
+	}
+
+	return path, nil
+}
+
+// nextPowerOf2 returns the next lowest power of 2 unless the input is a power
+// of two, in which case it returns the input
+func nextPowerOf2(v uint32) uint32 {
+	if v == 1 {
+		return 1
+	}
+	// keep track of the input
+	i := v
+
+	// find the next highest power using bit mashing
+	v--
+	v |= v >> 1
+	v |= v >> 2
+	v |= v >> 4
+	v |= v >> 8
+	v |= v >> 16
+	v++
+
+	// check if the input was the next highest power
+	if i == v {
+		return v
+	}
+
+	// return the next lowest power
+	return v / 2
+}

p2p/ipld/read_test.go

@@ -0,0 +1,201 @@
+package ipld
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	cid "github.com/ipfs/go-cid"
+	"github.com/ipfs/go-ipfs/core/coreapi"
+
+	coremock "github.com/ipfs/go-ipfs/core/mock"
+	format "github.com/ipfs/go-ipld-format"
+	"github.com/lazyledger/lazyledger-core/p2p/ipld/plugin/nodes"
+	"github.com/lazyledger/lazyledger-core/types"
+	"github.com/lazyledger/nmt"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCalcCIDPath(t *testing.T) {
+	type test struct {
+		name         string
+		index, total uint32
+		expected     []string
+	}
+
+	// test cases
+	tests := []test{
+		{"nil", 0, 0, []string(nil)},
+		{"0 index 16 total leaves", 0, 16, strings.Split("0/0/0/0", "/")},
+		{"1 index 16 total leaves", 1, 16, strings.Split("0/0/0/1", "/")},
+		{"9 index 16 total leaves", 9, 16, strings.Split("1/0/0/1", "/")},
+		{"15 index 16 total leaves", 15, 16, strings.Split("1/1/1/1", "/")},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			result, err := calcCIDPath(tt.index, tt.total)
+			if err != nil {
+				t.Error(err)
+			}
+			assert.Equal(t, tt.expected, result)
+		},
+		)
+	}
+}
+
+func TestNextPowerOf2(t *testing.T) {
+	type test struct {
+		input    uint32
+		expected uint32
+	}
+	tests := []test{
+		{
+			input:    2,
+			expected: 2,
+		},
+		{
+			input:    11,
+			expected: 8,
+		},
+		{
+			input:    511,
+			expected: 256,
+		},
+		{
+			input:    1,
+			expected: 1,
+		},
+		{
+			input:    0,
+			expected: 0,
+		},
+	}
+	for _, tt := range tests {
+		res := nextPowerOf2(tt.input)
+		assert.Equal(t, tt.expected, res)
+	}
+}
+
+func TestGetLeafData(t *testing.T) {
+	type test struct {
+		name    string
+		timeout time.Duration
+		rootCid cid.Cid
+		leaves  [][]byte
+	}
+
+	// create a mock node
+	ipfsNode, err := coremock.NewMockNode()
+	if err != nil {
+		t.Error(err)
+	}
+
+	// issue a new API object
+	ipfsAPI, err := coreapi.NewCoreAPI(ipfsNode)
+	if err != nil {
+		t.Error(err)
+	}
+
+	// create the context and batch needed for node collection from the tree
+	ctx := context.Background()
+	batch := format.NewBatch(ctx, ipfsAPI.Dag().Pinning())
+
+	// generate random data for the nmt
+	data := generateRandNamespacedRawData(16, types.NamespaceSize, types.ShareSize)
+
+	// create a random tree
+	tree, err := createNmtTree(ctx, batch, data)
+	if err != nil {
+		t.Error(err)
+	}
+
+	// calculate the root
+	root := tree.Root()
+
+	// commit the data to IPFS
+	err = batch.Commit()
+	if err != nil {
+		t.Error(err)
+	}
+
+	// compute the root and create a cid for the root hash
+	rootCid, err := nodes.CidFromNamespacedSha256(root.Bytes())
+	if err != nil {
+		t.Error(err)
+	}
+
+	// test cases
+	tests := []test{
+		{"16 leaves", time.Second, rootCid, data},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(context.Background(), tt.timeout)
+			defer cancel()
+			for i, leaf := range tt.leaves {
+				data, err := GetLeafData(ctx, tt.rootCid, uint32(i), uint32(len(tt.leaves)), ipfsAPI)
+				if err != nil {
+					t.Error(err)
+				}
+				assert.Equal(t, leaf, data)
+			}
+		},
+		)
+	}
+}
+
+// nmtcommitment generates the nmt root of some namespaced data
+func createNmtTree(
+	ctx context.Context,
+	batch *format.Batch,
+	namespacedData [][]byte,
+) (*nmt.NamespacedMerkleTree, error) {
+	na := nodes.NewNmtNodeAdder(ctx, batch)
+	tree := nmt.New(sha256.New(), nmt.NamespaceIDSize(types.NamespaceSize), nmt.NodeVisitor(na.Visit))
+	for _, leaf := range namespacedData {
+		err := tree.Push(leaf[:types.NamespaceSize], leaf[types.NamespaceSize:])
+		if err != nil {
+			return tree, err
+		}
+	}
+
+	return tree, nil
+}
+
+// this code is copy pasted from the plugin, and should likely be exported in the plugin instead
+func generateRandNamespacedRawData(total int, nidSize int, leafSize int) [][]byte {
+	data := make([][]byte, total)
+	for i := 0; i < total; i++ {
+		nid := make([]byte, nidSize)
+		_, err := rand.Read(nid)
+		if err != nil {
+			panic(err)
+		}
+		data[i] = nid
+	}
+
+	sortByteArrays(data)
+	for i := 0; i < total; i++ {
+		d := make([]byte, leafSize)
+		_, err := rand.Read(d)
+		if err != nil {
+			panic(err)
+		}
+		data[i] = append(data[i], d...)
+	}
+
+	return data
+}
+
+func sortByteArrays(src [][]byte) {
+	sort.Slice(src, func(i, j int) bool { return bytes.Compare(src[i], src[j]) < 0 })
+}