feat: support parsing out of context compact shares

pkg/shares/compact_shares_test.go

@@ -112,6 +112,48 @@ func Test_processCompactShares(t *testing.T) {
 	}
 }
 
+func TestAllSplit(t *testing.T) {
+	txs := testfactory.GenerateRandomlySizedTxs(1000, 150)
+	txShares, _, _, err := SplitTxs(txs)
+	require.NoError(t, err)
+	resTxs, err := ParseTxs(txShares)
+	require.NoError(t, err)
+	assert.Equal(t, resTxs, txs)
+}
+
+func TestParseRandomOutOfContextShares(t *testing.T) {
+	txs := testfactory.GenerateRandomlySizedTxs(1000, 150)
+	txShares, _, _, err := SplitTxs(txs)
+	require.NoError(t, err)
+
+	for i := 0; i < 1000; i++ {
+		start, length := testfactory.GetRandomSubSlice(len(txShares))
+		randomRange := NewRange(start, start+length)
+		resTxs, err := ParseTxs(txShares[randomRange.Start:randomRange.End])
+		require.NoError(t, err)
+		assert.True(t, testfactory.CheckSubArray(txs, resTxs))
+	}
+}
+
+func TestParseOutOfContextSharesUsingShareRanges(t *testing.T) {
+	txs := testfactory.GenerateRandomlySizedTxs(1000, 150)
+	txShares, _, shareRanges, err := SplitTxs(txs)
+	require.NoError(t, err)
+
+	for key, r := range shareRanges {
+		resTxs, err := ParseTxs(txShares[r.Start:r.End])
+		require.NoError(t, err)
+		has := false
+		for _, tx := range resTxs {
+			if tx.Key() == key {
+				has = true
+				break
+			}
+		}
+		assert.True(t, has)
+	}
+}
+
 func TestCompactShareContainsInfoByte(t *testing.T) {
 	css := NewCompactShareSplitter(appns.TxNamespace, appconsts.ShareVersionZero)
 	txs := testfactory.GenerateRandomTxs(1, appconsts.ContinuationCompactShareContentSize/4)
@@ -178,10 +220,6 @@ func Test_parseCompactSharesErrors(t *testing.T) {
 	}
 
 	testCases := []testCase{
-		{
-			"share with start indicator false",
-			txShares[1:], // set the first share to the second share which has the start indicator set to false
-		},
 		{
 			"share with unsupported share version",
 			[]Share{*shareWithUnsupportedShareVersion},

pkg/shares/parse_compact_shares.go

@@ -1,7 +1,5 @@
 package shares
 
-import "errors"
-
 // parseCompactShares returns data (transactions or intermediate state roots
 // based on the contents of rawShares and supportedShareVersions. If rawShares
 // contains a share with a version that isn't present in supportedShareVersions,
@@ -13,14 +11,6 @@ func parseCompactShares(shares []Share, supportedShareVersions []uint8) (data []
 		return nil, nil
 	}
 
-	seqStart, err := shares[0].IsSequenceStart()
-	if err != nil {
-		return nil, err
-	}
-	if !seqStart {
-		return nil, errors.New("first share is not the start of a sequence")
-	}
-
 	err = validateShareVersions(shares, supportedShareVersions)
 	if err != nil {
 		return nil, err
@@ -61,19 +51,32 @@ func parseRawData(rawData []byte) (units [][]byte, err error) {
 		if err != nil {
 			return nil, err
 		}
+		// the rest of raw data is padding
 		if unitLen == 0 {
 			return units, nil
 		}
+		// the rest of actual data contains only part of the next transaction so
+		// we stop parsing raw data
+		if unitLen > uint64(len(actualData)) {
+			return units, nil
+		}
 		rawData = actualData[unitLen:]
 		units = append(units, actualData[:unitLen])
 	}
 }
 
-// extractRawData returns the raw data contained in the shares. The raw data does
-// not contain the namespace ID, info byte, sequence length, or reserved bytes.
+// extractRawData returns the raw data representing complete transactions
+// contained in the shares. The raw data does not contain the namespace, info
+// byte, sequence length, or reserved bytes. Starts reading raw data based on
+// the reserved bytes in the first share.
 func extractRawData(shares []Share) (rawData []byte, err error) {
 	for i := 0; i < len(shares); i++ {
-		raw, err := shares[i].RawData()
+		var raw []byte
+		if i == 0 {
+			raw, err = shares[i].RawDataUsingReserved()
+		} else {
+			raw, err = shares[i].RawData()
+		}
 		if err != nil {
 			return nil, err
 		}

pkg/shares/shares.go

@@ -191,6 +191,51 @@ func (s *Share) rawDataStartIndex() int {
 	return index
 }
 
+// RawDataWithReserved returns the raw share data while taking reserved bytes into account.
+func (s *Share) RawDataUsingReserved() (rawData []byte, err error) {
+	rawDataStartIndexUsingReserved, err := s.rawDataStartIndexUsingReserved()
+	if err != nil {
+		return nil, err
+	}
+
+	// This means share is the last share and does not have any transaction beginning in it
+	if rawDataStartIndexUsingReserved == 0 {
+		return []byte{}, nil
+	}
+	if len(s.data) < rawDataStartIndexUsingReserved {
+		return rawData, fmt.Errorf("share %s is too short to contain raw data", s)
+	}
+
+	return s.data[rawDataStartIndexUsingReserved:], nil
+}
+
+// rawDataStartIndexUsingReserved returns the start index of raw data while accounting for
+// reserved bytes, if it exists in the share.
+func (s *Share) rawDataStartIndexUsingReserved() (int, error) {
+	isStart, err := s.IsSequenceStart()
+	if err != nil {
+		return 0, err
+	}
+	isCompact, err := s.IsCompactShare()
+	if err != nil {
+		return 0, err
+	}
+
+	index := appconsts.NamespaceSize + appconsts.ShareInfoBytes
+	if isStart {
+		index += appconsts.SequenceLenBytes
+	}
+
+	if isCompact {
+		reservedBytes, err := ParseReservedBytes(s.data[index : index+appconsts.CompactShareReservedBytes])
+		if err != nil {
+			return 0, err
+		}
+		return int(reservedBytes), nil
+	}
+	return index, nil
+}
+
 func ToBytes(shares []Share) (bytes [][]byte) {
 	bytes = make([][]byte, len(shares))
 	for i, share := range shares {

test/util/testfactory/txs.go

@@ -1,6 +1,7 @@
 package testfactory
 
 import (
+	"bytes"
 	crand "crypto/rand"
 	"math/rand"
 
@@ -31,3 +32,28 @@ func GenerateRandomTxs(count, size int) types.Txs {
 	}
 	return txs
 }
+
+// GetRandomSubSlice returns two integers representing a randomly sized range in the interval [0, size]
+func GetRandomSubSlice(size int) (start int, length int) {
+	length = rand.Intn(size + 1)
+	start = rand.Intn(size - length + 1)
+	return start, length
+}
+
+// CheckSubArray returns whether subTxList is a subarray of txList
+func CheckSubArray(txList []types.Tx, subTxList []types.Tx) bool {
+	for i := 0; i <= len(txList)-len(subTxList); i++ {
+		j := 0
+		for j = 0; j < len(subTxList); j++ {
+			tx := txList[i+j]
+			subTx := subTxList[j]
+			if !bytes.Equal([]byte(tx), []byte(subTx)) {
+				break
+			}
+		}
+		if j == len(subTxList) {
+			return true
+		}
+	}
+	return false
+}