feat: support parsing out of context compact shares

pkg/shares/compact_shares_test.go

@@ -112,6 +112,23 @@ func Test_processCompactShares(t *testing.T) {
 	}
 }
 
+func TestParseOutOfContextShares(t *testing.T) {
+	txs := testfactory.GenerateRandomlySizedTxs(1000, 150)
+	txShares, _, _, err := SplitTxs(txs)
+	require.NoError(t, err)
+
+	for i := 0; i < 1000; i++ {
+		start, length := testfactory.GetRandomSlice(len(txShares))
+
+		rawResTxs, err := parseCompactShares(txShares[start:start+length], appconsts.SupportedShareVersions)
+		require.NoError(t, err)
+
+		resTxs := coretypes.ToTxs(rawResTxs)
+		require.NoError(t, err)
+		assert.True(t, testfactory.CheckSubArray(txs, resTxs))
+	}
+}
+
 func TestCompactShareContainsInfoByte(t *testing.T) {
 	css := NewCompactShareSplitter(appns.TxNamespace, appconsts.ShareVersionZero)
 	txs := testfactory.GenerateRandomTxs(1, appconsts.ContinuationCompactShareContentSize/4)
@@ -178,10 +195,6 @@ func Test_parseCompactSharesErrors(t *testing.T) {
 	}
 
 	testCases := []testCase{
-		{
-			"share with start indicator false",
-			txShares[1:], // set the first share to the second share which has the start indicator set to false
-		},
 		{
 			"share with unsupported share version",
 			[]Share{*shareWithUnsupportedShareVersion},

pkg/shares/parse_compact_shares.go

@@ -1,7 +1,5 @@
 package shares
 
-import "errors"
-
 // parseCompactShares returns data (transactions or intermediate state roots
 // based on the contents of rawShares and supportedShareVersions. If rawShares
 // contains a share with a version that isn't present in supportedShareVersions,
@@ -13,14 +11,6 @@ func parseCompactShares(shares []Share, supportedShareVersions []uint8) (data []
 		return nil, nil
 	}
 
-	seqStart, err := shares[0].IsSequenceStart()
-	if err != nil {
-		return nil, err
-	}
-	if !seqStart {
-		return nil, errors.New("first share is not the start of a sequence")
-	}
-
 	err = validateShareVersions(shares, supportedShareVersions)
 	if err != nil {
 		return nil, err
@@ -61,7 +51,7 @@ func parseRawData(rawData []byte) (units [][]byte, err error) {
 		if err != nil {
 			return nil, err
 		}
-		if unitLen == 0 {
+		if unitLen == 0 || unitLen > uint64(len(actualData)) {
 			return units, nil
 		}
 		rawData = actualData[unitLen:]
@@ -73,7 +63,12 @@ func parseRawData(rawData []byte) (units [][]byte, err error) {
 // not contain the namespace ID, info byte, sequence length, or reserved bytes.
 func extractRawData(shares []Share) (rawData []byte, err error) {
 	for i := 0; i < len(shares); i++ {
-		raw, err := shares[i].RawData()
+		var raw []byte
+		if i == 0 {
+			raw, err = shares[i].RawDataUsingReserved()
+		} else {
+			raw, err = shares[i].RawData()
+		}
 		if err != nil {
 			return nil, err
 		}

pkg/shares/shares.go

@@ -191,6 +191,53 @@ func (s *Share) rawDataStartIndex() int {
 	return index
 }
 
+// RawDataWithReserved returns the raw share data while taking reserved bytes into account.
+func (s *Share) RawDataUsingReserved() (rawData []byte, err error) {
+	rawDataStartIndexUsingReserved, err := s.rawDataStartIndexUsingReserved()
+	if err != nil {
+		return nil, err
+	}
+
+	// This means share is the last share and does not have any transaction beginning in it
+	if rawDataStartIndexUsingReserved == 0 {
+		return []byte{}, nil
+	}
+	if len(s.data) < rawDataStartIndexUsingReserved {
+		return rawData, fmt.Errorf("share %s is too short to contain raw data", s)
+	}
+
+	return s.data[rawDataStartIndexUsingReserved:], nil
+}
+
+func (s *Share) rawDataStartIndexUsingReserved() (int, error) {
+	isStart, err := s.IsSequenceStart()
+	if err != nil {
+		panic(err)
+	}
+	isCompact, err := s.IsCompactShare()
+	if err != nil {
+		panic(err)
+	}
+
+	index := appconsts.NamespaceSize + appconsts.ShareInfoBytes
+	if isStart {
+		index += appconsts.SequenceLenBytes
+	}
+
+	if !isStart && isCompact {
+		reservedBytes, err := ParseReservedBytes(s.data[index : index+appconsts.CompactShareReservedBytes])
+		if err != nil {
+			return 0, err
+		}
+		return int(reservedBytes), nil
+	}
+
+	if isCompact {
+		index += appconsts.CompactShareReservedBytes
+	}
+	return index, nil
+}
+
 func ToBytes(shares []Share) (bytes [][]byte) {
 	bytes = make([][]byte, len(shares))
 	for i, share := range shares {

test/util/testfactory/txs.go

@@ -1,6 +1,7 @@
 package testfactory
 
 import (
+	"bytes"
 	crand "crypto/rand"
 	"math/rand"
 
@@ -31,3 +32,27 @@ func GenerateRandomTxs(count, size int) types.Txs {
 	}
 	return txs
 }
+
+func GetRandomSlice(size int) (start int, length int) {
+	length = rand.Intn(size + 1)
+	start = rand.Intn(size - length + 1)
+	return
+}
+
+// Returns whether subTxList is a subarray of txList
+func CheckSubArray(txList []types.Tx, subTxList []types.Tx) bool {
+	for i := 0; i <= len(txList)-len(subTxList); i++ {
+		j := 0
+		for j = 0; j < len(subTxList); j++ {
+			tx := txList[i+j]
+			subTx := subTxList[j]
+			if !bytes.Equal([]byte(tx), []byte(subTx)) {
+				break
+			}
+		}
+		if j == len(subTxList) {
+			return true
+		}
+	}
+	return false
+}