Feat/retrieve unsealed (#340)

* expose listing of cid infos and piece infos

* allow data to be retrieved from sectors before its fully sealed

* fixup types for piecestore and unsealing calls

* fix rebasea

* address review feedback

* feat(storagemarket): cleanup handling of record piece

* feat(storagemarket): add AvailableForRetrieval flag

* fix(lint): fix lint issues

Co-authored-by: hannahhoward <hannah@hannahhoward.net>

docs/retrievalclient.mmd

@@ -76,9 +76,6 @@ stateDiagram-v2
 	19 --> 15 : ClientEventComplete
 	8 --> 17 : <invalid Value>
 
-	note left of 3 : The following events only record in this state.<br><br>ClientEventLastPaymentRequested<br>ClientEventPaymentRequested<br>ClientEventAllBlocksReceived<br>ClientEventBlocksReceived
-
-
 	note left of 4 : The following events only record in this state.<br><br>ClientEventLastPaymentRequested<br>ClientEventPaymentRequested<br>ClientEventAllBlocksReceived<br>ClientEventBlocksReceived
 
 
@@ -87,3 +84,6 @@ stateDiagram-v2
 
 	note left of 6 : The following events only record in this state.<br><br>ClientEventLastPaymentRequested<br>ClientEventPaymentRequested<br>ClientEventAllBlocksReceived<br>ClientEventBlocksReceived
 
+
+	note left of 3 : The following events only record in this state.<br><br>ClientEventLastPaymentRequested<br>ClientEventPaymentRequested<br>ClientEventAllBlocksReceived<br>ClientEventBlocksReceived
+

docs/storageprovider.mmd

@@ -2,7 +2,7 @@ stateDiagram-v2
 	state "StorageDealUnknown" as 0
 	state "StorageDealStaged" as 4
 	state "StorageDealSealing" as 5
-	state "<invalid Value>" as 6
+	state "StorageDealFinalizing" as 6
 	state "StorageDealActive" as 7
 	state "StorageDealExpired" as 8
 	state "StorageDealSlashed" as 9
@@ -20,7 +20,7 @@ stateDiagram-v2
 	state "StorageDealError" as 26
 	4 : On entry runs HandoffDeal
 	5 : On entry runs VerifyDealActivated
-	6 : On entry runs RecordPieceInfo
+	6 : On entry runs CleanupDeal
 	7 : On entry runs WaitForDealCompletion
 	10 : On entry runs RejectDeal
 	11 : On entry runs FailDeal
@@ -67,10 +67,7 @@ stateDiagram-v2
 	4 --> 5 : ProviderEventDealHandedOff
 	5 --> 11 : ProviderEventDealActivationFailed
 	5 --> 6 : ProviderEventDealActivated
-	6 --> 11 : ProviderEventPieceStoreErrored
-	6 --> 11 : ProviderEventUnableToLocatePiece
-	6 --> 11 : ProviderEventReadMetadataErrored
-	6 --> 7 : ProviderEventPieceRecorded
+	6 --> 7 : ProviderEventCleanupFinished
 	7 --> 9 : ProviderEventDealSlashed
 	7 --> 8 : ProviderEventDealExpired
 	7 --> 26 : ProviderEventDealCompletionFailed
@@ -80,6 +77,9 @@ stateDiagram-v2
 	15 --> 26 : ProviderEventRestart
 	20 --> 11 : ProviderEventTrackFundsFailed
 
+	note left of 4 : The following events only record in this state.<br><br>ProviderEventPieceStoreErrored
+
+
 	note left of 20 : The following events only record in this state.<br><br>ProviderEventFundsReserved
 
 

piecestore/piecestore.go

@@ -59,6 +59,34 @@ func (ps *pieceStore) AddPieceBlockLocations(pieceCID cid.Cid, blockLocations ma
 	return nil
 }
 
+func (ps *pieceStore) ListPieceInfoKeys() ([]cid.Cid, error) {
+	var pis []PieceInfo
+	if err := ps.pieces.List(&pis); err != nil {
+		return nil, err
+	}
+
+	out := make([]cid.Cid, 0, len(pis))
+	for _, pi := range pis {
+		out = append(out, pi.PieceCID)
+	}
+
+	return out, nil
+}
+
+func (ps *pieceStore) ListCidInfoKeys() ([]cid.Cid, error) {
+	var cis []CIDInfo
+	if err := ps.cidInfos.List(&cis); err != nil {
+		return nil, err
+	}
+
+	out := make([]cid.Cid, 0, len(cis))
+	for _, ci := range cis {
+		out = append(out, ci.CID)
+	}
+
+	return out, nil
+}
+
 // Retrieve the PieceInfo associated with `pieceCID` from the piece info store.
 func (ps *pieceStore) GetPieceInfo(pieceCID cid.Cid) (PieceInfo, error) {
 	var out PieceInfo

piecestore/piecestore_test.go

@@ -29,9 +29,9 @@ func TestStorePieceInfo(t *testing.T) {
 		ps := initializePieceStore(t)
 		dealInfo := piecestore.DealInfo{
 			DealID:   abi.DealID(rand.Uint64()),
-			SectorID: rand.Uint64(),
-			Offset:   rand.Uint64(),
-			Length:   rand.Uint64(),
+			SectorID: abi.SectorNumber(rand.Uint64()),
+			Offset:   abi.PaddedPieceSize(rand.Uint64()),
+			Length:   abi.PaddedPieceSize(rand.Uint64()),
 		}
 		err := ps.AddDealForPiece(pieceCid, dealInfo)
 		assert.NoError(t, err)
@@ -46,9 +46,9 @@ func TestStorePieceInfo(t *testing.T) {
 		ps := initializePieceStore(t)
 		dealInfo := piecestore.DealInfo{
 			DealID:   abi.DealID(rand.Uint64()),
-			SectorID: rand.Uint64(),
-			Offset:   rand.Uint64(),
-			Length:   rand.Uint64(),
+			SectorID: abi.SectorNumber(rand.Uint64()),
+			Offset:   abi.PaddedPieceSize(rand.Uint64()),
+			Length:   abi.PaddedPieceSize(rand.Uint64()),
 		}
 		err := ps.AddDealForPiece(pieceCid, dealInfo)
 		assert.NoError(t, err)

piecestore/types.go

@@ -11,9 +11,9 @@ import (
 // DealInfo is information about a single deal for a given piece
 type DealInfo struct {
 	DealID   abi.DealID
-	SectorID uint64
-	Offset   uint64
-	Length   uint64
+	SectorID abi.SectorNumber
+	Offset   abi.PaddedPieceSize
+	Length   abi.PaddedPieceSize
 }
 
 // BlockLocation is information about where a given block is relative to the overall piece
@@ -55,4 +55,6 @@ type PieceStore interface {
 	AddPieceBlockLocations(pieceCID cid.Cid, blockLocations map[cid.Cid]BlockLocation) error
 	GetPieceInfo(pieceCID cid.Cid) (PieceInfo, error)
 	GetCIDInfo(payloadCID cid.Cid) (CIDInfo, error)
+	ListCidInfoKeys() ([]cid.Cid, error)
+	ListPieceInfoKeys() ([]cid.Cid, error)
 }

retrievalmarket/impl/integration_test.go

@@ -120,7 +120,7 @@ func requireSetupTestClientAndProvider(bgCtx context.Context, t *testing.T, payC
 		pieceStore.ExpectPiece(piece, piecestore.PieceInfo{
 			Deals: []piecestore.DealInfo{
 				{
-					Length: expectedQR.Size * uint64(i+1),
+					Length: abi.PaddedPieceSize(expectedQR.Size * uint64(i+1)),
 				},
 			},
 		})
@@ -135,8 +135,12 @@ func requireSetupTestClientAndProvider(bgCtx context.Context, t *testing.T, payC
 	provider, err := retrievalimpl.NewProvider(paymentAddress, providerNode, nw2, pieceStore, testData.MultiStore2, dt2, testData.Ds2)
 	require.NoError(t, err)
 
-	provider.SetPaymentInterval(expectedQR.MaxPaymentInterval, expectedQR.MaxPaymentIntervalIncrease)
-	provider.SetPricePerByte(expectedQR.MinPricePerByte)
+	ask := provider.GetAsk()
+	ask.PaymentInterval = expectedQR.MaxPaymentInterval
+	ask.PaymentIntervalIncrease = expectedQR.MaxPaymentIntervalIncrease
+	ask.PricePerByte = expectedQR.MinPricePerByte
+	ask.UnsealPrice = expectedQR.UnsealPrice
+	provider.SetAsk(ask)
 	require.NoError(t, provider.Start())
 
 	retrievalPeer := retrievalmarket.RetrievalPeer{
@@ -268,19 +272,19 @@ func TestClientCanMakeDealWithProvider(t *testing.T) {
 			err = cio.WriteCar(bgCtx, store.Bstore, payloadCID, shared.AllSelector(), &buf)
 			require.NoError(t, err)
 			carData := buf.Bytes()
-			sectorID := uint64(100000)
-			offset := uint64(1000)
+			sectorID := abi.SectorNumber(100000)
+			offset := abi.PaddedPieceSize(1000)
 			pieceInfo = piecestore.PieceInfo{
 				PieceCID: tut.GenerateCids(1)[0],
 				Deals: []piecestore.DealInfo{
 					{
 						SectorID: sectorID,
 						Offset:   offset,
-						Length:   uint64(len(carData)),
+						Length:   abi.UnpaddedPieceSize(len(carData)).Padded(),
 					},
 				},
 			}
-			providerNode.ExpectUnseal(sectorID, offset, uint64(len(carData)), carData)
+			providerNode.ExpectUnseal(sectorID, offset.Unpadded(), abi.UnpaddedPieceSize(len(carData)), carData)
 			// clearout provider blockstore
 			err = testData.MultiStore2.Delete(storeID)
 			require.NoError(t, err)
@@ -507,9 +511,15 @@ func setupProvider(
 		pieceStore, testData.MultiStore2, dt2, testData.Ds2,
 		retrievalimpl.DealDeciderOpt(decider))
 	require.NoError(t, err)
-	provider.SetPaymentInterval(expectedQR.MaxPaymentInterval, expectedQR.MaxPaymentIntervalIncrease)
-	provider.SetPricePerByte(expectedQR.MinPricePerByte)
-	provider.SetPricePerUnseal(expectedQR.UnsealPrice)
+
+	ask := provider.GetAsk()
+
+	ask.PaymentInterval = expectedQR.MaxPaymentInterval
+	ask.PaymentIntervalIncrease = expectedQR.MaxPaymentIntervalIncrease
+	ask.PricePerByte = expectedQR.MinPricePerByte
+	ask.UnsealPrice = expectedQR.UnsealPrice
+	provider.SetAsk(ask)
+
 	require.NoError(t, provider.Start())
 	return provider
 }