Nonblocking storage deals [#80] (#194)

* Add more states in Client and Provider FSM representing async ops:

- Waiting for storage market funds to appear
- Waiting for deals to be published

* Publishing deals doesn't block

* WaitForFunding state tests for Client, Provider

* Remove deal id from the provider node api PublishDeals

shared_testutil/test_types.go

@@ -113,15 +113,18 @@ func MakeTestDealPayment() retrievalmarket.DealPayment {
 
 // MakeTestUnsignedDealProposal generates a deal proposal with no signature
 func MakeTestUnsignedDealProposal() market.DealProposal {
+	start := uint64(rand.Int31())
+	end := start + uint64(rand.Int31())
+
 	return market.DealProposal{
 		PieceCID:  GenerateCids(1)[0],
 		PieceSize: abi.PaddedPieceSize(rand.Int63()),
 
 		Client:   address.TestAddress,
 		Provider: address.TestAddress2,
 
-		StartEpoch: abi.ChainEpoch(rand.Int63()),
-		EndEpoch:   abi.ChainEpoch(rand.Int63()),
+		StartEpoch: abi.ChainEpoch(start),
+		EndEpoch:   abi.ChainEpoch(end),
 
 		StoragePricePerEpoch: MakeTestTokenAmount(),
 		ProviderCollateral:   MakeTestTokenAmount(),

storagemarket/impl/client.go

@@ -11,6 +11,7 @@ import (
 	"github.com/filecoin-project/specs-actors/actors/abi"
 	"github.com/filecoin-project/specs-actors/actors/abi/big"
 	"github.com/filecoin-project/specs-actors/actors/builtin/market"
+	"github.com/filecoin-project/specs-actors/actors/runtime/exitcode"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-datastore"
 	blockstore "github.com/ipfs/go-ipfs-blockstore"
@@ -275,7 +276,27 @@ func (c *Client) GetPaymentEscrow(ctx context.Context, addr address.Address) (st
 }
 
 func (c *Client) AddPaymentEscrow(ctx context.Context, addr address.Address, amount abi.TokenAmount) error {
-	return c.node.AddFunds(ctx, addr, amount)
+	done := make(chan error)
+
+	mcid, err := c.node.AddFunds(ctx, addr, amount)
+	if err != nil {
+		return err
+	}
+
+	err = c.node.WaitForMessage(mcid, storagemarket.ChainConfidence, func(code exitcode.ExitCode, bytes []byte) error {
+		if code == exitcode.Ok {
+			done <- nil
+		} else {
+			done <- xerrors.Errorf("AddFunds error, exit code: %w", code)
+		}
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+
+	return <-done
 }
 
 type clientDealEnvironment struct {

storagemarket/impl/clientstates/client_fsm.go

@@ -12,15 +12,21 @@ import (
 // ClientEvents are the events that can happen in a storage client
 var ClientEvents = fsm.Events{
 	fsm.Event(storagemarket.ClientEventOpen).
-		From(storagemarket.StorageDealUnknown).ToNoChange(),
+		From(storagemarket.StorageDealUnknown).To(storagemarket.StorageDealEnsureClientFunds),
+	fsm.Event(storagemarket.ClientEventFundingInitiated).
+		From(storagemarket.StorageDealEnsureClientFunds).To(storagemarket.StorageDealClientFunding).
+		Action(func(deal *storagemarket.ClientDeal, mcid cid.Cid) error {
+			deal.AddFundsCid = mcid
+			return nil
+		}),
 	fsm.Event(storagemarket.ClientEventEnsureFundsFailed).
-		From(storagemarket.StorageDealUnknown).To(storagemarket.StorageDealFailing).
+		FromMany(storagemarket.StorageDealClientFunding, storagemarket.StorageDealEnsureClientFunds).To(storagemarket.StorageDealFailing).
 		Action(func(deal *storagemarket.ClientDeal, err error) error {
 			deal.Message = xerrors.Errorf("adding market funds failed: %w", err).Error()
 			return nil
 		}),
 	fsm.Event(storagemarket.ClientEventFundsEnsured).
-		From(storagemarket.StorageDealUnknown).To(storagemarket.StorageDealFundsEnsured),
+		FromMany(storagemarket.StorageDealEnsureClientFunds, storagemarket.StorageDealClientFunding).To(storagemarket.StorageDealFundsEnsured),
 	fsm.Event(storagemarket.ClientEventWriteProposalFailed).
 		From(storagemarket.StorageDealFundsEnsured).To(storagemarket.StorageDealError).
 		Action(func(deal *storagemarket.ClientDeal, err error) error {
@@ -97,10 +103,11 @@ var ClientEvents = fsm.Events{
 
 // ClientStateEntryFuncs are the handlers for different states in a storage client
 var ClientStateEntryFuncs = fsm.StateEntryFuncs{
-	storagemarket.StorageDealUnknown:          EnsureFunds,
-	storagemarket.StorageDealFundsEnsured:     ProposeDeal,
-	storagemarket.StorageDealValidating:       VerifyDealResponse,
-	storagemarket.StorageDealProposalAccepted: ValidateDealPublished,
-	storagemarket.StorageDealSealing:          VerifyDealActivated,
-	storagemarket.StorageDealFailing:          FailDeal,
+	storagemarket.StorageDealEnsureClientFunds: EnsureClientFunds,
+	storagemarket.StorageDealClientFunding:     WaitForFunding,
+	storagemarket.StorageDealFundsEnsured:      ProposeDeal,
+	storagemarket.StorageDealValidating:        VerifyDealResponse,
+	storagemarket.StorageDealProposalAccepted:  ValidateDealPublished,
+	storagemarket.StorageDealSealing:           VerifyDealActivated,
+	storagemarket.StorageDealFailing:           FailDeal,
 }

storagemarket/impl/clientstates/client_states.go

@@ -2,8 +2,10 @@ package clientstates
 
 import (
 	"github.com/filecoin-project/go-statemachine/fsm"
+	"github.com/filecoin-project/specs-actors/actors/runtime/exitcode"
 	"github.com/ipfs/go-cid"
 	logging "github.com/ipfs/go-log/v2"
+	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-fil-markets/storagemarket"
 	"github.com/filecoin-project/go-fil-markets/storagemarket/impl/clientutils"
@@ -24,19 +26,39 @@ type ClientDealEnvironment interface {
 // ClientStateEntryFunc is the type for all state entry functions on a storage client
 type ClientStateEntryFunc func(ctx fsm.Context, environment ClientDealEnvironment, deal storagemarket.ClientDeal) error
 
-// EnsureFunds attempts to ensure the client has enough funds for the deal being proposed
-func EnsureFunds(ctx fsm.Context, environment ClientDealEnvironment, deal storagemarket.ClientDeal) error {
-	tok, _, err := environment.Node().GetChainHead(ctx.Context())
+// EnsureClientFunds attempts to ensure the client has enough funds for the deal being proposed
+func EnsureClientFunds(ctx fsm.Context, environment ClientDealEnvironment, deal storagemarket.ClientDeal) error {
+	node := environment.Node()
+
+	tok, _, err := node.GetChainHead(ctx.Context())
 	if err != nil {
-		return ctx.Trigger(storagemarket.ClientEventEnsureFundsFailed, err)
+		return ctx.Trigger(storagemarket.ClientEventEnsureFundsFailed, xerrors.Errorf("acquiring chain head: %w", err))
 	}
 
-	if err := environment.Node().EnsureFunds(
-		ctx.Context(), deal.Proposal.Client, deal.Proposal.Client, deal.Proposal.ClientBalanceRequirement(), tok); err != nil {
+	mcid, err := node.EnsureFunds(ctx.Context(), deal.Proposal.Client, deal.Proposal.Client, deal.Proposal.ClientBalanceRequirement(), tok)
+
+	if err != nil {
 		return ctx.Trigger(storagemarket.ClientEventEnsureFundsFailed, err)
 	}
 
-	return ctx.Trigger(storagemarket.ClientEventFundsEnsured)
+	// if no message was sent, and there was no error, funds were already available
+	if mcid == cid.Undef {
+		return ctx.Trigger(storagemarket.ClientEventFundsEnsured)
+	}
+
+	return ctx.Trigger(storagemarket.ClientEventFundingInitiated, mcid)
+}
+
+// WaitForFunding waits for an AddFunds message to appear on the chain
+func WaitForFunding(ctx fsm.Context, environment ClientDealEnvironment, deal storagemarket.ClientDeal) error {
+	node := environment.Node()
+
+	return node.WaitForMessage(deal.AddFundsCid, storagemarket.ChainConfidence, func(code exitcode.ExitCode, bytes []byte) error {
+		if code == exitcode.Ok {
+			return ctx.Trigger(storagemarket.ClientEventFundsEnsured)
+		}
+		return ctx.Trigger(storagemarket.ClientEventEnsureFundsFailed, xerrors.Errorf("AddFunds exit code: %w", code))
+	})
 }
 
 // ProposeDeal sends the deal proposal to the provider