From 67fca94b427620a2954b86d9e68c2021110d5952 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 6 Feb 2020 15:36:23 -0800 Subject: [PATCH 1/3] update interfaces to work with offline deal flow --- shared_testutil/test_types.go | 2 +- storagemarket/impl/client.go | 10 ++++----- storagemarket/impl/client_cbor_gen.go | 11 ---------- storagemarket/impl/client_storagemarket.go | 4 ++-- storagemarket/impl/client_utils.go | 5 +++-- storagemarket/impl/provider.go | 2 +- storagemarket/impl/provider_states.go | 11 +++++++--- storagemarket/impl/provider_utils.go | 5 +++-- storagemarket/impl/request_validation_test.go | 20 +++++++++-------- storagemarket/network/types.go | 2 +- storagemarket/network/types_cbor_gen.go | 11 ---------- storagemarket/types.go | 16 +++++++++++--- storagemarket/types_cbor_gen.go | 22 ------------------- 13 files changed, 48 insertions(+), 73 deletions(-) diff --git a/shared_testutil/test_types.go b/shared_testutil/test_types.go index 4f8b6452..7e989513 100644 --- a/shared_testutil/test_types.go +++ b/shared_testutil/test_types.go @@ -156,7 +156,7 @@ func MakeTestSignedStorageAsk() *types.SignedStorageAsk { func MakeTestStorageNetworkProposal() smnet.Proposal { return smnet.Proposal{ DealProposal: MakeTestStorageDealProposal(), - Piece: GenerateCids(1)[0], + Piece: &storagemarket.DataRef{Root: GenerateCids(1)[0]}, } } diff --git a/storagemarket/impl/client.go b/storagemarket/impl/client.go index 2c9258a0..4811c588 100644 --- a/storagemarket/impl/client.go +++ b/storagemarket/impl/client.go @@ -4,7 +4,6 @@ import ( "context" "github.com/filecoin-project/go-fil-markets/storagemarket/network" - "github.com/filecoin-project/go-data-transfer" cborutil "github.com/filecoin-project/go-cbor-util" "github.com/ipfs/go-cid" @@ -14,6 +13,7 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-address" + datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-fil-markets/filestore" "github.com/filecoin-project/go-fil-markets/pieceio" "github.com/filecoin-project/go-fil-markets/pieceio/cario" @@ -170,7 +170,7 @@ func (c *Client) onUpdated(ctx context.Context, update clientDealUpdate) { } type ClientDealProposal struct { - Data cid.Cid + Data *storagemarket.DataRef PricePerEpoch tokenamount.TokenAmount ProposalExpiration uint64 @@ -188,7 +188,7 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro return cid.Undef, xerrors.Errorf("adding market funds failed: %w", err) } - commP, pieceSize, err := c.commP(ctx, p.Data) + commP, pieceSize, err := c.commP(ctx, p.Data.Root) if err != nil { return cid.Undef, xerrors.Errorf("computing commP failed: %w", err) } @@ -230,7 +230,7 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro State: storagemarket.StorageDealUnknown, Miner: p.MinerID, MinerWorker: p.MinerWorker, - PayloadCid: p.Data, + DataRef: p.Data, }, s: s, @@ -238,7 +238,7 @@ func (c *Client) Start(ctx context.Context, p ClientDealProposal) (cid.Cid, erro c.incoming <- deal - return deal.ProposalCid, c.discovery.AddPeer(p.Data, retrievalmarket.RetrievalPeer{ + return deal.ProposalCid, c.discovery.AddPeer(p.Data.Root, retrievalmarket.RetrievalPeer{ Address: dealProposal.Provider, ID: deal.Miner, }) diff --git a/storagemarket/impl/client_cbor_gen.go b/storagemarket/impl/client_cbor_gen.go index 22aca1b0..0d2e939d 100644 --- a/storagemarket/impl/client_cbor_gen.go +++ b/storagemarket/impl/client_cbor_gen.go @@ -67,10 +67,6 @@ func (t *ClientDealProposal) MarshalCBOR(w io.Writer) error { // t.Data (cid.Cid) (struct) - if err := cbg.WriteCid(w, t.Data); err != nil { - return xerrors.Errorf("failed to write cid field t.Data: %w", err) - } - // t.PricePerEpoch (tokenamount.TokenAmount) (struct) if err := t.PricePerEpoch.MarshalCBOR(w); err != nil { return err @@ -134,13 +130,6 @@ func (t *ClientDealProposal) UnmarshalCBOR(r io.Reader) error { { - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Data: %w", err) - } - - t.Data = c - } // t.PricePerEpoch (tokenamount.TokenAmount) (struct) diff --git a/storagemarket/impl/client_storagemarket.go b/storagemarket/impl/client_storagemarket.go index 45d3b6af..68065359 100644 --- a/storagemarket/impl/client_storagemarket.go +++ b/storagemarket/impl/client_storagemarket.go @@ -82,10 +82,10 @@ func (c *Client) GetAsk(ctx context.Context, info storagemarket.StorageProviderI return c.QueryAsk(ctx, info.PeerID, info.Address) } -func (c *Client) ProposeStorageDeal(ctx context.Context, addr address.Address, info *storagemarket.StorageProviderInfo, payloadCid cid.Cid, proposalExpiration storagemarket.Epoch, duration storagemarket.Epoch, price tokenamount.TokenAmount, collateral tokenamount.TokenAmount) (*storagemarket.ProposeStorageDealResult, error) { +func (c *Client) ProposeStorageDeal(ctx context.Context, addr address.Address, info *storagemarket.StorageProviderInfo, data *storagemarket.DataRef, proposalExpiration storagemarket.Epoch, duration storagemarket.Epoch, price tokenamount.TokenAmount, collateral tokenamount.TokenAmount) (*storagemarket.ProposeStorageDealResult, error) { proposal := ClientDealProposal{ - Data: payloadCid, + Data: data, PricePerEpoch: price, ProposalExpiration: uint64(proposalExpiration), Duration: uint64(duration), diff --git a/storagemarket/impl/client_utils.go b/storagemarket/impl/client_utils.go index 9b8a6f43..b9d8010f 100644 --- a/storagemarket/impl/client_utils.go +++ b/storagemarket/impl/client_utils.go @@ -13,9 +13,9 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "golang.org/x/xerrors" + datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-fil-markets/storagemarket/network" "github.com/filecoin-project/go-statestore" - "github.com/filecoin-project/go-data-transfer" ) func (c *Client) failDeal(id cid.Cid, cerr error) { @@ -134,10 +134,11 @@ func (c *ClientRequestValidator) ValidatePull( if err != nil { return xerrors.Errorf("Proposal CID %s: %w", dealVoucher.Proposal.String(), ErrNoDeal) } + if deal.Miner != receiver { return xerrors.Errorf("Deal Peer %s, Data Transfer Peer %s: %w", deal.Miner.String(), receiver.String(), ErrWrongPeer) } - if !deal.PayloadCid.Equals(baseCid) { + if !deal.DataRef.Root.Equals(baseCid) { return xerrors.Errorf("Deal Payload CID %s, Data Transfer CID %s: %w", string(deal.Proposal.PieceRef), baseCid.String(), ErrWrongPiece) } for _, state := range DataTransferStates { diff --git a/storagemarket/impl/provider.go b/storagemarket/impl/provider.go index fc7e276e..b35cef07 100644 --- a/storagemarket/impl/provider.go +++ b/storagemarket/impl/provider.go @@ -13,6 +13,7 @@ import ( "github.com/filecoin-project/go-address" cborutil "github.com/filecoin-project/go-cbor-util" + datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-fil-markets/filestore" "github.com/filecoin-project/go-fil-markets/pieceio" "github.com/filecoin-project/go-fil-markets/pieceio/cario" @@ -22,7 +23,6 @@ import ( "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-fil-markets/storagemarket/network" "github.com/filecoin-project/go-statestore" - "github.com/filecoin-project/go-data-transfer" ) var ProviderDsPrefix = "/deals/provider" diff --git a/storagemarket/impl/provider_states.go b/storagemarket/impl/provider_states.go index c3051047..2959a03a 100644 --- a/storagemarket/impl/provider_states.go +++ b/storagemarket/impl/provider_states.go @@ -3,6 +3,7 @@ package storageimpl import ( "bytes" "context" + "fmt" "github.com/filecoin-project/go-padreader" "github.com/ipfs/go-cid" @@ -78,6 +79,10 @@ func (p *Provider) validating(ctx context.Context, deal MinerDeal) (func(*MinerD // State: StorageDealTransferring func (p *Provider) transferring(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { + if deal.Ref.TransferType == storagemarket.TTManual { + return nil, fmt.Errorf("attempted to graphsync a deal marked as manual transfer") + } + ssb := builder.NewSelectorSpecBuilder(ipldfree.NodeBuilder()) // this is the selector for "get the whole DAG" @@ -93,7 +98,7 @@ func (p *Provider) transferring(ctx context.Context, deal MinerDeal) (func(*Mine _, err := p.dataTransfer.OpenPullDataChannel(ctx, deal.Client, &StorageDataTransferVoucher{Proposal: deal.ProposalCid}, - deal.Ref, + deal.Ref.Root, allSelector, ) if err != nil { @@ -110,7 +115,7 @@ func (p *Provider) verifydata(ctx context.Context, deal MinerDeal) (func(*MinerD allSelector := ssb.ExploreRecursive(selector.RecursionLimitNone(), ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node() - commp, path, _, err := p.pio.GeneratePieceCommitmentToFile(deal.Ref, allSelector) + commp, path, _, err := p.pio.GeneratePieceCommitmentToFile(deal.Ref.Root, allSelector) if err != nil { return nil, err } @@ -225,7 +230,7 @@ func (p *Provider) complete(ctx context.Context, deal MinerDeal) (func(*MinerDea } // TODO: Record actual block locations for all CIDs in piece by improving car writing err = p.pieceStore.AddPieceBlockLocations(deal.Proposal.PieceRef, map[cid.Cid]piecestore.BlockLocation{ - deal.Ref: {}, + deal.Ref.Root: {}, }) if err != nil { return nil, err diff --git a/storagemarket/impl/provider_utils.go b/storagemarket/impl/provider_utils.go index a7e8c2b0..836e28c8 100644 --- a/storagemarket/impl/provider_utils.go +++ b/storagemarket/impl/provider_utils.go @@ -6,9 +6,9 @@ import ( "github.com/ipld/go-ipld-prime" + datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-fil-markets/storagemarket" "github.com/filecoin-project/go-fil-markets/storagemarket/network" - "github.com/filecoin-project/go-data-transfer" cborutil "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/go-statestore" @@ -139,6 +139,7 @@ func NewProviderRequestValidator(deals *statestore.StateStore) *ProviderRequestV // - referenced deal matches the client // - referenced deal matches the given base CID // - referenced deal is in an acceptable state +// TODO: maybe this should accept a dataref? func (m *ProviderRequestValidator) ValidatePush( sender peer.ID, voucher datatransfer.Voucher, @@ -158,7 +159,7 @@ func (m *ProviderRequestValidator) ValidatePush( return xerrors.Errorf("Deal Peer %s, Data Transfer Peer %s: %w", deal.Client.String(), sender.String(), ErrWrongPeer) } - if !deal.Ref.Equals(baseCid) { + if !deal.Ref.Root.Equals(baseCid) { return xerrors.Errorf("Deal Payload CID %s, Data Transfer CID %s: %w", string(deal.Proposal.PieceRef), baseCid.String(), ErrWrongPiece) } for _, state := range DataTransferStates { diff --git a/storagemarket/impl/request_validation_test.go b/storagemarket/impl/request_validation_test.go index 5985811f..cf62aeae 100644 --- a/storagemarket/impl/request_validation_test.go +++ b/storagemarket/impl/request_validation_test.go @@ -14,7 +14,7 @@ import ( xerrors "golang.org/x/xerrors" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-cbor-util" + cborutil "github.com/filecoin-project/go-cbor-util" "github.com/filecoin-project/go-fil-markets/shared/types" "github.com/filecoin-project/go-fil-markets/storagemarket" deals "github.com/filecoin-project/go-fil-markets/storagemarket/impl" @@ -76,7 +76,9 @@ func newClientDeal(minerID peer.ID, state storagemarket.StorageDealStatus) (deal ClientDeal: storagemarket.ClientDeal{ Proposal: newProposal, ProposalCid: proposalNd.Cid(), - PayloadCid: blockGenerator.Next().Cid(), + DataRef: &storagemarket.DataRef{ + Root: blockGenerator.Next().Cid(), + }, Miner: minerID, MinerWorker: minerAddr, State: state, @@ -101,7 +103,7 @@ func newMinerDeal(clientID peer.ID, state storagemarket.StorageDealStatus) (deal ProposalCid: proposalNd.Cid(), Client: clientID, State: state, - Ref: ref, + Ref: &storagemarket.DataRef{Root: ref}, }, }, nil } @@ -144,7 +146,7 @@ func TestClientRequestValidation(t *testing.T) { if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil { t.Fatal("deal tracking failed") } - payloadCid := clientDeal.PayloadCid + payloadCid := clientDeal.DataRef.Root if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid}, payloadCid, nil), deals.ErrWrongPeer) { t.Fatal("Pull should fail if miner address is incorrect") } @@ -169,7 +171,7 @@ func TestClientRequestValidation(t *testing.T) { if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil { t.Fatal("deal tracking failed") } - payloadCid := clientDeal.PayloadCid + payloadCid := clientDeal.DataRef.Root if !xerrors.Is(crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid}, payloadCid, nil), deals.ErrInacceptableDealState) { t.Fatal("Pull should fail if deal is in a state that cannot be data transferred") } @@ -182,7 +184,7 @@ func TestClientRequestValidation(t *testing.T) { if err := state.Begin(clientDeal.ProposalCid, &clientDeal); err != nil { t.Fatal("deal tracking failed") } - payloadCid := clientDeal.PayloadCid + payloadCid := clientDeal.DataRef.Root if crv.ValidatePull(minerID, &deals.StorageDataTransferVoucher{clientDeal.ProposalCid}, payloadCid, nil) != nil { t.Fatal("Pull should should succeed when all parameters are correct") } @@ -229,7 +231,7 @@ func TestProviderRequestValidation(t *testing.T) { t.Fatal("deal tracking failed") } ref := minerDeal.Ref - if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid}, ref, nil), deals.ErrWrongPeer) { + if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid}, ref.Root, nil), deals.ErrWrongPeer) { t.Fatal("Push should fail if miner address is incorrect") } }) @@ -254,7 +256,7 @@ func TestProviderRequestValidation(t *testing.T) { t.Fatal("deal tracking failed") } ref := minerDeal.Ref - if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid}, ref, nil), deals.ErrInacceptableDealState) { + if !xerrors.Is(mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid}, ref.Root, nil), deals.ErrInacceptableDealState) { t.Fatal("Push should fail if deal is in a state that cannot be data transferred") } }) @@ -267,7 +269,7 @@ func TestProviderRequestValidation(t *testing.T) { t.Fatal("deal tracking failed") } ref := minerDeal.Ref - if mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid}, ref, nil) != nil { + if mrv.ValidatePush(clientID, &deals.StorageDataTransferVoucher{minerDeal.ProposalCid}, ref.Root, nil) != nil { t.Fatal("Push should should succeed when all parameters are correct") } }) diff --git a/storagemarket/network/types.go b/storagemarket/network/types.go index f28597e7..8fd688e7 100644 --- a/storagemarket/network/types.go +++ b/storagemarket/network/types.go @@ -16,7 +16,7 @@ import ( type Proposal struct { DealProposal *storagemarket.StorageDealProposal - Piece cid.Cid // Used for retrieving from the client + Piece *storagemarket.DataRef } var ProposalUndefined = Proposal{} diff --git a/storagemarket/network/types_cbor_gen.go b/storagemarket/network/types_cbor_gen.go index a807d1ed..15bd75ef 100644 --- a/storagemarket/network/types_cbor_gen.go +++ b/storagemarket/network/types_cbor_gen.go @@ -128,10 +128,6 @@ func (t *Proposal) MarshalCBOR(w io.Writer) error { // t.Piece (cid.Cid) (struct) - if err := cbg.WriteCid(w, t.Piece); err != nil { - return xerrors.Errorf("failed to write cid field t.Piece: %w", err) - } - return nil } @@ -175,13 +171,6 @@ func (t *Proposal) UnmarshalCBOR(r io.Reader) error { { - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Piece: %w", err) - } - - t.Piece = c - } return nil } diff --git a/storagemarket/types.go b/storagemarket/types.go index 3efe28f8..7bac119b 100644 --- a/storagemarket/types.go +++ b/storagemarket/types.go @@ -168,7 +168,7 @@ type MinerDeal struct { State StorageDealStatus PiecePath filestore.Path - Ref cid.Cid + Ref *DataRef DealID uint64 } @@ -180,7 +180,7 @@ type ClientDeal struct { Miner peer.ID MinerWorker address.Address DealID uint64 - PayloadCid cid.Cid + DataRef *DataRef PublishMessage *cid.Cid } @@ -301,6 +301,16 @@ type ProposeStorageDealResult struct { ProposalCid cid.Cid } +const ( + TTGraphsync = "graphsync" + TTManual = "manual" +) + +type DataRef struct { + TransferType string + Root cid.Cid +} + // The interface provided by the module to the outside world for storage clients. type StorageClient interface { Run(ctx context.Context) @@ -326,7 +336,7 @@ type StorageClient interface { //FindStorageOffers(criteria AskCriteria, limit uint) []*StorageOffer // ProposeStorageDeal initiates deal negotiation with a Storage Provider - ProposeStorageDeal(ctx context.Context, addr address.Address, info *StorageProviderInfo, payloadCid cid.Cid, proposalExpiration Epoch, duration Epoch, price tokenamount.TokenAmount, collateral tokenamount.TokenAmount) (*ProposeStorageDealResult, error) + ProposeStorageDeal(ctx context.Context, addr address.Address, info *StorageProviderInfo, data *DataRef, proposalExpiration Epoch, duration Epoch, price tokenamount.TokenAmount, collateral tokenamount.TokenAmount) (*ProposeStorageDealResult, error) // GetPaymentEscrow returns the current funds available for deal payment GetPaymentEscrow(ctx context.Context, addr address.Address) (Balance, error) diff --git a/storagemarket/types_cbor_gen.go b/storagemarket/types_cbor_gen.go index 1d353847..e86f55ad 100644 --- a/storagemarket/types_cbor_gen.go +++ b/storagemarket/types_cbor_gen.go @@ -64,10 +64,6 @@ func (t *ClientDeal) MarshalCBOR(w io.Writer) error { // t.PayloadCid (cid.Cid) (struct) - if err := cbg.WriteCid(w, t.PayloadCid); err != nil { - return xerrors.Errorf("failed to write cid field t.PayloadCid: %w", err) - } - // t.PublishMessage (cid.Cid) (struct) if t.PublishMessage == nil { @@ -162,13 +158,6 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { { - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.PayloadCid: %w", err) - } - - t.PayloadCid = c - } // t.PublishMessage (cid.Cid) (struct) @@ -260,10 +249,6 @@ func (t *MinerDeal) MarshalCBOR(w io.Writer) error { // t.Ref (cid.Cid) (struct) - if err := cbg.WriteCid(w, t.Ref); err != nil { - return xerrors.Errorf("failed to write cid field t.Ref: %w", err) - } - // t.DealID (uint64) (uint64) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil { return err @@ -351,13 +336,6 @@ func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { { - c, err := cbg.ReadCid(br) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Ref: %w", err) - } - - t.Ref = c - } // t.DealID (uint64) (uint64) From b38c8a79bedb2bb73b9c7da3d7304bda1bd316ab Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 7 Feb 2020 17:47:18 -0800 Subject: [PATCH 2/3] add functions needed to import data on the miner side --- storagemarket/impl/provider.go | 50 ++++++++++ storagemarket/impl/provider_states.go | 4 +- storagemarket/network/types_cbor_gen.go | 28 +++++- storagemarket/types.go | 4 +- storagemarket/types_cbor_gen.go | 116 +++++++++++++++++++++++- 5 files changed, 190 insertions(+), 12 deletions(-) diff --git a/storagemarket/impl/provider.go b/storagemarket/impl/provider.go index b35cef07..6e4d0b40 100644 --- a/storagemarket/impl/provider.go +++ b/storagemarket/impl/provider.go @@ -1,8 +1,11 @@ package storageimpl import ( + "bytes" "context" "errors" + "io" + "os" "sync" "github.com/ipfs/go-cid" @@ -292,3 +295,50 @@ func (p *Provider) Stop() error { <-p.stopped return p.net.StopHandlingRequests() } + +func (p *Provider) ImportDataForDeal(ctx context.Context, propCid cid.Cid, data io.Reader) error { + // TODO: be able to check if we have enough disk space + var d MinerDeal + if err := p.deals.Get(propCid).Get(&d); err != nil { + return xerrors.Errorf("failed getting deal %s: %w", propCid, err) + } + + tempfi, err := p.fs.CreateTemp() + if err != nil { + return xerrors.Errorf("failed to create temp file for data import: %w", err) + } + + n, err := io.Copy(tempfi, data) + if err != nil { + return xerrors.Errorf("importing deal data failed: %w", err) + } + _ = n // TODO: verify n? + + _, err = tempfi.Seek(0, os.SEEK_SET) + if err != nil { + return xerrors.Errorf("failed to seek through temp imported file: %w", err) + } + + commP, err := p.pio.ReadPiece(tempfi) + if err != nil { + return xerrors.Errorf("failed to generate commP") + } + + if !bytes.Equal(commP.Bytes(), d.Proposal.PieceRef) { + return xerrors.Errorf("given data does not match expected commP (got: %x, expected %x)", commP.Bytes(), d.Proposal.PieceRef) + } + + select { + case p.updated <- minerDealUpdate{ + newState: storagemarket.StorageDealPublishing, + id: propCid, + mut: func(deal *MinerDeal) { + deal.PiecePath = tempfi.Path() + }, + }: + case <-ctx.Done(): + return ctx.Err() + } + + return nil +} diff --git a/storagemarket/impl/provider_states.go b/storagemarket/impl/provider_states.go index 2959a03a..8d71b815 100644 --- a/storagemarket/impl/provider_states.go +++ b/storagemarket/impl/provider_states.go @@ -3,7 +3,6 @@ package storageimpl import ( "bytes" "context" - "fmt" "github.com/filecoin-project/go-padreader" "github.com/ipfs/go-cid" @@ -80,7 +79,8 @@ func (p *Provider) validating(ctx context.Context, deal MinerDeal) (func(*MinerD // State: StorageDealTransferring func (p *Provider) transferring(ctx context.Context, deal MinerDeal) (func(*MinerDeal), error) { if deal.Ref.TransferType == storagemarket.TTManual { - return nil, fmt.Errorf("attempted to graphsync a deal marked as manual transfer") + log.Info("deal entering manual transfer state") + return nil, nil } ssb := builder.NewSelectorSpecBuilder(ipldfree.NodeBuilder()) diff --git a/storagemarket/network/types_cbor_gen.go b/storagemarket/network/types_cbor_gen.go index 15bd75ef..07c3f388 100644 --- a/storagemarket/network/types_cbor_gen.go +++ b/storagemarket/network/types_cbor_gen.go @@ -1,3 +1,5 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + package network import ( @@ -10,8 +12,6 @@ import ( xerrors "golang.org/x/xerrors" ) -// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. - var _ = xerrors.Errorf func (t *AskRequest) MarshalCBOR(w io.Writer) error { @@ -126,8 +126,10 @@ func (t *Proposal) MarshalCBOR(w io.Writer) error { return err } - // t.Piece (cid.Cid) (struct) - + // t.Piece (storagemarket.DataRef) (struct) + if err := t.Piece.MarshalCBOR(w); err != nil { + return err + } return nil } @@ -167,10 +169,26 @@ func (t *Proposal) UnmarshalCBOR(r io.Reader) error { } } - // t.Piece (cid.Cid) (struct) + // t.Piece (storagemarket.DataRef) (struct) { + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + t.Piece = new(storagemarket.DataRef) + if err := t.Piece.UnmarshalCBOR(br); err != nil { + return err + } + } + } return nil } diff --git a/storagemarket/types.go b/storagemarket/types.go index 7bac119b..b8fca753 100644 --- a/storagemarket/types.go +++ b/storagemarket/types.go @@ -16,7 +16,7 @@ import ( "github.com/filecoin-project/go-fil-markets/shared/types" ) -//go:generate cbor-gen-for ClientDeal MinerDeal StorageDeal Balance StorageDealProposal +//go:generate cbor-gen-for ClientDeal MinerDeal StorageDeal Balance StorageDealProposal DataRef const DealProtocolID = "/fil/storage/mk/1.0.1" const AskProtocolID = "/fil/storage/ask/1.0.1" @@ -207,6 +207,8 @@ type StorageProvider interface { // GetStorageCollateral returns the current collateral balance GetStorageCollateral(ctx context.Context) (Balance, error) + + ImportDataForDeal(ctx context.Context, propCid cid.Cid, data io.Reader) error } // Node dependencies for a StorageProvider diff --git a/storagemarket/types_cbor_gen.go b/storagemarket/types_cbor_gen.go index e86f55ad..4d3da936 100644 --- a/storagemarket/types_cbor_gen.go +++ b/storagemarket/types_cbor_gen.go @@ -62,7 +62,10 @@ func (t *ClientDeal) MarshalCBOR(w io.Writer) error { return err } - // t.PayloadCid (cid.Cid) (struct) + // t.DataRef (storagemarket.DataRef) (struct) + if err := t.DataRef.MarshalCBOR(w); err != nil { + return err + } // t.PublishMessage (cid.Cid) (struct) @@ -154,10 +157,26 @@ func (t *ClientDeal) UnmarshalCBOR(r io.Reader) error { return fmt.Errorf("wrong type for uint64 field") } t.DealID = uint64(extra) - // t.PayloadCid (cid.Cid) (struct) + // t.DataRef (storagemarket.DataRef) (struct) { + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + t.DataRef = new(DataRef) + if err := t.DataRef.UnmarshalCBOR(br); err != nil { + return err + } + } + } // t.PublishMessage (cid.Cid) (struct) @@ -247,7 +266,10 @@ func (t *MinerDeal) MarshalCBOR(w io.Writer) error { return err } - // t.Ref (cid.Cid) (struct) + // t.Ref (storagemarket.DataRef) (struct) + if err := t.Ref.MarshalCBOR(w); err != nil { + return err + } // t.DealID (uint64) (uint64) if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.DealID))); err != nil { @@ -332,10 +354,26 @@ func (t *MinerDeal) UnmarshalCBOR(r io.Reader) error { t.PiecePath = filestore.Path(sval) } - // t.Ref (cid.Cid) (struct) + // t.Ref (storagemarket.DataRef) (struct) { + pb, err := br.PeekByte() + if err != nil { + return err + } + if pb == cbg.CborNull[0] { + var nbuf [1]byte + if _, err := br.Read(nbuf[:]); err != nil { + return err + } + } else { + t.Ref = new(DataRef) + if err := t.Ref.UnmarshalCBOR(br); err != nil { + return err + } + } + } // t.DealID (uint64) (uint64) @@ -765,3 +803,73 @@ func (t *StorageDealProposal) UnmarshalCBOR(r io.Reader) error { } return nil } + +func (t *DataRef) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if _, err := w.Write([]byte{130}); err != nil { + return err + } + + // t.TransferType (string) (string) + if len(t.TransferType) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.TransferType was too long") + } + + if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.TransferType)))); err != nil { + return err + } + if _, err := w.Write([]byte(t.TransferType)); err != nil { + return err + } + + // t.Root (cid.Cid) (struct) + + if err := cbg.WriteCid(w, t.Root); err != nil { + return xerrors.Errorf("failed to write cid field t.Root: %w", err) + } + + return nil +} + +func (t *DataRef) UnmarshalCBOR(r io.Reader) error { + br := cbg.GetPeeker(r) + + maj, extra, err := cbg.CborReadHeader(br) + if err != nil { + return err + } + if maj != cbg.MajArray { + return fmt.Errorf("cbor input should be of type array") + } + + if extra != 2 { + return fmt.Errorf("cbor input had wrong number of fields") + } + + // t.TransferType (string) (string) + + { + sval, err := cbg.ReadString(br) + if err != nil { + return err + } + + t.TransferType = string(sval) + } + // t.Root (cid.Cid) (struct) + + { + + c, err := cbg.ReadCid(br) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Root: %w", err) + } + + t.Root = c + + } + return nil +} From a22dafaacc6e6af558c8df60d7f3173ddbac3ed2 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Mon, 10 Feb 2020 16:44:09 -0800 Subject: [PATCH 3/3] address linter --- storagemarket/impl/provider.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/storagemarket/impl/provider.go b/storagemarket/impl/provider.go index 6e4d0b40..b1ef54ab 100644 --- a/storagemarket/impl/provider.go +++ b/storagemarket/impl/provider.go @@ -5,7 +5,6 @@ import ( "context" "errors" "io" - "os" "sync" "github.com/ipfs/go-cid" @@ -314,7 +313,7 @@ func (p *Provider) ImportDataForDeal(ctx context.Context, propCid cid.Cid, data } _ = n // TODO: verify n? - _, err = tempfi.Seek(0, os.SEEK_SET) + _, err = tempfi.Seek(0, io.SeekStart) if err != nil { return xerrors.Errorf("failed to seek through temp imported file: %w", err) }