diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f7e60690..e19d4faf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,7 +56,6 @@ import ( ipld "github.com/ipfs/go-ipld-format" "github.com/stretchr/testify/assert" - "github.com/filecoin-project/go-fil-components/datatransfer/impl" "github.com/filecoin-project/go-fil-components/filestore/file" ) ``` diff --git a/README.md b/README.md index 1c8bf63c..71555e6c 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,6 @@ Separating an implementation into a blockchain component and one or more mining ## Components -* [datatransfer](./datatransfer), a module that enables sending data - over [libp2p](https://github.com/libp2p) using - [ipfs/go-graphsync](https://github.com/ipfs/go-graphsync) * [filestore](https://github.com/filecoin-project/go-fil-components/filestore), ... ## Contributing diff --git a/datatransfer/Makefile b/datatransfer/Makefile deleted file mode 100644 index 1f6e2060..00000000 --- a/datatransfer/Makefile +++ /dev/null @@ -1,17 +0,0 @@ -all: build -.PHONY: all - -GOVERSION:=$(shell go version | cut -d' ' -f 3 | cut -d. -f 2) -ifeq ($(shell expr $(GOVERSION) \< 13), 1) -$(warning Your Golang version is go 1.$(GOVERSION)) -$(error Update Golang to version $(shell grep '^go' go.mod)) -endif - -build: - go build ./... - -test: - go test ./... - -type-gen: build - go run ./cbor-gen/main.go diff --git a/datatransfer/README.md b/datatransfer/README.md deleted file mode 100644 index 532337ec..00000000 --- a/datatransfer/README.md +++ /dev/null @@ -1,177 +0,0 @@ -# go-fil-components/datatransfer - -A go module to perform data transfers over [ipfs/go-graphsync](https://github.com/ipfs/go-graphsync) - -[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) - -## Description -This module encapsulates protocols for exchanging piece data between storage clients and miners, both when consummating a storage deal and when retrieving the piece later. - - -## Table of Contents -* [Background](https://github.com/filecoin-project/go-fil-components/tree/master/datatransfer#background) -* [Usage](https://github.com/filecoin-project/go-fil-components/tree/master/datatransfer#usage) - * [Initialize a data transfer module](https://github.com/filecoin-project/go-fil-components/tree/master/datatransfer#initialize-a-data-transfer-module) - * [Register a validator](https://github.com/filecoin-project/go-fil-components/tree/master/datatransfer#register-a-validator) - * [Open a Push or Pull Request](https://github.com/filecoin-project/go-fil-components/tree/master/datatransfer#open-a-push-or-pull-request) - * [Subscribe to Events](https://github.com/filecoin-project/go-fil-components/tree/master/datatransfer#subscribe-to-events) -* [Contribute](https://github.com/filecoin-project/go-fil-components/tree/master/datatransfer#contribute) - -## Background - -Please see the [design documentation](https://github.com/filecoin-project/go-fil-components/tree/master/datatransfer/docs/DESIGNDOC) -for this module for a high-level overview and and explanation of the terms and concepts. - -## Usage - -**Requires go 1.13** - -Install the module in your package or app with `go get "github.com/filecoin-project/go-fil-components/datatransfer"` - - -### Initialize a data transfer module -1. Set up imports. You need, minimally, the following imports: - ```go - package mypackage - - import ( - gsimpl "github.com/ipfs/go-graphsync/impl" - "github.com/filecoin-project/go-fil-components/datatransfer" - "github.com/libp2p/go-libp2p-core/host" - ) - - ``` -1. Provide or create a [libp2p host.Host](https://github.com/libp2p/go-libp2p-examples/tree/master/libp2p-host) -1. Provide or create a [go-graphsync GraphExchange](https://github.com/ipfs/go-graphsync#initializing-a-graphsync-exchange) -1. Create a new instance of GraphsyncDataTransfer - ```go - func NewGraphsyncDatatransfer(h host.Host, gs graphsync.GraphExchange) { - dt := datatransfer.NewGraphSyncDataTransfer(h, gs) - } - ``` - -1. If needed, build out your voucher struct and its validator. - - A push or pull request must include a voucher. The voucher's type must have been registered with - the node receiving the request before it's sent, otherwise the request will be rejected. - - [datatransfer.Voucher](https://github.com/filecoin-project/go-fil-components/blob/21dd66ba370176224114b13030ee68cb785fadb2/datatransfer/types.go#L17) - and [datatransfer.Validator](https://github.com/filecoin-project/go-fil-components/blob/21dd66ba370176224114b13030ee68cb785fadb2/datatransfer/types.go#L153) - are the interfaces used for validation of graphsync datatransfer messages. Voucher types plus a Validator for them must be registered - with the peer to whom requests will be sent. - -#### Example Toy Voucher and Validator -```go -type myVoucher struct { - data string -} - -func (v *myVoucher) ToBytes() ([]byte, error) { - return []byte(v.data), nil -} - -func (v *myVoucher) FromBytes(data []byte) error { - v.data = string(data) - return nil -} - -func (v *myVoucher) Type() string { - return "FakeDTType" -} - -type myValidator struct { - ctx context.Context - validationsReceived chan receivedValidation -} - -func (vl *myValidator) ValidatePush( - sender peer.ID, - voucher datatransfer.Voucher, - baseCid cid.Cid, - selector ipld.Node) error { - - v := voucher.(*myVoucher) - if v.data == "" || v.data != "validpush" { - return errors.New("invalid") - } - - return nil -} - -func (vl *myValidator) ValidatePull( - receiver peer.ID, - voucher datatransfer.Voucher, - baseCid cid.Cid, - selector ipld.Node) error { - - v := voucher.(*myVoucher) - if v.data == "" || v.data != "validpull" { - return errors.New("invalid") - } - - return nil -} - -``` - - -Please see -[go-fil-components/blob/master/datatransfer/types.go](https://github.com/filecoin-project/go-fil-components/blob/master/datatransfer/types.go) -for more detail. - - -### Register a validator -Before sending push or pull requests, you must register a `datatransfer.Voucher` -by its `reflect.Type` and `dataTransfer.RequestValidator` for vouchers that -must be sent with the request. Using the trivial examples above: -```go - func NewGraphsyncDatatransfer(h host.Host, gs graphsync.GraphExchange) { - dt := datatransfer.NewGraphSyncDataTransfer(h, gs) - - vouch := &myVoucher{} - mv := &myValidator{} - dt.RegisterVoucherType(reflect.TypeOf(vouch), mv) - } -``` - -For more detail, please see the [unit tests](https://github.com/filecoin-project/go-fil-components/blob/master/datatransfer/impl/graphsync/graphsync_impl_test.go). - -### Open a Push or Pull Request -For a push or pull request, provide a context, a `datatransfer.Voucher`, a host recipient `peer.ID`, a baseCID `cid.CID` and a selector `ipld.Node`. These -calls return a `datatransfer.ChannelID` and any error: -```go - channelID, err := dtm.OpenPullDataChannel(ctx, recipient, voucher, baseCid, selector) - // OR - channelID, err := dtm.OpenPushDataChannel(ctx, recipient, voucher, baseCid, selector) - -``` - -### Subscribe to Events - -The module allows the consumer to be notified when a graphsync Request is sent or a datatransfer push or pull request response is received: - -```go - func ToySubscriberFunc (event Event, channelState ChannelState) { - if event.Code == datatransfer.Error { - // log error, flail about helplessly - return - } - // - if channelState.Recipient() == our.PeerID && channelState.Received() > 0 { - // log some stuff, update some state somewhere, send data to a channel, etc. - } - } - - dtm := SetupDataTransferManager(ctx, h, gs, baseCid, snode) - unsubFunc := dtm.SubscribeToEvents(ToySubscriberFunc) - - // . . . later, when you don't need to know about events any more: - unsubFunc() -``` - -## Contributing -PRs are welcome! Please first read the design docs and look over the current code. PRs against -master require approval of at least two maintainers. For the rest, please see our -[CONTRIBUTING](https://github.com/filecoin-project/go-fil-components/CONTRIBUTING.md) guide. - -Copyright 2019. Protocol Labs, Inc. \ No newline at end of file diff --git a/datatransfer/docs/DESIGNDOC.md b/datatransfer/docs/DESIGNDOC.md deleted file mode 100644 index 69424bd3..00000000 --- a/datatransfer/docs/DESIGNDOC.md +++ /dev/null @@ -1,280 +0,0 @@ -# Data Transfer Module Design - -This module encapsulates protocols for exchanging piece data between storage clients and miners, both when consummating a storage deal and when retrieving the piece later. - -### Goals - -1. Abstraction of the underlying exchange protocols (e.g. bitswap, truck full of disks) from the rest of a Filecoin system. -2. A common negotiation protocol independent of the underlying data exchange protocol. -3. A single module that may be linked into both client and provider applications, producing and consuming an RPC-friendly API -4. Neutral as to the context of the transfer operations being performed (storage deal, retrieval deal, etc). -5. Data transfer can be initiated by either the sender or the receiver. -6. Data transfer can happen for either all or part of a piece - -### Concepts - -- **Push Request** - A request to send data to the other party - -- **Pull Request** - A request to have the other party send data - -- **Requestor** - The party that initiates the data transfer request (whether Push or Pull) - -- **Responder** - The party that receives the data transfer request - -- **Data Transfer Voucher** - A wrapper around information that can identify and verify the transfer request to the other party - -- **PullValidator** - When a pull request is received by a responder, looks at the -voucher sent with the request to verify if it's valid. Provided by the responder - -- **PushValidator** - When a push request is received by a responder, looks at the -data transfer voucher with the request to verify if it's valid. Provided by the -responder - -- **Scheduler** - Once a request is negotiated and verified, actually schedules and performs the transfer. The scheduler has access to an underlying verifiable protocol - -- **Listener** - A callback that receives notifications when different data transfer events happen - -- **Graphsync** - The default transfer protocol used by the Scheduler. The full graphsync specification can be found at [https://github.com/ipld/specs/blob/master/block-layer/graphsync/graphsync.md](https://github.com/ipld/specs/blob/master/block-layer/graphsync/graphsync.md) - -### Basic Phases - -There are two basic phases to any data transfer: - -1. Negotiation - the requestor and responder agree to the transfer by validating with the data transfer voucher -2. Transfer - The transfer is actually initiated by the party that will receive data. The default protocol used to do the transfer is Graphsync - -### Push Flow - -![Push Flow Diagram](./push-flow.png) - -1. A requestor initiates a Push transfer when it wants to send data to another party. -2. The requestors' data transfer module will send a push request to the responder along with the data transfer voucher. It also puts the data transfer -in the scheduler queue, meaning it expects the responder to initiate a transfer -once the request is verified -3. The responder's data transfer module validates the data transfer request via a PushValidator provided as a dependency by the responder -4. The responder's data transfer module schedules the transfer -5. The responder makes a graphsync request for the data -6. The requestor receives the graphsync request, verifies it's in the scheduler -and begins sending data -7. The responder receives data and can produce an indication of progress -8. The responder completes receiving data, and notifies any listeners - -The push flow is ideal for storage deals, where the client initiates the push -once it verifies the the deal is signed and on chain. - -### Pull Flow - -![Pull Flow Diagram](./pull-flow.png) - -1. A requestor initiates a Pull transfer when it wants to receive data from another party. -2. The requestors' data transfer module will send a pull request to the responder along with the data transfer voucher. -3. The responder's data transfer module validates the data transfer request via a PullValidator provided as a dependency by the responder -4. The responder's data transfer module schedules the transfer (meaning it is expecting the requestor to initiate the actual transfer) -5. The responder's data transfer module sends a response to the requestor saying it has accepted the transfer and is waiting for the requestor to initate -the transfer -6. The requestor schedules the data transfer -7. The requestor makes a graphsync request for the data -8. The responder receives the graphsync request, verifies it's in the scheduler -and begins sending data -9. The requestor receives data and can produce an indication of progress -10. The requestor completes receiving data, and notifies any listeners - -### Protocol - -A data transfer is negotiated over the network via the data transfer protocol. - -A Pull request expects a response. The requestor does not initiate the transfer -until they know the request is accepted. - -A Push request does expect a response. If the Responder accepts the request, -they initiate the transfer. - -#### Format - -```protobuf -message DataTransferMessage { - message Request { - int32 channelID = 1 - bool isPull = 2 - bytes voucher = 3 - bytes pieceID = 4 - bytes selector = 5 - bool isPartial = 6 - bool isCancel = 7 - } - - message Response { - int32 channelID = 1 - boolean accepted = 2 - } - - bool isResponse = 1 - Request request = 2 - Response response = 3 -} -``` - -### Prequisite Dependency Work - -There are two pieces of pre-work needed to build the data transfer module - -1. *go-ipld-prime* - needs to be augmented to read dag-protobuf encoded IPLD nodes (for unixFS files) -2. *go-graphsync* - needs to be augmented to allow validating requests before responses are sent, and to send validation information in a request - -### API & Data Structures - -```golang -import ipld "github.com/filecoin-project/specs/libraries/ipld" - -type Address struct {} -type PieceStore struct {} -type StorageDealID struct {} -type RetrievalDealID struct {} -type Function struct {} -type Piece struct {} -type ChannelID UInt -type LibP2PHost struct {} - -type DataTransferSubsystem struct @(mutable) { - host libp2p.Host - dataTransfers {ChannelID: DataTransferChannel} - pushValidator PushValidator - pullValidator PullValidator - pieceStore PieceStore - - // open a data transfer that will send data to the recipient address and - // transfer the whole piece - OpenPushDataTransfer(address Address, voucher DataTransferVoucher, PieceRef CID) ChannelID - - // open a data transfer that will send data to the recipient address and - // transfer parts of the piece that match the selector - OpenPartialPushDataTransfer(address Address, voucher DataTransferVoucher, PieceRef CID - Selector ipld.Selector) ChannelID - - // open a data transfer that will request data from the sending address - // and transfer the whole piece - OpenPullDataTransfer(address Address, voucher DataTransferVoucher, PieceRef CID) ChannelID - - // open a data transfer that will request data from the sending address and - // transfer parts of the piece that match the selector - OpenPartialPullDataTransfer(address Address, voucher DataTransferVoucher, PieceRef CID - Selector ipld.Selector) ChannelID - - // close an open channel (effectively a cancel) - CloseDataTransferChannel(x ChannelID) struct{} - - // get status of a transfer - TransferChannelStatus(x ChannelID) DataTransferStatus - - // get notified when certain types of events happen - RegisterDataTransferListener(listener DataTransferListener) - -} - -// A DataTransferVoucher is a ticket that is used to validate -// a data transfer request against the underlying storage or retrieval deal -// that precipitated it -type DataTransferVoucher union { - StorageDealVoucher - RetrievalDealVoucher -} - -type StorageDealVoucher struct { - id StorageDealID -} - -type RetrievalDealVoucher struct { - id RetrievalDealID -} - -// A PushValidator validates an incoming push request from the network -type PushValidator struct { - ValidatePush( - sender Address, - voucher DataTransferVoucher, - PieceRef CID, - Selector ipld.Selector) -} - -// A PullValidator validates an incoming pull request from the network -type PullValidator struct { - ValidatePull( - receiver Address, - voucher DataTransferVoucher, - PieceRef CID, - Selector ipld.Selector) -} - -// a data transfer listener specifies a callback that happens -// when certain types of events occur -type DataTransferListener struct { - eventType DataTransferEvent - Callback(dataTransfer DataTransferChannel, metadata EventMetadata) struct{} -} - -type Open struct {} -type Progress struct {} -type Error struct {} -type Complete struct {} - -type DataTransferEvent union { - Open - Progress - Error - Complete -} - -type EventMetadata struct {} - -type SchedulerCallback func (result ResultCode) - -type TransferProgress { - Id ChannelID - Progress float // 0...1 -} - -// A Scheduler manages actually scheduling data transfer requests on the network -// Assumes access to error-checked, ordered, reliable transmission protocol -type Scheduler struct { - ScheduleTransfer(channelID ChannelID, onComplete SchedulerCallback) struct{} - InProgressTransfers() [TransferProgress] -} - -// Data tracked for a data transfer -// TODO: Needs work -type DataTransferChannel struct { - channelID ChannelID - PieceRef CID - Selector ipld.Selector - voucher DataTransferVoucher - sender Address - recipient Address - totalSize UVarint - sent UVarint - received UVarint - - channelType() DataTransferType @(cached) - transferNum() Float @(cached) -} - -type Sender struct {} -type Recipient struct {} - -type DataTransferType union { - Sender - Recipient -} - -type Ongoing struct {} -type Completed struct {} -type Failed struct {} -type ChannelNotFoundError struct {} - -type DataTransferStatus union { - Ongoing - Completed - Failed - ChannelNotFoundError -} -``` - diff --git a/datatransfer/docs/markets-proposal.md b/datatransfer/docs/markets-proposal.md deleted file mode 100644 index 37e2892b..00000000 --- a/datatransfer/docs/markets-proposal.md +++ /dev/null @@ -1,220 +0,0 @@ -# Auxilary: Markets Shared Component Proposal - -### Goals - -1. StorageMarket & RetrievalMarket can operate in a seperate process from the chain, storageminer, and data transfer (but don't need to) -2. Retrievals can be done in chunks to allow building trust in miner as data is retrieved. But on the client side they can be tracked as a whole. -3. StorageDeals go on chain. RetrievalDeals operate off chain, only interacting with chain through PaymentChannels -4. All chain components have no responsibility other than updating the chain (the StorageMarketActor does not coordinate next steps in chain) - -### Concepts - -**StorageClient** - The party requesting storage - -**StorageProvider** - The party offering storage - -**StorageDealProposal** - A storage deal made and signed by the client, not accepted - -**StorageDeal** - A storage deal that has been accepted and signed by the StorageProvider - -**StorageMarketActor** - The component that puts storage deals on chain and keeps them updated. - -**RetrievalClient** - The party requesting retrieval - -**RetrievalProvider** - The party retrieving data and sending to the client - -**RetrievalRequest** - a concept local to the RetrievalClient, for a piece, that -may be transferred through one or more RetrievalDeals - -**RetrievalDeal** - a deal to transfer some or all of a piece. Not signed, simply a proposal - -**PaymentChannel** - the only on chain component of a retrieval deal, and the canonical way to reference a retrieval that has been agreed upon by parties - -### TODO - -- Asks/Bids - -### StorageMarket Flow - -![StorageMarket Flow](./storage-market-flow.png) - -1. A storage deal is initiated by the StorageClient sending a StorageDealProposal to the StorageProvider -2. If the StorageProvider accepts the deal, it signs the deal and returns the signed StorageDeal to the StorageClient -3. At this point, either party may place the deal on chain through the StorageMarketActor, though canonically, the StorageProvider should always do this as soon as it signs the deal -4. The StorageClient opens a push request through the data transfer system -5. The StorageProvider receives the data transfer request and verifies it throught he StorageMarketActor -6. The StorageProvider initiates data transfer -7. The StorageClient sends data -8. The StorageProvider is notified transfer is complete -9. The StorageProvider tells the StorageMarketActor to update the deal -10. The StorageProvider tells the StorageMiner to add the piece and begin sealing - -### RetrievalMarket Flow - -![RetrievalMarket Flow](./retrieval-market-flow.png) - -1. The user (cmd line, api, etc) sends a RetrievalRequest to the RetrievalClient -2. For n partial pieces of the piece id: - a. RetrievalClient sends a RetrievalDeal to the RetrievalProvider for the partial piece - b. RetrievalProvider accepts the deal and opens a Payment Channel - c. RetrievalProvider sends the payment channel to RetrievalClient - d. RetrievalClient adds funds to the PaymentChannel (? - not sure how payment channels work) - e. RetrievalClient opens a pull request through the data transfer system - f. The RetrievalProvider receives the data transfer request and verifies it throught the PaymentChannel - g. The RetrievalProvider notifes the RetrievalClient that it accepts the data transfer - h. The RetrievalClient initiates data transfer - i. The RetrievalProvider sends data - j. The RetrievalClient is notified transfer is complete - k. The RetrievalClient tells the PaymentChannel to transfer funds -3. The RetrievalClient notifies the user transfer is complete - -### Protocol - -TODO - -#### StorageDealProtocol -#### RetrievalDealProtocol - -Should these be seperate? - -### Data Structures - -```golang -type StorageClient struct { - dataTransfer DataTransferSubsystem - storageMarketActor StorageMarketActor - host LibP2PHost - - // starts a storage deal - InitiateDeal(StorageDealParams) ClientDealRef - - // internal method called when a signed proposal - // is received - onStorageDealReceived(StorageDeal) - - // registered with data transfer subsystem as a listener - // to notify any higher level listeners - onTransferComplete(DataTransferChannel) - - // query the current storage deal status - GetStorageDealStatus(ClientDealRef) DealStatus - - // to tell use when evens happen with a deal - RegisterListener(Listener) -} - -type StorageProvider struct { - dataTransfer DataTransferSubsystem - storageMarketActor StorageMarkeyActor - storageMiner StorageMiner - host LibP2PHost - - // internal method called when proposals are received - onStorageDealProposalReceived(StorageDealProposal) - - // registered with data transfer subsystem as a listener - onTransferComplete(DataTransferChannel) - - // StorageProvider is a PushValidator for the Data Transfer Subsystem - VerifyPushRequest(Voucher) -} - -type RetrievalDeal struct { - pieceID CID - part ipld.Selector - paymentChannel PaymentChannel - client address.Address - provider address.Address -} - -type RetrievalRequestData struct { - pieceID CID - miner address.Address - deals [RetrievalDeal] // ? - all the deals for this retrieval... -} - -type RetrievalClient struct { - retrievals map[ClientRef]RetrievalRequestData - dataTransfer DataTransferSubsystem - paymentChannelActor PaymentChannelActor - host LibP2PHost - - // starts a retrieval - InitiateRetrieval(RetrievalRequest) ClientRef - - // internal method called when accepted deal received - // from provider - onRetrievalDealReceived(PaymentChannel) - - // registered with data transfer subsystem as a listener - // to notify any higher level listeners - onTransferComplete(DataTransferChannel) - - // query the current retrieval status - GetRetrievalStatus(ClientRef) RetrievalStatus - - // to tell use when events happen with a retrieval - // including when the whole piece is complete - RegisterListener(Listener) -} - -type RetrievalProvider struct { - dataTransfer DataTransferSubsystem - storageMarketActor StorageMarkeyActor - storageMiner StorageMiner - host LibP2PHost - - // internal method called when proposals are received - onRetrievalDealReceived(RetrievalDeal) - - // registered with data transfer subsystem as a listener - onTransferComplete(DataTransferChannel) - - // RetrievalProvider is a PullValidator for the Data Transfer Subsystem - VerifyPullRequest(Voucher) -} - -type StorageMarketActor struct { - // need access to: - // - BlockchainSubsystem - // - needs access to StateTree - // to verify storage capacity and pledge amount - // and slash storage providers for storage faults - // - needs access to access to MessagePool if deals on chain - // - NetworkSubsystem - // - needs access to MessagePubsub - - ParticipantBalances { Address : StorageParticipantBalance } - StorageDeals { StorageDealID : StorageDeal } - - RegisterParticipant(lockedBalance TokenAmount) bool - WithdrawBalance(balance TokenAmount) struct {} - AddBalance(balance TokenAmount) struct {} - CheckLockedBalance(participantAddr Address) TokenAmount - - // TODO: There should be some punishment for publishing a deal too late - // call by CommitSector in StorageMiningSubsystem - // a StorageDeal is only published on chain when it passes VerifyStorageDeal - PublishStorageDeal(newStorageDeal StorageDeal) PublishStorageDealResponse - - // check if StorageDeal is still valid - // check if StorageDeal has the right signature - // check if provider and client have sufficient balances - VerifyStorageDeal(newStorageDeal StorageDeal) bool - - // TODO: StorageDeals should be renewable - UpdateStorageDeal(newStorageDeal StorageDeal) struct {} - - // call by StorageMarketActor on ExpiredSet - // remove StorageDeal from StorageMarketActor - // return StorageCollateral to miners - CloseExpiredStorageDeal(targetStorageDeal StorageDeal) struct {} - - // call by StorageMinerActor on successful PoSt Submission - HandleStorageDealPayment(storageDeal StorageDeal, currEpoch Epoch) struct {} - - // call by StorageMinerActor or CronActor to slash deal collateral - SlashStorageDealCollateral(targetStorageDeal StorageDeal) struct {} - -} -``` \ No newline at end of file diff --git a/datatransfer/docs/pull-flow.mmd b/datatransfer/docs/pull-flow.mmd deleted file mode 100644 index 6e4fdca6..00000000 --- a/datatransfer/docs/pull-flow.mmd +++ /dev/null @@ -1,27 +0,0 @@ -sequenceDiagram - - participant Requestor - participant ReqDT as Data Transfer Module (Requestor) - participant ReqSC as Scheduler (Requestor) - participant ReqGS as Graphsync (Requestor) - participant ResGS as Graphsync (Requestor) - participant ResSC as Scheduler (Requestor) - participant ResDT as Data Transfer Module (Responder) - participant Responder - - Note over Requestor: Likely A Client - Note over Responder: Likely A Miner - - Requestor ->> ReqDT : Initiate Pull - ReqDT ->> ReqSC : Schedule Transfer - ReqSC ->> ReqGS : Make Graphsync Request w/ Data Transfer Request Piggy Backed - ReqGS ->> ResGS : Send Graphsync Request (w/ Data Transfer Request) - ResGS ->> ResDT : Verify Request (validate & schedule) - ResDT ->> Responder : Validate Pull Request - Responder ->> ResDT : Pull Request validated - ResDT ->> ResSC : Schedule Transfer - ResSC ->> ResGS : Send response w/ DTR Accepted Piggy Backed - ResGS ->> ReqGS : Send response w/ DTR Accepted Piggy Backed - ReqGS ->> ReqSC : Response Progress (to end) (include DT Accepted) - ReqSC ->> ReqDT : Request Complete - ReqDT ->> Requestor : Request Completed (if listening) \ No newline at end of file diff --git a/datatransfer/docs/pull-flow.png b/datatransfer/docs/pull-flow.png deleted file mode 100644 index 3b2dce13..00000000 Binary files a/datatransfer/docs/pull-flow.png and /dev/null differ diff --git a/datatransfer/docs/pull-flow.svg b/datatransfer/docs/pull-flow.svg deleted file mode 100644 index 19c4647c..00000000 --- a/datatransfer/docs/pull-flow.svg +++ /dev/null @@ -1,5 +0,0 @@ -RequestorData Transfer Module (Requestor)Scheduler (Requestor)Graphsync (Requestor)Graphsync (Requestor)Scheduler (Requestor)Data Transfer Module (Responder)ResponderLikely A ClientLikely A MinerInitiate PullSchedule TransferMake Graphsync Request w/ Data Transfer Request Piggy BackedSend Graphsync Request (w/ Data Transfer Request)Verify Request (validate & schedule)Validate Pull RequestPull Request validatedSchedule TransferSend response w/ DTR Accepted Piggy BackedSend response w/ DTR Accepted Piggy BackedResponse Progress (to end) (include DT Accepted)Request CompleteRequest Completed (if listening)RequestorData Transfer Module (Requestor)Scheduler (Requestor)Graphsync (Requestor)Graphsync (Requestor)Scheduler (Requestor)Data Transfer Module (Responder)Responder \ No newline at end of file diff --git a/datatransfer/docs/push-flow.mmd b/datatransfer/docs/push-flow.mmd deleted file mode 100644 index 9f92df7f..00000000 --- a/datatransfer/docs/push-flow.mmd +++ /dev/null @@ -1,28 +0,0 @@ -sequenceDiagram - - participant Requestor - participant ReqDT as Data Transfer Module (Requestor) - participant ReqSC as Scheduler (Requestor) - participant ReqGS as Graphsync (Requestor) - participant ResGS as Graphsync (Requestor) - participant ResSC as Scheduler (Requestor) - participant ResDT as Data Transfer Module (Responder) - participant Responder - - Note over Requestor: Likely A Client - Note over Responder: Likely A Miner - - Requestor ->> ReqDT : Initiate Push - ReqDT ->> ReqSC : Schedule Transfer - ReqDT ->> ResDT : Send Data Transfer Request - ResDT ->> Responder : Validate Push Request - Responder ->> ResDT : Push Request validated - ResDT ->> ResSC : Schedule Transfer - ResSC ->> ResGS : Make Graphsync Request - ResGS ->> ReqGS : Send Graphsync Request - ReqGS ->> ReqSC : Verify Transfer Scheduled - ReqSC ->> ReqGS : Request is scheduled - ReqGS ->> ResGS : Send Response - ResGS ->> ResSC : Response Progress (to end) - ResSC ->> ResDT : Request Complete - ResDT ->> Responder : Request Completed (if listening) diff --git a/datatransfer/docs/push-flow.png b/datatransfer/docs/push-flow.png deleted file mode 100644 index 11e1f517..00000000 Binary files a/datatransfer/docs/push-flow.png and /dev/null differ diff --git a/datatransfer/docs/retrieval-market-flow.mmd b/datatransfer/docs/retrieval-market-flow.mmd deleted file mode 100644 index f7ad5148..00000000 --- a/datatransfer/docs/retrieval-market-flow.mmd +++ /dev/null @@ -1,38 +0,0 @@ -sequenceDiagram - participant PMC as PaymentChannel (Client) - participant RetrievalClient - participant DTC as Data Transfer (Client) - participant DTM as Data Transfer (Provider) - participant RetrievalProvider - participant PMM as PaymentChannel (Provider) - participant StorageMiner - - opt Querying - RetrievalClient ->>+ RetrievalProvider: NewRetrievalQuery(RetreivalQuery) - RetrievalProvider -->>- RetrievalClient: RetrievalQueryResponse - end - - RetrievalClient ->>+ RetrievalClient: RetrievePiece(PieceID) - loop PieceRetrieval - RetrievalClient ->>+ RetrievalProvider: NewRetrievalDealProposal(RetrievalDealProposal) - RetrievalProvider -->> RetrievalProvider: AcceptRetrievalDealProposal(RetrievalDealPropsal) - RetrievalProvider ->>+ PMM: OpenPaymenChannel - PMM -->>- RetrievalProvider: paymentChannel - RetrievalProvider -->>- RetrievalClient: NewPaymentChannel - RetrievalClient ->>+ PMC: AddFunds - PMC -->>- RetrievalClient: fundsAdded - RetrievalClient ->> DTC: OpenPartialPullRequest(StorageProvider.Address, Voucher(PaymentChannel), PieceID, Selector) - DTC ->>+ DTM: OpenPartialPullRequest - DTM ->>+ RetrievalProvider: ValidateTransfer(Voucher) - RetrievalProvider ->>+ PMM: VerifyPaymentChannel(PaymentChannel) - PMM -->>- RetrievalProvider: paymentChannelVerified - RetrievalProvider ->>+ StorageMiner: UnsealSector - StorageMiner -->>- RetrievalProvider: sectorUnsealed - RetrievalProvider -->>- DTM: tranferValidated - DTM -->>- DTC: Transfer Accepted - DTC ->>+ DTM: StartTransfer - DTM -->>- DTC: Send Data - DTC ->> RetrievalClient: Transfer Complete (via listener) - RetrievalClient ->> PMC: CloseDeal - end - RetrievalClient -->>- RetrievalClient: pieceTransferred \ No newline at end of file diff --git a/datatransfer/docs/retrieval-market-flow.png b/datatransfer/docs/retrieval-market-flow.png deleted file mode 100644 index 649ae80a..00000000 Binary files a/datatransfer/docs/retrieval-market-flow.png and /dev/null differ diff --git a/datatransfer/docs/storage-market-flow.mmd b/datatransfer/docs/storage-market-flow.mmd deleted file mode 100644 index 876b3cd3..00000000 --- a/datatransfer/docs/storage-market-flow.mmd +++ /dev/null @@ -1,48 +0,0 @@ -sequenceDiagram - participant SMAC as StorageMarketActor (Client) - participant StorageClient - participant DTC as Data Transfer (Client) - participant DTM as Data Transfer (Provider) - participant StorageProvider - participant SMAM as StorageMarketActor (Provider) - participant StorageMiner - - opt ClientSetup - StorageClient ->> SMAC: RegisterParticipant(TokenAmount) - StorageClient ->> SMAC: AddBalance(TokenAmount) - StorageClient ->> SMAC: WithdrawBalance(TokenAmount) - StorageClient ->>+ SMAC: CheckLockedBalance(Address) - SMAC -->>- StorageClient: TokenAmount - end - - opt ProviderSetup - StorageProvider ->> SMAM: RegisterParticipant(TokenAmount) - StorageProvider ->> SMAM: AddBalance(TokenAmount) - StorageProvider ->> SMAM: WithdrawBalance(TokenAmount) - StorageProvider ->+ SMAM: CheckLockedBalance(Address) - SMAM -->>- StorageProvider: TokenAmount - StorageProvider ->> DTM: RegisterListener - end - - StorageClient -->>+ StorageProvider: NewStorageDealProposal(StorageDealProposal) - StorageProvider ->>+ SMAM: VerifyBalance(StorageMarketParticipant1) - SMAM -->>- StorageProvider: balance - StorageProvider ->> StorageProvider: SignStorageDealProposal(StorageDealProposal) - StorageProvider -->- StorageClient: NewStorageDeal(StorageDeal) - StorageProvider ->> SMAM: PublishStorageDeal(StorageDeal) - StorageClient ->> SMAC: PublishStorageDeal(StorageDeal) - - StorageClient ->>+ SMAC: VerifyStorageDeal(StorageDeal) - SMAC -->>- StorageClient: storageDealVerified - StorageClient ->> DTC: OpenPushRequest(StorageProvider.Address, Voucher(StorageDealID), PieceID) - DTC ->>+ DTM: OpenPushRequest - DTM ->>+ StorageProvider: ValidateTransfer(Voucher) - StorageProvider ->>+ SMAM: VerifyStorageDeal(StorageDeal) - SMAM -->>- StorageProvider: storageDealVerified - StorageProvider -->>- DTM: DealValidated - DTM -->>- DTC: Start transfer - DTC ->> DTM: Send Data - DTM ->> StorageProvider: Transfer Complete (via listener) - StorageProvider ->>+ SMAM: UpdateStorageDeal - SMAM -->>- StorageProvider: storageDealUpdated - StorageProvider ->> StorageMiner: HandleStorageDeal(Deal) diff --git a/datatransfer/docs/storage-market-flow.png b/datatransfer/docs/storage-market-flow.png deleted file mode 100644 index 143796b7..00000000 Binary files a/datatransfer/docs/storage-market-flow.png and /dev/null differ diff --git a/datatransfer/impl/graphsync/fixtures/lorem.txt b/datatransfer/impl/graphsync/fixtures/lorem.txt deleted file mode 100644 index c5589235..00000000 --- a/datatransfer/impl/graphsync/fixtures/lorem.txt +++ /dev/null @@ -1,49 +0,0 @@ -Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Vitae semper quis lectus nulla at volutpat diam ut venenatis. Ac tortor dignissim convallis aenean et tortor at. Faucibus ornare suspendisse sed nisi lacus sed. Commodo ullamcorper a lacus vestibulum sed arcu non. Est pellentesque elit ullamcorper dignissim. Quam quisque id diam vel quam. Pretium aenean pharetra magna ac. In nulla posuere sollicitudin aliquam ultrices. Sed arcu non odio euismod lacinia at. Suspendisse ultrices gravida dictum fusce ut placerat orci nulla pellentesque. Feugiat vivamus at augue eget arcu. - -Pellentesque nec nam aliquam sem et tortor. Vitae tortor condimentum lacinia quis vel. Cras pulvinar mattis nunc sed. In massa tempor nec feugiat. Ornare arcu odio ut sem nulla. Diam maecenas sed enim ut sem. Pretium vulputate sapien nec sagittis. Bibendum arcu vitae elementum curabitur vitae nunc sed velit dignissim. Duis ut diam quam nulla porttitor massa. Viverra mauris in aliquam sem fringilla ut morbi. Ullamcorper eget nulla facilisi etiam dignissim. Vulputate mi sit amet mauris commodo quis imperdiet massa tincidunt. Nunc consequat interdum varius sit. Nunc mi ipsum faucibus vitae aliquet nec ullamcorper. Nunc sed augue lacus viverra. Lobortis scelerisque fermentum dui faucibus in ornare quam. Urna neque viverra justo nec ultrices. Varius vel pharetra vel turpis nunc eget lorem dolor sed. - -Feugiat nisl pretium fusce id velit ut tortor pretium. Lorem dolor sed viverra ipsum nunc aliquet bibendum. Ultrices vitae auctor eu augue ut lectus. Pharetra massa massa ultricies mi quis. Nibh cras pulvinar mattis nunc sed blandit libero. Ac felis donec et odio pellentesque diam volutpat. Lectus proin nibh nisl condimentum id venenatis. Quis vel eros donec ac odio. Commodo sed egestas egestas fringilla phasellus faucibus scelerisque eleifend donec. Adipiscing diam donec adipiscing tristique. - -Tempus imperdiet nulla malesuada pellentesque elit eget gravida cum sociis. Libero nunc consequat interdum varius sit. Et pharetra pharetra massa massa. Feugiat pretium nibh ipsum consequat. Amet commodo nulla facilisi nullam vehicula. Ornare arcu dui vivamus arcu felis bibendum ut tristique. At erat pellentesque adipiscing commodo elit at imperdiet dui. Auctor neque vitae tempus quam pellentesque nec nam aliquam sem. Eget velit aliquet sagittis id consectetur. Enim diam vulputate ut pharetra sit amet aliquam id diam. Eget velit aliquet sagittis id consectetur purus ut faucibus pulvinar. Amet porttitor eget dolor morbi. Felis eget velit aliquet sagittis id. Facilisis magna etiam tempor orci eu. Lacus suspendisse faucibus interdum posuere lorem. Pharetra et ultrices neque ornare aenean euismod. Platea dictumst quisque sagittis purus. - -Quis varius quam quisque id diam vel quam elementum. Augue mauris augue neque gravida in fermentum et sollicitudin. Sapien nec sagittis aliquam malesuada bibendum arcu. Urna duis convallis convallis tellus id interdum velit. Tellus in hac habitasse platea dictumst vestibulum. Fames ac turpis egestas maecenas pharetra convallis. Diam volutpat commodo sed egestas egestas fringilla phasellus faucibus. Placerat orci nulla pellentesque dignissim enim sit amet venenatis. Sed adipiscing diam donec adipiscing. Praesent elementum facilisis leo vel fringilla est. Sed enim ut sem viverra aliquet eget sit amet tellus. Proin sagittis nisl rhoncus mattis rhoncus urna neque viverra. Turpis egestas pretium aenean pharetra magna ac placerat vestibulum. Massa id neque aliquam vestibulum morbi blandit cursus risus. Vitae congue eu consequat ac. Egestas erat imperdiet sed euismod nisi porta lorem mollis aliquam. Dolor purus non enim praesent elementum facilisis. Ultrices mi tempus imperdiet nulla malesuada pellentesque elit. In est ante in nibh. - -Facilisis gravida neque convallis a. Urna nunc id cursus metus aliquam eleifend mi. Lacus luctus accumsan tortor posuere ac. Molestie nunc non blandit massa. Iaculis urna id volutpat lacus laoreet non. Cursus vitae congue mauris rhoncus aenean. Nunc vel risus commodo viverra maecenas. A pellentesque sit amet porttitor eget dolor morbi. Leo vel orci porta non pulvinar neque laoreet suspendisse. Sit amet facilisis magna etiam tempor. Consectetur a erat nam at lectus urna duis convallis convallis. Vestibulum morbi blandit cursus risus at ultrices. Dolor purus non enim praesent elementum. Adipiscing elit pellentesque habitant morbi tristique senectus et netus et. Et odio pellentesque diam volutpat commodo sed egestas egestas fringilla. Leo vel fringilla est ullamcorper eget nulla. Dui ut ornare lectus sit amet. Erat pellentesque adipiscing commodo elit at imperdiet dui accumsan sit. - -Tristique senectus et netus et. Pellentesque diam volutpat commodo sed egestas egestas fringilla. Mauris pharetra et ultrices neque ornare aenean. Amet tellus cras adipiscing enim. Convallis aenean et tortor at risus viverra adipiscing at. Proin sagittis nisl rhoncus mattis rhoncus urna neque viverra justo. Dictumst vestibulum rhoncus est pellentesque elit. Fringilla ut morbi tincidunt augue interdum velit euismod in pellentesque. Dictum at tempor commodo ullamcorper a lacus vestibulum. Sed viverra tellus in hac habitasse platea. Sed id semper risus in hendrerit. In hendrerit gravida rutrum quisque non tellus orci ac. Sit amet risus nullam eget. Sit amet est placerat in egestas erat imperdiet sed. In nisl nisi scelerisque eu ultrices. Sit amet mattis vulputate enim nulla aliquet. - -Dignissim suspendisse in est ante in nibh mauris cursus. Vitae proin sagittis nisl rhoncus. Id leo in vitae turpis massa sed elementum. Lobortis elementum nibh tellus molestie nunc non blandit massa enim. Arcu dictum varius duis at consectetur. Suspendisse faucibus interdum posuere lorem ipsum dolor sit amet consectetur. Imperdiet nulla malesuada pellentesque elit eget gravida cum sociis. Sed adipiscing diam donec adipiscing. Purus sit amet volutpat consequat mauris nunc congue nisi vitae. Elementum nisi quis eleifend quam adipiscing vitae proin sagittis nisl. Mattis ullamcorper velit sed ullamcorper morbi tincidunt ornare massa. Sit amet nisl purus in mollis nunc sed. Turpis tincidunt id aliquet risus feugiat in ante. Id diam maecenas ultricies mi eget mauris pharetra et ultrices. - -Aliquam purus sit amet luctus venenatis lectus magna fringilla urna. Id diam vel quam elementum pulvinar. Elementum sagittis vitae et leo duis. Viverra aliquet eget sit amet tellus cras adipiscing enim eu. Et tortor at risus viverra adipiscing at in tellus integer. Purus in massa tempor nec feugiat. Augue neque gravida in fermentum et sollicitudin ac orci. Sodales ut eu sem integer vitae justo eget magna fermentum. Netus et malesuada fames ac. Augue interdum velit euismod in. Sed elementum tempus egestas sed sed risus pretium. Mattis vulputate enim nulla aliquet porttitor lacus luctus. Dui vivamus arcu felis bibendum ut tristique et egestas quis. - -Viverra justo nec ultrices dui sapien. Quisque egestas diam in arcu cursus euismod quis viverra nibh. Nam libero justo laoreet sit amet cursus sit amet. Lacus sed viverra tellus in hac habitasse. Blandit aliquam etiam erat velit scelerisque in. Ut sem nulla pharetra diam sit amet nisl suscipit adipiscing. Diam sollicitudin tempor id eu nisl nunc. Eget duis at tellus at urna condimentum mattis. Urna porttitor rhoncus dolor purus non enim praesent elementum facilisis. Sed turpis tincidunt id aliquet risus feugiat. Est velit egestas dui id ornare arcu odio ut sem. Nibh sit amet commodo nulla facilisi nullam vehicula. Sit amet consectetur adipiscing elit duis tristique sollicitudin. Eu facilisis sed odio morbi. Massa id neque aliquam vestibulum morbi. In eu mi bibendum neque egestas congue quisque egestas. Massa sed elementum tempus egestas sed sed risus. Quam elementum pulvinar etiam non. At augue eget arcu dictum varius duis at consectetur lorem. - -Penatibus et magnis dis parturient montes nascetur ridiculus. Dictumst quisque sagittis purus sit amet volutpat consequat. Bibendum at varius vel pharetra. Sed adipiscing diam donec adipiscing tristique risus nec feugiat in. Phasellus faucibus scelerisque eleifend donec pretium. Vitae tortor condimentum lacinia quis vel eros. Ac tincidunt vitae semper quis lectus nulla at volutpat diam. Eget sit amet tellus cras adipiscing. Morbi tristique senectus et netus. Nullam vehicula ipsum a arcu cursus vitae congue mauris rhoncus. Auctor urna nunc id cursus metus aliquam eleifend. Ultrices vitae auctor eu augue. Eu non diam phasellus vestibulum lorem sed risus ultricies. Fames ac turpis egestas sed tempus. Volutpat blandit aliquam etiam erat. Dictum varius duis at consectetur lorem. Sit amet volutpat consequat mauris nunc congue. Volutpat sed cras ornare arcu dui vivamus arcu felis. - -Scelerisque fermentum dui faucibus in ornare quam viverra. Interdum velit laoreet id donec ultrices tincidunt arcu. Netus et malesuada fames ac. Netus et malesuada fames ac turpis. Suscipit tellus mauris a diam maecenas sed enim ut sem. Id velit ut tortor pretium. Neque aliquam vestibulum morbi blandit cursus risus at. Cum sociis natoque penatibus et magnis dis parturient. Lobortis elementum nibh tellus molestie nunc non blandit. Ipsum dolor sit amet consectetur adipiscing elit duis tristique. Amet nisl purus in mollis. Amet massa vitae tortor condimentum lacinia quis vel eros donec. Proin sagittis nisl rhoncus mattis rhoncus urna neque viverra justo. - -Nullam ac tortor vitae purus faucibus. Dis parturient montes nascetur ridiculus mus mauris. Molestie at elementum eu facilisis sed odio morbi. Scelerisque felis imperdiet proin fermentum leo vel orci porta. Lectus proin nibh nisl condimentum id venenatis a. Eget nullam non nisi est sit amet facilisis. Hendrerit gravida rutrum quisque non tellus orci ac auctor. Ut faucibus pulvinar elementum integer enim. Rhoncus dolor purus non enim praesent elementum facilisis. Enim sed faucibus turpis in eu mi bibendum. Faucibus nisl tincidunt eget nullam. - -Cursus risus at ultrices mi tempus imperdiet nulla malesuada pellentesque. Pretium nibh ipsum consequat nisl vel pretium lectus quam. Semper viverra nam libero justo laoreet sit amet cursus sit. Augue eget arcu dictum varius duis at consectetur lorem donec. Et malesuada fames ac turpis. Erat nam at lectus urna duis convallis convallis. Dictum sit amet justo donec enim. Urna condimentum mattis pellentesque id nibh tortor id. Morbi tempus iaculis urna id. Lectus proin nibh nisl condimentum id venenatis a condimentum. Nibh sit amet commodo nulla facilisi nullam vehicula. Dui faucibus in ornare quam. Gravida arcu ac tortor dignissim convallis aenean. Consectetur adipiscing elit pellentesque habitant morbi tristique. Pulvinar elementum integer enim neque volutpat ac tincidunt vitae. Pharetra pharetra massa massa ultricies mi quis hendrerit. Dictum at tempor commodo ullamcorper a lacus vestibulum sed. Mattis pellentesque id nibh tortor id. Ultricies integer quis auctor elit sed vulputate. Pretium vulputate sapien nec sagittis aliquam malesuada. - -Auctor augue mauris augue neque gravida. Porttitor lacus luctus accumsan tortor posuere ac ut. Urna neque viverra justo nec ultrices dui. Sit amet est placerat in egestas. Urna nec tincidunt praesent semper feugiat nibh sed pulvinar. Tincidunt eget nullam non nisi est sit amet facilisis magna. Elementum tempus egestas sed sed risus pretium quam vulputate dignissim. Fermentum posuere urna nec tincidunt praesent semper feugiat nibh sed. Porttitor eget dolor morbi non arcu risus quis. Non quam lacus suspendisse faucibus interdum. Venenatis cras sed felis eget velit aliquet sagittis id. Arcu ac tortor dignissim convallis aenean et. Morbi tincidunt ornare massa eget egestas purus. Ac feugiat sed lectus vestibulum mattis ullamcorper velit sed ullamcorper. Vestibulum morbi blandit cursus risus at ultrices. Volutpat blandit aliquam etiam erat velit scelerisque. - -Et egestas quis ipsum suspendisse. Amet consectetur adipiscing elit duis. Purus ut faucibus pulvinar elementum integer enim neque. Cursus vitae congue mauris rhoncus aenean vel elit scelerisque mauris. Tincidunt eget nullam non nisi est. Aliquam purus sit amet luctus. Dui ut ornare lectus sit amet est placerat in. Fringilla ut morbi tincidunt augue interdum velit euismod in. Felis eget nunc lobortis mattis aliquam faucibus purus in. Suspendisse interdum consectetur libero id faucibus nisl. - -Scelerisque fermentum dui faucibus in ornare quam. Lectus proin nibh nisl condimentum id venenatis a condimentum vitae. Fames ac turpis egestas integer eget aliquet nibh praesent tristique. Arcu non sodales neque sodales ut etiam sit. Pharetra convallis posuere morbi leo urna. Nec dui nunc mattis enim ut tellus. Nunc sed augue lacus viverra vitae. Consequat id porta nibh venenatis cras sed felis. Dolor sit amet consectetur adipiscing. Tellus rutrum tellus pellentesque eu tincidunt tortor aliquam nulla. - -Metus aliquam eleifend mi in nulla posuere. Blandit massa enim nec dui nunc mattis enim. Aliquet nibh praesent tristique magna. In aliquam sem fringilla ut. Magna fermentum iaculis eu non. Eget aliquet nibh praesent tristique magna sit amet purus. Ultrices gravida dictum fusce ut placerat orci. Fermentum posuere urna nec tincidunt praesent. Enim tortor at auctor urna nunc. Ridiculus mus mauris vitae ultricies leo integer malesuada nunc vel. Sed id semper risus in hendrerit gravida rutrum. Vestibulum lectus mauris ultrices eros in cursus turpis. Et sollicitudin ac orci phasellus egestas tellus rutrum. Pellentesque elit ullamcorper dignissim cras tincidunt lobortis feugiat vivamus at. Metus vulputate eu scelerisque felis imperdiet proin fermentum leo. Porta non pulvinar neque laoreet suspendisse. Suscipit adipiscing bibendum est ultricies integer quis auctor elit sed. Euismod in pellentesque massa placerat duis ultricies lacus sed. Pellentesque adipiscing commodo elit at imperdiet dui accumsan sit amet. - -Pellentesque eu tincidunt tortor aliquam nulla facilisi. Commodo nulla facilisi nullam vehicula ipsum a arcu. Commodo quis imperdiet massa tincidunt nunc pulvinar sapien et. Faucibus purus in massa tempor. Purus semper eget duis at tellus at urna condimentum. Vivamus at augue eget arcu dictum. Lacus vel facilisis volutpat est velit egestas dui id. Malesuada fames ac turpis egestas maecenas pharetra. Nunc faucibus a pellentesque sit amet porttitor eget dolor. Ultricies tristique nulla aliquet enim. Vel risus commodo viverra maecenas accumsan lacus vel facilisis volutpat. Dignissim diam quis enim lobortis scelerisque. Donec ultrices tincidunt arcu non sodales neque sodales ut etiam. - -Vitae proin sagittis nisl rhoncus mattis rhoncus urna neque. Fermentum leo vel orci porta non. At elementum eu facilisis sed. Quis enim lobortis scelerisque fermentum. Fermentum odio eu feugiat pretium nibh ipsum consequat. Habitant morbi tristique senectus et netus et. Enim praesent elementum facilisis leo vel fringilla est ullamcorper. Egestas quis ipsum suspendisse ultrices gravida dictum. Nam libero justo laoreet sit amet cursus sit amet. Viverra tellus in hac habitasse platea dictumst vestibulum. Varius vel pharetra vel turpis nunc eget. Nullam non nisi est sit amet facilisis magna. Ullamcorper eget nulla facilisi etiam dignissim diam. Ante metus dictum at tempor commodo ullamcorper a lacus. - -Etiam non quam lacus suspendisse. Ut venenatis tellus in metus vulputate eu scelerisque felis. Pulvinar sapien et ligula ullamcorper malesuada proin libero. Consequat interdum varius sit amet mattis. Nunc eget lorem dolor sed viverra ipsum nunc aliquet. Potenti nullam ac tortor vitae purus faucibus ornare. Urna et pharetra pharetra massa massa ultricies mi quis hendrerit. Purus in mollis nunc sed id. Pharetra vel turpis nunc eget lorem dolor sed viverra. Et netus et malesuada fames ac turpis. Libero id faucibus nisl tincidunt eget nullam non nisi. Cursus sit amet dictum sit amet. Porttitor lacus luctus accumsan tortor. - -Volutpat diam ut venenatis tellus in metus vulputate eu scelerisque. Sed viverra tellus in hac habitasse. Aliquam sem et tortor consequat id. Pellentesque habitant morbi tristique senectus et netus et. Consectetur purus ut faucibus pulvinar elementum. Aliquam malesuada bibendum arcu vitae elementum curabitur vitae nunc sed. Malesuada bibendum arcu vitae elementum curabitur vitae nunc sed. Sollicitudin tempor id eu nisl nunc mi ipsum. Fringilla phasellus faucibus scelerisque eleifend donec pretium vulputate sapien nec. Quis eleifend quam adipiscing vitae proin sagittis nisl rhoncus. Bibendum neque egestas congue quisque egestas. A iaculis at erat pellentesque adipiscing commodo elit at imperdiet. Pulvinar etiam non quam lacus. Adipiscing commodo elit at imperdiet. Scelerisque eu ultrices vitae auctor. Sed cras ornare arcu dui vivamus arcu felis bibendum ut. Ornare lectus sit amet est. - -Consequat semper viverra nam libero justo laoreet sit. Imperdiet sed euismod nisi porta lorem mollis aliquam ut porttitor. Cras sed felis eget velit aliquet sagittis id consectetur. Dolor morbi non arcu risus quis. Adipiscing tristique risus nec feugiat in fermentum posuere urna. Dolor magna eget est lorem ipsum dolor. Mauris pharetra et ultrices neque ornare aenean euismod. Nulla facilisi etiam dignissim diam quis. Ultrices tincidunt arcu non sodales. Fames ac turpis egestas maecenas pharetra convallis posuere morbi leo. Interdum varius sit amet mattis vulputate. Tincidunt praesent semper feugiat nibh sed pulvinar. Quisque sagittis purus sit amet volutpat. - -Sed vulputate odio ut enim blandit. Vitae auctor eu augue ut lectus arcu bibendum. Consectetur adipiscing elit pellentesque habitant morbi tristique senectus et. Scelerisque eu ultrices vitae auctor eu augue. Etiam dignissim diam quis enim lobortis scelerisque fermentum dui faucibus. Tellus integer feugiat scelerisque varius. Vulputate enim nulla aliquet porttitor lacus luctus accumsan tortor. Amet nisl purus in mollis. Scelerisque viverra mauris in aliquam sem fringilla ut morbi tincidunt. Semper eget duis at tellus at. Erat velit scelerisque in dictum non consectetur a erat nam. Gravida rutrum quisque non tellus orci. Morbi blandit cursus risus at. Mauris sit amet massa vitae. Non odio euismod lacinia at quis risus sed vulputate. Fermentum posuere urna nec tincidunt praesent. Ut eu sem integer vitae justo eget magna fermentum iaculis. Ullamcorper velit sed ullamcorper morbi tincidunt ornare massa. Arcu cursus euismod quis viverra nibh. Arcu dui vivamus arcu felis bibendum. - -Eros in cursus turpis massa tincidunt dui ut. Urna condimentum mattis pellentesque id nibh tortor id aliquet lectus. Nibh venenatis cras sed felis. Ac felis donec et odio pellentesque diam. Ultricies lacus sed turpis tincidunt id aliquet risus. Diam volutpat commodo sed egestas. Dignissim sodales ut eu sem integer vitae. Pellentesque eu tincidunt tortor aliquam nulla facilisi. Et tortor consequat id porta nibh venenatis cras sed felis. Aliquam ultrices sagittis orci a. Id interdum velit laoreet id donec ultrices tincidunt arcu non. Dictum fusce ut placerat orci nulla pellentesque dignissim. At erat pellentesque adipiscing commodo elit. Ut venenatis tellus in metus vulputate eu scelerisque felis imperdiet. Lectus sit amet est placerat in. Montes nascetur ridiculus mus mauris. Libero volutpat sed cras ornare arcu. Mi sit amet mauris commodo quis imperdiet massa. Sed id semper risus in hendrerit. \ No newline at end of file diff --git a/datatransfer/impl/graphsync/graphsync_impl.go b/datatransfer/impl/graphsync/graphsync_impl.go deleted file mode 100644 index 4f1adc1b..00000000 --- a/datatransfer/impl/graphsync/graphsync_impl.go +++ /dev/null @@ -1,365 +0,0 @@ -package graphsyncimpl - -import ( - "bytes" - "context" - "errors" - "fmt" - "reflect" - "sync" - "time" - - "github.com/ipfs/go-cid" - "github.com/ipfs/go-graphsync" - "github.com/ipld/go-ipld-prime" - cidlink "github.com/ipld/go-ipld-prime/linking/cid" - "github.com/libp2p/go-libp2p-core/host" - "github.com/libp2p/go-libp2p-core/peer" - - "github.com/filecoin-project/go-fil-components/datatransfer" - "github.com/filecoin-project/go-fil-components/datatransfer/message" - "github.com/filecoin-project/go-fil-components/datatransfer/network" -) - -const ( - // ExtensionDataTransfer is the identifier for the data transfer extension to graphsync - ExtensionDataTransfer = graphsync.ExtensionName("fil/data-transfer") -) - -//go:generate cbor-gen-for ExtensionDataTransferData - -// ExtensionDataTransferData is the extension data for -// the graphsync extension. -type ExtensionDataTransferData struct { - TransferID uint64 - Initiator peer.ID - IsPull bool -} - -var EmptyExtensionDataTransferData = ExtensionDataTransferData{} - -// This file implements a VERY simple, incomplete version of the data transfer -// module that allows us to make the necessary insertions of data transfer -// functionality into the storage market -// It does not: -// -- support multiple subscribers -// -- do any actual network coordination or use Graphsync - -type validateType struct { - voucherType reflect.Type // nolint: structcheck - validator datatransfer.RequestValidator // nolint: structcheck -} - -type graphsyncImpl struct { - dataTransferNetwork network.DataTransferNetwork - subscribers []datatransfer.Subscriber - validatedTypes map[string]validateType - channelsLk sync.RWMutex - channels map[datatransfer.ChannelID]datatransfer.ChannelState - gs graphsync.GraphExchange - peerID peer.ID - lastTIDLk sync.Mutex - lastTID int64 -} - -// NewGraphSyncDataTransfer initializes a new graphsync based data transfer manager -func NewGraphSyncDataTransfer(host host.Host, gs graphsync.GraphExchange) datatransfer.Manager { - dataTransferNetwork := network.NewFromLibp2pHost(host) - impl := &graphsyncImpl{ - dataTransferNetwork, - nil, - make(map[string]validateType), - sync.RWMutex{}, - make(map[datatransfer.ChannelID]datatransfer.ChannelState), - gs, - host.ID(), - sync.Mutex{}, - 0, - } - if err := gs.RegisterRequestReceivedHook(impl.gsReqRecdHook); err != nil { - log.Error(err) - return nil - } - dtReceiver := &graphsyncReceiver{impl} - dataTransferNetwork.SetDelegate(dtReceiver) - return impl -} - -// gsReqRecdHook is a graphsync.OnRequestReceivedHook hook -// if an incoming request does not match a previous push request, it returns an error. -func (impl *graphsyncImpl) gsReqRecdHook(p peer.ID, request graphsync.RequestData, hookActions graphsync.RequestReceivedHookActions) { - - // if this is a push request the sender is us. - transferData, err := impl.getExtensionData(request) - if err != nil { - hookActions.TerminateWithError(err) - return - } - - raw, _ := request.Extension(ExtensionDataTransfer) - respData := graphsync.ExtensionData{Name: ExtensionDataTransfer, Data: raw} - - // extension not found; probably not our request. - if transferData == EmptyExtensionDataTransferData { - return - } - - sender := impl.peerID - initiator := impl.peerID - if transferData.IsPull { - // if it's a pull request: the initiator is them - initiator = p - } - chid := datatransfer.ChannelID{Initiator: initiator, ID: datatransfer.TransferID(transferData.TransferID)} - - if impl.getChannelByIDAndSender(chid, sender) == datatransfer.EmptyChannelState { - hookActions.TerminateWithError(err) - return - } - - hookActions.ValidateRequest() - hookActions.SendExtensionData(respData) -} - -// gsExtended is a small interface used by getExtensionData -type gsExtended interface { - Extension(name graphsync.ExtensionName) ([]byte, bool) -} - -// getExtensionData unmarshals extension data. -// Returns: -// * EmptyExtensionDataTransferData + nil if the extension is not found -// * EmptyExtensionDataTransferData + error if the extendedData fails to unmarshal -// * unmarshaled ExtensionDataTransferData + nil if all goes well -func (impl *graphsyncImpl) getExtensionData(extendedData gsExtended) (ExtensionDataTransferData, error) { - data, ok := extendedData.Extension(ExtensionDataTransfer) - if !ok { - return EmptyExtensionDataTransferData, nil - } - var extStruct ExtensionDataTransferData - - reader := bytes.NewReader(data) - if err := extStruct.UnmarshalCBOR(reader); err != nil { - return EmptyExtensionDataTransferData, err - } - return extStruct, nil -} - -// RegisterVoucherType registers a validator for the given voucher type -// returns error if: -// * voucher type does not implement voucher -// * there is a voucher type registered with an identical identifier -// * voucherType's Kind is not reflect.Ptr -func (impl *graphsyncImpl) RegisterVoucherType(voucherType reflect.Type, validator datatransfer.RequestValidator) error { - if voucherType.Kind() != reflect.Ptr { - return fmt.Errorf("voucherType must be a reflect.Ptr Kind") - } - v := reflect.New(voucherType.Elem()) - voucher, ok := v.Interface().(datatransfer.Voucher) - if !ok { - return fmt.Errorf("voucher does not implement Voucher interface") - } - - _, isReg := impl.validatedTypes[voucher.Type()] - if isReg { - return fmt.Errorf("voucher type already registered: %s", voucherType.String()) - } - - impl.validatedTypes[voucher.Type()] = validateType{ - voucherType: voucherType, - validator: validator, - } - return nil -} - -// OpenPushDataChannel opens a data transfer that will send data to the recipient peer and -// transfer parts of the piece that match the selector -func (impl *graphsyncImpl) OpenPushDataChannel(ctx context.Context, requestTo peer.ID, voucher datatransfer.Voucher, baseCid cid.Cid, selector ipld.Node) (datatransfer.ChannelID, error) { - tid, err := impl.sendDtRequest(ctx, selector, false, voucher, baseCid, requestTo) - if err != nil { - return datatransfer.ChannelID{}, err - } - - chid, err := impl.createNewChannel(tid, baseCid, selector, voucher, - impl.peerID, impl.peerID, requestTo) // initiator = us, sender = us, receiver = them - if err != nil { - return chid, err - } - return chid, nil -} - -// OpenPullDataChannel opens a data transfer that will request data from the sending peer and -// transfer parts of the piece that match the selector -func (impl *graphsyncImpl) OpenPullDataChannel(ctx context.Context, requestTo peer.ID, voucher datatransfer.Voucher, baseCid cid.Cid, selector ipld.Node) (datatransfer.ChannelID, error) { - - tid, err := impl.sendDtRequest(ctx, selector, true, voucher, baseCid, requestTo) - if err != nil { - return datatransfer.ChannelID{}, err - } - // initiator = us, sender = them, receiver = us - chid, err := impl.createNewChannel(tid, baseCid, selector, voucher, - impl.peerID, requestTo, impl.peerID) - if err != nil { - return chid, err - } - return chid, nil -} - -// createNewChannel creates a new channel id and channel state and saves to channels. -// returns error if the channel exists already. -func (impl *graphsyncImpl) createNewChannel(tid datatransfer.TransferID, baseCid cid.Cid, selector ipld.Node, voucher datatransfer.Voucher, initiator, dataSender, dataReceiver peer.ID) (datatransfer.ChannelID, error) { - chid := datatransfer.ChannelID{Initiator: initiator, ID: tid} - chst := datatransfer.ChannelState{Channel: datatransfer.NewChannel(0, baseCid, selector, voucher, dataSender, dataReceiver, 0)} - impl.channelsLk.Lock() - defer impl.channelsLk.Unlock() - _, ok := impl.channels[chid] - if ok { - return chid, errors.New("tried to create channel but it already exists") - } - impl.channels[chid] = chst - return chid, nil -} - -// sendDtRequest encapsulates message creation and posting to the data transfer network with the provided parameters -func (impl *graphsyncImpl) sendDtRequest(ctx context.Context, selector ipld.Node, isPull bool, voucher datatransfer.Voucher, baseCid cid.Cid, to peer.ID) (datatransfer.TransferID, error) { - sbytes, err := nodeAsBytes(selector) - if err != nil { - return 0, err - } - vbytes, err := voucher.ToBytes() - if err != nil { - return 0, err - } - tid := impl.generateTransferID() - req := message.NewRequest(tid, isPull, voucher.Type(), vbytes, baseCid, sbytes) - - if err := impl.dataTransferNetwork.SendMessage(ctx, to, req); err != nil { - return 0, err - } - return tid, nil -} - -func (impl *graphsyncImpl) sendResponse(ctx context.Context, isAccepted bool, to peer.ID, tid datatransfer.TransferID) { - resp := message.NewResponse(tid, isAccepted) - if err := impl.dataTransferNetwork.SendMessage(ctx, to, resp); err != nil { - log.Error(err) - } -} - -// close an open channel (effectively a cancel) -func (impl *graphsyncImpl) CloseDataTransferChannel(x datatransfer.ChannelID) {} - -// get status of a transfer -func (impl *graphsyncImpl) TransferChannelStatus(x datatransfer.ChannelID) datatransfer.Status { - return datatransfer.ChannelNotFoundError -} - -// Subscribers returns a copy of the list of subscribers. -func (impl *graphsyncImpl) Subscribers() []datatransfer.Subscriber { - subscribersCopy := make([]datatransfer.Subscriber, len(impl.subscribers)) - copy(subscribersCopy, impl.subscribers) - return subscribersCopy -} - -// get notified when certain types of events happen -func (impl *graphsyncImpl) SubscribeToEvents(subscriber datatransfer.Subscriber) datatransfer.Unsubscribe { - impl.subscribers = append(impl.subscribers, subscriber) - return impl.unsubscribeAt(subscriber) -} - -// unsubscribeAt returns a function that removes an item from impl.subscribers by comparing -// their reflect.ValueOf before pulling the item out of the slice. Does not preserve order. -// Subsequent, repeated calls to the func with the same Subscriber are a no-op. -func (impl *graphsyncImpl) unsubscribeAt(sub datatransfer.Subscriber) datatransfer.Unsubscribe { - return func() { - curLen := len(impl.subscribers) - for i, el := range impl.subscribers { - if reflect.ValueOf(sub) == reflect.ValueOf(el) { - impl.subscribers[i] = impl.subscribers[curLen-1] - impl.subscribers = impl.subscribers[:curLen-1] - return - } - } - } -} - -func (impl *graphsyncImpl) notifySubscribers(evt datatransfer.Event, cs datatransfer.ChannelState) { - for _, cb := range impl.Subscribers() { - cb(evt, cs) - } -} - -// get all in progress transfers -func (impl *graphsyncImpl) InProgressChannels() map[datatransfer.ChannelID]datatransfer.ChannelState { - impl.channelsLk.RLock() - defer impl.channelsLk.RUnlock() - channelsCopy := make(map[datatransfer.ChannelID]datatransfer.ChannelState, len(impl.channels)) - for channelID, channelState := range impl.channels { - channelsCopy[channelID] = channelState - } - return channelsCopy -} - -// getChannelByIDAndSender searches for a channel in the slice of channels with id `chid`. -// Returns datatransfer.EmptyChannelState if there is no channel with that id -func (impl *graphsyncImpl) getChannelByIDAndSender(chid datatransfer.ChannelID, sender peer.ID) datatransfer.ChannelState { - impl.channelsLk.RLock() - channelState, ok := impl.channels[chid] - impl.channelsLk.RUnlock() - if !ok || channelState.Sender() != sender { - return datatransfer.EmptyChannelState - } - return channelState -} - -// generateTransferID() generates a unique-to-runtime TransferID for use in creating -// ChannelIDs -func (impl *graphsyncImpl) generateTransferID() datatransfer.TransferID { - impl.lastTIDLk.Lock() - impl.lastTID++ - impl.lastTIDLk.Unlock() - return datatransfer.TransferID(impl.lastTID) -} - -// sendGsRequest assembles a graphsync request and determines if the transfer was completed/successful. -// notifies subscribers of final request status. -func (impl *graphsyncImpl) sendGsRequest(ctx context.Context, initiator peer.ID, transferID datatransfer.TransferID, isPull bool, dataSender peer.ID, root cidlink.Link, stor ipld.Node) { - extDtData := ExtensionDataTransferData{ - TransferID: uint64(transferID), - Initiator: initiator, - IsPull: isPull, - } - var buf bytes.Buffer - if err := extDtData.MarshalCBOR(&buf); err != nil { - log.Error(err) - } - extData := buf.Bytes() - _, errChan := impl.gs.Request(ctx, dataSender, root, stor, - graphsync.ExtensionData{ - Name: ExtensionDataTransfer, - Data: extData, - }) - go func() { - var lastError error - for err := range errChan { - lastError = err - } - evt := datatransfer.Event{ - Code: datatransfer.Error, - Timestamp: time.Now(), - } - chid := datatransfer.ChannelID{Initiator: initiator, ID: transferID} - chst := impl.getChannelByIDAndSender(chid, dataSender) - if chst == datatransfer.EmptyChannelState { - msg := "cannot find a matching channel for this request" - evt.Message = msg - } else { - if lastError == nil { - evt.Code = datatransfer.Complete - } else { - evt.Message = lastError.Error() - } - } - impl.notifySubscribers(evt, chst) - }() -} diff --git a/datatransfer/impl/graphsync/graphsync_impl_cbor_gen.go b/datatransfer/impl/graphsync/graphsync_impl_cbor_gen.go deleted file mode 100644 index f61acf15..00000000 --- a/datatransfer/impl/graphsync/graphsync_impl_cbor_gen.go +++ /dev/null @@ -1,98 +0,0 @@ -package graphsyncimpl - -import ( - "fmt" - "io" - - "github.com/libp2p/go-libp2p-core/peer" - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" -) - -// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. - -var _ = xerrors.Errorf - -func (t *ExtensionDataTransferData) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{131}); err != nil { - return err - } - - // t.TransferID (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.TransferID))); err != nil { - return err - } - - // t.Initiator (peer.ID) (string) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.Initiator)))); err != nil { - return err - } - if _, err := w.Write([]byte(t.Initiator)); err != nil { - return err - } - - // t.IsPull (bool) (bool) - if err := cbg.WriteBool(w, t.IsPull); err != nil { - return err - } - return nil -} - -func (t *ExtensionDataTransferData) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 3 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.TransferID (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.TransferID = uint64(extra) - // t.Initiator (peer.ID) (string) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - t.Initiator = peer.ID(sval) - } - // t.IsPull (bool) (bool) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajOther { - return fmt.Errorf("booleans must be major type 7") - } - switch extra { - case 20: - t.IsPull = false - case 21: - t.IsPull = true - default: - return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) - } - return nil -} diff --git a/datatransfer/impl/graphsync/graphsync_impl_test.go b/datatransfer/impl/graphsync/graphsync_impl_test.go deleted file mode 100644 index 2f84ae5a..00000000 --- a/datatransfer/impl/graphsync/graphsync_impl_test.go +++ /dev/null @@ -1,1283 +0,0 @@ -package graphsyncimpl_test - -import ( - "bytes" - "context" - "errors" - "io" - "io/ioutil" - "math/rand" - "os" - "path/filepath" - "reflect" - "testing" - "time" - - "github.com/ipfs/go-cid" - "github.com/ipfs/go-graphsync" - gsimpl "github.com/ipfs/go-graphsync/impl" - gsmsg "github.com/ipfs/go-graphsync/message" - gsnet "github.com/ipfs/go-graphsync/network" - chunker "github.com/ipfs/go-ipfs-chunker" - files "github.com/ipfs/go-ipfs-files" - ipldformat "github.com/ipfs/go-ipld-format" - unixfile "github.com/ipfs/go-unixfs/file" - "github.com/ipfs/go-unixfs/importer/balanced" - ihelper "github.com/ipfs/go-unixfs/importer/helpers" - "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/encoding/dagcbor" - ipldfree "github.com/ipld/go-ipld-prime/impl/free" - cidlink "github.com/ipld/go-ipld-prime/linking/cid" - "github.com/ipld/go-ipld-prime/traversal/selector" - "github.com/ipld/go-ipld-prime/traversal/selector/builder" - "github.com/libp2p/go-libp2p-core/peer" - mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/filecoin-project/go-fil-components/datatransfer" - . "github.com/filecoin-project/go-fil-components/datatransfer/impl/graphsync" - "github.com/filecoin-project/go-fil-components/datatransfer/message" - "github.com/filecoin-project/go-fil-components/datatransfer/network" - "github.com/filecoin-project/go-fil-components/datatransfer/testutil" - "github.com/filecoin-project/go-fil-components/shared_testutil" -) - -type receivedMessage struct { - message message.DataTransferMessage - sender peer.ID -} - -// Receiver is an interface for receiving messages from the GraphSyncNetwork. -type receiver struct { - messageReceived chan receivedMessage -} - -func (r *receiver) ReceiveRequest( - ctx context.Context, - sender peer.ID, - incoming message.DataTransferRequest) { - - select { - case <-ctx.Done(): - case r.messageReceived <- receivedMessage{incoming, sender}: - } -} - -func (r *receiver) ReceiveResponse( - ctx context.Context, - sender peer.ID, - incoming message.DataTransferResponse) { - - select { - case <-ctx.Done(): - case r.messageReceived <- receivedMessage{incoming, sender}: - } -} - -func (r *receiver) ReceiveError(err error) { -} - -type fakeDTType struct { - data string -} - -func (ft *fakeDTType) ToBytes() ([]byte, error) { - return []byte(ft.data), nil -} - -func (ft *fakeDTType) FromBytes(data []byte) error { - ft.data = string(data) - return nil -} - -func (ft *fakeDTType) Type() string { - return "FakeDTType" -} - -func TestDataTransferOneWay(t *testing.T) { - // create network - ctx := context.Background() - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - gsData := newGraphsyncTestingData(ctx, t) - host1 := gsData.Td.Host1 - host2 := gsData.Td.Host2 - // setup receiving peer to just record message coming in - dtnet2 := network.NewFromLibp2pHost(host2) - r := &receiver{ - messageReceived: make(chan receivedMessage), - } - dtnet2.SetDelegate(r) - - gs := gsData.setupGraphsyncHost1() - dt := NewGraphSyncDataTransfer(host1, gs) - - t.Run("OpenPushDataTransfer", func(t *testing.T) { - ssb := builder.NewSelectorSpecBuilder(ipldfree.NodeBuilder()) - - // this is the selector for "get the whole DAG" - // TODO: support storage deals with custom payload selectors - stor := ssb.ExploreRecursive(selector.RecursionLimitNone(), - ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node() - - voucher := fakeDTType{"applesauce"} - baseCid := testutil.GenerateCids(1)[0] - channelID, err := dt.OpenPushDataChannel(ctx, host2.ID(), &voucher, baseCid, stor) - require.NoError(t, err) - require.NotNil(t, channelID) - require.Equal(t, channelID.Initiator, host1.ID()) - require.NoError(t, err) - - var messageReceived receivedMessage - select { - case <-ctx.Done(): - t.Fatal("did not receive message sent") - case messageReceived = <-r.messageReceived: - } - - sender := messageReceived.sender - require.Equal(t, sender, host1.ID()) - - received := messageReceived.message - require.True(t, received.IsRequest()) - receivedRequest, ok := received.(message.DataTransferRequest) - require.True(t, ok) - - require.Equal(t, receivedRequest.TransferID(), channelID.ID) - require.Equal(t, receivedRequest.BaseCid(), baseCid) - require.False(t, receivedRequest.IsCancel()) - require.False(t, receivedRequest.IsPull()) - reader := bytes.NewReader(receivedRequest.Selector()) - receivedSelector, err := dagcbor.Decoder(ipldfree.NodeBuilder(), reader) - require.NoError(t, err) - require.Equal(t, receivedSelector, stor) - receivedVoucher := new(fakeDTType) - err = receivedVoucher.FromBytes(receivedRequest.Voucher()) - require.NoError(t, err) - require.Equal(t, *receivedVoucher, voucher) - require.Equal(t, receivedRequest.VoucherType(), voucher.Type()) - }) - - t.Run("OpenPullDataTransfer", func(t *testing.T) { - ssb := builder.NewSelectorSpecBuilder(ipldfree.NodeBuilder()) - - stor := ssb.ExploreRecursive(selector.RecursionLimitNone(), - ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node() - - voucher := fakeDTType{"applesauce"} - baseCid := testutil.GenerateCids(1)[0] - channelID, err := dt.OpenPullDataChannel(ctx, host2.ID(), &voucher, baseCid, stor) - require.NoError(t, err) - require.NotNil(t, channelID) - require.Equal(t, channelID.Initiator, host1.ID()) - require.NoError(t, err) - - var messageReceived receivedMessage - select { - case <-ctx.Done(): - t.Fatal("did not receive message sent") - case messageReceived = <-r.messageReceived: - } - - sender := messageReceived.sender - require.Equal(t, sender, host1.ID()) - - received := messageReceived.message - require.True(t, received.IsRequest()) - receivedRequest, ok := received.(message.DataTransferRequest) - require.True(t, ok) - - require.Equal(t, receivedRequest.TransferID(), channelID.ID) - require.Equal(t, receivedRequest.BaseCid(), baseCid) - require.False(t, receivedRequest.IsCancel()) - require.True(t, receivedRequest.IsPull()) - reader := bytes.NewReader(receivedRequest.Selector()) - receivedSelector, err := dagcbor.Decoder(ipldfree.NodeBuilder(), reader) - require.NoError(t, err) - require.Equal(t, receivedSelector, stor) - receivedVoucher := new(fakeDTType) - err = receivedVoucher.FromBytes(receivedRequest.Voucher()) - require.NoError(t, err) - require.Equal(t, *receivedVoucher, voucher) - require.Equal(t, receivedRequest.VoucherType(), voucher.Type()) - }) -} - -type receivedValidation struct { - isPull bool - other peer.ID - voucher datatransfer.Voucher - baseCid cid.Cid - selector ipld.Node -} - -type fakeValidator struct { - ctx context.Context - validationsReceived chan receivedValidation -} - -func (fv *fakeValidator) ValidatePush( - sender peer.ID, - voucher datatransfer.Voucher, - baseCid cid.Cid, - selector ipld.Node) error { - - select { - case <-fv.ctx.Done(): - case fv.validationsReceived <- receivedValidation{false, sender, voucher, baseCid, selector}: - } - return nil -} - -func (fv *fakeValidator) ValidatePull( - receiver peer.ID, - voucher datatransfer.Voucher, - baseCid cid.Cid, - selector ipld.Node) error { - - select { - case <-fv.ctx.Done(): - case fv.validationsReceived <- receivedValidation{true, receiver, voucher, baseCid, selector}: - } - return nil -} - -func TestDataTransferValidation(t *testing.T) { - // create network - ctx := context.Background() - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - gsData := newGraphsyncTestingData(ctx, t) - host1 := gsData.Td.Host1 - host2 := gsData.Td.Host2 - dtnet1 := network.NewFromLibp2pHost(host1) - r := &receiver{ - messageReceived: make(chan receivedMessage), - } - dtnet1.SetDelegate(r) - - gs2 := &fakeGraphSync{ - requests: make(chan receivedGraphSyncRequest, 1), - } - - fv := &fakeValidator{ctx, make(chan receivedValidation)} - - id := datatransfer.TransferID(rand.Int31()) - var buffer bytes.Buffer - require.NoError(t, dagcbor.Encoder(gsData.Td.AllSelector, &buffer)) - - t.Run("ValidatePush", func(t *testing.T) { - dt2 := NewGraphSyncDataTransfer(host2, gs2) - err := dt2.RegisterVoucherType(reflect.TypeOf(&fakeDTType{}), fv) - require.NoError(t, err) - // create push request - voucher, baseCid, request := createDTRequest(t, false, id, buffer.Bytes()) - - require.NoError(t, dtnet1.SendMessage(ctx, host2.ID(), request)) - - var validation receivedValidation - select { - case <-ctx.Done(): - t.Fatal("did not receive message sent") - case validation = <-fv.validationsReceived: - assert.False(t, validation.isPull) - } - - select { - case <-ctx.Done(): - t.Fatal("did not receive message sent") - case <-r.messageReceived: - } - - assert.False(t, validation.isPull) - assert.Equal(t, host1.ID(), validation.other) - assert.Equal(t, &voucher, validation.voucher) - assert.Equal(t, baseCid, validation.baseCid) - assert.Equal(t, gsData.Td.AllSelector, validation.selector) - }) - - t.Run("ValidatePull", func(t *testing.T) { - // create pull request - voucher, baseCid, request := createDTRequest(t, true, id, buffer.Bytes()) - require.NoError(t, dtnet1.SendMessage(ctx, host2.ID(), request)) - - var validation receivedValidation - select { - case <-ctx.Done(): - t.Fatal("did not receive message sent") - case validation = <-fv.validationsReceived: - } - select { - case <-ctx.Done(): - t.Fatal("did not receive message sent") - case <-r.messageReceived: - } - - assert.True(t, validation.isPull) - assert.Equal(t, validation.other, host1.ID()) - assert.Equal(t, &voucher, validation.voucher) - assert.Equal(t, baseCid, validation.baseCid) - assert.Equal(t, gsData.Td.AllSelector, validation.selector) - }) -} - -func createDTRequest(t *testing.T, isPull bool, id datatransfer.TransferID, selectorBytes []byte) (fakeDTType, cid.Cid, message.DataTransferRequest) { - voucher := fakeDTType{"applesauce"} - baseCid := testutil.GenerateCids(1)[0] - voucherBytes, err := voucher.ToBytes() - require.NoError(t, err) - request := message.NewRequest(id, isPull, voucher.Type(), voucherBytes, baseCid, selectorBytes) - return voucher, baseCid, request -} - -type stubbedValidator struct { - didPush bool - didPull bool - expectPush bool - expectPull bool - pushError error - pullError error -} - -func newSV() *stubbedValidator { - return &stubbedValidator{false, false, false, false, nil, nil} -} - -func (sv *stubbedValidator) ValidatePush( - sender peer.ID, - voucher datatransfer.Voucher, - baseCid cid.Cid, - selector ipld.Node) error { - sv.didPush = true - return sv.pushError -} - -func (sv *stubbedValidator) ValidatePull( - receiver peer.ID, - voucher datatransfer.Voucher, - baseCid cid.Cid, - selector ipld.Node) error { - sv.didPull = true - return sv.pullError -} - -func (sv *stubbedValidator) stubErrorPush() { - sv.pushError = errors.New("something went wrong") -} - -func (sv *stubbedValidator) stubSuccessPush() { - sv.pullError = nil -} - -func (sv *stubbedValidator) expectSuccessPush() { - sv.expectPush = true - sv.stubSuccessPush() -} - -func (sv *stubbedValidator) expectErrorPush() { - sv.expectPush = true - sv.stubErrorPush() -} - -func (sv *stubbedValidator) stubErrorPull() { - sv.pullError = errors.New("something went wrong") -} - -func (sv *stubbedValidator) stubSuccessPull() { - sv.pullError = nil -} - -func (sv *stubbedValidator) expectSuccessPull() { - sv.expectPull = true - sv.stubSuccessPull() -} - -func (sv *stubbedValidator) expectErrorPull() { - sv.expectPull = true - sv.stubErrorPull() -} - -func (sv *stubbedValidator) verifyExpectations(t *testing.T) { - if sv.expectPush { - require.True(t, sv.didPush) - } - if sv.expectPull { - require.True(t, sv.didPull) - } -} - -func TestGraphsyncImpl_RegisterVoucherType(t *testing.T) { - ctx := context.Background() - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - mn := mocknet.New(ctx) - // setup network - host1, err := mn.GenPeer() - require.NoError(t, err) - - gs1 := &fakeGraphSync{ - requests: make(chan receivedGraphSyncRequest, 1), - } - dt := NewGraphSyncDataTransfer(host1, gs1) - fv := &fakeValidator{ctx, make(chan receivedValidation)} - - // a voucher type can be registered - assert.NoError(t, dt.RegisterVoucherType(reflect.TypeOf(&fakeDTType{}), fv)) - - // it cannot be re-registered - assert.EqualError(t, dt.RegisterVoucherType(reflect.TypeOf(&fakeDTType{}), fv), "voucher type already registered: *graphsyncimpl_test.fakeDTType") - - // it must be registered as a pointer - assert.EqualError(t, dt.RegisterVoucherType(reflect.TypeOf(fakeDTType{}), fv), - "voucherType must be a reflect.Ptr Kind") -} - -func TestDataTransferSubscribing(t *testing.T) { - // create network - ctx := context.Background() - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - gsData := newGraphsyncTestingData(ctx, t) - host1 := gsData.Td.Host1 - host2 := gsData.Td.Host2 - - gs1 := &fakeGraphSync{ - requests: make(chan receivedGraphSyncRequest, 1), - } - gs2 := &fakeGraphSync{ - requests: make(chan receivedGraphSyncRequest, 1), - } - sv := newSV() - sv.stubErrorPull() - sv.stubErrorPush() - dt2 := NewGraphSyncDataTransfer(host2, gs2) - require.NoError(t, dt2.RegisterVoucherType(reflect.TypeOf(&fakeDTType{}), sv)) - voucher := fakeDTType{"applesauce"} - baseCid := testutil.GenerateCids(1)[0] - - dt1 := NewGraphSyncDataTransfer(host1, gs1) - - subscribe1Calls := make(chan struct{}, 1) - subscribe1 := func(event datatransfer.Event, channelState datatransfer.ChannelState) { - if event.Code == datatransfer.Error { - subscribe1Calls <- struct{}{} - } - } - subscribe2Calls := make(chan struct{}, 1) - subscribe2 := func(event datatransfer.Event, channelState datatransfer.ChannelState) { - if event.Code == datatransfer.Error { - subscribe2Calls <- struct{}{} - } - } - unsub1 := dt1.SubscribeToEvents(subscribe1) - unsub2 := dt1.SubscribeToEvents(subscribe2) - _, err := dt1.OpenPushDataChannel(ctx, host2.ID(), &voucher, baseCid, gsData.Td.AllSelector) - require.NoError(t, err) - select { - case <-ctx.Done(): - t.Fatal("subscribed events not received") - case <-subscribe1Calls: - } - select { - case <-ctx.Done(): - t.Fatal("subscribed events not received") - case <-subscribe2Calls: - } - unsub1() - unsub2() - - subscribe3Calls := make(chan struct{}, 1) - subscribe3 := func(event datatransfer.Event, channelState datatransfer.ChannelState) { - if event.Code == datatransfer.Error { - subscribe3Calls <- struct{}{} - } - } - subscribe4Calls := make(chan struct{}, 1) - subscribe4 := func(event datatransfer.Event, channelState datatransfer.ChannelState) { - if event.Code == datatransfer.Error { - subscribe4Calls <- struct{}{} - } - } - unsub3 := dt1.SubscribeToEvents(subscribe3) - unsub4 := dt1.SubscribeToEvents(subscribe4) - _, err = dt1.OpenPullDataChannel(ctx, host2.ID(), &voucher, baseCid, gsData.Td.AllSelector) - require.NoError(t, err) - select { - case <-ctx.Done(): - t.Fatal("subscribed events not received") - case <-subscribe1Calls: - t.Fatal("received channel that should have been unsubscribed") - case <-subscribe2Calls: - t.Fatal("received channel that should have been unsubscribed") - case <-subscribe3Calls: - } - select { - case <-ctx.Done(): - t.Fatal("subscribed events not received") - case <-subscribe1Calls: - t.Fatal("received channel that should have been unsubscribed") - case <-subscribe2Calls: - t.Fatal("received channel that should have been unsubscribed") - case <-subscribe4Calls: - } - unsub3() - unsub4() -} - -func TestDataTransferInitiatingPushGraphsyncRequests(t *testing.T) { - // create network - ctx := context.Background() - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - gsData := newGraphsyncTestingData(ctx, t) - host1 := gsData.Td.Host1 - host2 := gsData.Td.Host2 - - gs2 := &fakeGraphSync{ - requests: make(chan receivedGraphSyncRequest, 1), - } - - // setup receiving peer to just record message coming in - dtnet1 := network.NewFromLibp2pHost(host1) - r := &receiver{ - messageReceived: make(chan receivedMessage), - } - dtnet1.SetDelegate(r) - - id := datatransfer.TransferID(rand.Int31()) - var buffer bytes.Buffer - - err := dagcbor.Encoder(gsData.Td.AllSelector, &buffer) - require.NoError(t, err) - - _, baseCid, request := createDTRequest(t, false, id, buffer.Bytes()) - - t.Run("with successful validation", func(t *testing.T) { - sv := newSV() - sv.expectSuccessPush() - - dt2 := NewGraphSyncDataTransfer(host2, gs2) - require.NoError(t, dt2.RegisterVoucherType(reflect.TypeOf(&fakeDTType{}), sv)) - - require.NoError(t, dtnet1.SendMessage(ctx, host2.ID(), request)) - select { - case <-ctx.Done(): - t.Fatal("did not receive message sent") - case <-r.messageReceived: - } - sv.verifyExpectations(t) - - var requestReceived receivedGraphSyncRequest - select { - case <-ctx.Done(): - t.Fatal("did not receive message sent") - case requestReceived = <-gs2.requests: - } - - sv.verifyExpectations(t) - - receiver := requestReceived.p - require.Equal(t, receiver, host1.ID()) - - cl, ok := requestReceived.root.(cidlink.Link) - require.True(t, ok) - require.Equal(t, baseCid, cl.Cid) - - require.Equal(t, gsData.Td.AllSelector, requestReceived.selector) - - }) - - t.Run("with error validation", func(t *testing.T) { - sv := newSV() - sv.expectErrorPush() - - dt2 := NewGraphSyncDataTransfer(host2, gs2) - require.NoError(t, dt2.RegisterVoucherType(reflect.TypeOf(&fakeDTType{}), sv)) - - require.NoError(t, dtnet1.SendMessage(ctx, host2.ID(), request)) - select { - case <-ctx.Done(): - t.Fatal("did not receive message sent") - case <-r.messageReceived: - } - sv.verifyExpectations(t) - - // no graphsync request should be scheduled - require.Empty(t, gs2.requests) - - }) - -} - -func TestDataTransferInitiatingPullGraphsyncRequests(t *testing.T) { - ctx := context.Background() - gsData := newGraphsyncTestingData(ctx, t) - host1 := gsData.Td.Host1 // initiates the pull request - host2 := gsData.Td.Host2 // sends the data - - voucher := fakeDTType{"applesauce"} - baseCid := testutil.GenerateCids(1)[0] - - t.Run("with successful validation", func(t *testing.T) { - gs1Init := &fakeGraphSync{ - requests: make(chan receivedGraphSyncRequest, 1), - } - gs2Sender := &fakeGraphSync{ - requests: make(chan receivedGraphSyncRequest, 1), - } - - sv := newSV() - sv.expectSuccessPull() - - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - - dtInit := NewGraphSyncDataTransfer(host1, gs1Init) - dtSender := NewGraphSyncDataTransfer(host2, gs2Sender) - err := dtSender.RegisterVoucherType(reflect.TypeOf(&fakeDTType{}), sv) - require.NoError(t, err) - - _, err = dtInit.OpenPullDataChannel(ctx, host2.ID(), &voucher, baseCid, gsData.Td.AllSelector) - require.NoError(t, err) - - var requestReceived receivedGraphSyncRequest - select { - case <-ctx.Done(): - t.Fatal("did not receive message sent") - case requestReceived = <-gs1Init.requests: - } - sv.verifyExpectations(t) - - receiver := requestReceived.p - require.Equal(t, receiver, host2.ID()) - - cl, ok := requestReceived.root.(cidlink.Link) - require.True(t, ok) - require.Equal(t, baseCid.String(), cl.Cid.String()) - - require.Equal(t, gsData.Td.AllSelector, requestReceived.selector) - }) - - t.Run("with error validation", func(t *testing.T) { - gs1 := &fakeGraphSync{ - requests: make(chan receivedGraphSyncRequest, 1), - } - gs2 := &fakeGraphSync{ - requests: make(chan receivedGraphSyncRequest, 1), - } - - dt1 := NewGraphSyncDataTransfer(host1, gs1) - sv := newSV() - sv.expectErrorPull() - - dt2 := NewGraphSyncDataTransfer(host2, gs2) - err := dt2.RegisterVoucherType(reflect.TypeOf(&fakeDTType{}), sv) - require.NoError(t, err) - - subscribeCalls := make(chan struct{}, 1) - subscribe := func(event datatransfer.Event, channelState datatransfer.ChannelState) { - if event.Code == datatransfer.Error { - subscribeCalls <- struct{}{} - } - } - unsub := dt1.SubscribeToEvents(subscribe) - _, err = dt1.OpenPullDataChannel(ctx, host2.ID(), &voucher, baseCid, gsData.Td.AllSelector) - require.NoError(t, err) - - select { - case <-ctx.Done(): - t.Fatal("subscribed events not received") - case <-subscribeCalls: - } - - sv.verifyExpectations(t) - - // no graphsync request should be scheduled - require.Empty(t, gs1.requests) - unsub() - }) - - t.Run("does not schedule graphsync request if is push request", func(t *testing.T) { - gs1 := &fakeGraphSync{ - requests: make(chan receivedGraphSyncRequest, 1), - } - gs2 := &fakeGraphSync{ - requests: make(chan receivedGraphSyncRequest, 1), - } - - sv := newSV() - sv.expectSuccessPush() - - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - - dt1 := NewGraphSyncDataTransfer(host1, gs1) - dt2 := NewGraphSyncDataTransfer(host2, gs2) - err := dt2.RegisterVoucherType(reflect.TypeOf(&fakeDTType{}), sv) - require.NoError(t, err) - - subscribeCalls := make(chan struct{}, 1) - subscribe := func(event datatransfer.Event, channelState datatransfer.ChannelState) { - if event.Code == datatransfer.Error { - subscribeCalls <- struct{}{} - } - } - unsub := dt1.SubscribeToEvents(subscribe) - _, err = dt1.OpenPushDataChannel(ctx, host2.ID(), &voucher, baseCid, gsData.Td.AllSelector) - require.NoError(t, err) - - select { - case <-ctx.Done(): - t.Fatal("subscribed events not received") - case <-subscribeCalls: - } - sv.verifyExpectations(t) - - // no graphsync request should be scheduled - require.Empty(t, gs1.requests) - unsub() - }) -} - -type receivedGraphSyncMessage struct { - message gsmsg.GraphSyncMessage - p peer.ID -} - -type fakeGraphSyncReceiver struct { - receivedMessages chan receivedGraphSyncMessage -} - -func (fgsr *fakeGraphSyncReceiver) ReceiveMessage(ctx context.Context, sender peer.ID, incoming gsmsg.GraphSyncMessage) { - select { - case <-ctx.Done(): - case fgsr.receivedMessages <- receivedGraphSyncMessage{incoming, sender}: - } -} - -func (fgsr *fakeGraphSyncReceiver) ReceiveError(_ error) { -} -func (fgsr *fakeGraphSyncReceiver) Connected(p peer.ID) { -} -func (fgsr *fakeGraphSyncReceiver) Disconnected(p peer.ID) { -} - -func (fgsr *fakeGraphSyncReceiver) consumeResponses(ctx context.Context, t *testing.T) graphsync.ResponseStatusCode { - var gsMessageReceived receivedGraphSyncMessage - for { - select { - case <-ctx.Done(): - t.Fail() - case gsMessageReceived = <-fgsr.receivedMessages: - responses := gsMessageReceived.message.Responses() - if (len(responses) > 0) && gsmsg.IsTerminalResponseCode(responses[0].Status()) { - return responses[0].Status() - } - } - } -} - -func TestRespondingToPushGraphsyncRequests(t *testing.T) { - // create network - ctx := context.Background() - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - gsData := newGraphsyncTestingData(ctx, t) - host1 := gsData.Td.Host1 // initiator and data sender - host2 := gsData.Td.Host2 // data recipient, makes graphsync request for data - voucher := fakeDTType{"applesauce"} - link := gsData.loadUnixFSFile(t, false) - - // setup receiving peer to just record message coming in - dtnet2 := network.NewFromLibp2pHost(host2) - r := &receiver{ - messageReceived: make(chan receivedMessage), - } - dtnet2.SetDelegate(r) - - gsr := &fakeGraphSyncReceiver{ - receivedMessages: make(chan receivedGraphSyncMessage), - } - gsData.GsNet2.SetDelegate(gsr) - - gs1 := gsData.setupGraphsyncHost1() - dt1 := NewGraphSyncDataTransfer(host1, gs1) - - t.Run("when request is initiated", func(t *testing.T) { - _, err := dt1.OpenPushDataChannel(ctx, host2.ID(), &voucher, link.(cidlink.Link).Cid, gsData.Td.AllSelector) - require.NoError(t, err) - - var messageReceived receivedMessage - select { - case <-ctx.Done(): - t.Fatal("did not receive message sent") - case messageReceived = <-r.messageReceived: - } - requestReceived := messageReceived.message.(message.DataTransferRequest) - - var buf bytes.Buffer - extStruct := &ExtensionDataTransferData{TransferID: uint64(requestReceived.TransferID())} - err = extStruct.MarshalCBOR(&buf) - require.NoError(t, err) - extData := buf.Bytes() - - var selBuf bytes.Buffer - err = dagcbor.Encoder(gsData.Td.AllSelector, &selBuf) - require.NoError(t, err) - selectorBytes := selBuf.Bytes() - - request := gsmsg.NewRequest(graphsync.RequestID(rand.Int31()), link.(cidlink.Link).Cid, selectorBytes, graphsync.Priority(rand.Int31()), graphsync.ExtensionData{ - Name: ExtensionDataTransfer, - Data: extData, - }) - gsmessage := gsmsg.New() - gsmessage.AddRequest(request) - require.NoError(t, gsData.GsNet2.SendMessage(ctx, host1.ID(), gsmessage)) - - status := gsr.consumeResponses(ctx, t) - require.False(t, gsmsg.IsTerminalFailureCode(status)) - }) - - t.Run("when no request is initiated", func(t *testing.T) { - var buf bytes.Buffer - extStruct := &ExtensionDataTransferData{TransferID: rand.Uint64()} - err := extStruct.MarshalCBOR(&buf) - require.NoError(t, err) - extData := buf.Bytes() - - var selBuf bytes.Buffer - err = dagcbor.Encoder(gsData.Td.AllSelector, &selBuf) - require.NoError(t, err) - selectorBytes := selBuf.Bytes() - request := gsmsg.NewRequest(graphsync.RequestID(rand.Int31()), link.(cidlink.Link).Cid, selectorBytes, graphsync.Priority(rand.Int31()), graphsync.ExtensionData{ - Name: ExtensionDataTransfer, - Data: extData, - }) - gsmessage := gsmsg.New() - gsmessage.AddRequest(request) - require.NoError(t, gsData.GsNet2.SendMessage(ctx, host1.ID(), gsmessage)) - - status := gsr.consumeResponses(ctx, t) - require.True(t, gsmsg.IsTerminalFailureCode(status)) - }) -} - -func TestResponseHookWhenExtensionNotFound(t *testing.T) { - // create network - ctx := context.Background() - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - gsData := newGraphsyncTestingData(ctx, t) - host1 := gsData.Td.Host1 // initiator and data sender - host2 := gsData.Td.Host2 // data recipient, makes graphsync request for data - voucher := fakeDTType{"applesauce"} - link := gsData.loadUnixFSFile(t, false) - - // setup receiving peer to just record message coming in - dtnet2 := network.NewFromLibp2pHost(host2) - r := &receiver{ - messageReceived: make(chan receivedMessage), - } - dtnet2.SetDelegate(r) - - gsr := &fakeGraphSyncReceiver{ - receivedMessages: make(chan receivedGraphSyncMessage), - } - gsData.GsNet2.SetDelegate(gsr) - - gs1 := gsData.setupGraphsyncHost1() - dt1 := NewGraphSyncDataTransfer(host1, gs1) - - t.Run("when it's not our extension, does not error and does not validate", func(t *testing.T) { - //register a hook that validates the request so we don't fail in gs because the request - //never gets processed - validateHook := func(p peer.ID, req graphsync.RequestData, ha graphsync.RequestReceivedHookActions) { - ha.ValidateRequest() - } - require.NoError(t, gs1.RegisterRequestReceivedHook(validateHook)) - - _, err := dt1.OpenPushDataChannel(ctx, host2.ID(), &voucher, link.(cidlink.Link).Cid, gsData.Td.AllSelector) - require.NoError(t, err) - - select { - case <-ctx.Done(): - t.Fatal("did not receive message sent") - case <-r.messageReceived: - } - - var selBuf bytes.Buffer - err = dagcbor.Encoder(gsData.Td.AllSelector, &selBuf) - require.NoError(t, err) - selectorBytes := selBuf.Bytes() - - request := gsmsg.NewRequest(graphsync.RequestID(rand.Int31()), link.(cidlink.Link).Cid, selectorBytes, graphsync.Priority(rand.Int31())) - gsmessage := gsmsg.New() - gsmessage.AddRequest(request) - require.NoError(t, gsData.GsNet2.SendMessage(ctx, host1.ID(), gsmessage)) - - status := gsr.consumeResponses(ctx, t) - assert.False(t, gsmsg.IsTerminalFailureCode(status)) - }) -} - -func TestRespondingToPullGraphsyncRequests(t *testing.T) { - //create network - ctx := context.Background() - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - - gsData := newGraphsyncTestingData(ctx, t) - host1 := gsData.Td.Host1 // initiator, and recipient, makes graphync request - host2 := gsData.Td.Host2 // data sender - - // setup receiving peer to just record message coming in - dtnet1 := network.NewFromLibp2pHost(host1) - r := &receiver{ - messageReceived: make(chan receivedMessage), - } - dtnet1.SetDelegate(r) - - gsr := &fakeGraphSyncReceiver{ - receivedMessages: make(chan receivedGraphSyncMessage), - } - gsData.GsNet1.SetDelegate(gsr) - - gs2 := gsData.setupGraphsyncHost2() - - link := gsData.loadUnixFSFile(t, true) - - id := datatransfer.TransferID(rand.Int31()) - var buf bytes.Buffer - err := dagcbor.Encoder(gsData.Td.AllSelector, &buf) - require.NoError(t, err) - selectorBytes := buf.Bytes() - - t.Run("When a pull request is initiated and validated", func(t *testing.T) { - sv := newSV() - sv.expectSuccessPull() - - dt1 := NewGraphSyncDataTransfer(host2, gs2) - require.NoError(t, dt1.RegisterVoucherType(reflect.TypeOf(&fakeDTType{}), sv)) - - _, _, request := createDTRequest(t, true, id, selectorBytes) - require.NoError(t, dtnet1.SendMessage(ctx, host2.ID(), request)) - var messageReceived receivedMessage - select { - case <-ctx.Done(): - t.Fatal("did not receive message sent") - case messageReceived = <-r.messageReceived: - } - sv.verifyExpectations(t) - receivedResponse, ok := messageReceived.message.(message.DataTransferResponse) - require.True(t, ok) - require.True(t, receivedResponse.Accepted()) - extStruct := &ExtensionDataTransferData{ - TransferID: uint64(receivedResponse.TransferID()), - Initiator: host1.ID(), - IsPull: true, - } - - var buf2 = bytes.Buffer{} - err = extStruct.MarshalCBOR(&buf2) - require.NoError(t, err) - extData := buf2.Bytes() - - gsRequest := gsmsg.NewRequest(graphsync.RequestID(rand.Int31()), link.(cidlink.Link).Cid, selectorBytes, graphsync.Priority(rand.Int31()), graphsync.ExtensionData{ - Name: ExtensionDataTransfer, - Data: extData, - }) - - // initiator requests data over graphsync network - gsmessage := gsmsg.New() - gsmessage.AddRequest(gsRequest) - require.NoError(t, gsData.GsNet1.SendMessage(ctx, host2.ID(), gsmessage)) - status := gsr.consumeResponses(ctx, t) - require.False(t, gsmsg.IsTerminalFailureCode(status)) - }) - - t.Run("When request is not initiated, graphsync response is error", func(t *testing.T) { - _ = NewGraphSyncDataTransfer(host2, gs2) - extStruct := &ExtensionDataTransferData{TransferID: rand.Uint64()} - - var buf2 bytes.Buffer - err = extStruct.MarshalCBOR(&buf2) - require.NoError(t, err) - extData := buf2.Bytes() - request := gsmsg.NewRequest(graphsync.RequestID(rand.Int31()), link.(cidlink.Link).Cid, selectorBytes, graphsync.Priority(rand.Int31()), graphsync.ExtensionData{ - Name: ExtensionDataTransfer, - Data: extData, - }) - gsmessage := gsmsg.New() - gsmessage.AddRequest(request) - - // non-initiator requests data over graphsync network, but should not get it - // because there was no previous request - require.NoError(t, gsData.GsNet1.SendMessage(ctx, host2.ID(), gsmessage)) - status := gsr.consumeResponses(ctx, t) - require.True(t, gsmsg.IsTerminalFailureCode(status)) - }) -} - -func TestDataTransferPushRoundTrip(t *testing.T) { - ctx := context.Background() - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - - gsData := newGraphsyncTestingData(ctx, t) - host1 := gsData.Td.Host1 // initiator, data sender - host2 := gsData.Td.Host2 // data recipient - - root := gsData.loadUnixFSFile(t, false) - rootCid := root.(cidlink.Link).Cid - gs1 := gsData.setupGraphsyncHost1() - gs2 := gsData.setupGraphsyncHost2() - - dt1 := NewGraphSyncDataTransfer(host1, gs1) - dt2 := NewGraphSyncDataTransfer(host2, gs2) - - finished := make(chan struct{}, 1) - var subscriber datatransfer.Subscriber = func(event datatransfer.Event, channelState datatransfer.ChannelState) { - if event.Code == datatransfer.Complete { - finished <- struct{}{} - } - } - unsub := dt2.SubscribeToEvents(subscriber) - voucher := fakeDTType{"applesauce"} - sv := newSV() - sv.expectSuccessPull() - require.NoError(t, dt2.RegisterVoucherType(reflect.TypeOf(&fakeDTType{}), sv)) - - chid, err := dt1.OpenPushDataChannel(ctx, host2.ID(), &voucher, rootCid, gsData.Td.AllSelector) - require.NoError(t, err) - select { - case <-ctx.Done(): - t.Fatal("Did not complete succcessful data transfer") - case <-finished: - gsData.verifyFileTransferred(t, root, true) - } - assert.Equal(t, chid.Initiator, host1.ID()) - unsub() -} - -func TestDataTransferPullRoundTrip(t *testing.T) { - ctx := context.Background() - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - - gsData := newGraphsyncTestingData(ctx, t) - host1 := gsData.Td.Host1 - host2 := gsData.Td.Host2 - - root := gsData.loadUnixFSFile(t, false) - rootCid := root.(cidlink.Link).Cid - gs1 := gsData.setupGraphsyncHost1() - gs2 := gsData.setupGraphsyncHost2() - - dt1 := NewGraphSyncDataTransfer(host1, gs1) - dt2 := NewGraphSyncDataTransfer(host2, gs2) - - finished := make(chan struct{}, 1) - var subscriber datatransfer.Subscriber = func(event datatransfer.Event, channelState datatransfer.ChannelState) { - if event.Code == datatransfer.Complete { - finished <- struct{}{} - } - } - dt2.SubscribeToEvents(subscriber) - voucher := fakeDTType{"applesauce"} - sv := newSV() - sv.expectSuccessPull() - require.NoError(t, dt1.RegisterVoucherType(reflect.TypeOf(&fakeDTType{}), sv)) - - _, err := dt2.OpenPullDataChannel(ctx, host1.ID(), &voucher, rootCid, gsData.Td.AllSelector) - require.NoError(t, err) - select { - case <-ctx.Done(): - t.Fatal("Did not complete succcessful data transfer") - case <-finished: - gsData.verifyFileTransferred(t, root, true) - } -} - -const unixfsChunkSize uint64 = 1 << 10 -const unixfsLinksPerLevel = 1024 - -type GraphsyncTestingData struct { - Td *shared_testutil.Libp2pTestData - GsNet1 gsnet.GraphSyncNetwork - GsNet2 gsnet.GraphSyncNetwork -} - -func newGraphsyncTestingData(ctx context.Context, t *testing.T) *GraphsyncTestingData { - - testData := shared_testutil.NewLibp2pTestData(context.Background(), t) - - gsData := &GraphsyncTestingData{Td: testData} - gsData.GsNet1 = gsnet.NewFromLibp2pHost(gsData.Td.Host1) - gsData.GsNet2 = gsnet.NewFromLibp2pHost(gsData.Td.Host2) - - return gsData -} - -func (gsData *GraphsyncTestingData) setupGraphsyncHost1() graphsync.GraphExchange { - // setup graphsync - return gsimpl.New(gsData.Td.Ctx, gsData.GsNet1, gsData.Td.Bridge1, gsData.Td.Loader1, gsData.Td.Storer1) -} - -func (gsData *GraphsyncTestingData) setupGraphsyncHost2() graphsync.GraphExchange { - // setup graphsync - return gsimpl.New(gsData.Td.Ctx, gsData.GsNet2, gsData.Td.Bridge2, gsData.Td.Loader2, gsData.Td.Storer2) -} - -func (gsData *GraphsyncTestingData) loadUnixFSFile(t *testing.T, useSecondNode bool) ipld.Link { - - // read in a fixture file - path, err := filepath.Abs(filepath.Join("fixtures", "lorem.txt")) - require.NoError(t, err) - - f, err := os.Open(path) - require.NoError(t, err) - - var buf bytes.Buffer - tr := io.TeeReader(f, &buf) - file := files.NewReaderFile(tr) - - // import to UnixFS - var dagService ipldformat.DAGService - if useSecondNode { - dagService = gsData.Td.DagService2 - } else { - dagService = gsData.Td.DagService1 - } - bufferedDS := ipldformat.NewBufferedDAG(gsData.Td.Ctx, dagService) - - params := ihelper.DagBuilderParams{ - Maxlinks: unixfsLinksPerLevel, - RawLeaves: true, - CidBuilder: nil, - Dagserv: bufferedDS, - } - - db, err := params.New(chunker.NewSizeSplitter(file, int64(unixfsChunkSize))) - require.NoError(t, err) - - nd, err := balanced.Layout(db) - require.NoError(t, err) - - err = bufferedDS.Commit() - require.NoError(t, err) - - // save the original files bytes - gsData.Td.OrigBytes = buf.Bytes() - - return cidlink.Link{Cid: nd.Cid()} -} - -func (gsData *GraphsyncTestingData) verifyFileTransferred(t *testing.T, link ipld.Link, useSecondNode bool) { - var dagService ipldformat.DAGService - if useSecondNode { - dagService = gsData.Td.DagService2 - } else { - dagService = gsData.Td.DagService1 - } - - c := link.(cidlink.Link).Cid - - // load the root of the UnixFS DAG from the new blockstore - otherNode, err := dagService.Get(gsData.Td.Ctx, c) - require.NoError(t, err) - - // Setup a UnixFS file reader - n, err := unixfile.NewUnixfsFile(gsData.Td.Ctx, dagService, otherNode) - require.NoError(t, err) - - fn, ok := n.(files.File) - require.True(t, ok) - - // Read the bytes for the UnixFS File - finalBytes, err := ioutil.ReadAll(fn) - require.NoError(t, err) - - // verify original bytes match final bytes! - require.EqualValues(t, gsData.Td.OrigBytes, finalBytes) -} - -type receivedGraphSyncRequest struct { - p peer.ID - root ipld.Link - selector ipld.Node - extensions []graphsync.ExtensionData -} - -type fakeGraphSync struct { - requests chan receivedGraphSyncRequest // records calls to fakeGraphSync.Request -} - -// Request initiates a new GraphSync request to the given peer using the given selector spec. -func (fgs *fakeGraphSync) Request(ctx context.Context, p peer.ID, root ipld.Link, selector ipld.Node, extensions ...graphsync.ExtensionData) (<-chan graphsync.ResponseProgress, <-chan error) { - - fgs.requests <- receivedGraphSyncRequest{p, root, selector, extensions} - responses := make(chan graphsync.ResponseProgress) - errors := make(chan error) - close(responses) - close(errors) - return responses, errors -} - -// RegisterResponseReceivedHook adds a hook that runs when a request is received -func (fgs *fakeGraphSync) RegisterRequestReceivedHook(hook graphsync.OnRequestReceivedHook) error { - return nil -} - -// RegisterResponseReceivedHook adds a hook that runs when a response is received -func (fgs *fakeGraphSync) RegisterResponseReceivedHook(graphsync.OnResponseReceivedHook) error { - return nil -} - -type subscriptionProvider interface { - Subscribers() []datatransfer.Subscriber - SubscribeToEvents(subscriber datatransfer.Subscriber) datatransfer.Unsubscribe -} - -func TestGraphsyncImpl_SubscribeToEvents(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - gsData := newGraphsyncTestingData(ctx, t) - host1 := gsData.Td.Host1 - gs1 := &fakeGraphSync{ - requests: make(chan receivedGraphSyncRequest, 1), - } - dt1 := NewGraphSyncDataTransfer(host1, gs1) - - subscribe1Calls := make(chan struct{}, 1) - subscriber := func(event datatransfer.Event, channelState datatransfer.ChannelState) { - if event.Code == datatransfer.Error { - subscribe1Calls <- struct{}{} - } - } - - subscribe2Calls := make(chan struct{}, 1) - subscriber2 := func(event datatransfer.Event, cst datatransfer.ChannelState) { - if event.Code != datatransfer.Error { - subscribe2Calls <- struct{}{} - } - } - - unsubFunc := dt1.SubscribeToEvents(subscriber) - impl := dt1.(subscriptionProvider) - assert.Equal(t, 1, len(impl.Subscribers())) - - unsubFunc2 := impl.SubscribeToEvents(subscriber2) - assert.Equal(t, 2, len(impl.Subscribers())) - - // ensure subsequent calls don't cause errors, and also check that the right item - // is removed, i.e. no false positives. - unsubFunc() - unsubFunc() - assert.Equal(t, 1, len(impl.Subscribers())) - - // ensure it can delete all elems - unsubFunc2() - assert.Equal(t, 0, len(impl.Subscribers())) -} diff --git a/datatransfer/impl/graphsync/graphsync_receiver.go b/datatransfer/impl/graphsync/graphsync_receiver.go deleted file mode 100644 index 41a52665..00000000 --- a/datatransfer/impl/graphsync/graphsync_receiver.go +++ /dev/null @@ -1,143 +0,0 @@ -package graphsyncimpl - -import ( - "context" - "fmt" - "reflect" - "time" - - "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime" - cidlink "github.com/ipld/go-ipld-prime/linking/cid" - "github.com/libp2p/go-libp2p-core/peer" - - "github.com/filecoin-project/go-fil-components/datatransfer" - "github.com/filecoin-project/go-fil-components/datatransfer/message" -) - -type graphsyncReceiver struct { - impl *graphsyncImpl -} - -// ReceiveRequest takes an incoming data transfer request, validates the voucher and -// processes the message. -func (receiver *graphsyncReceiver) ReceiveRequest( - ctx context.Context, - initiator peer.ID, - incoming message.DataTransferRequest) { - - voucher, err := receiver.validateVoucher(initiator, incoming) - if err != nil { - receiver.impl.sendResponse(ctx, false, initiator, incoming.TransferID()) - return - } - stor, _ := nodeFromBytes(incoming.Selector()) - root := cidlink.Link{Cid: incoming.BaseCid()} - - var dataSender, dataReceiver peer.ID - if incoming.IsPull() { - dataSender = receiver.impl.peerID - dataReceiver = initiator - } else { - dataSender = initiator - dataReceiver = receiver.impl.peerID - receiver.impl.sendGsRequest(ctx, initiator, incoming.TransferID(), incoming.IsPull(), dataSender, root, stor) - } - - _, err = receiver.impl.createNewChannel(incoming.TransferID(), incoming.BaseCid(), stor, voucher, initiator, dataSender, dataReceiver) - if err != nil { - log.Error(err) - receiver.impl.sendResponse(ctx, false, initiator, incoming.TransferID()) - return - } - receiver.impl.sendResponse(ctx, true, initiator, incoming.TransferID()) -} - -// validateVoucher converts a voucher in an incoming message to its appropriate -// voucher struct, then runs the validator and returns the results. -// returns error if: -// * voucherFromRequest fails -// * deserialization of selector fails -// * validation fails -func (receiver *graphsyncReceiver) validateVoucher(sender peer.ID, incoming message.DataTransferRequest) (datatransfer.Voucher, error) { - - vtypStr := incoming.VoucherType() - vouch, err := receiver.voucherFromRequest(incoming) - if err != nil { - return vouch, err - } - - var validatorFunc func(peer.ID, datatransfer.Voucher, cid.Cid, ipld.Node) error - if incoming.IsPull() { - validatorFunc = receiver.impl.validatedTypes[vtypStr].validator.ValidatePull - } else { - validatorFunc = receiver.impl.validatedTypes[vtypStr].validator.ValidatePush - } - - stor, err := nodeFromBytes(incoming.Selector()) - if err != nil { - return vouch, err - } - - if err = validatorFunc(sender, vouch, incoming.BaseCid(), stor); err != nil { - return nil, err - } - - return vouch, nil -} - -// voucherFromRequest takes an incoming request and attempts to create a -// voucher struct from it using the registered validated types. It returns -// a deserialized voucher and any error. It returns error if: -// * the voucher type has no validator registered -// * the voucher cannot be instantiated via reflection -// * request voucher bytes cannot be deserialized via .FromBytes() -func (receiver *graphsyncReceiver) voucherFromRequest(incoming message.DataTransferRequest) (datatransfer.Voucher, error) { - vtypStr := incoming.VoucherType() - - validatedType, ok := receiver.impl.validatedTypes[vtypStr] - if !ok { - return nil, fmt.Errorf("unregistered voucher type %s", vtypStr) - } - vStructVal := reflect.New(validatedType.voucherType.Elem()) - voucher, ok := vStructVal.Interface().(datatransfer.Voucher) - if !ok || reflect.ValueOf(voucher).IsNil() { - return nil, fmt.Errorf("problem instantiating type %s, voucher: %v", vtypStr, voucher) - } - if err := voucher.FromBytes(incoming.Voucher()); err != nil { - return voucher, err - } - return voucher, nil -} - -// ReceiveResponse handles responses to our Push or Pull data transfer request. -// It schedules a graphsync transfer only if our Pull Request is accepted. -func (receiver *graphsyncReceiver) ReceiveResponse( - ctx context.Context, - sender peer.ID, - incoming message.DataTransferResponse) { - evt := datatransfer.Event{ - Code: datatransfer.Error, - Message: "", - Timestamp: time.Now(), - } - chst := datatransfer.EmptyChannelState - if incoming.Accepted() { - // if we are handling a response to a pull request then they are sending data and the - // initiator is us. construct a channel id for a pull request that we initiated and see - // if there is one in our saved channel list. otherwise we should not respond. - chid := datatransfer.ChannelID{Initiator: receiver.impl.peerID, ID: incoming.TransferID()} - - // if we are handling a response to a pull request then they are sending data and the - // initiator is us - if chst = receiver.impl.getChannelByIDAndSender(chid, sender); chst != datatransfer.EmptyChannelState { - baseCid := chst.BaseCID() - root := cidlink.Link{Cid: baseCid} - receiver.impl.sendGsRequest(ctx, receiver.impl.peerID, incoming.TransferID(), true, sender, root, chst.Selector()) - evt.Code = datatransfer.Progress - } - } - receiver.impl.notifySubscribers(evt, chst) -} - -func (receiver *graphsyncReceiver) ReceiveError(error) {} diff --git a/datatransfer/impl/graphsync/graphsync_receiver_test.go b/datatransfer/impl/graphsync/graphsync_receiver_test.go deleted file mode 100644 index bb4dc34a..00000000 --- a/datatransfer/impl/graphsync/graphsync_receiver_test.go +++ /dev/null @@ -1,193 +0,0 @@ -package graphsyncimpl_test - -import ( - "bytes" - "context" - "math/rand" - "reflect" - "testing" - "time" - - "github.com/ipld/go-ipld-prime/encoding/dagcbor" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/filecoin-project/go-fil-components/datatransfer" - . "github.com/filecoin-project/go-fil-components/datatransfer/impl/graphsync" - "github.com/filecoin-project/go-fil-components/datatransfer/message" - "github.com/filecoin-project/go-fil-components/datatransfer/network" - "github.com/filecoin-project/go-fil-components/datatransfer/testutil" -) - -func TestSendResponseToIncomingRequest(t *testing.T) { - // create network - ctx := context.Background() - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - - gsData := newGraphsyncTestingData(ctx, t) - host1 := gsData.Td.Host1 - host2 := gsData.Td.Host2 - - // setup receiving peer to just record message coming in - dtnet1 := network.NewFromLibp2pHost(host1) - r := &receiver{ - messageReceived: make(chan receivedMessage), - } - dtnet1.SetDelegate(r) - - gs2 := &fakeGraphSync{ - requests: make(chan receivedGraphSyncRequest, 1), - } - - voucher := fakeDTType{"applesauce"} - baseCid := testutil.GenerateCids(1)[0] - var buffer bytes.Buffer - err := dagcbor.Encoder(gsData.Td.AllSelector, &buffer) - require.NoError(t, err) - - t.Run("Response to push with successful validation", func(t *testing.T) { - id := datatransfer.TransferID(rand.Int31()) - sv := newSV() - sv.expectSuccessPush() - - dt := NewGraphSyncDataTransfer(host2, gs2) - require.NoError(t, dt.RegisterVoucherType(reflect.TypeOf(&fakeDTType{}), sv)) - - isPull := false - voucherBytes, err := voucher.ToBytes() - require.NoError(t, err) - _ = message.NewRequest(id, isPull, voucher.Type(), voucherBytes, baseCid, buffer.Bytes()) - request := message.NewRequest(id, isPull, voucher.Type(), voucherBytes, baseCid, buffer.Bytes()) - require.NoError(t, dtnet1.SendMessage(ctx, host2.ID(), request)) - var messageReceived receivedMessage - select { - case <-ctx.Done(): - t.Fatal("did not receive message sent") - case messageReceived = <-r.messageReceived: - } - - sv.verifyExpectations(t) - - sender := messageReceived.sender - require.Equal(t, sender, host2.ID()) - - received := messageReceived.message - require.False(t, received.IsRequest()) - receivedResponse, ok := received.(message.DataTransferResponse) - require.True(t, ok) - - assert.Equal(t, receivedResponse.TransferID(), id) - require.True(t, receivedResponse.Accepted()) - - }) - - t.Run("Response to push with error validation", func(t *testing.T) { - id := datatransfer.TransferID(rand.Int31()) - sv := newSV() - sv.expectErrorPush() - dt := NewGraphSyncDataTransfer(host2, gs2) - err = dt.RegisterVoucherType(reflect.TypeOf(&fakeDTType{}), sv) - require.NoError(t, err) - - isPull := false - - voucherBytes, err := voucher.ToBytes() - require.NoError(t, err) - request := message.NewRequest(id, isPull, voucher.Type(), voucherBytes, baseCid, buffer.Bytes()) - require.NoError(t, dtnet1.SendMessage(ctx, host2.ID(), request)) - - var messageReceived receivedMessage - select { - case <-ctx.Done(): - t.Fatal("did not receive message sent") - case messageReceived = <-r.messageReceived: - } - - sv.verifyExpectations(t) - - sender := messageReceived.sender - require.Equal(t, sender, host2.ID()) - - received := messageReceived.message - require.False(t, received.IsRequest()) - receivedResponse, ok := received.(message.DataTransferResponse) - require.True(t, ok) - - require.Equal(t, receivedResponse.TransferID(), id) - require.False(t, receivedResponse.Accepted()) - }) - - t.Run("Response to pull with successful validation", func(t *testing.T) { - id := datatransfer.TransferID(rand.Int31()) - sv := newSV() - sv.expectSuccessPull() - - dt := NewGraphSyncDataTransfer(host2, gs2) - err = dt.RegisterVoucherType(reflect.TypeOf(&fakeDTType{}), sv) - require.NoError(t, err) - - isPull := true - - voucherBytes, err := voucher.ToBytes() - require.NoError(t, err) - request := message.NewRequest(id, isPull, voucher.Type(), voucherBytes, baseCid, buffer.Bytes()) - - require.NoError(t, dtnet1.SendMessage(ctx, host2.ID(), request)) - var messageReceived receivedMessage - select { - case <-ctx.Done(): - t.Fatal("did not receive message sent") - case messageReceived = <-r.messageReceived: - } - - sv.verifyExpectations(t) - - sender := messageReceived.sender - require.Equal(t, sender, host2.ID()) - - received := messageReceived.message - require.False(t, received.IsRequest()) - receivedResponse, ok := received.(message.DataTransferResponse) - require.True(t, ok) - - require.Equal(t, receivedResponse.TransferID(), id) - require.True(t, receivedResponse.Accepted()) - }) - - t.Run("Response to push with error validation", func(t *testing.T) { - id := datatransfer.TransferID(rand.Int31()) - sv := newSV() - sv.expectErrorPull() - - dt := NewGraphSyncDataTransfer(host2, gs2) - err = dt.RegisterVoucherType(reflect.TypeOf(&fakeDTType{}), sv) - require.NoError(t, err) - - isPull := true - voucherBytes, err := voucher.ToBytes() - require.NoError(t, err) - request := message.NewRequest(id, isPull, voucher.Type(), voucherBytes, baseCid, buffer.Bytes()) - require.NoError(t, dtnet1.SendMessage(ctx, host2.ID(), request)) - - var messageReceived receivedMessage - select { - case <-ctx.Done(): - t.Fatal("did not receive message sent") - case messageReceived = <-r.messageReceived: - } - - sv.verifyExpectations(t) - - sender := messageReceived.sender - require.Equal(t, sender, host2.ID()) - - received := messageReceived.message - require.False(t, received.IsRequest()) - receivedResponse, ok := received.(message.DataTransferResponse) - require.True(t, ok) - - require.Equal(t, receivedResponse.TransferID(), id) - require.False(t, receivedResponse.Accepted()) - }) -} diff --git a/datatransfer/impl/graphsync/utils.go b/datatransfer/impl/graphsync/utils.go deleted file mode 100644 index 900058f4..00000000 --- a/datatransfer/impl/graphsync/utils.go +++ /dev/null @@ -1,28 +0,0 @@ -package graphsyncimpl - -import ( - "bytes" - - logging "github.com/ipfs/go-log" - "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/encoding/dagcbor" - ipldfree "github.com/ipld/go-ipld-prime/impl/free" -) - -var log = logging.Logger("graphsync-impl") - -// nodeAsBytes serializes an ipld.Node -func nodeAsBytes(node ipld.Node) ([]byte, error) { - var buffer bytes.Buffer - err := dagcbor.Encoder(node, &buffer) - if err != nil { - return nil, err - } - return buffer.Bytes(), nil -} - -// nodeFromBytes deserializes an ipld.Node -func nodeFromBytes(from []byte) (ipld.Node, error) { - reader := bytes.NewReader(from) - return dagcbor.Decoder(ipldfree.NodeBuilder(), reader) -} diff --git a/datatransfer/message/message.go b/datatransfer/message/message.go deleted file mode 100644 index d55c11a2..00000000 --- a/datatransfer/message/message.go +++ /dev/null @@ -1,76 +0,0 @@ -package message - -import ( - "io" - - "github.com/ipfs/go-cid" - cborgen "github.com/whyrusleeping/cbor-gen" - - "github.com/filecoin-project/go-fil-components/datatransfer" -) - -// Reference file: https://github.com/ipfs/go-graphsync/blob/master/message/message.go -// though here we have a simpler message type that serializes/deserializes to two -// different types that share an interface, and we serialize to CBOR and not Protobuf. - -// DataTransferMessage is a message for the data transfer protocol -// (either request or response) that can serialize to a protobuf -type DataTransferMessage interface { - IsRequest() bool - TransferID() datatransfer.TransferID - cborgen.CBORMarshaler - cborgen.CBORUnmarshaler - ToNet(w io.Writer) error -} - -// DataTransferRequest is a response message for the data transfer protocol -type DataTransferRequest interface { - DataTransferMessage - IsPull() bool - VoucherType() string - Voucher() []byte - BaseCid() cid.Cid - Selector() []byte - IsCancel() bool -} - -// DataTransferResponse is a response message for the data transfer protocol -type DataTransferResponse interface { - DataTransferMessage - Accepted() bool -} - -// NewRequest generates a new request for the data transfer protocol -func NewRequest(id datatransfer.TransferID, isPull bool, voucherIdentifier string, voucher []byte, baseCid cid.Cid, selector []byte) DataTransferRequest { - return &transferRequest{ - Pull: isPull, - Vouch: voucher, - Stor: selector, - BCid: baseCid.String(), - VTyp: voucherIdentifier, - XferID: uint64(id), - } -} - -// CancelRequest request generates a request to cancel an in progress request -func CancelRequest(id datatransfer.TransferID) DataTransferRequest { - return &transferRequest{ - Canc: true, - XferID: uint64(id), - } -} - -// NewResponse builds a new Data Transfer response -func NewResponse(id datatransfer.TransferID, accepted bool) DataTransferResponse { - return &transferResponse{Acpt: accepted, XferID: uint64(id)} -} - -// FromNet can read a network stream to deserialize a GraphSyncMessage -func FromNet(r io.Reader) (DataTransferMessage, error) { - tresp := transferMessage{} - err := tresp.UnmarshalCBOR(r) - if tresp.IsRequest() { - return tresp.Request, nil - } - return tresp.Response, err -} diff --git a/datatransfer/message/message_test.go b/datatransfer/message/message_test.go deleted file mode 100644 index 55eae6f1..00000000 --- a/datatransfer/message/message_test.go +++ /dev/null @@ -1,196 +0,0 @@ -package message_test - -import ( - "bytes" - "math/rand" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/filecoin-project/go-fil-components/datatransfer" - . "github.com/filecoin-project/go-fil-components/datatransfer/message" - "github.com/filecoin-project/go-fil-components/datatransfer/testutil" -) - -func TestNewRequest(t *testing.T) { - baseCid := testutil.GenerateCids(1)[0] - selector := testutil.RandomBytes(100) - isPull := true - id := datatransfer.TransferID(rand.Int31()) - vtype := "FakeVoucherType" - voucher := testutil.RandomBytes(100) - - request := NewRequest(id, isPull, vtype, voucher, baseCid, selector) - assert.Equal(t, id, request.TransferID()) - assert.False(t, request.IsCancel()) - assert.True(t, request.IsPull()) - assert.True(t, request.IsRequest()) - assert.Equal(t, baseCid.String(), request.BaseCid().String()) - assert.Equal(t, vtype, request.VoucherType()) - assert.Equal(t, voucher, request.Voucher()) - assert.Equal(t, selector, request.Selector()) - - // Sanity check to make sure we can cast to DataTransferMessage - msg, ok := request.(DataTransferMessage) - require.True(t, ok) - - assert.True(t, msg.IsRequest()) - assert.Equal(t, request.TransferID(), msg.TransferID()) -} -func TestTransferRequest_MarshalCBOR(t *testing.T) { - // sanity check MarshalCBOR does its thing w/o error - req := NewTestTransferRequest() - wbuf := new(bytes.Buffer) - require.NoError(t, req.MarshalCBOR(wbuf)) - assert.Greater(t, wbuf.Len(), 0) -} -func TestTransferRequest_UnmarshalCBOR(t *testing.T) { - req := NewTestTransferRequest() - wbuf := new(bytes.Buffer) - // use ToNet / FromNet - require.NoError(t, req.ToNet(wbuf)) - - desMsg, err := FromNet(wbuf) - require.NoError(t, err) - - // Verify round-trip - assert.Equal(t, req.TransferID(), desMsg.TransferID()) - assert.Equal(t, req.IsRequest(), desMsg.IsRequest()) - - desReq := desMsg.(DataTransferRequest) - assert.Equal(t, req.IsPull(), desReq.IsPull()) - assert.Equal(t, req.IsCancel(), desReq.IsCancel()) - assert.Equal(t, req.BaseCid(), desReq.BaseCid()) - assert.Equal(t, req.VoucherType(), desReq.VoucherType()) - assert.Equal(t, req.Voucher(), desReq.Voucher()) - assert.Equal(t, req.Selector(), desReq.Selector()) -} - -func TestResponses(t *testing.T) { - id := datatransfer.TransferID(rand.Int31()) - response := NewResponse(id, false) // not accepted - assert.Equal(t, response.TransferID(), id) - assert.False(t, response.Accepted()) - assert.False(t, response.IsRequest()) - - // Sanity check to make sure we can cast to DataTransferMessage - msg, ok := response.(DataTransferMessage) - require.True(t, ok) - - assert.False(t, msg.IsRequest()) - assert.Equal(t, response.TransferID(), msg.TransferID()) -} - -func TestTransferResponse_MarshalCBOR(t *testing.T) { - id := datatransfer.TransferID(rand.Int31()) - response := NewResponse(id, true) // accepted - - // sanity check that we can marshal data - wbuf := new(bytes.Buffer) - require.NoError(t, response.ToNet(wbuf)) - assert.Greater(t, wbuf.Len(), 0) -} - -func TestTransferResponse_UnmarshalCBOR(t *testing.T) { - id := datatransfer.TransferID(rand.Int31()) - response := NewResponse(id, true) // accepted - - wbuf := new(bytes.Buffer) - require.NoError(t, response.ToNet(wbuf)) - - // verify round trip - desMsg, err := FromNet(wbuf) - require.NoError(t, err) - assert.False(t, desMsg.IsRequest()) - assert.Equal(t, id, desMsg.TransferID()) - - desResp, ok := desMsg.(DataTransferResponse) - require.True(t, ok) - assert.True(t, desResp.Accepted()) -} - -func TestRequestCancel(t *testing.T) { - id := datatransfer.TransferID(rand.Int31()) - req := CancelRequest(id) - require.Equal(t, req.TransferID(), id) - require.True(t, req.IsRequest()) - require.True(t, req.IsCancel()) - - wbuf := new(bytes.Buffer) - require.NoError(t, req.ToNet(wbuf)) - - deserialized, err := FromNet(wbuf) - require.NoError(t, err) - - deserializedRequest, ok := deserialized.(DataTransferRequest) - require.True(t, ok) - require.Equal(t, deserializedRequest.TransferID(), req.TransferID()) - require.Equal(t, deserializedRequest.IsCancel(), req.IsCancel()) - require.Equal(t, deserializedRequest.IsRequest(), req.IsRequest()) -} - -func TestToNetFromNetEquivalency(t *testing.T) { - baseCid := testutil.GenerateCids(1)[0] - selector := testutil.RandomBytes(100) - isPull := false - id := datatransfer.TransferID(rand.Int31()) - accepted := false - voucherType := "FakeVoucherType" - voucher := testutil.RandomBytes(100) - request := NewRequest(id, isPull, voucherType, voucher, baseCid, selector) - buf := new(bytes.Buffer) - err := request.ToNet(buf) - require.NoError(t, err) - require.Greater(t, buf.Len(), 0) - deserialized, err := FromNet(buf) - require.NoError(t, err) - - deserializedRequest, ok := deserialized.(DataTransferRequest) - require.True(t, ok) - - require.Equal(t, deserializedRequest.TransferID(), request.TransferID()) - require.Equal(t, deserializedRequest.IsCancel(), request.IsCancel()) - require.Equal(t, deserializedRequest.IsPull(), request.IsPull()) - require.Equal(t, deserializedRequest.IsRequest(), request.IsRequest()) - require.Equal(t, deserializedRequest.BaseCid(), request.BaseCid()) - require.Equal(t, deserializedRequest.VoucherType(), request.VoucherType()) - require.Equal(t, deserializedRequest.Voucher(), request.Voucher()) - require.Equal(t, deserializedRequest.Selector(), request.Selector()) - - response := NewResponse(id, accepted) - err = response.ToNet(buf) - require.NoError(t, err) - deserialized, err = FromNet(buf) - require.NoError(t, err) - - deserializedResponse, ok := deserialized.(DataTransferResponse) - require.True(t, ok) - - require.Equal(t, deserializedResponse.TransferID(), response.TransferID()) - require.Equal(t, deserializedResponse.Accepted(), response.Accepted()) - require.Equal(t, deserializedResponse.IsRequest(), response.IsRequest()) - - request = CancelRequest(id) - err = request.ToNet(buf) - require.NoError(t, err) - deserialized, err = FromNet(buf) - require.NoError(t, err) - - deserializedRequest, ok = deserialized.(DataTransferRequest) - require.True(t, ok) - - require.Equal(t, deserializedRequest.TransferID(), request.TransferID()) - require.Equal(t, deserializedRequest.IsCancel(), request.IsCancel()) - require.Equal(t, deserializedRequest.IsRequest(), request.IsRequest()) -} - -func NewTestTransferRequest() DataTransferRequest { - bcid := testutil.GenerateCids(1)[0] - selector := testutil.RandomBytes(100) - isPull := false - id := datatransfer.TransferID(rand.Int31()) - vtype := "FakeVoucherType" - v := testutil.RandomBytes(100) - return NewRequest(id, isPull, vtype, v, bcid, selector) -} diff --git a/datatransfer/message/transfer_message.go b/datatransfer/message/transfer_message.go deleted file mode 100644 index 87c5638f..00000000 --- a/datatransfer/message/transfer_message.go +++ /dev/null @@ -1,36 +0,0 @@ -package message - -import ( - "io" - - "github.com/filecoin-project/go-fil-components/datatransfer" -) - -//go:generate cbor-gen-for transferMessage -type transferMessage struct { - IsRq bool - - Request *transferRequest - Response *transferResponse -} - -// ========= DataTransferMessage interface - -// IsRequest returns true if this message is a data request -func (tm *transferMessage) IsRequest() bool { - return tm.IsRq -} - -// TransferID returns the TransferID of this message -func (tm *transferMessage) TransferID() datatransfer.TransferID { - if tm.IsRequest() { - return tm.Request.TransferID() - } - return tm.Response.TransferID() -} - -// ToNet serializes a transfer message type. It is simply a wrapper for MarshalCBOR, to provide -// symmetry with FromNet -func (tm *transferMessage) ToNet(w io.Writer) error { - return tm.MarshalCBOR(w) -} diff --git a/datatransfer/message/transfer_message_cbor_gen.go b/datatransfer/message/transfer_message_cbor_gen.go deleted file mode 100644 index f53464e8..00000000 --- a/datatransfer/message/transfer_message_cbor_gen.go +++ /dev/null @@ -1,116 +0,0 @@ -package message - -import ( - "fmt" - "io" - - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" -) - -// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. - -var _ = xerrors.Errorf - -func (t *transferMessage) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{131}); err != nil { - return err - } - - // t.IsRq (bool) (bool) - if err := cbg.WriteBool(w, t.IsRq); err != nil { - return err - } - - // t.Request (message.transferRequest) (struct) - if err := t.Request.MarshalCBOR(w); err != nil { - return err - } - - // t.Response (message.transferResponse) (struct) - if err := t.Response.MarshalCBOR(w); err != nil { - return err - } - return nil -} - -func (t *transferMessage) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 3 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.IsRq (bool) (bool) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajOther { - return fmt.Errorf("booleans must be major type 7") - } - switch extra { - case 20: - t.IsRq = false - case 21: - t.IsRq = true - default: - return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) - } - // t.Request (message.transferRequest) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - t.Request = new(transferRequest) - if err := t.Request.UnmarshalCBOR(br); err != nil { - return err - } - } - - } - // t.Response (message.transferResponse) (struct) - - { - - pb, err := br.PeekByte() - if err != nil { - return err - } - if pb == cbg.CborNull[0] { - var nbuf [1]byte - if _, err := br.Read(nbuf[:]); err != nil { - return err - } - } else { - t.Response = new(transferResponse) - if err := t.Response.UnmarshalCBOR(br); err != nil { - return err - } - } - - } - return nil -} diff --git a/datatransfer/message/transfer_request.go b/datatransfer/message/transfer_request.go deleted file mode 100644 index 4a00ea98..00000000 --- a/datatransfer/message/transfer_request.go +++ /dev/null @@ -1,91 +0,0 @@ -package message - -import ( - "io" - - "github.com/filecoin-project/go-fil-components/datatransfer" - "github.com/ipfs/go-cid" -) - -//go:generate cbor-gen-for transferRequest - -// transferRequest is a struct that fulfills the DataTransferRequest interface. -// its members are exported to be used by cbor-gen -type transferRequest struct { - BCid string - Canc bool - PID []byte - Part bool - Pull bool - Stor []byte - Vouch []byte - VTyp string - XferID uint64 -} - -// IsRequest always returns true in this case because this is a transfer request -func (trq *transferRequest) IsRequest() bool { - return true -} - -func (trq *transferRequest) TransferID() datatransfer.TransferID { - return datatransfer.TransferID(trq.XferID) -} - -// ========= DataTransferRequest interface -// IsPull returns true if this is a data pull request -func (trq *transferRequest) IsPull() bool { - return trq.Pull -} - -// VoucherType returns the Voucher ID -func (trq *transferRequest) VoucherType() string { - return trq.VTyp -} - -// Voucher returns the Voucher bytes -func (trq *transferRequest) Voucher() []byte { - return trq.Vouch -} - -// BaseCid returns the Base CID -func (trq *transferRequest) BaseCid() cid.Cid { - res, err := cid.Decode(trq.BCid) - if err != nil { - return cid.Undef - } - return res -} - -// Selector returns the message Selector bytes -func (trq *transferRequest) Selector() []byte { - return trq.Stor -} - -// IsCancel returns true if this is a cancel request -func (trq *transferRequest) IsCancel() bool { - return trq.Canc -} - -// IsPartial returns true if this is a partial request -func (trq *transferRequest) IsPartial() bool { - return trq.Part -} - -// Cancel cancels a transfer request -func (trq *transferRequest) Cancel() error { - // do other stuff ? - trq.Canc = true - return nil -} - -// ToNet serializes a transfer request. It's a wrapper for MarshalCBOR to provide -// symmetry with FromNet -func (trq *transferRequest) ToNet(w io.Writer) error { - msg := transferMessage{ - IsRq: true, - Request: trq, - Response: nil, - } - return msg.MarshalCBOR(w) -} diff --git a/datatransfer/message/transfer_request_cbor_gen.go b/datatransfer/message/transfer_request_cbor_gen.go deleted file mode 100644 index 31742b96..00000000 --- a/datatransfer/message/transfer_request_cbor_gen.go +++ /dev/null @@ -1,234 +0,0 @@ -package message - -import ( - "fmt" - "io" - - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" -) - -// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. - -var _ = xerrors.Errorf - -func (t *transferRequest) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{137}); err != nil { - return err - } - - // t.BCid (string) (string) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.BCid)))); err != nil { - return err - } - if _, err := w.Write([]byte(t.BCid)); err != nil { - return err - } - - // t.Canc (bool) (bool) - if err := cbg.WriteBool(w, t.Canc); err != nil { - return err - } - - // t.PID ([]uint8) (slice) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.PID)))); err != nil { - return err - } - if _, err := w.Write(t.PID); err != nil { - return err - } - - // t.Part (bool) (bool) - if err := cbg.WriteBool(w, t.Part); err != nil { - return err - } - - // t.Pull (bool) (bool) - if err := cbg.WriteBool(w, t.Pull); err != nil { - return err - } - - // t.Stor ([]uint8) (slice) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Stor)))); err != nil { - return err - } - if _, err := w.Write(t.Stor); err != nil { - return err - } - - // t.Vouch ([]uint8) (slice) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajByteString, uint64(len(t.Vouch)))); err != nil { - return err - } - if _, err := w.Write(t.Vouch); err != nil { - return err - } - - // t.VTyp (string) (string) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajTextString, uint64(len(t.VTyp)))); err != nil { - return err - } - if _, err := w.Write([]byte(t.VTyp)); err != nil { - return err - } - - // t.XferID (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.XferID))); err != nil { - return err - } - return nil -} - -func (t *transferRequest) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 9 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.BCid (string) (string) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - t.BCid = string(sval) - } - // t.Canc (bool) (bool) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajOther { - return fmt.Errorf("booleans must be major type 7") - } - switch extra { - case 20: - t.Canc = false - case 21: - t.Canc = true - default: - return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) - } - // t.PID ([]uint8) (slice) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - - if extra > cbg.ByteArrayMaxLen { - return fmt.Errorf("t.PID: byte array too large (%d)", extra) - } - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.PID = make([]byte, extra) - if _, err := io.ReadFull(br, t.PID); err != nil { - return err - } - // t.Part (bool) (bool) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajOther { - return fmt.Errorf("booleans must be major type 7") - } - switch extra { - case 20: - t.Part = false - case 21: - t.Part = true - default: - return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) - } - // t.Pull (bool) (bool) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajOther { - return fmt.Errorf("booleans must be major type 7") - } - switch extra { - case 20: - t.Pull = false - case 21: - t.Pull = true - default: - return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) - } - // t.Stor ([]uint8) (slice) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - - if extra > cbg.ByteArrayMaxLen { - return fmt.Errorf("t.Stor: byte array too large (%d)", extra) - } - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.Stor = make([]byte, extra) - if _, err := io.ReadFull(br, t.Stor); err != nil { - return err - } - // t.Vouch ([]uint8) (slice) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - - if extra > cbg.ByteArrayMaxLen { - return fmt.Errorf("t.Vouch: byte array too large (%d)", extra) - } - if maj != cbg.MajByteString { - return fmt.Errorf("expected byte array") - } - t.Vouch = make([]byte, extra) - if _, err := io.ReadFull(br, t.Vouch); err != nil { - return err - } - // t.VTyp (string) (string) - - { - sval, err := cbg.ReadString(br) - if err != nil { - return err - } - - t.VTyp = string(sval) - } - // t.XferID (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.XferID = uint64(extra) - return nil -} diff --git a/datatransfer/message/transfer_response.go b/datatransfer/message/transfer_response.go deleted file mode 100644 index 9f897102..00000000 --- a/datatransfer/message/transfer_response.go +++ /dev/null @@ -1,40 +0,0 @@ -package message - -import ( - "io" - - "github.com/filecoin-project/go-fil-components/datatransfer" -) - -//go:generate cbor-gen-for transferResponse - -// transferResponse is a private struct that satisfies the DataTransferResponse interface -type transferResponse struct { - Acpt bool - XferID uint64 -} - -func (trsp *transferResponse) TransferID() datatransfer.TransferID { - return datatransfer.TransferID(trsp.XferID) -} - -// IsRequest always returns false in this case because this is a transfer response -func (trsp *transferResponse) IsRequest() bool { - return false -} - -// Accepted returns true if the request is accepted in the response -func (trsp *transferResponse) Accepted() bool { - return trsp.Acpt -} - -// ToNet serializes a transfer response. It's a wrapper for MarshalCBOR to provide -// symmetry with FromNet -func (trsp *transferResponse) ToNet(w io.Writer) error { - msg := transferMessage{ - IsRq: false, - Request: nil, - Response: trsp, - } - return msg.MarshalCBOR(w) -} diff --git a/datatransfer/message/transfer_response_cbor_gen.go b/datatransfer/message/transfer_response_cbor_gen.go deleted file mode 100644 index 04f2db7e..00000000 --- a/datatransfer/message/transfer_response_cbor_gen.go +++ /dev/null @@ -1,79 +0,0 @@ -package message - -import ( - "fmt" - "io" - - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" -) - -// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. - -var _ = xerrors.Errorf - -func (t *transferResponse) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if _, err := w.Write([]byte{130}); err != nil { - return err - } - - // t.Acpt (bool) (bool) - if err := cbg.WriteBool(w, t.Acpt); err != nil { - return err - } - - // t.XferID (uint64) (uint64) - if _, err := w.Write(cbg.CborEncodeMajorType(cbg.MajUnsignedInt, uint64(t.XferID))); err != nil { - return err - } - return nil -} - -func (t *transferResponse) UnmarshalCBOR(r io.Reader) error { - br := cbg.GetPeeker(r) - - maj, extra, err := cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajArray { - return fmt.Errorf("cbor input should be of type array") - } - - if extra != 2 { - return fmt.Errorf("cbor input had wrong number of fields") - } - - // t.Acpt (bool) (bool) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajOther { - return fmt.Errorf("booleans must be major type 7") - } - switch extra { - case 20: - t.Acpt = false - case 21: - t.Acpt = true - default: - return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) - } - // t.XferID (uint64) (uint64) - - maj, extra, err = cbg.CborReadHeader(br) - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint64 field") - } - t.XferID = uint64(extra) - return nil -} diff --git a/datatransfer/network/interface.go b/datatransfer/network/interface.go deleted file mode 100644 index add880e8..00000000 --- a/datatransfer/network/interface.go +++ /dev/null @@ -1,56 +0,0 @@ -package network - -import ( - "context" - - "github.com/libp2p/go-libp2p-core/peer" - "github.com/libp2p/go-libp2p-core/protocol" - - "github.com/filecoin-project/go-fil-components/datatransfer/message" -) - -var ( - // ProtocolDataTransfer is the protocol identifier for graphsync messages - ProtocolDataTransfer protocol.ID = "/fil/datatransfer/1.0.0" -) - -// DataTransferNetwork provides network connectivity for GraphSync. -type DataTransferNetwork interface { - - // SendMessage sends a GraphSync message to a peer. - SendMessage( - context.Context, - peer.ID, - message.DataTransferMessage) error - - // SetDelegate registers the Reciver to handle messages received from the - // network. - SetDelegate(Receiver) - - // ConnectTo establishes a connection to the given peer - ConnectTo(context.Context, peer.ID) error - - NewMessageSender(context.Context, peer.ID) (MessageSender, error) -} - -// MessageSender is an interface to send messages to a peer -type MessageSender interface { - SendMsg(context.Context, message.DataTransferMessage) error - Close() error - Reset() error -} - -// Receiver is an interface for receiving messages from the GraphSyncNetwork. -type Receiver interface { - ReceiveRequest( - ctx context.Context, - sender peer.ID, - incoming message.DataTransferRequest) - - ReceiveResponse( - ctx context.Context, - sender peer.ID, - incoming message.DataTransferResponse) - - ReceiveError(error) -} diff --git a/datatransfer/network/libp2p_impl.go b/datatransfer/network/libp2p_impl.go deleted file mode 100644 index a3641d0d..00000000 --- a/datatransfer/network/libp2p_impl.go +++ /dev/null @@ -1,165 +0,0 @@ -package network - -import ( - "context" - "fmt" - "io" - "time" - - logging "github.com/ipfs/go-log" - "github.com/libp2p/go-libp2p-core/helpers" - "github.com/libp2p/go-libp2p-core/host" - "github.com/libp2p/go-libp2p-core/network" - "github.com/libp2p/go-libp2p-core/peer" - - "github.com/filecoin-project/go-fil-components/datatransfer/message" -) - -var log = logging.Logger("data_transfer_network") - -var sendMessageTimeout = time.Minute * 10 - -// NewFromLibp2pHost returns a GraphSyncNetwork supported by underlying Libp2p host. -func NewFromLibp2pHost(host host.Host) DataTransferNetwork { - dataTransferNetwork := libp2pDataTransferNetwork{ - host: host, - } - - return &dataTransferNetwork -} - -// libp2pDataTransferNetwork transforms the libp2p host interface, which sends and receives -// NetMessage objects, into the graphsync network interface. -type libp2pDataTransferNetwork struct { - host host.Host - // inbound messages from the network are forwarded to the receiver - receiver Receiver -} - -type streamMessageSender struct { - s network.Stream -} - -func (s *streamMessageSender) Close() error { - return helpers.FullClose(s.s) -} - -func (s *streamMessageSender) Reset() error { - return s.s.Reset() -} - -func (s *streamMessageSender) SendMsg(ctx context.Context, msg message.DataTransferMessage) error { - return msgToStream(ctx, s.s, msg) -} - -func msgToStream(ctx context.Context, s network.Stream, msg message.DataTransferMessage) error { - if msg.IsRequest() { - log.Debugf("Outgoing request message for transfer ID: %d", msg.TransferID()) - } - - deadline := time.Now().Add(sendMessageTimeout) - if dl, ok := ctx.Deadline(); ok { - deadline = dl - } - if err := s.SetWriteDeadline(deadline); err != nil { - log.Warnf("error setting deadline: %s", err) - } - - switch s.Protocol() { - case ProtocolDataTransfer: - if err := msg.ToNet(s); err != nil { - log.Debugf("error: %s", err) - return err - } - default: - return fmt.Errorf("unrecognized protocol on remote: %s", s.Protocol()) - } - - if err := s.SetWriteDeadline(time.Time{}); err != nil { - log.Warnf("error resetting deadline: %s", err) - } - return nil -} - -func (dtnet *libp2pDataTransferNetwork) NewMessageSender(ctx context.Context, p peer.ID) (MessageSender, error) { - s, err := dtnet.newStreamToPeer(ctx, p) - if err != nil { - return nil, err - } - - return &streamMessageSender{s: s}, nil -} - -func (dtnet *libp2pDataTransferNetwork) newStreamToPeer(ctx context.Context, p peer.ID) (network.Stream, error) { - return dtnet.host.NewStream(ctx, p, ProtocolDataTransfer) -} - -func (dtnet *libp2pDataTransferNetwork) SendMessage( - ctx context.Context, - p peer.ID, - outgoing message.DataTransferMessage) error { - - s, err := dtnet.newStreamToPeer(ctx, p) - if err != nil { - return err - } - - if err = msgToStream(ctx, s, outgoing); err != nil { - if err2 := s.Reset(); err2 != nil { - log.Error(err) - return err2 - } - return err - } - - // TODO(https://github.com/libp2p/go-libp2p-net/issues/28): Avoid this goroutine. - go helpers.AwaitEOF(s) // nolint: errcheck,gosec - return s.Close() - -} - -func (dtnet *libp2pDataTransferNetwork) SetDelegate(r Receiver) { - dtnet.receiver = r - dtnet.host.SetStreamHandler(ProtocolDataTransfer, dtnet.handleNewStream) -} - -func (dtnet *libp2pDataTransferNetwork) ConnectTo(ctx context.Context, p peer.ID) error { - return dtnet.host.Connect(ctx, peer.AddrInfo{ID: p}) -} - -// handleNewStream receives a new stream from the network. -func (dtnet *libp2pDataTransferNetwork) handleNewStream(s network.Stream) { - defer s.Close() // nolint: errcheck,gosec - - if dtnet.receiver == nil { - s.Reset() // nolint: errcheck,gosec - return - } - - for { - received, err := message.FromNet(s) - if err != nil { - if err != io.EOF { - s.Reset() // nolint: errcheck,gosec - go dtnet.receiver.ReceiveError(err) - log.Debugf("graphsync net handleNewStream from %s error: %s", s.Conn().RemotePeer(), err) - } - return - } - - p := s.Conn().RemotePeer() - ctx := context.Background() - log.Debugf("graphsync net handleNewStream from %s", s.Conn().RemotePeer()) - if received.IsRequest() { - receivedRequest, ok := received.(message.DataTransferRequest) - if ok { - dtnet.receiver.ReceiveRequest(ctx, p, receivedRequest) - } - } else { - receivedResponse, ok := received.(message.DataTransferResponse) - if ok { - dtnet.receiver.ReceiveResponse(ctx, p, receivedResponse) - } - } - } -} diff --git a/datatransfer/network/libp2p_impl_test.go b/datatransfer/network/libp2p_impl_test.go deleted file mode 100644 index bb739b07..00000000 --- a/datatransfer/network/libp2p_impl_test.go +++ /dev/null @@ -1,137 +0,0 @@ -package network_test - -import ( - "context" - "math/rand" - "testing" - "time" - - "github.com/libp2p/go-libp2p-core/peer" - mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/filecoin-project/go-fil-components/datatransfer" - "github.com/filecoin-project/go-fil-components/datatransfer/message" - "github.com/filecoin-project/go-fil-components/datatransfer/network" - "github.com/filecoin-project/go-fil-components/datatransfer/testutil" -) - -// Receiver is an interface for receiving messages from the DataTransferNetwork. -type receiver struct { - messageReceived chan struct{} - lastRequest message.DataTransferRequest - lastResponse message.DataTransferResponse - lastSender peer.ID - connectedPeers chan peer.ID -} - -func (r *receiver) ReceiveRequest( - ctx context.Context, - sender peer.ID, - incoming message.DataTransferRequest) { - r.lastSender = sender - r.lastRequest = incoming - select { - case <-ctx.Done(): - case r.messageReceived <- struct{}{}: - } -} - -func (r *receiver) ReceiveResponse( - ctx context.Context, - sender peer.ID, - incoming message.DataTransferResponse) { - r.lastSender = sender - r.lastResponse = incoming - select { - case <-ctx.Done(): - case r.messageReceived <- struct{}{}: - } -} - -func (r *receiver) ReceiveError(err error) { -} - -func TestMessageSendAndReceive(t *testing.T) { - // create network - ctx := context.Background() - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - mn := mocknet.New(ctx) - - host1, err := mn.GenPeer() - require.NoError(t, err) - host2, err := mn.GenPeer() - require.NoError(t, err) - err = mn.LinkAll() - require.NoError(t, err) - - dtnet1 := network.NewFromLibp2pHost(host1) - dtnet2 := network.NewFromLibp2pHost(host2) - r := &receiver{ - messageReceived: make(chan struct{}), - connectedPeers: make(chan peer.ID, 2), - } - dtnet1.SetDelegate(r) - dtnet2.SetDelegate(r) - - err = dtnet1.ConnectTo(ctx, host2.ID()) - require.NoError(t, err) - - t.Run("Send Request", func(t *testing.T) { - baseCid := testutil.GenerateCids(1)[0] - selector := testutil.RandomBytes(100) - isPull := false - id := datatransfer.TransferID(rand.Int31()) - vType := "FakeVoucherType" - voucher := testutil.RandomBytes(100) - request := message.NewRequest(id, isPull, vType, voucher, baseCid, selector) - require.NoError(t, dtnet1.SendMessage(ctx, host2.ID(), request)) - - select { - case <-ctx.Done(): - t.Fatal("did not receive message sent") - case <-r.messageReceived: - } - - sender := r.lastSender - require.Equal(t, sender, host1.ID()) - - receivedRequest := r.lastRequest - require.NotNil(t, receivedRequest) - - assert.Equal(t, request.TransferID(), receivedRequest.TransferID()) - assert.Equal(t, request.IsCancel(), receivedRequest.IsCancel()) - assert.Equal(t, request.IsPull(), receivedRequest.IsPull()) - assert.Equal(t, request.IsRequest(), receivedRequest.IsRequest()) - assert.True(t, receivedRequest.BaseCid().Equals(request.BaseCid())) - assert.Equal(t, request.VoucherType(), receivedRequest.VoucherType()) - assert.Equal(t, request.Voucher(), receivedRequest.Voucher()) - assert.Equal(t, request.Selector(), receivedRequest.Selector()) - }) - - t.Run("Send Response", func(t *testing.T) { - accepted := false - id := datatransfer.TransferID(rand.Int31()) - response := message.NewResponse(id, accepted) - require.NoError(t, dtnet2.SendMessage(ctx, host1.ID(), response)) - - select { - case <-ctx.Done(): - t.Fatal("did not receive message sent") - case <-r.messageReceived: - } - - sender := r.lastSender - require.NotNil(t, sender) - assert.Equal(t, sender, host2.ID()) - - receivedResponse := r.lastResponse - - assert.Equal(t, response.TransferID(), receivedResponse.TransferID()) - assert.Equal(t, response.Accepted(), receivedResponse.Accepted()) - assert.Equal(t, response.IsRequest(), receivedResponse.IsRequest()) - - }) -} diff --git a/datatransfer/testutil/testutil.go b/datatransfer/testutil/testutil.go deleted file mode 100644 index da2af32d..00000000 --- a/datatransfer/testutil/testutil.go +++ /dev/null @@ -1,83 +0,0 @@ -package testutil - -import ( - "bytes" - - blocks "github.com/ipfs/go-block-format" - "github.com/ipfs/go-cid" - blocksutil "github.com/ipfs/go-ipfs-blocksutil" - "github.com/jbenet/go-random" - "github.com/libp2p/go-libp2p-core/peer" -) - -var blockGenerator = blocksutil.NewBlockGenerator() - -//var prioritySeq int -var seedSeq int64 - -// RandomBytes returns a byte array of the given size with random values. -func RandomBytes(n int64) []byte { - data := new(bytes.Buffer) - random.WritePseudoRandomBytes(n, data, seedSeq) // nolint: gosec,errcheck - seedSeq++ - return data.Bytes() -} - -// GenerateBlocksOfSize generates a series of blocks of the given byte size -func GenerateBlocksOfSize(n int, size int64) []blocks.Block { - generatedBlocks := make([]blocks.Block, 0, n) - for i := 0; i < n; i++ { - b := blocks.NewBlock(RandomBytes(size)) - generatedBlocks = append(generatedBlocks, b) - - } - return generatedBlocks -} - -// GenerateCids produces n content identifiers. -func GenerateCids(n int) []cid.Cid { - cids := make([]cid.Cid, 0, n) - for i := 0; i < n; i++ { - c := blockGenerator.Next().Cid() - cids = append(cids, c) - } - return cids -} - -var peerSeq int - -// GeneratePeers creates n peer ids. -func GeneratePeers(n int) []peer.ID { - peerIds := make([]peer.ID, 0, n) - for i := 0; i < n; i++ { - peerSeq++ - p := peer.ID(peerSeq) - peerIds = append(peerIds, p) - } - return peerIds -} - -// ContainsPeer returns true if a peer is found n a list of peers. -func ContainsPeer(peers []peer.ID, p peer.ID) bool { - for _, n := range peers { - if p == n { - return true - } - } - return false -} - -// IndexOf returns the index of a given cid in an array of blocks -func IndexOf(blks []blocks.Block, c cid.Cid) int { - for i, n := range blks { - if n.Cid() == c { - return i - } - } - return -1 -} - -// ContainsBlock returns true if a block is found n a list of blocks -func ContainsBlock(blks []blocks.Block, block blocks.Block) bool { - return IndexOf(blks, block.Cid()) != -1 -} diff --git a/datatransfer/types.go b/datatransfer/types.go deleted file mode 100644 index 776776b9..00000000 --- a/datatransfer/types.go +++ /dev/null @@ -1,195 +0,0 @@ -package datatransfer - -import ( - "context" - "reflect" - "time" - - "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime" - "github.com/libp2p/go-libp2p-core/peer" -) - -// Voucher is used to validate -// a data transfer request against the underlying storage or retrieval deal -// that precipitated it. The only requirement is a voucher can read and write -// from bytes, and has a string identifier type -type Voucher interface { - // ToBytes converts the Voucher to raw bytes - ToBytes() ([]byte, error) - // FromBytes reads a Voucher from raw bytes - FromBytes([]byte) error - // Type is a unique string identifier for this voucher type - Type() string -} - -// Status is the status of transfer for a given channel -type Status int - -const ( - // Ongoing means the data transfer is in progress - Ongoing Status = iota - - // Completed means the data transfer is completed successfully - Completed - - // Failed means the data transfer failed - Failed - - // ChannelNotFoundError means the searched for data transfer does not exist - ChannelNotFoundError -) - -// TransferID is an identifier for a data transfer, shared between -// request/responder and unique to the requester -type TransferID uint64 - -// ChannelID is a unique identifier for a channel, distinct by both the other -// party's peer ID + the transfer ID -type ChannelID struct { - Initiator peer.ID - ID TransferID -} - -// Channel represents all the parameters for a single data transfer -type Channel struct { - // an identifier for this channel shared by request and responder, set by requester through protocol - transferID TransferID - // base CID for the piece being transferred - baseCid cid.Cid - // portion of Piece to return, specified by an IPLD selector - selector ipld.Node - // used to verify this channel - voucher Voucher - // the party that is sending the data (not who initiated the request) - sender peer.ID - // the party that is receiving the data (not who initiated the request) - recipient peer.ID - // expected amount of data to be transferred - totalSize uint64 -} - -// NewChannel makes a new channel -func NewChannel(transferID TransferID, baseCid cid.Cid, - selector ipld.Node, - voucher Voucher, - sender peer.ID, - recipient peer.ID, - totalSize uint64) Channel { - return Channel{transferID, baseCid, selector, voucher, sender, recipient, totalSize} -} - -// TransferID returns the transfer id for this channel -func (c Channel) TransferID() TransferID { return c.transferID } - -// BaseCID returns the CID that is at the root of this data transfer -func (c Channel) BaseCID() cid.Cid { return c.baseCid } - -// Selector returns the IPLD selector for this data transfer (represented as -// an IPLD node) -func (c Channel) Selector() ipld.Node { return c.selector } - -// Voucher returns the voucher for this data transfer -func (c Channel) Voucher() Voucher { return c.voucher } - -// Sender returns the peer id for the node that is sending data -func (c Channel) Sender() peer.ID { return c.sender } - -// Recipient returns the peer id for the node that is receiving data -func (c Channel) Recipient() peer.ID { return c.recipient } - -// TotalSize returns the total size for the data being transferred -func (c Channel) TotalSize() uint64 { return c.totalSize } - -// ChannelState is immutable channel data plus mutable state -type ChannelState struct { - Channel - // total bytes sent from this node (0 if receiver) - sent uint64 - // total bytes received by this node (0 if sender) - received uint64 -} - -var EmptyChannelState = ChannelState{} - -// Sent returns the number of bytes sent -func (c ChannelState) Sent() uint64 { return c.sent } - -// Received returns the number of bytes received -func (c ChannelState) Received() uint64 { return c.received } - -// EventCode is a name for an event that occurs on a data transfer channel -type EventCode int - -const ( - // Open is an event occurs when a channel is first opened - Open EventCode = iota - - // Progress is an event that gets emitted every time more data is transferred - Progress - - // Error is an event that emits when an error occurs in a data transfer - Error - - // Complete is emitted when a data transfer is complete - Complete -) - -// Event is a struct containing information about a data transfer event -type Event struct { - Code EventCode // What type of event it is - Message string // Any clarifying information about the event - Timestamp time.Time // when the event happened -} - -// Subscriber is a callback that is called when events are emitted -type Subscriber func(event Event, channelState ChannelState) - -// Unsubscribe is a function that gets called to unsubscribe from data transfer events -type Unsubscribe func() - -// RequestValidator is an interface implemented by the client of the -// data transfer module to validate requests -type RequestValidator interface { - // ValidatePush validates a push request received from the peer that will send data - ValidatePush( - sender peer.ID, - voucher Voucher, - baseCid cid.Cid, - selector ipld.Node) error - // ValidatePull validates a pull request received from the peer that will receive data - ValidatePull( - receiver peer.ID, - voucher Voucher, - baseCid cid.Cid, - selector ipld.Node) error -} - -// Manager is the core interface presented by all implementations of -// of the data transfer sub system -type Manager interface { - // RegisterVoucherType registers a validator for the given voucher type - // will error if voucher type does not implement voucher - // or if there is a voucher type registered with an identical identifier - RegisterVoucherType(voucherType reflect.Type, validator RequestValidator) error - - // open a data transfer that will send data to the recipient peer and - // transfer parts of the piece that match the selector - OpenPushDataChannel(ctx context.Context, to peer.ID, voucher Voucher, baseCid cid.Cid, selector ipld.Node) (ChannelID, error) - - // open a data transfer that will request data from the sending peer and - // transfer parts of the piece that match the selector - OpenPullDataChannel(ctx context.Context, to peer.ID, voucher Voucher, baseCid cid.Cid, selector ipld.Node) (ChannelID, error) - - // close an open channel (effectively a cancel) - CloseDataTransferChannel(x ChannelID) - - // get status of a transfer - TransferChannelStatus(x ChannelID) Status - - // get notified when certain types of events happen - SubscribeToEvents(subscriber Subscriber) Unsubscribe - - // get all in progress transfers - InProgressChannels() map[ChannelID]ChannelState -} diff --git a/go.mod b/go.mod index 4706ac58..93941ad0 100644 --- a/go.mod +++ b/go.mod @@ -7,11 +7,9 @@ require ( github.com/filecoin-project/go-address v0.0.0-20191219011437-af739c490b4f github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 + github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce github.com/filecoin-project/go-statestore v0.0.0-20191219195854-7a95521e8f15 - github.com/google/go-cmp v0.3.1 // indirect github.com/hannahhoward/cbor-gen-for v0.0.0-20191216214420-3e450425c40c - github.com/hashicorp/golang-lru v0.5.3 // indirect - github.com/ipfs/go-bitswap v0.1.8 // indirect github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.3-0.20190908200855-f22eea50656c github.com/ipfs/go-car v0.0.3-0.20191203022317-23b0a85fd1b1 @@ -20,7 +18,6 @@ require ( github.com/ipfs/go-graphsync v0.0.4 github.com/ipfs/go-ipfs-blockstore v0.1.0 github.com/ipfs/go-ipfs-blocksutil v0.0.1 - github.com/ipfs/go-ipfs-chunker v0.0.1 github.com/ipfs/go-ipfs-ds-help v0.0.1 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 github.com/ipfs/go-ipfs-files v0.0.4 @@ -30,16 +27,9 @@ require ( github.com/ipfs/go-merkledag v0.2.4 github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785 - github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c github.com/libp2p/go-libp2p v0.3.0 - github.com/libp2p/go-libp2p-blankhost v0.1.4 // indirect github.com/libp2p/go-libp2p-core v0.2.4 - github.com/libp2p/go-libp2p-record v0.1.1 // indirect - github.com/libp2p/go-libp2p-swarm v0.2.2 // indirect github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 - github.com/multiformats/go-multiaddr-dns v0.2.0 // indirect - github.com/onsi/ginkgo v1.9.0 // indirect - github.com/onsi/gomega v1.6.0 // indirect github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a github.com/stretchr/testify v1.4.0 github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0 diff --git a/go.sum b/go.sum index f91f1c2e..0477405a 100644 --- a/go.sum +++ b/go.sum @@ -55,6 +55,8 @@ github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2 h1:a github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03 h1:2pMXdBnCiXjfCYx/hLqFxccPoqsSveQFxVLvNxy9bus= github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= +github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce h1:Jdejrx6XVSTRy2PiX08HCU5y68p3wx2hNMJJc/J7kZY= +github.com/filecoin-project/go-data-transfer v0.0.0-20191219005021-4accf56bd2ce/go.mod h1:b14UWxhxVCAjrQUYvVGrQRRsjAh79wXYejw9RbUcAww= github.com/filecoin-project/go-statestore v0.0.0-20191219195854-7a95521e8f15 h1:6uEnOV0KO1jzART47HpCrnpAiMfDXHv7RoT/qCfcN9Q= github.com/filecoin-project/go-statestore v0.0.0-20191219195854-7a95521e8f15/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -242,7 +244,6 @@ github.com/libp2p/go-conn-security v0.0.1/go.mod h1:bGmu51N0KU9IEjX7kl2PQjgZa40J github.com/libp2p/go-conn-security-multistream v0.1.0 h1:aqGmto+ttL/uJgX0JtQI0tD21CIEy5eYd1Hlp0juHY0= github.com/libp2p/go-conn-security-multistream v0.1.0/go.mod h1:aw6eD7LOsHEX7+2hJkDxw1MteijaVcI+/eP2/x3J1xc= github.com/libp2p/go-eventbus v0.0.2/go.mod h1:Hr/yGlwxA/stuLnpMiu82lpNKpvRy3EaJxPu40XYOwk= -github.com/libp2p/go-eventbus v0.0.3 h1:4sB0NrwnWr6qGeq2RWUp/JG1wNajf6gyILInId72hrw= github.com/libp2p/go-eventbus v0.0.3/go.mod h1:Hr/yGlwxA/stuLnpMiu82lpNKpvRy3EaJxPu40XYOwk= github.com/libp2p/go-eventbus v0.1.0 h1:mlawomSAjjkk97QnYiEmHsLu7E136+2oCWSHRUvMfzQ= github.com/libp2p/go-eventbus v0.1.0/go.mod h1:vROgu5cs5T7cv7POWlWxBaVLxfSegC5UGQf8A2eEmx4= @@ -386,7 +387,6 @@ github.com/multiformats/go-multiaddr v0.1.1 h1:rVAztJYMhCQ7vEFr8FvxW3mS+HF2eY/oP github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= -github.com/multiformats/go-multiaddr-dns v0.0.3 h1:P19q/k9jwmtgh+qXFkKfgFM7rCg/9l5AVqh7VNxSXhs= github.com/multiformats/go-multiaddr-dns v0.0.3/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.2.0 h1:YWJoIDwLePniH7OU5hBnDZV6SWuvJqJ0YtN6pLeH9zA= github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0= diff --git a/storagemarket/impl/client.go b/storagemarket/impl/client.go index d0bba5a0..6073dc9a 100644 --- a/storagemarket/impl/client.go +++ b/storagemarket/impl/client.go @@ -3,7 +3,7 @@ package storageimpl import ( "context" - "github.com/filecoin-project/go-fil-components/datatransfer" + "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-fil-components/shared/tokenamount" "github.com/ipfs/go-cid" diff --git a/storagemarket/impl/client_utils.go b/storagemarket/impl/client_utils.go index cc30969d..a5257aea 100644 --- a/storagemarket/impl/client_utils.go +++ b/storagemarket/impl/client_utils.go @@ -14,7 +14,7 @@ import ( "golang.org/x/xerrors" "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-fil-components/datatransfer" + "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-fil-components/filestore" "github.com/filecoin-project/go-fil-components/pieceio/padreader" "github.com/filecoin-project/go-fil-components/pieceio/sectorcalculator" diff --git a/storagemarket/impl/provider.go b/storagemarket/impl/provider.go index c2c8c424..bb07a156 100644 --- a/storagemarket/impl/provider.go +++ b/storagemarket/impl/provider.go @@ -15,7 +15,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-cbor-util" - "github.com/filecoin-project/go-fil-components/datatransfer" + "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-fil-components/shared/tokenamount" "github.com/filecoin-project/go-fil-components/shared/types" "github.com/filecoin-project/go-fil-components/storagemarket" diff --git a/storagemarket/impl/provider_utils.go b/storagemarket/impl/provider_utils.go index 2b9c983f..10a1caaf 100644 --- a/storagemarket/impl/provider_utils.go +++ b/storagemarket/impl/provider_utils.go @@ -6,7 +6,7 @@ import ( "github.com/ipld/go-ipld-prime" - "github.com/filecoin-project/go-fil-components/datatransfer" + "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-fil-components/storagemarket" "github.com/filecoin-project/go-cbor-util"